Implements the approved plan to replace the broken regex/index-master extraction with an LLM-subagent pipeline. Four parallel lanes: Lane A — scripts/extract_common.py (PDF/docx/doc/pptx/html/zip, no max_pages truncation), normalize_sources.py, chunk_sources.py (~20pg chunks + overlap, manifest registry), activity_schema.json. Lane B — app/config_taxonomy.py (16 fixed category slugs), schema rebuilt from scratch in app/models/ with content_type, language, source_files, source_excerpt, normalized_name, extraction_confidence, needs_review; FTS5 + 3 triggers extended with materials_list and skills_developed. Lane C — build_database.py (--rebuild, atomic swap, schema + fuzzy source_excerpt validation, dedup with needs_review band), validate_extractions.py, review_queue.py, new run_extraction.py orchestrator, SUBAGENT_PROMPT.md. Lane D — search.py content_type/language filters (default search excludes non-game content), E7 schema-compat audit; fixed a NULL keywords AttributeError in _boost_search_relevance. Removes 8 orphaned/dead scripts and app/services/parser.py + indexer.py. Adds tests/ (70 passing, 1 skipped — libreoffice absent). Note: Lane D made one additive edit to app/models/database.py (_update_category_counts) to surface content_type/language in get_filter_options, outside its nominal lane boundary but after Lane B completed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
141 lines
5.7 KiB
Python
141 lines
5.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
CRITICAL REGRESSION TEST (plan §6, §7).
|
|
|
|
`search.py` changed the result sets of /search and /api/search: the default
|
|
search now EXCLUDES the non-game content types (rețetă / cântec / ceremonie),
|
|
which surface only when the user explicitly filters that content_type or picks
|
|
a non-game category. This test guards that behaviour.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from app.models.activity import Activity
|
|
from app.models.database import DatabaseManager
|
|
from app.services.search import SearchService
|
|
from app.config_taxonomy import NON_GAME_CONTENT_TYPES
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# fixtures
|
|
# --------------------------------------------------------------------------
|
|
def _activity(name, content_type, category="altele", language="ro"):
|
|
return Activity(
|
|
name=name,
|
|
description=f"Descriere pentru {name}, un conținut de tip {content_type}.",
|
|
category=category,
|
|
content_type=content_type,
|
|
language=language,
|
|
source_file="test/fixture.txt",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def search_service(tmp_path):
|
|
"""A SearchService over a temp DB seeded with one row per content_type."""
|
|
db = DatabaseManager(str(tmp_path / "activities.db"))
|
|
db.clear_database()
|
|
db.bulk_insert_activities([
|
|
_activity("Vanatoarea de comori", "joc", category="wide-games"),
|
|
_activity("Cercul de cunoastere", "activitate", category="icebreakers"),
|
|
_activity("Reteta de paine la ceaun", "reteta", category="retete"),
|
|
_activity("Cantecul de tabara", "cantec", category="cantece-ceremonii"),
|
|
_activity("Ceremonia de inchidere", "ceremonie", category="cantece-ceremonii"),
|
|
_activity("Game in English", "joc", category="wide-games", language="en"),
|
|
])
|
|
return SearchService(db)
|
|
|
|
|
|
def _content_types(results):
|
|
return {r.get("content_type") for r in results}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# the regression: default search excludes non-game content types
|
|
# --------------------------------------------------------------------------
|
|
def test_default_search_excludes_non_game_content(search_service):
|
|
"""No filters → rețete / cântece / ceremonii must NOT appear."""
|
|
results = search_service.search_activities()
|
|
types = _content_types(results)
|
|
|
|
assert types, "default search returned nothing"
|
|
for non_game in NON_GAME_CONTENT_TYPES:
|
|
assert non_game not in types, (
|
|
f"default search leaked non-game content_type '{non_game}'"
|
|
)
|
|
# game content is still present
|
|
assert "joc" in types
|
|
assert "activitate" in types
|
|
|
|
|
|
def test_default_search_with_text_excludes_non_game(search_service):
|
|
"""A text query still excludes non-game content by default."""
|
|
results = search_service.search_activities(search_text="conținut")
|
|
assert NON_GAME_CONTENT_TYPES[0] not in _content_types(results)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# explicit content_type filter INCLUDES the non-game rows
|
|
# --------------------------------------------------------------------------
|
|
def test_explicit_content_type_filter_includes_non_game(search_service):
|
|
"""Filtering content_type=reteta returns exactly the rețete."""
|
|
results = search_service.search_activities(filters={"content_type": "reteta"})
|
|
types = _content_types(results)
|
|
|
|
assert types == {"reteta"}, f"expected only rețete, got {types}"
|
|
assert len(results) == 1
|
|
|
|
|
|
def test_explicit_content_type_filter_for_cantec(search_service):
|
|
results = search_service.search_activities(filters={"content_type": "cantec"})
|
|
assert _content_types(results) == {"cantec"}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# a non-game CATEGORY filter also lifts the exclusion
|
|
# --------------------------------------------------------------------------
|
|
def test_non_game_category_filter_includes_non_game(search_service):
|
|
"""Picking category=cantece-ceremonii surfaces cântece + ceremonii."""
|
|
results = search_service.search_activities(
|
|
filters={"category": "cantece-ceremonii"})
|
|
types = _content_types(results)
|
|
|
|
assert "cantec" in types
|
|
assert "ceremonie" in types
|
|
|
|
|
|
def test_game_category_filter_still_excludes_non_game(search_service):
|
|
"""A normal (game) category filter keeps the non-game exclusion."""
|
|
results = search_service.search_activities(filters={"category": "wide-games"})
|
|
types = _content_types(results)
|
|
for non_game in NON_GAME_CONTENT_TYPES:
|
|
assert non_game not in types
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# language filter
|
|
# --------------------------------------------------------------------------
|
|
def test_language_filter_ro(search_service):
|
|
results = search_service.search_activities(filters={"language": "ro"})
|
|
assert results
|
|
assert all(r.get("language") == "ro" for r in results)
|
|
|
|
|
|
def test_language_filter_en(search_service):
|
|
results = search_service.search_activities(filters={"language": "en"})
|
|
assert results
|
|
assert all(r.get("language") == "en" for r in results)
|
|
assert {r.get("name") for r in results} == {"Game in English"}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# get_filter_options surfaces the new axes
|
|
# --------------------------------------------------------------------------
|
|
def test_filter_options_include_content_type_and_language(search_service):
|
|
"""The dynamic-filter mechanism now exposes content_type + language."""
|
|
options = search_service.db.get_filter_options()
|
|
assert "content_type" in options
|
|
assert "language" in options
|
|
assert "joc" in options["content_type"]
|
|
assert set(options["language"]) == {"ro", "en"}
|