Rebuild extraction pipeline infrastructure (Faza 0 prep)

Implements the approved plan to replace the broken regex/index-master
extraction with an LLM-subagent pipeline. Four parallel lanes:

Lane A — scripts/extract_common.py (PDF/docx/doc/pptx/html/zip, no
  max_pages truncation), normalize_sources.py, chunk_sources.py
  (~20pg chunks + overlap, manifest registry), activity_schema.json.
Lane B — app/config_taxonomy.py (16 fixed category slugs), schema
  rebuilt from scratch in app/models/ with content_type, language,
  source_files, source_excerpt, normalized_name, extraction_confidence,
  needs_review; FTS5 + 3 triggers extended with materials_list and
  skills_developed.
Lane C — build_database.py (--rebuild, atomic swap, schema + fuzzy
  source_excerpt validation, dedup with needs_review band),
  validate_extractions.py, review_queue.py, new run_extraction.py
  orchestrator, SUBAGENT_PROMPT.md.
Lane D — search.py content_type/language filters (default search
  excludes non-game content), E7 schema-compat audit; fixed a NULL
  keywords AttributeError in _boost_search_relevance.

Removes 8 orphaned/dead scripts and app/services/parser.py +
indexer.py. Adds tests/ (70 passing, 1 skipped — libreoffice absent).

Note: Lane D made one additive edit to app/models/database.py
(_update_category_counts) to surface content_type/language in
get_filter_options, outside its nominal lane boundary but after
Lane B completed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

app/models/activity.py

@@ -5,6 +5,22 @@ Activity data model for INDEX-SISTEM-JOCURI v2.0
 from dataclasses import dataclass, field
 from typing import List, Optional, Dict, Any
 import json
+import re
+import unicodedata
+
+
+def normalize_name(name: str) -> str:
+    """Diacritic-free, lowercased, whitespace-collapsed form of a name.
+
+    Used as the exact-match key for dedup grouping (see plan §4).
+    """
+    if not name:
+        return ""
+    decomposed = unicodedata.normalize("NFKD", name)
+    ascii_str = "".join(c for c in decomposed if not unicodedata.combining(c))
+    ascii_str = ascii_str.lower().strip()
+    ascii_str = re.sub(r"\s+", " ", ascii_str)
+    return ascii_str
 
 @dataclass
 class Activity:
@@ -19,10 +35,19 @@ class Activity:
     # Categories
     category: str = ""
     subcategory: Optional[str] = None
-    
+    # content_type is an axis INDEPENDENT of category:
+    # one of joc/activitate/reteta/cantec/ceremonie (see config_taxonomy).
+    content_type: Optional[str] = None
+
     # Source information
     source_file: str = ""
     page_reference: Optional[str] = None
+    # source_files: JSON-encoded list of every source the activity was seen in.
+    # `source_file` (singular) stays as the primary/original source; build_database
+    # (Lane C) accumulates the full list here on dedup-merge.
+    source_files: List[str] = field(default_factory=list)
+    # Short verbatim quote from the source — anti-hallucination anchor.
+    source_excerpt: Optional[str] = None
     
     # Age and participants
     age_group_min: Optional[int] = None
@@ -44,11 +69,22 @@ class Activity:
     keywords: Optional[str] = None
     tags: List[str] = field(default_factory=list)
     popularity_score: int = 0
-    
+
+    # Extraction / language metadata
+    language: Optional[str] = None          # 'ro' / 'en'
+    normalized_name: Optional[str] = None   # dedup key; auto-derived from name
+    extraction_confidence: Optional[str] = None  # 'high' / 'med' / 'low'
+    needs_review: int = 0
+
     # Database fields
     id: Optional[int] = None
     created_at: Optional[str] = None
     updated_at: Optional[str] = None
+
+    def __post_init__(self):
+        """Derive normalized_name from name when not explicitly provided."""
+        if not self.normalized_name:
+            self.normalized_name = normalize_name(self.name)
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert activity to dictionary for database storage"""
@@ -59,8 +95,11 @@ class Activity:
             'variations': self.variations,
             'category': self.category,
             'subcategory': self.subcategory,
+            'content_type': self.content_type,
             'source_file': self.source_file,
+            'source_files': json.dumps(self.source_files) if self.source_files else None,
             'page_reference': self.page_reference,
+            'source_excerpt': self.source_excerpt,
             'age_group_min': self.age_group_min,
             'age_group_max': self.age_group_max,
             'participants_min': self.participants_min,
@@ -73,7 +112,11 @@ class Activity:
             'difficulty_level': self.difficulty_level,
             'keywords': self.keywords,
             'tags': json.dumps(self.tags) if self.tags else None,
-            'popularity_score': self.popularity_score
+            'popularity_score': self.popularity_score,
+            'language': self.language,
+            'normalized_name': self.normalized_name or normalize_name(self.name),
+            'extraction_confidence': self.extraction_confidence,
+            'needs_review': self.needs_review,
         }
     
     @classmethod
@@ -86,7 +129,17 @@ class Activity:
                 tags = json.loads(data['tags'])
             except (json.JSONDecodeError, TypeError):
                 tags = []
-        
+
+        # source_files may arrive as a JSON string (DB) or a list (extraction)
+        source_files = data.get('source_files')
+        if isinstance(source_files, str):
+            try:
+                source_files = json.loads(source_files)
+            except (json.JSONDecodeError, TypeError):
+                source_files = []
+        elif source_files is None:
+            source_files = []
+
         return cls(
             id=data.get('id'),
             name=data.get('name', ''),
@@ -95,8 +148,11 @@ class Activity:
             variations=data.get('variations'),
             category=data.get('category', ''),
             subcategory=data.get('subcategory'),
+            content_type=data.get('content_type'),
             source_file=data.get('source_file', ''),
+            source_files=source_files,
             page_reference=data.get('page_reference'),
+            source_excerpt=data.get('source_excerpt'),
             age_group_min=data.get('age_group_min'),
             age_group_max=data.get('age_group_max'),
             participants_min=data.get('participants_min'),
@@ -110,6 +166,10 @@ class Activity:
             keywords=data.get('keywords'),
             tags=tags,
             popularity_score=data.get('popularity_score', 0),
+            language=data.get('language'),
+            normalized_name=data.get('normalized_name'),
+            extraction_confidence=data.get('extraction_confidence'),
+            needs_review=data.get('needs_review', 0) or 0,
             created_at=data.get('created_at'),
             updated_at=data.get('updated_at')
         )