romfast/game-library
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild extraction pipeline infrastructure (Faza 0 prep)

Browse Source 
Implements the approved plan to replace the broken regex/index-master
extraction with an LLM-subagent pipeline. Four parallel lanes:

Lane A — scripts/extract_common.py (PDF/docx/doc/pptx/html/zip, no
  max_pages truncation), normalize_sources.py, chunk_sources.py
  (~20pg chunks + overlap, manifest registry), activity_schema.json.
Lane B — app/config_taxonomy.py (16 fixed category slugs), schema
  rebuilt from scratch in app/models/ with content_type, language,
  source_files, source_excerpt, normalized_name, extraction_confidence,
  needs_review; FTS5 + 3 triggers extended with materials_list and
  skills_developed.
Lane C — build_database.py (--rebuild, atomic swap, schema + fuzzy
  source_excerpt validation, dedup with needs_review band),
  validate_extractions.py, review_queue.py, new run_extraction.py
  orchestrator, SUBAGENT_PROMPT.md.
Lane D — search.py content_type/language filters (default search
  excludes non-game content), E7 schema-compat audit; fixed a NULL
  keywords AttributeError in _boost_search_relevance.

Removes 8 orphaned/dead scripts and app/services/parser.py +
indexer.py. Adds tests/ (70 passing, 1 skipped — libreoffice absent).

Note: Lane D made one additive edit to app/models/database.py
(_update_category_counts) to surface content_type/language in
get_filter_options, outside its nominal lane boundary but after
Lane B completed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Agent
2026-05-19 17:43:38 +00:00
parent e0080edf85
commit 66ae831c36
37 changed files with 4101 additions and 1881 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
app/models/activity.py
View File

@@ -5,6 +5,22 @@ Activity data model for INDEX-SISTEM-JOCURI v2.0
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any
import json
import re
import unicodedata
def normalize_name(name: str) -> str:
"""Diacritic-free, lowercased, whitespace-collapsed form of a name.
Used as the exact-match key for dedup grouping (see plan §4).
"""
if not name:
return ""
decomposed = unicodedata.normalize("NFKD", name)
ascii_str = "".join(c for c in decomposed if not unicodedata.combining(c))
ascii_str = ascii_str.lower().strip()
ascii_str = re.sub(r"\s+", " ", ascii_str)
return ascii_str
@dataclass
class Activity:
@@ -19,10 +35,19 @@ class Activity:
# Categories
category: str = ""
subcategory: Optional[str] = None
# content_type is an axis INDEPENDENT of category:
# one of joc/activitate/reteta/cantec/ceremonie (see config_taxonomy).
content_type: Optional[str] = None
# Source information
source_file: str = ""
page_reference: Optional[str] = None
# source_files: JSON-encoded list of every source the activity was seen in.
# `source_file` (singular) stays as the primary/original source; build_database
# (Lane C) accumulates the full list here on dedup-merge.
source_files: List[str] = field(default_factory=list)
# Short verbatim quote from the source — anti-hallucination anchor.
source_excerpt: Optional[str] = None
# Age and participants
age_group_min: Optional[int] = None
@@ -44,11 +69,22 @@ class Activity:
keywords: Optional[str] = None
tags: List[str] = field(default_factory=list)
popularity_score: int = 0
# Extraction / language metadata
language: Optional[str] = None # 'ro' / 'en'
normalized_name: Optional[str] = None # dedup key; auto-derived from name
extraction_confidence: Optional[str] = None # 'high' / 'med' / 'low'
needs_review: int = 0
# Database fields
id: Optional[int] = None
created_at: Optional[str] = None
updated_at: Optional[str] = None
def __post_init__(self):
"""Derive normalized_name from name when not explicitly provided."""
if not self.normalized_name:
self.normalized_name = normalize_name(self.name)
def to_dict(self) -> Dict[str, Any]:
"""Convert activity to dictionary for database storage"""
@@ -59,8 +95,11 @@ class Activity:
'variations': self.variations,
'category': self.category,
'subcategory': self.subcategory,
'content_type': self.content_type,
'source_file': self.source_file,
'source_files': json.dumps(self.source_files) if self.source_files else None,
'page_reference': self.page_reference,
'source_excerpt': self.source_excerpt,
'age_group_min': self.age_group_min,
'age_group_max': self.age_group_max,
'participants_min': self.participants_min,
@@ -73,7 +112,11 @@ class Activity:
'difficulty_level': self.difficulty_level,
'keywords': self.keywords,
'tags': json.dumps(self.tags) if self.tags else None,
'popularity_score': self.popularity_score
'popularity_score': self.popularity_score,
'language': self.language,
'normalized_name': self.normalized_name or normalize_name(self.name),
'extraction_confidence': self.extraction_confidence,
'needs_review': self.needs_review,
}
@classmethod
@@ -86,7 +129,17 @@ class Activity:
tags = json.loads(data['tags'])
except (json.JSONDecodeError, TypeError):
tags = []
# source_files may arrive as a JSON string (DB) or a list (extraction)
source_files = data.get('source_files')
if isinstance(source_files, str):
try:
source_files = json.loads(source_files)
except (json.JSONDecodeError, TypeError):
source_files = []
elif source_files is None:
source_files = []
return cls(
id=data.get('id'),
name=data.get('name', ''),
@@ -95,8 +148,11 @@ class Activity:
variations=data.get('variations'),
category=data.get('category', ''),
subcategory=data.get('subcategory'),
content_type=data.get('content_type'),
source_file=data.get('source_file', ''),
source_files=source_files,
page_reference=data.get('page_reference'),
source_excerpt=data.get('source_excerpt'),
age_group_min=data.get('age_group_min'),
age_group_max=data.get('age_group_max'),
participants_min=data.get('participants_min'),
@@ -110,6 +166,10 @@ class Activity:
keywords=data.get('keywords'),
tags=tags,
popularity_score=data.get('popularity_score', 0),
language=data.get('language'),
normalized_name=data.get('normalized_name'),
extraction_confidence=data.get('extraction_confidence'),
needs_review=data.get('needs_review', 0) or 0,
created_at=data.get('created_at'),
updated_at=data.get('updated_at')
)

55
app/models/database.py
View File

@@ -30,6 +30,8 @@ class DatabaseManager:
"""Initialize database with v2.0 schema"""
with self._get_connection() as conn:
# Main activities table
# NOTE: schema is rebuilt from scratch (plan §6) — no in-place
# migration. The old DB is deleted and recreated by build_database.
conn.execute("""
CREATE TABLE IF NOT EXISTS activities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -39,9 +41,12 @@ class DatabaseManager:
variations TEXT,
category TEXT NOT NULL,
subcategory TEXT,
content_type TEXT,
source_file TEXT NOT NULL,
source_files TEXT,
page_reference TEXT,
source_excerpt TEXT,
-- Structured parameters
age_group_min INTEGER,
age_group_max INTEGER,
@@ -49,26 +54,34 @@ class DatabaseManager:
participants_max INTEGER,
duration_min INTEGER,
duration_max INTEGER,
-- Categories for filtering
materials_category TEXT,
materials_list TEXT,
skills_developed TEXT,
difficulty_level TEXT,
-- Metadata
keywords TEXT,
tags TEXT,
popularity_score INTEGER DEFAULT 0,
-- Extraction / language metadata
language TEXT,
normalized_name TEXT,
extraction_confidence TEXT,
needs_review INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# FTS5 virtual table for search
conn.execute("""
CREATE VIRTUAL TABLE IF NOT EXISTS activities_fts USING fts5(
name, description, rules, variations, keywords,
materials_list, skills_developed,
content='activities',
content_rowid='id'
)
@@ -92,6 +105,7 @@ class DatabaseManager:
"CREATE INDEX IF NOT EXISTS idx_activities_age ON activities(age_group_min, age_group_max)",
"CREATE INDEX IF NOT EXISTS idx_activities_participants ON activities(participants_min, participants_max)",
"CREATE INDEX IF NOT EXISTS idx_activities_duration ON activities(duration_min, duration_max)",
"CREATE INDEX IF NOT EXISTS idx_activities_normalized_name ON activities(normalized_name)",
"CREATE INDEX IF NOT EXISTS idx_categories_type ON categories(type)"
]
@@ -102,24 +116,34 @@ class DatabaseManager:
conn.execute("""
CREATE TRIGGER IF NOT EXISTS activities_fts_insert AFTER INSERT ON activities
BEGIN
INSERT INTO activities_fts(rowid, name, description, rules, variations, keywords)
VALUES (new.id, new.name, new.description, new.rules, new.variations, new.keywords);
INSERT INTO activities_fts(rowid, name, description, rules, variations,
keywords, materials_list, skills_developed)
VALUES (new.id, new.name, new.description, new.rules, new.variations,
new.keywords, new.materials_list, new.skills_developed);
END
""")
conn.execute("""
CREATE TRIGGER IF NOT EXISTS activities_fts_delete AFTER DELETE ON activities
BEGIN
DELETE FROM activities_fts WHERE rowid = old.id;
INSERT INTO activities_fts(activities_fts, rowid, name, description, rules,
variations, keywords, materials_list, skills_developed)
VALUES ('delete', old.id, old.name, old.description, old.rules,
old.variations, old.keywords, old.materials_list, old.skills_developed);
END
""")
conn.execute("""
CREATE TRIGGER IF NOT EXISTS activities_fts_update AFTER UPDATE ON activities
BEGIN
DELETE FROM activities_fts WHERE rowid = old.id;
INSERT INTO activities_fts(rowid, name, description, rules, variations, keywords)
VALUES (new.id, new.name, new.description, new.rules, new.variations, new.keywords);
INSERT INTO activities_fts(activities_fts, rowid, name, description, rules,
variations, keywords, materials_list, skills_developed)
VALUES ('delete', old.id, old.name, old.description, old.rules,
old.variations, old.keywords, old.materials_list, old.skills_developed);
INSERT INTO activities_fts(rowid, name, description, rules, variations,
keywords, materials_list, skills_developed)
VALUES (new.id, new.name, new.description, new.rules, new.variations,
new.keywords, new.materials_list, new.skills_developed);
END
""")
@@ -179,6 +203,8 @@ class DatabaseManager:
"""Update category usage counts"""
categories_to_update = [
('category', activity.category),
('content_type', activity.content_type),
('language', activity.language),
('age_group', activity.get_age_range_display()),
('participants', activity.get_participants_display()),
('duration', activity.get_duration_display()),
@@ -332,8 +358,11 @@ class DatabaseManager:
def clear_database(self):
"""Clear all data from database"""
with self._get_connection() as conn:
# Deleting from activities fires the delete trigger, which removes
# the matching FTS rows. The explicit 'delete-all' command then
# guarantees the external-content FTS index is fully cleared.
conn.execute("DELETE FROM activities")
conn.execute("DELETE FROM activities_fts")
conn.execute("INSERT INTO activities_fts(activities_fts) VALUES('delete-all')")
conn.execute("DELETE FROM categories")
conn.commit()