Rebuild extraction pipeline infrastructure (Faza 0 prep)
Implements the approved plan to replace the broken regex/index-master extraction with an LLM-subagent pipeline. Four parallel lanes: Lane A — scripts/extract_common.py (PDF/docx/doc/pptx/html/zip, no max_pages truncation), normalize_sources.py, chunk_sources.py (~20pg chunks + overlap, manifest registry), activity_schema.json. Lane B — app/config_taxonomy.py (16 fixed category slugs), schema rebuilt from scratch in app/models/ with content_type, language, source_files, source_excerpt, normalized_name, extraction_confidence, needs_review; FTS5 + 3 triggers extended with materials_list and skills_developed. Lane C — build_database.py (--rebuild, atomic swap, schema + fuzzy source_excerpt validation, dedup with needs_review band), validate_extractions.py, review_queue.py, new run_extraction.py orchestrator, SUBAGENT_PROMPT.md. Lane D — search.py content_type/language filters (default search excludes non-game content), E7 schema-compat audit; fixed a NULL keywords AttributeError in _boost_search_relevance. Removes 8 orphaned/dead scripts and app/services/parser.py + indexer.py. Adds tests/ (70 passing, 1 skipped — libreoffice absent). Note: Lane D made one additive edit to app/models/database.py (_update_category_counts) to surface content_type/language in get_filter_options, outside its nominal lane boundary but after Lane B completed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,11 +7,17 @@ from flask import Blueprint, request, render_template, jsonify, current_app
|
||||
from app.models.database import DatabaseManager
|
||||
from app.models.activity import Activity
|
||||
from app.services.search import SearchService
|
||||
from app.config_taxonomy import CATEGORIES, CONTENT_TYPES
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
bp = Blueprint('main', __name__)
|
||||
|
||||
# Slug -> Romanian display name. Category and content_type slugs never collide,
|
||||
# so a single flat map is enough for the UI filter labels.
|
||||
LANGUAGE_NAMES = {'ro': 'Română', 'en': 'Engleză'}
|
||||
DISPLAY_NAMES = {**CATEGORIES, **CONTENT_TYPES, **LANGUAGE_NAMES}
|
||||
|
||||
# Initialize database manager (will be configured in application factory)
|
||||
def get_db_manager():
|
||||
"""Get database manager instance"""
|
||||
@@ -36,15 +42,17 @@ def index():
|
||||
# Get database statistics for the interface
|
||||
stats = db.get_statistics()
|
||||
|
||||
return render_template('index.html',
|
||||
return render_template('index.html',
|
||||
filters=filter_options,
|
||||
display_names=DISPLAY_NAMES,
|
||||
stats=stats)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading main page: {e}")
|
||||
# Fallback with empty filters
|
||||
return render_template('index.html',
|
||||
return render_template('index.html',
|
||||
filters={},
|
||||
display_names=DISPLAY_NAMES,
|
||||
stats={'total_activities': 0})
|
||||
|
||||
@bp.route('/search', methods=['GET', 'POST'])
|
||||
@@ -82,8 +90,9 @@ def search():
|
||||
search_query=search_query,
|
||||
applied_filters=filters,
|
||||
filters=filter_options,
|
||||
display_names=DISPLAY_NAMES,
|
||||
results_count=len(activities))
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"Search error: {e}")
|
||||
return render_template('results.html',
|
||||
@@ -91,6 +100,7 @@ def search():
|
||||
search_query='',
|
||||
applied_filters={},
|
||||
filters={},
|
||||
display_names=DISPLAY_NAMES,
|
||||
results_count=0,
|
||||
error=str(e))
|
||||
|
||||
@@ -121,6 +131,7 @@ def activity_detail(activity_id):
|
||||
|
||||
return render_template('activity.html',
|
||||
activity=activity,
|
||||
display_names=DISPLAY_NAMES,
|
||||
similar_activities=similar_activities)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user