Faza 1 complete: bilingual+enrichment plumbing, UI/filters, frozen DB
Extraction finished (575/588 chunks; 6 content-filter-blocked, 7 await re-extraction). DB rebuilt and frozen at 9418 activities — content_keys are now stable for the enrichment overlay. Part A (plumbing + UI): - database.py: name_ro/description_ro/rules_ro/variations_ro, indoor_outdoor, space_needed, estimated_fields, source_id/source_ids/chunk_key columns; FTS5 indexes the 4 *_ro columns across CREATE + all 3 triggers; new equality filters + category counts for both axes. - activity.py: new fields + bilingual display helpers (get_display_*, is_estimated, axis displays). - config_taxonomy.py: INDOOR_OUTDOOR/SPACE_NEEDED enums + normalizers (None on unrecognised, no fabrication). - search.py / routes.py / config.py / templates / css: new dropdowns, RO-primary rendering with "(estimat)" markers and collapsible original text, and a /source/<id> download route shipped DARK behind SOURCE_DOWNLOAD_ENABLED (copyright opt-in). - build_database.py: source_id/chunk_key in dict_to_activity; merge_cluster unions source_ids without touching enrichment fields. Part B (enrichment pipeline, built not yet run): - build_database.py: load_enrichment + apply_enrichment (post-dedup, keyed on content_key) + --enrichment CLI + stated-vs-estimated QA. - run_enrichment.py (resumable, --source/--limit pilot scoping, --collect), ENRICHMENT_PROMPT.md. Repair: scripts/repair_extractions.py fixes the subagents' systematic unescaped-ASCII-quote bug with a faithful char-scanner (escapes, never truncates) + schema validation + a strictly-more-text guard. json_repair was tried first, truncated silently, and is NOT used. build_database has no repair dependency. Tests: tests/test_enrichment.py added; 99 pass. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,7 @@ the UI displays the Romanian name. `category` (thematic domain) and
|
||||
|
||||
import unicodedata
|
||||
import re
|
||||
from typing import Dict, List
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# --- Categories (thematic domain) --------------------------------------------
|
||||
# slug -> Romanian display name. ~16 fixed slugs; `altele` is the mandatory
|
||||
@@ -215,6 +215,89 @@ def normalize_content_type(value: str) -> str:
|
||||
return aliases.get(slug, DEFAULT_CONTENT_TYPE)
|
||||
|
||||
|
||||
# --- Indoor / outdoor (enrichment axis) --------------------------------------
|
||||
# Where the activity is run. Inferred during enrichment when the source is
|
||||
# silent — such inferences are flagged in `estimated_fields`. slug -> RO label.
|
||||
INDOOR_OUTDOOR: Dict[str, str] = {
|
||||
"indoor": "Interior",
|
||||
"outdoor": "Exterior",
|
||||
"either": "Interior sau exterior",
|
||||
}
|
||||
|
||||
# --- Space needed (enrichment axis) ------------------------------------------
|
||||
# Rough footprint the activity requires. slug -> RO label.
|
||||
SPACE_NEEDED: Dict[str, str] = {
|
||||
"mic": "Spațiu mic",
|
||||
"mediu": "Spațiu mediu",
|
||||
"mare": "Spațiu mare",
|
||||
}
|
||||
|
||||
# Aliases for robustness against LLM output variation. Keys are _slugify'd.
|
||||
_INDOOR_OUTDOOR_ALIASES: Dict[str, str] = {
|
||||
"interior": "indoor",
|
||||
"inside": "indoor",
|
||||
"in": "indoor",
|
||||
"exterior": "outdoor",
|
||||
"outside": "outdoor",
|
||||
"out": "outdoor",
|
||||
"aer-liber": "outdoor",
|
||||
"both": "either",
|
||||
"any": "either",
|
||||
"ambele": "either",
|
||||
"interior-exterior": "either",
|
||||
"indoor-outdoor": "either",
|
||||
}
|
||||
|
||||
_SPACE_NEEDED_ALIASES: Dict[str, str] = {
|
||||
"small": "mic",
|
||||
"redus": "mic",
|
||||
"putin": "mic",
|
||||
"medium": "mediu",
|
||||
"moderat": "mediu",
|
||||
"large": "mare",
|
||||
"big": "mare",
|
||||
"mult": "mare",
|
||||
"spatiu-mic": "mic",
|
||||
"spatiu-mediu": "mediu",
|
||||
"spatiu-mare": "mare",
|
||||
}
|
||||
|
||||
|
||||
def normalize_indoor_outdoor(value: str) -> Optional[str]:
|
||||
"""Map an arbitrary string to an indoor_outdoor slug, or None.
|
||||
|
||||
Unlike categories, this has NO mandatory fallback: an unrecognised or
|
||||
empty value yields None (field simply absent), so we never fabricate a
|
||||
location the enrichment did not assert.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
slug = _slugify(str(value))
|
||||
if slug in INDOOR_OUTDOOR:
|
||||
return slug
|
||||
return _INDOOR_OUTDOOR_ALIASES.get(slug)
|
||||
|
||||
|
||||
def normalize_space_needed(value: str) -> Optional[str]:
|
||||
"""Map an arbitrary string to a space_needed slug, or None (no fallback)."""
|
||||
if not value:
|
||||
return None
|
||||
slug = _slugify(str(value))
|
||||
if slug in SPACE_NEEDED:
|
||||
return slug
|
||||
return _SPACE_NEEDED_ALIASES.get(slug)
|
||||
|
||||
|
||||
def indoor_outdoor_display_name(slug: str) -> str:
|
||||
"""RO display name for an indoor_outdoor slug."""
|
||||
return INDOOR_OUTDOOR.get(slug, slug)
|
||||
|
||||
|
||||
def space_needed_display_name(slug: str) -> str:
|
||||
"""RO display name for a space_needed slug."""
|
||||
return SPACE_NEEDED.get(slug, slug)
|
||||
|
||||
|
||||
def is_valid_category(slug: str) -> bool:
|
||||
"""True if `slug` is a valid category slug."""
|
||||
return slug in CATEGORIES
|
||||
|
||||
Reference in New Issue
Block a user