Faza 1 complete: bilingual+enrichment plumbing, UI/filters, frozen DB

Extraction finished (575/588 chunks; 6 content-filter-blocked, 7 await
re-extraction). DB rebuilt and frozen at 9418 activities — content_keys
are now stable for the enrichment overlay.

Part A (plumbing + UI):
- database.py: name_ro/description_ro/rules_ro/variations_ro, indoor_outdoor,
  space_needed, estimated_fields, source_id/source_ids/chunk_key columns;
  FTS5 indexes the 4 *_ro columns across CREATE + all 3 triggers; new equality
  filters + category counts for both axes.
- activity.py: new fields + bilingual display helpers (get_display_*,
  is_estimated, axis displays).
- config_taxonomy.py: INDOOR_OUTDOOR/SPACE_NEEDED enums + normalizers
  (None on unrecognised, no fabrication).
- search.py / routes.py / config.py / templates / css: new dropdowns,
  RO-primary rendering with "(estimat)" markers and collapsible original
  text, and a /source/<id> download route shipped DARK behind
  SOURCE_DOWNLOAD_ENABLED (copyright opt-in).
- build_database.py: source_id/chunk_key in dict_to_activity; merge_cluster
  unions source_ids without touching enrichment fields.

Part B (enrichment pipeline, built not yet run):
- build_database.py: load_enrichment + apply_enrichment (post-dedup, keyed on
  content_key) + --enrichment CLI + stated-vs-estimated QA.
- run_enrichment.py (resumable, --source/--limit pilot scoping, --collect),
  ENRICHMENT_PROMPT.md.

Repair: scripts/repair_extractions.py fixes the subagents' systematic
unescaped-ASCII-quote bug with a faithful char-scanner (escapes, never
truncates) + schema validation + a strictly-more-text guard. json_repair was
tried first, truncated silently, and is NOT used. build_database has no repair
dependency.

Tests: tests/test_enrichment.py added; 99 pass.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Agent
2026-05-29 18:10:13 +00:00
parent 46d9592a55
commit bcfb6841eb
18 changed files with 1579 additions and 167 deletions

View File

@@ -76,6 +76,25 @@ class Activity:
extraction_confidence: Optional[str] = None # 'high' / 'med' / 'low'
needs_review: int = 0
# Enrichment overlay (applied at build time from data/enrichment.json; see
# plan Part B). Bilingual: the EN/source text stays in name/description/...
# and the Romanian rendering lands in the *_ro twins. Absent fields leave
# the underlying DB value untouched.
name_ro: Optional[str] = None
description_ro: Optional[str] = None
rules_ro: Optional[str] = None
variations_ro: Optional[str] = None
indoor_outdoor: Optional[str] = None # slug: indoor / outdoor / either
space_needed: Optional[str] = None # slug: mic / mediu / mare
# Names of fields whose value was INFERRED by enrichment (source was
# silent) rather than stated in the source — surfaced as "(estimat)" in UI.
estimated_fields: List[str] = field(default_factory=list)
# Source provenance for the download route + enrichment keying.
source_id: Optional[str] = None # e.g. "876d1a2d_marcaje_turistice"
source_ids: List[str] = field(default_factory=list) # all source_ids merged
chunk_key: Optional[str] = None # e.g. "<source_id>.part01"
# Database fields
id: Optional[int] = None
created_at: Optional[str] = None
@@ -117,6 +136,16 @@ class Activity:
'normalized_name': self.normalized_name or normalize_name(self.name),
'extraction_confidence': self.extraction_confidence,
'needs_review': self.needs_review,
'name_ro': self.name_ro,
'description_ro': self.description_ro,
'rules_ro': self.rules_ro,
'variations_ro': self.variations_ro,
'indoor_outdoor': self.indoor_outdoor,
'space_needed': self.space_needed,
'estimated_fields': json.dumps(self.estimated_fields) if self.estimated_fields else None,
'source_id': self.source_id,
'source_ids': json.dumps(self.source_ids) if self.source_ids else None,
'chunk_key': self.chunk_key,
}
@classmethod
@@ -140,6 +169,19 @@ class Activity:
elif source_files is None:
source_files = []
# estimated_fields / source_ids: JSON string (DB) or list (in-memory)
def _json_list(value):
if isinstance(value, str):
try:
parsed = json.loads(value)
return parsed if isinstance(parsed, list) else []
except (json.JSONDecodeError, TypeError):
return []
return list(value) if value else []
estimated_fields = _json_list(data.get('estimated_fields'))
source_ids = _json_list(data.get('source_ids'))
return cls(
id=data.get('id'),
name=data.get('name', ''),
@@ -170,6 +212,16 @@ class Activity:
normalized_name=data.get('normalized_name'),
extraction_confidence=data.get('extraction_confidence'),
needs_review=data.get('needs_review', 0) or 0,
name_ro=data.get('name_ro'),
description_ro=data.get('description_ro'),
rules_ro=data.get('rules_ro'),
variations_ro=data.get('variations_ro'),
indoor_outdoor=data.get('indoor_outdoor'),
space_needed=data.get('space_needed'),
estimated_fields=estimated_fields,
source_id=data.get('source_id'),
source_ids=source_ids,
chunk_key=data.get('chunk_key'),
created_at=data.get('created_at'),
updated_at=data.get('updated_at')
)
@@ -210,4 +262,44 @@ class Activity:
return self.materials_category
elif self.materials_list:
return self.materials_list[:100] + "..." if len(self.materials_list) > 100 else self.materials_list
return "nu specificate"
return "nu specificate"
# --- Enrichment / bilingual display helpers ------------------------------
def get_display_name(self) -> str:
"""Romanian name when enriched, else the original."""
return self.name_ro or self.name
def get_display_description(self) -> str:
"""Romanian description when enriched, else the original."""
return self.description_ro or self.description
def get_display_rules(self) -> Optional[str]:
"""Romanian rules when enriched, else the original."""
return self.rules_ro or self.rules
def get_display_variations(self) -> Optional[str]:
"""Romanian variations when enriched, else the original."""
return self.variations_ro or self.variations
def has_translation(self) -> bool:
"""True if any Romanian enrichment text is present."""
return bool(self.name_ro or self.description_ro
or self.rules_ro or self.variations_ro)
def is_estimated(self, field_name: str) -> bool:
"""True if `field_name` was inferred by enrichment (source was silent)."""
return field_name in (self.estimated_fields or [])
def get_indoor_outdoor_display(self) -> Optional[str]:
"""RO label for indoor_outdoor, or None when unset."""
if not self.indoor_outdoor:
return None
from app.config_taxonomy import indoor_outdoor_display_name
return indoor_outdoor_display_name(self.indoor_outdoor)
def get_space_needed_display(self) -> Optional[str]:
"""RO label for space_needed, or None when unset."""
if not self.space_needed:
return None
from app.config_taxonomy import space_needed_display_name
return space_needed_display_name(self.space_needed)