Faza 1 complete: bilingual+enrichment plumbing, UI/filters, frozen DB
Extraction finished (575/588 chunks; 6 content-filter-blocked, 7 await re-extraction). DB rebuilt and frozen at 9418 activities — content_keys are now stable for the enrichment overlay. Part A (plumbing + UI): - database.py: name_ro/description_ro/rules_ro/variations_ro, indoor_outdoor, space_needed, estimated_fields, source_id/source_ids/chunk_key columns; FTS5 indexes the 4 *_ro columns across CREATE + all 3 triggers; new equality filters + category counts for both axes. - activity.py: new fields + bilingual display helpers (get_display_*, is_estimated, axis displays). - config_taxonomy.py: INDOOR_OUTDOOR/SPACE_NEEDED enums + normalizers (None on unrecognised, no fabrication). - search.py / routes.py / config.py / templates / css: new dropdowns, RO-primary rendering with "(estimat)" markers and collapsible original text, and a /source/<id> download route shipped DARK behind SOURCE_DOWNLOAD_ENABLED (copyright opt-in). - build_database.py: source_id/chunk_key in dict_to_activity; merge_cluster unions source_ids without touching enrichment fields. Part B (enrichment pipeline, built not yet run): - build_database.py: load_enrichment + apply_enrichment (post-dedup, keyed on content_key) + --enrichment CLI + stated-vs-estimated QA. - run_enrichment.py (resumable, --source/--limit pilot scoping, --collect), ENRICHMENT_PROMPT.md. Repair: scripts/repair_extractions.py fixes the subagents' systematic unescaped-ASCII-quote bug with a faithful char-scanner (escapes, never truncates) + schema validation + a strictly-more-text guard. json_repair was tried first, truncated silently, and is NOT used. build_database has no repair dependency. Tests: tests/test_enrichment.py added; 99 pass. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Agent
244
scripts/repair_extractions.py
Normal file
244
scripts/repair_extractions.py
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
repair_extractions.py — one-shot repair of malformed extraction JSON.
|
||||
|
||||
Subagents systematically emit unescaped ASCII double-quotes inside string
|
||||
values (Romanian text like „Unu" uses a closing " that terminates the JSON
|
||||
string early). Re-extraction reproduces the bug, so we repair instead.
|
||||
|
||||
IMPORTANT — why NOT json_repair: json_repair "recovers" an unescaped quote by
|
||||
ending the string at the stray quote and reinterpreting the trailing text as a
|
||||
new key, which (a) TRUNCATES the value and (b) injects garbage keys. The
|
||||
truncation is silent (the field is still non-empty) and slips past a naive
|
||||
presence check. So we use a faithful char-scanner that ESCAPES stray quotes
|
||||
(\\") instead of splitting on them, then validate the result against the real
|
||||
activity schema (additionalProperties:false also catches any residual split).
|
||||
|
||||
This is an OFFLINE maintenance tool. build_database.py must NOT depend on it —
|
||||
the "DB regenerable from data/extracted/" invariant requires plain valid JSON on
|
||||
disk. We write clean JSON back to data/extracted/ and the build reads vanilla
|
||||
json.
|
||||
|
||||
Source selection (faithful recovery needs the ORIGINAL malformed text):
|
||||
* a chunk is a candidate when a MALFORMED original exists — either the
|
||||
top-level data/extracted/<key>.json is itself invalid, or a malformed
|
||||
original sits in data/extracted/_rejected/<key>.json.
|
||||
* the malformed original is preferred as the repair source.
|
||||
* chunks whose only artifact is already-valid JSON (e.g. a prior json_repair
|
||||
output that lost the original) are NOT silently "repaired" — if such a chunk
|
||||
has no valid top-level file it is reported as needing RE-EXTRACTION.
|
||||
|
||||
Usage:
|
||||
python scripts/repair_extractions.py # report only (dry run)
|
||||
python scripts/repair_extractions.py --apply # write repaired JSON
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parent
|
||||
EXTRACTED = REPO_ROOT / "data" / "extracted"
|
||||
REJECTED = EXTRACTED / "_rejected"
|
||||
|
||||
if str(SCRIPT_DIR) not in __import__("sys").path:
|
||||
__import__("sys").path.insert(0, str(SCRIPT_DIR))
|
||||
from import_common import DEFAULT_SCHEMA_PATH, load_schema, validate_extraction # noqa: E402
|
||||
|
||||
|
||||
def escape_stray_quotes(s: str) -> str:
|
||||
"""Escape ASCII double-quotes that occur INSIDE a JSON string value.
|
||||
|
||||
A `"` inside a string is treated as a real string-close only when the next
|
||||
non-whitespace char is structural (`,` `}` `]` `:`) or EOF; otherwise it is
|
||||
content and is escaped to `\\"`. This preserves the full value instead of
|
||||
truncating it (the json_repair failure mode).
|
||||
"""
|
||||
out: list[str] = []
|
||||
in_str = False
|
||||
esc = False
|
||||
n = len(s)
|
||||
i = 0
|
||||
while i < n:
|
||||
c = s[i]
|
||||
if esc:
|
||||
out.append(c)
|
||||
esc = False
|
||||
i += 1
|
||||
continue
|
||||
if c == "\\":
|
||||
out.append(c)
|
||||
esc = True
|
||||
i += 1
|
||||
continue
|
||||
if c == '"':
|
||||
if not in_str:
|
||||
in_str = True
|
||||
out.append(c)
|
||||
else:
|
||||
j = i + 1
|
||||
while j < n and s[j] in " \t\r\n":
|
||||
j += 1
|
||||
nxt = s[j] if j < n else ""
|
||||
if nxt in ",}]:" or nxt == "":
|
||||
in_str = False
|
||||
out.append(c)
|
||||
else:
|
||||
out.append('\\"') # content quote → escape, keep value whole
|
||||
i += 1
|
||||
continue
|
||||
out.append(c)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _is_valid_json(path: Path) -> bool:
|
||||
try:
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
return True
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _malformed_source(key: str) -> Optional[Path]:
|
||||
"""Return the malformed-original file for a chunk, preferring top-level."""
|
||||
live = EXTRACTED / f"{key}.json"
|
||||
if live.exists() and not _is_valid_json(live):
|
||||
return live
|
||||
rej = REJECTED / f"{key}.json"
|
||||
if rej.exists() and not _is_valid_json(rej):
|
||||
return rej
|
||||
return None
|
||||
|
||||
|
||||
def _candidate_keys() -> tuple[dict[str, Path], list[str]]:
|
||||
"""
|
||||
(repair_candidates, needs_reextraction).
|
||||
|
||||
repair_candidates: key -> malformed source file (faithfully repairable).
|
||||
needs_reextraction: chunks with no malformed original AND no valid
|
||||
top-level file (their original was lost) — must be re-extracted.
|
||||
"""
|
||||
keys = set()
|
||||
for fn in glob.glob(str(EXTRACTED / "*.json")):
|
||||
keys.add(Path(fn).stem)
|
||||
for fn in glob.glob(str(REJECTED / "*.json")):
|
||||
keys.add(Path(fn).stem)
|
||||
|
||||
candidates: dict[str, Path] = {}
|
||||
needs_reextraction: list[str] = []
|
||||
for key in sorted(keys):
|
||||
# A malformed original anywhere is faithfully repairable, and is the
|
||||
# source of truth even if a (json_repair-produced, possibly truncated)
|
||||
# valid top-level file exists — escaping the original never truncates,
|
||||
# so re-repairing from it is always >= the json_repair output.
|
||||
src = _malformed_source(key)
|
||||
if src is not None:
|
||||
candidates[key] = src
|
||||
continue
|
||||
live = EXTRACTED / f"{key}.json"
|
||||
if live.exists() and _is_valid_json(live):
|
||||
continue # genuinely-valid extraction, nothing to do
|
||||
# no valid top-level and no malformed original to repair from
|
||||
needs_reextraction.append(key)
|
||||
return candidates, needs_reextraction
|
||||
|
||||
|
||||
def repair(apply: bool) -> int:
|
||||
schema = load_schema(DEFAULT_SCHEMA_PATH)
|
||||
candidates, needs_reextraction = _candidate_keys()
|
||||
|
||||
print("=" * 64)
|
||||
print(f"REPAIR EXTRACTIONS ({'APPLY' if apply else 'dry run'})")
|
||||
print("=" * 64)
|
||||
print(f"repair candidates: {len(candidates)}")
|
||||
|
||||
def _textlen(data: dict) -> int:
|
||||
total = 0
|
||||
for a in data.get("activities", []):
|
||||
if isinstance(a, dict):
|
||||
for v in a.values():
|
||||
if isinstance(v, str):
|
||||
total += len(v)
|
||||
return total
|
||||
|
||||
ok = 0
|
||||
kept_toplevel = 0
|
||||
still_bad: list[str] = []
|
||||
schema_fail: list[tuple[str, str]] = []
|
||||
|
||||
for key, src in candidates.items():
|
||||
live = EXTRACTED / f"{key}.json"
|
||||
live_valid = live.exists() and _is_valid_json(live)
|
||||
|
||||
raw = src.read_text(encoding="utf-8")
|
||||
fixed = escape_stray_quotes(raw)
|
||||
try:
|
||||
data = json.loads(fixed)
|
||||
except json.JSONDecodeError as exc:
|
||||
if live_valid:
|
||||
kept_toplevel += 1 # genuine top-level is fine; stale _rejected
|
||||
else:
|
||||
still_bad.append(f"{key}: still invalid after escape ({exc})")
|
||||
continue
|
||||
errors = validate_extraction(data, schema)
|
||||
if errors:
|
||||
if live_valid:
|
||||
kept_toplevel += 1
|
||||
else:
|
||||
schema_fail.append((key, errors[0]))
|
||||
print(f" {key[:50]:<50} SCHEMA-FAIL: {errors[0][:40]}")
|
||||
continue
|
||||
|
||||
# Faithfulness guard: only replace a valid top-level when the escaped
|
||||
# repair carries STRICTLY more text (i.e. the top-level was a truncated
|
||||
# json_repair output). Genuine extractions are kept untouched.
|
||||
if live_valid:
|
||||
try:
|
||||
live_data = json.loads(live.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
live_data = {}
|
||||
if _textlen(data) <= _textlen(live_data):
|
||||
kept_toplevel += 1
|
||||
continue
|
||||
|
||||
n = len(data.get("activities", []))
|
||||
print(f" {key[:50]:<50} {n:>3} acts REPAIR")
|
||||
if apply:
|
||||
live.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
ok += 1
|
||||
|
||||
print("-" * 64)
|
||||
print(f"repaired: {ok} | kept genuine top-level: {kept_toplevel} | "
|
||||
f"schema-fail: {len(schema_fail)} | still-bad: {len(still_bad)} | "
|
||||
f"needs re-extraction: {len(needs_reextraction)}")
|
||||
for key, err in schema_fail:
|
||||
print(f" ⚠ schema {key}: {err[:60]}")
|
||||
for msg in still_bad:
|
||||
print(f" ✘ {msg}")
|
||||
for key in needs_reextraction:
|
||||
print(f" ↻ re-extract: {key}")
|
||||
if not apply:
|
||||
print("\nDry run — re-run with --apply to write repaired JSON.")
|
||||
print("=" * 64)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Repair malformed extraction JSON.")
|
||||
parser.add_argument("--apply", action="store_true",
|
||||
help="write repaired JSON (default: dry run)")
|
||||
args = parser.parse_args(argv)
|
||||
return repair(args.apply)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user