756f77730f
Seed app/data/operatii-etichetate.json regenerat cu subagenti Haiku pe TOATE cele 17181 operatii distincte (ordine frecventa, 100%), inlocuind seed-ul Groq (3758). Validare Haiku vs Groq pe 157 op etichetate: la dezacorduri Haiku corect ~22/30, Groq ~0. Haiku prinde gunoiul ratat de Groq (ITP, chirie anvelope, nume piese fara actiune): NUL 2200 (12.8%) vs ~7.6% Groq; adaptare electronica OE-7 (nu OE-5), placute frana uzura OE-1 (nu OE-F avarie). US-001..006: prefiltru NUL determinist, etichetator offline, generator seed, seeder mapping_suggestions (in init_db, gated seed_operatii_enabled), embeddings indexeaza corpus etichetat, enrich NUL+kNN. Distributie seed: OE-1 80.1%, NUL 12.8%, OE-2 3.5%, restul rar (OE-4/3/7/8/R/I/5, AITLV, R-ODO). config: seed_operatii_enabled=True + embeddings_enabled=True implicit (SILVER populat + sugestii semantice; ambele suggestion-only, dezactivabile prin env). Suita: 1387 passed, 1 deselected (live). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
345 lines
13 KiB
Python
345 lines
13 KiB
Python
"""Generare seed etichetat operatie->cod (US-003, PRD 5.18).
|
|
|
|
Produce artefactul `app/data/operatii-etichetate.json` (comis in repo), consumat de
|
|
seeder (US-004) si de corpusul embeddings (US-005). NU cheama LLM la runtime — o
|
|
singura data, offline, pe LM Studio (backend implicit, D4).
|
|
|
|
Pipeline dedup OBLIGATORIU, in ordine, INAINTE de orice apel LLM (D5):
|
|
1. Agrega cele N CSV-uri -> freq pe denumire RAW (NR ne-numeric -> skip rand, F9).
|
|
2. `cheie = normalize_for_match(denumire)` (ACEEASI functie ca DB/k-NN, NU strip exact).
|
|
Arunca randurile cu `cheie == ""` inainte de dedup (coliziune pe slot UNIQUE gol, F6).
|
|
3. Dedup pe cheie: un reprezentant per cheie, `freq = suma NR`.
|
|
4. Harta `cheie -> cod` din TOATE etichetele existente: `labels-groq-partial.json` (cheiat
|
|
brut) + seedul comis anterior (cheiat normalizat). Conflict (acelasi cheie, coduri diferite
|
|
pe variante raw) -> castiga codul cu freq-max, tie-break pe cod sortat (F3).
|
|
5. `de_etichetat = corpus(in prag) - harta`. Sortat desc pe freq = SINGURUL input la LLM.
|
|
|
|
Idempotenta cross-run (F2/F7): seedul comis = cache de etichete -> re-run = 0 apeluri LLM.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import glob
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
|
|
# Functia de normalizare = sursa unica de adevar (consistenta cu DB/k-NN).
|
|
_APP_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
if _APP_ROOT not in sys.path:
|
|
sys.path.insert(0, _APP_ROOT)
|
|
from app.mapping import normalize_for_match # noqa: E402
|
|
|
|
|
|
def _load_eticheteaza():
|
|
path = os.path.join(os.path.dirname(__file__), "eticheteaza.py")
|
|
spec = importlib.util.spec_from_file_location("eticheteaza", path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
sys.modules.setdefault("eticheteaza", mod)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
# Cai implicite (relative la repo).
|
|
DEFAULT_CSV_GLOB = os.path.join(_APP_ROOT, "docs", "operatii-service", "*.csv")
|
|
DEFAULT_LABELS = os.path.join(_APP_ROOT, "tools", "mapare-llm", "labels-groq-partial.json")
|
|
DEFAULT_SEED = os.path.join(_APP_ROOT, "app", "data", "operatii-etichetate.json")
|
|
|
|
NUL_LABEL = "NUL"
|
|
DEFAULT_CONFIDENCE = 0.7
|
|
DEFAULT_SOURCE = "llm_seed"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Pasul 1-3: corpus agregat pe cheie normalizata #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _freq_raw(csv_paths: list[str]) -> Counter:
|
|
"""Counter denumire_raw -> suma NR. NR ne-numeric -> skip rand (F9), nu zero-weight."""
|
|
freq: Counter = Counter()
|
|
for f in csv_paths:
|
|
with open(f, encoding="utf-8", errors="replace") as fh:
|
|
for r in list(csv.reader(fh, delimiter=";"))[1:]:
|
|
if len(r) <= 2:
|
|
continue
|
|
den = r[1].strip()
|
|
if not den:
|
|
continue
|
|
nr_raw = (r[2] or "").strip()
|
|
try:
|
|
nr = int(nr_raw)
|
|
except ValueError:
|
|
continue # F9: skip rand cu NR ne-numeric
|
|
freq[den] += nr
|
|
return freq
|
|
|
|
|
|
def _corpus_din_freq(freq_raw: Counter) -> dict[str, dict]:
|
|
"""{cheie_normalizata -> {denumire, freq}}. Arunca cheile vide (F6).
|
|
|
|
`denumire` = varianta raw cu freq individual maxim (tie-break: raw sortat asc),
|
|
folosita ca text trimis la LLM si stocata in seed.
|
|
"""
|
|
grup: dict[str, list[tuple[str, int]]] = defaultdict(list)
|
|
for raw, n in freq_raw.items():
|
|
cheie = normalize_for_match(raw)
|
|
if not cheie:
|
|
continue # F6
|
|
grup[cheie].append((raw, n))
|
|
|
|
corpus: dict[str, dict] = {}
|
|
for cheie, variante in grup.items():
|
|
freq = sum(n for _, n in variante)
|
|
# reprezentant determinist: freq max, tie-break raw sortat.
|
|
denumire = sorted(variante, key=lambda rn: (-rn[1], rn[0]))[0][0]
|
|
corpus[cheie] = {"denumire": denumire, "freq": freq}
|
|
return corpus
|
|
|
|
|
|
def agrega_corpus(csv_paths: list[str]) -> dict[str, dict]:
|
|
"""{cheie_normalizata -> {denumire, freq}} din CSV-uri (pasii 1-3)."""
|
|
return _corpus_din_freq(_freq_raw(csv_paths))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Pasul 4: harta cheie -> cod din etichetele existente (reuse + conflict) #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _incarca_seed(seed_path: str | None) -> list[dict]:
|
|
if not seed_path or not os.path.exists(seed_path):
|
|
return []
|
|
try:
|
|
return json.loads(open(seed_path, encoding="utf-8").read())
|
|
except (ValueError, OSError):
|
|
return []
|
|
|
|
|
|
def construieste_harta_etichete(
|
|
freq_raw: Counter,
|
|
corpus: dict[str, dict],
|
|
labels_path: str | None,
|
|
seed_existent: list[dict],
|
|
) -> dict[str, str]:
|
|
"""Harta cheie_normalizata -> eticheta (cod RAR sau 'NUL'), reuse in spatiu normalizat.
|
|
|
|
Voturi ponderate pe freq; conflict pe acelasi cheie -> freq-max, tie-break cod sortat (F3).
|
|
"""
|
|
votes: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
|
|
|
|
# labels-groq-partial.json: cheiat pe text BRUT.
|
|
if labels_path and os.path.exists(labels_path):
|
|
labels = json.loads(open(labels_path, encoding="utf-8").read())
|
|
for raw, cod in labels.items():
|
|
cheie = normalize_for_match(raw)
|
|
if not cheie:
|
|
continue
|
|
cod = str(cod or "").strip().upper()
|
|
if not cod:
|
|
continue
|
|
votes[cheie][cod] += freq_raw.get(raw, 0)
|
|
|
|
# seed comis anterior: cheiat normalizat (cache cross-run).
|
|
for e in seed_existent:
|
|
cheie = e.get("denumire_normalizata")
|
|
if not cheie:
|
|
continue
|
|
eticheta = NUL_LABEL if e.get("is_nul") else str(e.get("cod") or "").strip().upper()
|
|
if not eticheta:
|
|
continue
|
|
votes[cheie][eticheta] += corpus.get(cheie, {}).get("freq", 0)
|
|
|
|
harta: dict[str, str] = {}
|
|
for cheie, codmap in votes.items():
|
|
# freq desc, apoi cod asc -> determinist.
|
|
harta[cheie] = sorted(codmap.items(), key=lambda kv: (-kv[1], kv[0]))[0][0]
|
|
return harta
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Pasul 5: selectie de_etichetat (prag de volum) + orchestrare #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def selecteaza_de_etichetat(
|
|
corpus: dict[str, dict],
|
|
harta: dict[str, str],
|
|
*,
|
|
target_volum: float,
|
|
etichetare_all: bool,
|
|
) -> list[str]:
|
|
"""Cheile ne-etichetate, sortate desc pe freq, in interiorul pragului de volum."""
|
|
ordered = sorted(corpus, key=lambda k: (-corpus[k]["freq"], k))
|
|
if etichetare_all:
|
|
in_prag = ordered
|
|
else:
|
|
total = sum(c["freq"] for c in corpus.values()) or 1
|
|
in_prag = []
|
|
cum = 0
|
|
for k in ordered:
|
|
in_prag.append(k)
|
|
cum += corpus[k]["freq"]
|
|
if cum / total >= target_volum:
|
|
break
|
|
return [k for k in in_prag if k not in harta]
|
|
|
|
|
|
def genereaza(
|
|
csv_paths: list[str],
|
|
*,
|
|
labels_path: str | None = DEFAULT_LABELS,
|
|
seed_path: str = DEFAULT_SEED,
|
|
target_volum: float = 0.9,
|
|
etichetare_all: bool = False,
|
|
clasifica=None,
|
|
batch: int = 32,
|
|
confidence: float = DEFAULT_CONFIDENCE,
|
|
source: str = DEFAULT_SOURCE,
|
|
progres=None,
|
|
checkpoint_every: int = 1,
|
|
pauza: float = 0.0,
|
|
) -> dict:
|
|
"""Genereaza/actualizeaza seedul. Intoarce statistici. Scrie `seed_path`.
|
|
|
|
`clasifica(batch_denumiri) -> list[cod]` e injectabil (teste); default = LM Studio.
|
|
`progres(mesaj)` e un callback optional de logare.
|
|
|
|
Checkpointing (`checkpoint_every` batch-uri): seedul se scrie pe disc periodic in
|
|
timpul rularii, NU doar la final. Esential pe GPU-box-ul instabil (shutdown termic
|
|
sub sarcina, memorie lmstudio-gpu-etichetare): un crash la batch-ul 80/104 pastreaza
|
|
progresul, iar re-run-ul continua din cache (idempotenta cross-run). 0 = doar la final.
|
|
"""
|
|
freq_raw = _freq_raw(csv_paths)
|
|
corpus = _corpus_din_freq(freq_raw)
|
|
seed_existent = _incarca_seed(seed_path)
|
|
harta = construieste_harta_etichete(freq_raw, corpus, labels_path, seed_existent)
|
|
de_etichetat = selecteaza_de_etichetat(
|
|
corpus, harta, target_volum=target_volum, etichetare_all=etichetare_all
|
|
)
|
|
reused = len(harta)
|
|
|
|
brute = int(sum(freq_raw.values()))
|
|
if progres:
|
|
progres(f"{len(freq_raw)} randuri brute distincte -> {len(corpus)} dupa normalizare "
|
|
f"-> {len(de_etichetat)} trimise la LLM (deja: {len(harta)})")
|
|
|
|
clasif = clasifica
|
|
if clasif is None:
|
|
et = _load_eticheteaza()
|
|
backend = et.get_backend()
|
|
if progres:
|
|
progres(f"backend={backend.name} url={backend.url} model={backend.model}")
|
|
|
|
def clasif(batch_denumiri):
|
|
return et.call(batch_denumiri, backend)[0]
|
|
|
|
apeluri = 0
|
|
valide = _valid_labels()
|
|
nr_batch = (len(de_etichetat) + batch - 1) // batch
|
|
for k in range(0, len(de_etichetat), batch):
|
|
chunk = de_etichetat[k:k + batch]
|
|
denumiri = [corpus[c]["denumire"] for c in chunk]
|
|
codes = clasif(denumiri)
|
|
apeluri += 1
|
|
for cheie, cod in zip(chunk, codes):
|
|
cod = str(cod or "").strip().upper()
|
|
if cod in valide: # '?' / cod invalid -> ramane ne-etichetat (retry la urmatorul run)
|
|
harta[cheie] = cod
|
|
if progres:
|
|
progres(f" batch {apeluri}/{nr_batch} "
|
|
f"-> total etichetat {sum(1 for c in harta if c in corpus)}")
|
|
# Checkpoint periodic: protejeaza progresul pe GPU-box instabil.
|
|
if checkpoint_every and apeluri % checkpoint_every == 0:
|
|
_scrie_seed(seed_path, _construieste_seed(corpus, harta, confidence=confidence, source=source))
|
|
# Pauza intre batch-uri: ragaz termic pentru GPU-box (shutdown sub sarcina sustinuta).
|
|
if pauza and k + batch < len(de_etichetat):
|
|
import time as _t
|
|
_t.sleep(pauza)
|
|
|
|
seed = _construieste_seed(corpus, harta, confidence=confidence, source=source)
|
|
_scrie_seed(seed_path, seed)
|
|
return {
|
|
"brute": brute,
|
|
"distincte": len(corpus),
|
|
"deja_etichetate": reused,
|
|
"de_etichetat": len(de_etichetat),
|
|
"apeluri_llm": apeluri,
|
|
"seed": len(seed),
|
|
}
|
|
|
|
|
|
def _valid_labels() -> set[str]:
|
|
et = _load_eticheteaza()
|
|
return set(et.ALL_LABELS)
|
|
|
|
|
|
def _construieste_seed(corpus, harta, *, confidence, source) -> list[dict]:
|
|
"""Seed ordonat determinist (pe cheie) -> byte-stabil intre rulari."""
|
|
out = []
|
|
for cheie in sorted(harta):
|
|
if cheie not in corpus:
|
|
continue # eticheta fara corespondent in corpusul curent
|
|
eticheta = harta[cheie]
|
|
is_nul = eticheta == NUL_LABEL
|
|
out.append({
|
|
"denumire": corpus[cheie]["denumire"],
|
|
"denumire_normalizata": cheie,
|
|
"cod": None if is_nul else eticheta,
|
|
"is_nul": is_nul,
|
|
"source": source,
|
|
"confidence": confidence,
|
|
})
|
|
return out
|
|
|
|
|
|
def _scrie_seed(seed_path: str, seed: list[dict]) -> None:
|
|
os.makedirs(os.path.dirname(os.path.abspath(seed_path)), exist_ok=True)
|
|
with open(seed_path, "w", encoding="utf-8") as fh:
|
|
json.dump(seed, fh, ensure_ascii=False, indent=2)
|
|
fh.write("\n")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CLI #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description="Genereaza seed etichetat operatie->cod (LM Studio).")
|
|
ap.add_argument("--target-volum", type=float, default=0.9,
|
|
help="prag de acoperire pe volum (default 0.9 = D1)")
|
|
ap.add_argument("--all", action="store_true", help="eticheteaza tot corpusul, ignora pragul")
|
|
ap.add_argument("--batch", type=int, default=32, help="dimensiune batch (conservator: 32-40)")
|
|
ap.add_argument("--pauza", type=float, default=1.5,
|
|
help="secunde de pauza intre batch-uri (ragaz termic GPU); 0 = fara")
|
|
ap.add_argument("--checkpoint-every", type=int, default=1,
|
|
help="scrie seedul la fiecare N batch-uri (1 = dupa fiecare, crash-safe)")
|
|
ap.add_argument("--confidence", type=float, default=DEFAULT_CONFIDENCE)
|
|
ap.add_argument("--csv-glob", default=DEFAULT_CSV_GLOB)
|
|
ap.add_argument("--labels", default=DEFAULT_LABELS)
|
|
ap.add_argument("--seed", default=DEFAULT_SEED)
|
|
args = ap.parse_args(argv)
|
|
|
|
csv_paths = sorted(glob.glob(args.csv_glob))
|
|
if not csv_paths:
|
|
ap.error(f"niciun CSV gasit la {args.csv_glob}")
|
|
|
|
stats = genereaza(
|
|
csv_paths,
|
|
labels_path=args.labels,
|
|
seed_path=args.seed,
|
|
target_volum=args.target_volum,
|
|
etichetare_all=args.all,
|
|
batch=args.batch,
|
|
pauza=args.pauza,
|
|
checkpoint_every=args.checkpoint_every,
|
|
confidence=args.confidence,
|
|
progres=lambda m: print(m, flush=True),
|
|
)
|
|
print("GATA:", json.dumps(stats, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|