rar-autopass/tools/mapare-llm/knn_calibrate.py

"""Calibrare politica k-NN pentru sugestia embeddings (top-1 vs vot ponderat top-k).

Metodologie: leave-one-out pe corpusul SILVER (`mapping_suggestions`) folosind
vectorii persistati in `embedding_cache` (zero re-embedding). Pentru fiecare
exemplu: se exclude din corpus, se prezice din vecinii ramasi, se compara cu
eticheta lui SILVER (cod sau NUL).

ATENTIE la interpretare: SILVER e etichetat de LLM (~95% calitate la scara),
NU ground-truth uman — valorile absolute sunt optimiste; comparatia RELATIVA
intre politici si calibrarea pragului raman valide (aceeasi tinta pentru toate).

Politici evaluate:
  top1       — eticheta primului vecin (comportamentul curent din enrich_suggestions)
  vote<k>    — vot ponderat cu similaritatea pe top-k vecini peste prag;
               NUL e eticheta proprie (castiga -> supresie)

Metrice (per politica x prag):
  coverage    — % din exemple pentru care se emite o predictie (cod sau NUL)
  precizie    — % predictii corecte din cele emise
  cod-gresit  — % din TOATE exemplele unde s-a emis un COD diferit de gold
                (critic: cod gresit preselectat = risc FINALIZATA eronata)

Rulare:
    python3 tools/mapare-llm/knn_calibrate.py --db data/autopass.db
    python3 tools/mapare-llm/knn_calibrate.py --db data/autopass.db --sample 5000 --seed 42
"""
from __future__ import annotations

import argparse
import os
import random
import sqlite3
import sys

import numpy as np

_HERE = os.path.dirname(os.path.abspath(__file__))
_ROOT = os.path.abspath(os.path.join(_HERE, "..", ".."))
if _ROOT not in sys.path:
    sys.path.insert(0, _ROOT)

from app import embedding_cache as cache  # noqa: E402
from app.embeddings import FASTEMBED_MODEL  # noqa: E402

NUL = "NUL"


def load_corpus(db_path: str):
    """Incarca (etichete, matrice vectori) aliniate pozitional din SQLite."""
    conn = sqlite3.connect(db_path)
    conn.row_factory = sqlite3.Row
    rows = conn.execute(
        "SELECT denumire_normalizata, cod_prestatie, is_nul FROM mapping_suggestions "
        "WHERE denumire_normalizata IS NOT NULL"
    ).fetchall()
    vecmap = {
        h: np.frombuffer(b, dtype=np.float32)
        for h, b in conn.execute(
            "SELECT text_hash, vector FROM embedding_cache WHERE model=?", (FASTEMBED_MODEL,)
        )
    }
    labels: list[str] = []
    vecs: list[np.ndarray] = []
    lipsa = 0
    for r in rows:
        v = vecmap.get(cache.text_hash(r["denumire_normalizata"]))
        if v is None:
            lipsa += 1
            continue
        labels.append(NUL if r["is_nul"] else str(r["cod_prestatie"]))
        vecs.append(v)
    if lipsa:
        print(f"AVERTISMENT: {lipsa} exemple fara vector in cache (excluse)")
    matrix = np.vstack(vecs)
    norms = np.linalg.norm(matrix, axis=1)
    norms[norms == 0.0] = 1.0
    return labels, matrix / norms[:, None]


def predict_top1(nbr_labels: list[str], nbr_sims: np.ndarray, prag: float) -> str | None:
    """Politica curenta: eticheta celui mai apropiat vecin, daca trece pragul."""
    if nbr_sims[0] < prag:
        return None
    return nbr_labels[0]


def predict_vote(nbr_labels: list[str], nbr_sims: np.ndarray, prag: float) -> str | None:
    """Vot ponderat cu similaritatea pe vecinii peste prag."""
    scoruri: dict[str, float] = {}
    for lab, sim in zip(nbr_labels, nbr_sims):
        if sim < prag:
            continue
        scoruri[lab] = scoruri.get(lab, 0.0) + float(sim)
    if not scoruri:
        return None
    return max(scoruri, key=lambda k: scoruri[k])


def evalueaza(labels, unit, indices, k_max, politici, praguri):
    """LOO: pentru fiecare index, top-k_max vecini (fara el insusi), apoi
    aplica fiecare (politica, prag) pe aceiasi vecini. Intoarce metrice agregate."""
    stats = {(p, prag): {"emis": 0, "corect": 0, "cod_gresit": 0} for p in politici for prag in praguri}
    n = len(indices)
    for count, i in enumerate(indices, 1):
        sims = unit @ unit[i]
        sims[i] = -1.0  # exclude exemplul insusi (leave-one-out)
        top = np.argpartition(-sims, k_max)[:k_max]
        top = top[np.argsort(-sims[top])]
        nbr_labels = [labels[j] for j in top]
        nbr_sims = sims[top]
        gold = labels[i]
        for (pname, prag), st in stats.items():
            pred = politici[pname](nbr_labels, nbr_sims, prag)
            if pred is None:
                continue
            st["emis"] += 1
            if pred == gold:
                st["corect"] += 1
            elif pred != NUL:
                st["cod_gresit"] += 1  # a emis un COD gresit (sau cod in loc de NUL)
        if count % 2000 == 0:
            print(f"  ...{count}/{n}", file=sys.stderr)
    return stats, n


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--db", default=os.path.join(_ROOT, "data", "autopass.db"))
    ap.add_argument("--sample", type=int, default=0, help="LOO doar pe un esantion (0 = tot corpusul)")
    ap.add_argument("--seed", type=int, default=42)
    ap.add_argument("--k", type=int, default=5, help="k pentru politica de vot")
    args = ap.parse_args()

    labels, unit = load_corpus(args.db)
    print(f"corpus: {len(labels)} exemple ({sum(1 for l in labels if l == NUL)} NUL)")

    indices = list(range(len(labels)))
    if args.sample and args.sample < len(indices):
        random.Random(args.seed).shuffle(indices)
        indices = indices[: args.sample]
    print(f"evaluare LOO pe {len(indices)} exemple, k={args.k}")

    politici = {
        "top1": predict_top1,
        f"vote{args.k}": predict_vote,
    }
    praguri = [0.5, 0.7, 0.8, 0.85, 0.88, 0.9, 0.92, 0.95]
    stats, n = evalueaza(labels, unit, indices, args.k, politici, praguri)

    print(f"\n{'politica':<8} {'prag':>5} {'coverage':>9} {'precizie':>9} {'cod-gresit':>11}")
    for pname in politici:
        for prag in praguri:
            st = stats[(pname, prag)]
            cov = st["emis"] / n
            prec = (st["corect"] / st["emis"]) if st["emis"] else 0.0
            wc = st["cod_gresit"] / n
            print(f"{pname:<8} {prag:>5.2f} {cov:>8.1%} {prec:>8.1%} {wc:>10.2%}")


if __name__ == "__main__":
    main()