perf+calitate sugestii k-NN: matvec numpy, vot top-5 cu prag calibrat, indicatori import

- embeddings: corpus ca matrice numpy cu norme precalculate; suggest_nearest
  = un matvec (~0.6ms/query fata de ~500ms cosine pur-Python la 17k vectori)
- enrich_suggestions: vot ponderat cu similaritatea pe top-5 vecini (NUL =
  eticheta proprie); prag 0.5 -> 0.88, calibrat LOO pe corpusul SILVER
  (tools/mapare-llm/knn_calibrate.py): precizie 90.5% -> 93.1%, cod gresit
  preselectat 7.2% -> 4.7%; sub prag abtinere -> preselectie fuzzy
- UI: codul sugerat de sistem afisat explicit cu sursa si scorul, separat de
  lista fuzzy ("potrivire text"); indicator de progres reparat pe upload
  (display:inline anula .htmx-indicator) si adaugat pe pasii 2->3 si
  "Salveaza maparile"

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Agent
2026-07-07 10:10:58 +00:00
parent 65b6e4694d
commit 809823a8ae
13 changed files with 358 additions and 24 deletions

View File

@@ -25,6 +25,11 @@ import math
import threading
from typing import Protocol, runtime_checkable
try:
import numpy as _np # garantat de fastembed/onnxruntime; fallback pur-Python daca lipseste
except ImportError: # pragma: no cover
_np = None
log = logging.getLogger(__name__)
# Modelul ales: paraphrase-multilingual-MiniLM-L12-v2
@@ -94,6 +99,10 @@ class EmbeddingEngine:
self._corpus_vecs: list[list[float]] = []
self._corpus_items: list[dict] = []
self._corpus_sig: str | None = None
# Matrice numpy + norme precalculate: cautarea NN devine un matvec
# (~ms) in loc de cosine pur-Python per item (~0.5s la 17k vectori).
self._corpus_matrix = None
self._corpus_norms = None
def is_available(self) -> bool:
"""True daca backend-ul e disponibil si gata de folosire."""
@@ -128,6 +137,8 @@ class EmbeddingEngine:
self._corpus_vecs = []
self._corpus_items = []
self._corpus_sig = None
self._corpus_matrix = None
self._corpus_norms = None
if not items or not self.is_available():
return
@@ -147,10 +158,32 @@ class EmbeddingEngine:
self._corpus_vecs = self._backend.embed(texts)
self._corpus_items = list(items)
self._corpus_sig = signature
self._build_matrix()
except Exception as exc:
log.warning("embeddings: index_corpus esuat: %s", exc)
# corpus ramane gol -- suggest_nearest va returna []
def _build_matrix(self) -> None:
"""Precalculeaza matricea numpy + normele corpusului pentru cautarea NN.
Esecul (numpy lipsa, vectori neregulati) lasa matricea None --
suggest_nearest cade pe scanarea pur-Python, corpusul ramane valid.
"""
if _np is None:
return
try:
matrix = _np.asarray(self._corpus_vecs, dtype=_np.float32)
if matrix.ndim != 2:
raise ValueError(f"vectori corpus neregulati (ndim={matrix.ndim})")
norms = _np.linalg.norm(matrix, axis=1)
norms[norms == 0.0] = 1.0 # vector nul -> similaritate 0, fara div/0
self._corpus_matrix = matrix
self._corpus_norms = norms
except Exception as exc:
log.warning("embeddings: matrice numpy esuata, fallback pur-Python: %s", exc)
self._corpus_matrix = None
self._corpus_norms = None
def embed(self, texts: list[str]) -> list[list[float]]:
"""Vectorizeaza texte brute prin backend (folosit la miss-uri de cache).
@@ -179,6 +212,24 @@ class EmbeddingEngine:
try:
query_vecs = self._backend.embed([str(denumire)])
query_vec = query_vecs[0]
if self._corpus_matrix is not None:
q = _np.asarray(query_vec, dtype=_np.float32)
qn = float(_np.linalg.norm(q)) or 1.0
sims = (self._corpus_matrix @ q) / (self._corpus_norms * qn)
k = min(top_k, sims.shape[0])
if k <= 0:
return []
# argpartition + sort doar pe top_k: O(n) in loc de O(n log n)
idx = _np.argpartition(-sims, k - 1)[:k]
idx = idx[_np.argsort(-sims[idx])]
return [
{
"cod": self._corpus_items[i]["cod"],
"is_nul": bool(self._corpus_items[i].get("is_nul", False)),
"similaritate": float(sims[i]),
}
for i in idx
]
scored = [
{
"cod": item["cod"],