perf+calitate sugestii k-NN: matvec numpy, vot top-5 cu prag calibrat, indicatori import
- embeddings: corpus ca matrice numpy cu norme precalculate; suggest_nearest
= un matvec (~0.6ms/query fata de ~500ms cosine pur-Python la 17k vectori)
- enrich_suggestions: vot ponderat cu similaritatea pe top-5 vecini (NUL =
eticheta proprie); prag 0.5 -> 0.88, calibrat LOO pe corpusul SILVER
(tools/mapare-llm/knn_calibrate.py): precizie 90.5% -> 93.1%, cod gresit
preselectat 7.2% -> 4.7%; sub prag abtinere -> preselectie fuzzy
- UI: codul sugerat de sistem afisat explicit cu sursa si scorul, separat de
lista fuzzy ("potrivire text"); indicator de progres reparat pe upload
(display:inline anula .htmx-indicator) si adaugat pe pasii 2->3 si
"Salveaza maparile"
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,11 @@ import math
|
||||
import threading
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
try:
|
||||
import numpy as _np # garantat de fastembed/onnxruntime; fallback pur-Python daca lipseste
|
||||
except ImportError: # pragma: no cover
|
||||
_np = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Modelul ales: paraphrase-multilingual-MiniLM-L12-v2
|
||||
@@ -94,6 +99,10 @@ class EmbeddingEngine:
|
||||
self._corpus_vecs: list[list[float]] = []
|
||||
self._corpus_items: list[dict] = []
|
||||
self._corpus_sig: str | None = None
|
||||
# Matrice numpy + norme precalculate: cautarea NN devine un matvec
|
||||
# (~ms) in loc de cosine pur-Python per item (~0.5s la 17k vectori).
|
||||
self._corpus_matrix = None
|
||||
self._corpus_norms = None
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""True daca backend-ul e disponibil si gata de folosire."""
|
||||
@@ -128,6 +137,8 @@ class EmbeddingEngine:
|
||||
self._corpus_vecs = []
|
||||
self._corpus_items = []
|
||||
self._corpus_sig = None
|
||||
self._corpus_matrix = None
|
||||
self._corpus_norms = None
|
||||
|
||||
if not items or not self.is_available():
|
||||
return
|
||||
@@ -147,10 +158,32 @@ class EmbeddingEngine:
|
||||
self._corpus_vecs = self._backend.embed(texts)
|
||||
self._corpus_items = list(items)
|
||||
self._corpus_sig = signature
|
||||
self._build_matrix()
|
||||
except Exception as exc:
|
||||
log.warning("embeddings: index_corpus esuat: %s", exc)
|
||||
# corpus ramane gol -- suggest_nearest va returna []
|
||||
|
||||
def _build_matrix(self) -> None:
|
||||
"""Precalculeaza matricea numpy + normele corpusului pentru cautarea NN.
|
||||
|
||||
Esecul (numpy lipsa, vectori neregulati) lasa matricea None --
|
||||
suggest_nearest cade pe scanarea pur-Python, corpusul ramane valid.
|
||||
"""
|
||||
if _np is None:
|
||||
return
|
||||
try:
|
||||
matrix = _np.asarray(self._corpus_vecs, dtype=_np.float32)
|
||||
if matrix.ndim != 2:
|
||||
raise ValueError(f"vectori corpus neregulati (ndim={matrix.ndim})")
|
||||
norms = _np.linalg.norm(matrix, axis=1)
|
||||
norms[norms == 0.0] = 1.0 # vector nul -> similaritate 0, fara div/0
|
||||
self._corpus_matrix = matrix
|
||||
self._corpus_norms = norms
|
||||
except Exception as exc:
|
||||
log.warning("embeddings: matrice numpy esuata, fallback pur-Python: %s", exc)
|
||||
self._corpus_matrix = None
|
||||
self._corpus_norms = None
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Vectorizeaza texte brute prin backend (folosit la miss-uri de cache).
|
||||
|
||||
@@ -179,6 +212,24 @@ class EmbeddingEngine:
|
||||
try:
|
||||
query_vecs = self._backend.embed([str(denumire)])
|
||||
query_vec = query_vecs[0]
|
||||
if self._corpus_matrix is not None:
|
||||
q = _np.asarray(query_vec, dtype=_np.float32)
|
||||
qn = float(_np.linalg.norm(q)) or 1.0
|
||||
sims = (self._corpus_matrix @ q) / (self._corpus_norms * qn)
|
||||
k = min(top_k, sims.shape[0])
|
||||
if k <= 0:
|
||||
return []
|
||||
# argpartition + sort doar pe top_k: O(n) in loc de O(n log n)
|
||||
idx = _np.argpartition(-sims, k - 1)[:k]
|
||||
idx = idx[_np.argsort(-sims[idx])]
|
||||
return [
|
||||
{
|
||||
"cod": self._corpus_items[i]["cod"],
|
||||
"is_nul": bool(self._corpus_items[i].get("is_nul", False)),
|
||||
"similaritate": float(sims[i]),
|
||||
}
|
||||
for i in idx
|
||||
]
|
||||
scored = [
|
||||
{
|
||||
"cod": item["cod"],
|
||||
|
||||
Reference in New Issue
Block a user