romfast/rar-autopass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf+calitate sugestii k-NN: matvec numpy, vot top-5 cu prag calibrat, indicatori import

Browse Source 
- embeddings: corpus ca matrice numpy cu norme precalculate; suggest_nearest
  = un matvec (~0.6ms/query fata de ~500ms cosine pur-Python la 17k vectori)
- enrich_suggestions: vot ponderat cu similaritatea pe top-5 vecini (NUL =
  eticheta proprie); prag 0.5 -> 0.88, calibrat LOO pe corpusul SILVER
  (tools/mapare-llm/knn_calibrate.py): precizie 90.5% -> 93.1%, cod gresit
  preselectat 7.2% -> 4.7%; sub prag abtinere -> preselectie fuzzy
- UI: codul sugerat de sistem afisat explicit cu sursa si scorul, separat de
  lista fuzzy ("potrivire text"); indicator de progres reparat pe upload
  (display:inline anula .htmx-indicator) si adaugat pe pasii 2->3 si
  "Salveaza maparile"

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Agent
2026-07-07 10:10:58 +00:00
parent 65b6e4694d
commit 809823a8ae
13 changed files with 358 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
app/embeddings.py
View File

@@ -25,6 +25,11 @@ import math
import threading
from typing import Protocol, runtime_checkable
try:
import numpy as _np # garantat de fastembed/onnxruntime; fallback pur-Python daca lipseste
except ImportError: # pragma: no cover
_np = None
log = logging.getLogger(__name__)
# Modelul ales: paraphrase-multilingual-MiniLM-L12-v2
@@ -94,6 +99,10 @@ class EmbeddingEngine:
self._corpus_vecs: list[list[float]] = []
self._corpus_items: list[dict] = []
self._corpus_sig: str | None = None
# Matrice numpy + norme precalculate: cautarea NN devine un matvec
# (~ms) in loc de cosine pur-Python per item (~0.5s la 17k vectori).
self._corpus_matrix = None
self._corpus_norms = None
def is_available(self) -> bool:
"""True daca backend-ul e disponibil si gata de folosire."""
@@ -128,6 +137,8 @@ class EmbeddingEngine:
self._corpus_vecs = []
self._corpus_items = []
self._corpus_sig = None
self._corpus_matrix = None
self._corpus_norms = None
if not items or not self.is_available():
return
@@ -147,10 +158,32 @@ class EmbeddingEngine:
self._corpus_vecs = self._backend.embed(texts)
self._corpus_items = list(items)
self._corpus_sig = signature
self._build_matrix()
except Exception as exc:
log.warning("embeddings: index_corpus esuat: %s", exc)
# corpus ramane gol -- suggest_nearest va returna []
def _build_matrix(self) -> None:
"""Precalculeaza matricea numpy + normele corpusului pentru cautarea NN.
Esecul (numpy lipsa, vectori neregulati) lasa matricea None --
suggest_nearest cade pe scanarea pur-Python, corpusul ramane valid.
"""
if _np is None:
return
try:
matrix = _np.asarray(self._corpus_vecs, dtype=_np.float32)
if matrix.ndim != 2:
raise ValueError(f"vectori corpus neregulati (ndim={matrix.ndim})")
norms = _np.linalg.norm(matrix, axis=1)
norms[norms == 0.0] = 1.0 # vector nul -> similaritate 0, fara div/0
self._corpus_matrix = matrix
self._corpus_norms = norms
except Exception as exc:
log.warning("embeddings: matrice numpy esuata, fallback pur-Python: %s", exc)
self._corpus_matrix = None
self._corpus_norms = None
def embed(self, texts: list[str]) -> list[list[float]]:
"""Vectorizeaza texte brute prin backend (folosit la miss-uri de cache).
@@ -179,6 +212,24 @@ class EmbeddingEngine:
try:
query_vecs = self._backend.embed([str(denumire)])
query_vec = query_vecs[0]
if self._corpus_matrix is not None:
q = _np.asarray(query_vec, dtype=_np.float32)
qn = float(_np.linalg.norm(q)) or 1.0
sims = (self._corpus_matrix @ q) / (self._corpus_norms * qn)
k = min(top_k, sims.shape[0])
if k <= 0:
return []
# argpartition + sort doar pe top_k: O(n) in loc de O(n log n)
idx = _np.argpartition(-sims, k - 1)[:k]
idx = idx[_np.argsort(-sims[idx])]
return [
{
"cod": self._corpus_items[i]["cod"],
"is_nul": bool(self._corpus_items[i].get("is_nul", False)),
"similaritate": float(sims[i]),
}
for i in idx
]
scored = [
{
"cod": item["cod"],

52
app/mapping.py
View File

@@ -631,10 +631,17 @@ def delete_text_rule(conn, account_id: int | None, pattern: str) -> None:
)
# Prag minim de similaritate cosine pentru sugestia din embeddings NN.
# Sub acest scor, sugestia NN e prea incerta si nu o afisam (previne recomandari
# irelevante cand corpus-ul e mic sau neindexat corect).
EMB_MIN_SIMILARITATE = 0.5
# Prag minim de similaritate cosine pentru un vecin k-NN luat in calcul la vot.
# Calibrat prin leave-one-out pe corpusul SILVER (tools/mapare-llm/knn_calibrate.py):
# modelul e anizotrop (perechi nelegate scoreaza 0.79-0.87), deci sub ~0.88 vecinul
# e zgomot. La 0.88: coverage 83%, precizie 93%; la 0.5 (vechiul prag): coverage
# 100% dar 7.4% cod gresit preselectat.
EMB_MIN_SIMILARITATE = 0.88
# Cati vecini intra in votul ponderat cu similaritatea. Vot > top-1: corpusul SILVER
# are etichete contradictorii pe denumiri aproape identice; votul e imun la ele
# (+~1pp precizie la acelasi coverage, masurat LOO).
EMB_VOTE_TOP_K = 5
# Protejeaza secventa hash->load->embed->save->purge->index (embedding_cache) de
# executie concurenta intre warmup-ul de fundal (block=True) si calea de request
@@ -769,7 +776,8 @@ def enrich_suggestions(
Returneaza:
{
'sugestie_principala': {'cod_prestatie': str, 'sursa': str} | None,
'surse': {'gold_partajat': str|None, 'silver': str|None, 'embedding': str|None, 'nul': bool}
'surse': {'gold_partajat': str|None, 'silver': str|None, 'embedding': str|None,
'embedding_similaritate': float|None, 'nul': bool}
}
INVARIANTE:
@@ -783,7 +791,10 @@ def enrich_suggestions(
importa normalize_for_match din mapping).
"""
sugestie_principala: dict | None = None
surse: dict = {"gold_partajat": None, "silver": None, "embedding": None, "nul": False}
surse: dict = {
"gold_partajat": None, "silver": None,
"embedding": None, "embedding_similaritate": None, "nul": False,
}
if not denumire:
return {"sugestie_principala": sugestie_principala, "surse": surse}
@@ -829,15 +840,28 @@ def enrich_suggestions(
# Corpusul k-NN e text NORMALIZAT (denumire_normalizata),
# deci query-ul TREBUIE normalizat la fel — altfel cosine degradeaza si
# nu mai e configul sub care s-a masurat 94.3%.
nn = _emb.suggest_nearest(normalize_for_match(denumire), top_k=1)
# Prag minim: similaritate prea mica = sugestie inutila.
# Evita recomandari irelevante cand corpus-ul e mic/partial.
if nn and nn[0].get("similaritate", 0) >= EMB_MIN_SIMILARITATE:
if nn[0].get("is_nul"):
# Vecin NUL (non-operatie) = semnal de SUPRESIE, nu cod.
nn = _emb.suggest_nearest(normalize_for_match(denumire), top_k=EMB_VOTE_TOP_K)
# Vot ponderat cu similaritatea pe vecinii peste prag; NUL e eticheta
# proprie (castiga -> supresie, nu cod). Vecinii sub prag nu voteaza.
scoruri: dict[str, float] = {}
sim_max: dict[str, float] = {}
for v in nn:
sim = float(v.get("similaritate", 0))
if sim < EMB_MIN_SIMILARITATE:
continue
lab = "NUL" if v.get("is_nul") else (str(v["cod"]) if v.get("cod") else None)
if lab is None:
continue
scoruri[lab] = scoruri.get(lab, 0.0) + sim
sim_max[lab] = max(sim_max.get(lab, 0.0), sim)
if scoruri:
castigator = max(scoruri, key=lambda k: scoruri[k])
if castigator == "NUL":
# Vecinatate NUL (non-operatie) = semnal de SUPRESIE, nu cod.
surse["nul"] = True
elif nn[0].get("cod"):
surse["embedding"] = str(nn[0]["cod"])
else:
surse["embedding"] = castigator
surse["embedding_similaritate"] = sim_max[castigator]
except Exception:
pass # degradare gratioasa (#16b): motorul absent nu blocheaza

10
app/web/routes.py
View File

@@ -1392,7 +1392,10 @@ def _nemapate_pentru_submission(row, nomenclator: list[dict], conn=None) -> list
"denumire": item.get("denumire"),
"suggestions": suggest_codes(item.get("denumire"), nomenclator, limit=5),
"sugestie_principala": None,
"surse_sugestie": {"gold_partajat": None, "silver": None, "embedding": None, "nul": False},
"surse_sugestie": {
"gold_partajat": None, "silver": None,
"embedding": None, "embedding_similaritate": None, "nul": False,
},
}
# L14-S6: imbogatire cu GOLD partajat > SILVER > embeddings (SUGGESTION-ONLY, #13)
if conn is not None:
@@ -3034,7 +3037,10 @@ def _collect_unmapped_ops(preview_rows: list[dict], nomenclator: list[dict], con
entry["suggestions"] = suggest_codes(entry["denumire"], nomenclator, limit=5)
# Init default pe FIECARE entry -> contract template identic (conn=None inclus).
entry["sugestie_principala"] = None
entry["surse_sugestie"] = {"gold_partajat": None, "silver": None, "embedding": None, "nul": False}
entry["surse_sugestie"] = {
"gold_partajat": None, "silver": None,
"embedding": None, "embedding_similaritate": None, "nul": False,
}
# L14-S6: imbogatire cu GOLD partajat > SILVER > embeddings (SUGGESTION-ONLY, #13)
if conn is not None:
enriched = enrich_suggestions(conn, entry["denumire"])

7
app/web/templates/_mapari.html
View File

@@ -62,10 +62,17 @@
{% else %}
<span class="sugg-sursa sugg-sursa--similar" title="operatie similara deja vazuta (k-NN/exact)">similar</span>
{% endif %}
{# Codul sugerat de sistem, explicit (nu doar preselectat in dropdown);
pentru k-NN si scorul de similaritate. #}
<strong>{{ e.sugestie_principala.cod_prestatie }}</strong>
{%- if e.sugestie_principala.sursa == 'embedding' and e.surse_sugestie.embedding_similaritate %}
({{ (e.surse_sugestie.embedding_similaritate * 100)|round|int }}%)
{%- endif %}
{% elif e.surse_sugestie and e.surse_sugestie.nul %}
<span class="sugg-sursa sugg-sursa--nul" title="pare non-operatie (ITP/plata/discount...)">non-operatie</span>
{% endif %}
{% if e.suggestions %}
{% if e.sugestie_principala or (e.surse_sugestie and e.surse_sugestie.nul) %}<span class="muted">· potrivire text:</span>{% endif %}
{% for s in e.suggestions[:3] %}
<span class="sugg">{{ s.cod_prestatie }} ({{ s.score|round|int }}%)</span>{% if not loop.last %}, {% endif %}
{% endfor %}

8
app/web/templates/_mapcoloane.html
View File

@@ -65,7 +65,8 @@
<form hx-post="/_import/{{ import_id }}/mapare-coloane"
hx-target="#import-section"
hx-swap="outerHTML">
hx-swap="outerHTML"
hx-indicator="#mapcol-spinner">
<input type="hidden" name="csrf_token" value="{{ csrf_token or '' }}">
<div style="margin-bottom:8px; display:flex; align-items:center; gap:10px; flex-wrap:wrap;">
@@ -162,6 +163,11 @@
style="min-height:44px; padding:10px 24px; font-size:var(--fs-md);{% if not prima_inreg %} opacity:0.5; cursor:not-allowed;{% endif %}">
Salveaza si continua la preview
</button>
<span id="mapcol-spinner" class="htmx-indicator muted"
style="font-size:var(--fs-sm);" role="status">
<span class="spin-dot" aria-hidden="true"></span>
se verifica randurile si se calculeaza sugestiile...
</span>
{% if not prima_inreg %}
<span style="font-size:var(--fs-xs); color:var(--err);">
Fisierul nu contine randuri de date — incarca un fisier cu cel putin o inregistrare.

17
app/web/templates/_preview_import.html
View File

@@ -81,7 +81,8 @@
<span class="s-ok">ok</span> si maparea se retine pentru fisierele viitoare.
</p>
<form hx-post="/_import/{{ import_id }}/mapare-operatii"
hx-target="#import-section" hx-swap="outerHTML">
hx-target="#import-section" hx-swap="outerHTML"
hx-indicator="#mapop-spinner">
<input type="hidden" name="csrf_token" value="{{ csrf_token or '' }}">
{% for e in unmapped_ops %}
{%- set top = e.suggestions[0] if e.suggestions else None -%}
@@ -106,9 +107,16 @@
{% else %}
<span class="sugg-sursa sugg-sursa--similar" title="operatie similara deja vazuta (k-NN/exact)">similar</span>
{% endif %}
{# Codul sugerat de sistem, explicit (nu doar preselectat in dropdown);
pentru k-NN si scorul de similaritate. #}
<strong>{{ e.sugestie_principala.cod_prestatie }}</strong>
{%- if e.sugestie_principala.sursa == 'embedding' and e.surse_sugestie.embedding_similaritate %}
({{ (e.surse_sugestie.embedding_similaritate * 100)|round|int }}%)
{%- endif %}
{% elif e.surse_sugestie and e.surse_sugestie.nul %}
<span class="sugg-sursa sugg-sursa--nul" title="pare non-operatie (ITP/plata/discount...)">non-operatie</span>
{% endif %}
{% if e.suggestions and (e.sugestie_principala or (e.surse_sugestie and e.surse_sugestie.nul)) %}<span class="muted">· potrivire text:</span>{% endif %}
{% for s in e.suggestions[:3] %}
<span class="sugg">{{ s.cod_prestatie }} ({{ s.score|round|int }}%)</span>{% if not loop.last %}, {% endif %}
{% endfor %}
@@ -127,8 +135,13 @@
</div>
</div>
{% endfor %}
<div style="margin-top:12px;">
<div style="margin-top:12px; display:flex; align-items:center; gap:12px;">
<button type="submit" style="min-height:44px;">Salveaza maparile</button>
<span id="mapop-spinner" class="htmx-indicator muted"
style="font-size:var(--fs-sm);" role="status">
<span class="spin-dot" aria-hidden="true"></span>
se salveaza si se recalculeaza preview-ul...
</span>
</div>
</form>
</div>

7
app/web/templates/_upload.html
View File

@@ -122,9 +122,12 @@
</p>
{% endif %}
{# FARA display in stilul inline: ar suprascrie .htmx-indicator{display:none}
si indicatorul ar ramane vizibil permanent (deci invizibil ca progres). #}
<span id="upload-spinner" class="htmx-indicator muted"
style="font-size:var(--fs-sm); margin-top:6px; display:inline;">
se parseaza fisierul...
style="font-size:var(--fs-sm); margin-top:6px;" role="status">
<span class="spin-dot" aria-hidden="true"></span>
se parseaza fisierul si se pregatesc sugestiile...
</span>
</form>
</div>

5
app/web/templates/base.html
View File

@@ -176,6 +176,11 @@
flex-wrap:wrap; z-index:10; }
/* Indicator HTMX — ascuns pana la request */
.htmx-indicator { display:none; }
/* Cerc rotitor pentru indicatorii de procesare (import: parsare + sugestii) */
.spin-dot { display:inline-block; width:13px; height:13px; border:2px solid var(--line);
border-top-color:var(--accent); border-radius:50%; vertical-align:-2px;
margin-right:6px; animation:spin-dot .7s linear infinite; }
@keyframes spin-dot { to { transform:rotate(360deg); } }
/* Selector tema stil pill — icon + eticheta temei curente.
Eticheta se ascunde pe <=560px (spatiu ingust), ramane iconita. */
.tema-btn { display:inline-flex; align-items:center; gap:6px; height:36px; padding:0 12px;