feat(embeddings): cache persistent de vectori in SQLite + warmup in fundal
Vectorii corpusului k-NN persista in tabela embedding_cache (PK model+text_hash,
blob float32 LE); la warmup se vectorizeaza doar textele lipsa din cache, deci
restartul cu corpus neschimbat nu mai plateste ~1-2 min de embed (embed=0).
- app/embedding_cache.py: serializare array('f'), load/save/purge chunk 500 cu
BEGIN/COMMIT explicit (conexiuni autocommit), validare dimensiune la scriere
si citire, orchestrare sync_corpus_vectors cu embed_fn injectat
- index_corpus(vectors=): vectori precalculati cu validare aliniere; mismatch
-> fallback embed complet
- ensure_embeddings_corpus: warmup in thread la startup (block=True), calea de
request ne-blocanta (acquire non-blocking pe lock; warmup in curs -> return
imediat); purjare orfane + modele vechi doar dupa indexare reusita
- log warmup: cache=N embed=M in Xs
- 31 teste noi (cold/warm/incremental, model schimbat, concurenta, ranking
exact, echivalenta float32); suita completa 1596 passed
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Agent
277
tests/test_embedding_cache.py
Normal file
277
tests/test_embedding_cache.py
Normal file
@@ -0,0 +1,277 @@
|
||||
"""Teste pentru app/embedding_cache.py -- cache persistent de vectori in SQLite.
|
||||
|
||||
Acopera: round-trip serializare, blob corupt = miss, validare dimensiune la scriere,
|
||||
chunking >500 randuri, save partial esuat, purge orfane + modele vechi, purge NU
|
||||
ruleaza cand embed_fn esueaza.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from array import array
|
||||
|
||||
import pytest
|
||||
|
||||
from app.embedding_cache import (
|
||||
EMB_DIM,
|
||||
blob_to_vector,
|
||||
load_cached_vectors,
|
||||
purge_stale,
|
||||
save_vectors,
|
||||
sync_corpus_vectors,
|
||||
text_hash,
|
||||
vector_to_blob,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def conn(monkeypatch):
|
||||
tmp = tempfile.mkdtemp()
|
||||
monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "embcache.db"))
|
||||
monkeypatch.setenv("AUTOPASS_WEB_AUTH_REQUIRED", "false")
|
||||
monkeypatch.setenv("AUTOPASS_EMBEDDINGS_ENABLED", "true") # A13d: anti-vacuos
|
||||
from app.config import get_settings
|
||||
get_settings.cache_clear()
|
||||
from app.db import init_db, get_connection
|
||||
init_db()
|
||||
c = get_connection()
|
||||
yield c
|
||||
c.close()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def _vec(seed: float = 1.0, dim: int = EMB_DIM) -> list[float]:
|
||||
return [seed + i * 0.001 for i in range(dim)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Serializare #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_roundtrip_vector_to_blob_blob_to_vector():
|
||||
v = _vec(3.5)
|
||||
blob = vector_to_blob(v)
|
||||
assert isinstance(blob, bytes)
|
||||
assert len(blob) == EMB_DIM * 4
|
||||
out = blob_to_vector(blob)
|
||||
assert isinstance(out, array)
|
||||
assert out == pytest.approx(v, rel=1e-6)
|
||||
|
||||
|
||||
def test_text_hash_deterministic_and_distinct():
|
||||
assert text_hash("SCHIMB ULEI") == text_hash("SCHIMB ULEI")
|
||||
assert text_hash("SCHIMB ULEI") != text_hash("SCHIMB FILTRU")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# load_cached_vectors #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_load_cached_vectors_roundtrip(conn):
|
||||
h = text_hash("SCHIMB ULEI")
|
||||
save_vectors(conn, "model-a", [(h, _vec(1.0))])
|
||||
out = load_cached_vectors(conn, "model-a", [h])
|
||||
assert h in out
|
||||
assert out[h] == pytest.approx(_vec(1.0), rel=1e-6)
|
||||
|
||||
|
||||
def test_load_cached_vectors_miss_on_missing_hash(conn):
|
||||
out = load_cached_vectors(conn, "model-a", [text_hash("NECUNOSCUT")])
|
||||
assert out == {}
|
||||
|
||||
|
||||
def test_load_cached_vectors_blob_corupt_e_miss(conn):
|
||||
h = text_hash("SCHIMB ULEI")
|
||||
conn.execute("BEGIN")
|
||||
conn.execute(
|
||||
"INSERT INTO embedding_cache (text_hash, model, vector) VALUES (?, ?, ?)",
|
||||
(h, "model-a", b"\x00\x01\x02"), # lungime gresita
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
out = load_cached_vectors(conn, "model-a", [h])
|
||||
assert h not in out
|
||||
|
||||
|
||||
def test_load_cached_vectors_scoped_pe_model(conn):
|
||||
h = text_hash("SCHIMB ULEI")
|
||||
save_vectors(conn, "model-a", [(h, _vec(1.0))])
|
||||
out = load_cached_vectors(conn, "model-b", [h])
|
||||
assert out == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# save_vectors: validare dimensiune + chunking #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_save_vectors_respinge_dimensiune_gresita(conn):
|
||||
h = text_hash("SCHIMB ULEI")
|
||||
save_vectors(conn, "model-a", [(h, [1.0, 2.0, 3.0])]) # nu e EMB_DIM
|
||||
out = load_cached_vectors(conn, "model-a", [h])
|
||||
assert h not in out
|
||||
|
||||
|
||||
def test_save_vectors_chunking_peste_500_randuri(conn):
|
||||
items = [(text_hash(f"text-{i}"), _vec(float(i))) for i in range(1200)]
|
||||
save_vectors(conn, "model-a", items)
|
||||
hashes = [h for h, _ in items]
|
||||
out = load_cached_vectors(conn, "model-a", hashes)
|
||||
assert len(out) == 1200
|
||||
for h, vec in items:
|
||||
assert out[h] == pytest.approx(vec, rel=1e-6)
|
||||
|
||||
|
||||
def test_save_vectors_insert_or_replace_idempotent(conn):
|
||||
h = text_hash("SCHIMB ULEI")
|
||||
save_vectors(conn, "model-a", [(h, _vec(1.0))])
|
||||
save_vectors(conn, "model-a", [(h, _vec(2.0))]) # populare intrerupta -> completare
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM embedding_cache WHERE model=? AND text_hash=?",
|
||||
("model-a", h),
|
||||
).fetchone()
|
||||
assert row["n"] == 1
|
||||
out = load_cached_vectors(conn, "model-a", [h])
|
||||
assert out[h] == pytest.approx(_vec(2.0), rel=1e-6)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# purge_stale #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_purge_stale_sterge_orfane_model_curent(conn):
|
||||
h1, h2 = text_hash("A"), text_hash("B")
|
||||
save_vectors(conn, "model-a", [(h1, _vec(1.0)), (h2, _vec(2.0))])
|
||||
purge_stale(conn, "model-a", corpus_hashes=[h1]) # h2 nu mai e in corpus
|
||||
out = load_cached_vectors(conn, "model-a", [h1, h2])
|
||||
assert h1 in out
|
||||
assert h2 not in out
|
||||
|
||||
|
||||
def test_purge_stale_sterge_modele_vechi(conn):
|
||||
h = text_hash("A")
|
||||
save_vectors(conn, "model-old", [(h, _vec(1.0))])
|
||||
save_vectors(conn, "model-a", [(h, _vec(2.0))])
|
||||
purge_stale(conn, "model-a", corpus_hashes=[h])
|
||||
assert load_cached_vectors(conn, "model-old", [h]) == {}
|
||||
assert h in load_cached_vectors(conn, "model-a", [h])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# sync_corpus_vectors #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_sync_corpus_vectors_cold_start(conn):
|
||||
calls = []
|
||||
|
||||
def embed_fn(texts):
|
||||
calls.append(list(texts))
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
texts = ["A", "B", "C"]
|
||||
vecs = sync_corpus_vectors(conn, "model-a", texts, embed_fn)
|
||||
assert len(vecs) == 3
|
||||
assert calls == [texts] # toate 3 lipsesc din cache -> toate trimise la embed
|
||||
hashes = [text_hash(t) for t in texts]
|
||||
cached = load_cached_vectors(conn, "model-a", hashes)
|
||||
assert len(cached) == 3
|
||||
|
||||
|
||||
def test_sync_corpus_vectors_warm_start_zero_embed_calls(conn):
|
||||
calls = []
|
||||
|
||||
def embed_fn(texts):
|
||||
calls.append(list(texts))
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
texts = ["A", "B", "C"]
|
||||
sync_corpus_vectors(conn, "model-a", texts, embed_fn)
|
||||
calls.clear()
|
||||
vecs2 = sync_corpus_vectors(conn, "model-a", texts, embed_fn)
|
||||
assert calls == [] # nimic nou de vectorizat
|
||||
assert len(vecs2) == 3
|
||||
|
||||
|
||||
def test_sync_corpus_vectors_incremental_un_text_nou(conn):
|
||||
calls = []
|
||||
|
||||
def embed_fn(texts):
|
||||
calls.append(list(texts))
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
sync_corpus_vectors(conn, "model-a", ["A", "B"], embed_fn)
|
||||
calls.clear()
|
||||
sync_corpus_vectors(conn, "model-a", ["A", "B", "C"], embed_fn)
|
||||
assert calls == [["C"]]
|
||||
|
||||
|
||||
def test_sync_corpus_vectors_nu_purjeaza_singur_apelantul_decide(conn):
|
||||
"""sync_corpus_vectors NU mai purjeaza -- e responsabilitatea apelantului, DUPA
|
||||
o indexare reusita (vezi ensure_embeddings_corpus). purge_stale ramane apelabil
|
||||
separat, explicit, de catre apelant."""
|
||||
def embed_fn(texts):
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
sync_corpus_vectors(conn, "model-a", ["A", "B"], embed_fn)
|
||||
sync_corpus_vectors(conn, "model-a", ["A"], embed_fn) # B disparut din corpus solicitat
|
||||
out = load_cached_vectors(conn, "model-a", [text_hash("A"), text_hash("B")])
|
||||
assert text_hash("A") in out
|
||||
assert text_hash("B") in out # nepurjat automat -- sync_corpus_vectors nu mai face asta
|
||||
|
||||
purge_stale(conn, "model-a", {text_hash("A")}) # apelantul purjeaza dupa indexare reusita
|
||||
out2 = load_cached_vectors(conn, "model-a", [text_hash("A"), text_hash("B")])
|
||||
assert text_hash("A") in out2
|
||||
assert text_hash("B") not in out2
|
||||
|
||||
|
||||
def test_sync_corpus_vectors_nu_purjeaza_cand_embed_fn_esueaza(conn):
|
||||
def embed_fn_ok(texts):
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
sync_corpus_vectors(conn, "model-a", ["A", "B"], embed_fn_ok)
|
||||
|
||||
def embed_fn_broken(texts):
|
||||
raise RuntimeError("model indisponibil")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
sync_corpus_vectors(conn, "model-a", ["A", "C"], embed_fn_broken)
|
||||
|
||||
# "B" nu a fost purjat -- sync_corpus_vectors nu purjeaza niciodata singur.
|
||||
out = load_cached_vectors(conn, "model-a", [text_hash("A"), text_hash("B")])
|
||||
assert text_hash("A") in out
|
||||
assert text_hash("B") in out
|
||||
|
||||
|
||||
class _LockingConn:
|
||||
"""Wrapper peste o conexiune reala: simuleaza `database is locked` la BEGIN
|
||||
(exercita try/except-ul din save_vectors, nu il ocoleste)."""
|
||||
|
||||
def __init__(self, real):
|
||||
self._real = real
|
||||
|
||||
def execute(self, sql, *a, **kw):
|
||||
if sql.strip() == "BEGIN":
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
return self._real.execute(sql, *a, **kw)
|
||||
|
||||
def executemany(self, *a, **kw):
|
||||
return self._real.executemany(*a, **kw)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
def test_sync_corpus_vectors_save_partial_esuat_continua_din_ram(conn):
|
||||
"""Daca save_vectors esueaza (ex. DB locked la BEGIN), vectorii noi tot se
|
||||
intorc din RAM -- indexarea continua, doar persistarea in cache rateaza."""
|
||||
locking = _LockingConn(conn)
|
||||
|
||||
def embed_fn(texts):
|
||||
return [_vec(float(i)) for i in range(len(texts))]
|
||||
|
||||
vecs = sync_corpus_vectors(locking, "model-a", ["A", "B"], embed_fn)
|
||||
assert len(vecs) == 2
|
||||
assert vecs[0] == pytest.approx(_vec(0.0), rel=1e-6)
|
||||
|
||||
# Nimic nu a fost persistat (BEGIN a esuat la fiecare chunk).
|
||||
out = load_cached_vectors(conn, "model-a", [text_hash("A"), text_hash("B")])
|
||||
assert out == {}
|
||||
@@ -166,6 +166,79 @@ def test_index_corpus_no_exception_on_backend_error():
|
||||
assert engine.suggest_nearest("CEVA") == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# index_corpus(vectors=) -- precalculati, aliniati cu items (A8) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_index_corpus_vectors_precalculati_nu_apeleaza_backend():
|
||||
"""Cand `vectors` e furnizat, backend-ul NU e apelat pentru corpus (embed=0)."""
|
||||
|
||||
class NoCallBackend:
|
||||
def embed(self, texts):
|
||||
raise AssertionError("backend.embed() NU trebuie apelat cand vectors e furnizat")
|
||||
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
vectors = [_vec("SCHIMB ULEI"), _vec("REPARATIE MOTOR")]
|
||||
engine = EmbeddingEngine(backend=NoCallBackend())
|
||||
engine.index_corpus(corpus, vectors=vectors)
|
||||
assert engine.has_corpus()
|
||||
|
||||
|
||||
def test_index_corpus_vectors_ranking_exact_warm_start():
|
||||
"""Vectori precalculati din 'cache' produc EXACT acelasi cod ca embed direct
|
||||
(nu doar non-empty) -- prinde o eventuala dezaliniere intre items si vectors."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI MOTOR", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE CUTIE VITEZE", "cod": "OE-1"},
|
||||
{"denumire": "VERIFICARE DIRECTIE VOLAN", "cod": "OE-4"},
|
||||
{"denumire": "INLOCUIT PLACUTE FRANA", "cod": "OE-2"},
|
||||
]
|
||||
vectors = [_vec(item["denumire"]) for item in corpus]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=vectors)
|
||||
|
||||
assert engine.suggest_nearest("SCHIMB ULEI MOTOR", top_k=1)[0]["cod"] == "OE-3"
|
||||
assert engine.suggest_nearest("VERIFICARE DIRECTIE VOLAN", top_k=1)[0]["cod"] == "OE-4"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_mismatch_lungime_fallback_embed_complet():
|
||||
"""len(vectors) != len(items) -> fallback pe embed complet (backend chemat), fara exceptie."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=[_vec("SCHIMB ULEI")]) # un singur vector pentru 2 itemi
|
||||
|
||||
assert engine.has_corpus()
|
||||
results = engine.suggest_nearest("SCHIMB ULEI", top_k=1)
|
||||
assert results and results[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_contine_none_fallback_embed_complet():
|
||||
"""Un `None` in `vectors` -> fallback pe embed complet, fara exceptie."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=[_vec("SCHIMB ULEI"), None])
|
||||
|
||||
assert engine.has_corpus()
|
||||
assert engine.suggest_nearest("SCHIMB ULEI", top_k=1)[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_none_e_comportamentul_existent():
|
||||
"""`vectors=None` (default) -> embed complet prin backend, ca inainte."""
|
||||
corpus = [{"denumire": "SCHIMB ULEI", "cod": "OE-3"}]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=None)
|
||||
assert engine.suggest_nearest("SCHIMB ULEI", top_k=1)[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# API la nivel de modul (singleton global) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
347
tests/test_embeddings_warmup_cache.py
Normal file
347
tests/test_embeddings_warmup_cache.py
Normal file
@@ -0,0 +1,347 @@
|
||||
"""Teste de flux warmup + cache persistent la nivelul `ensure_embeddings_corpus`
|
||||
(app/mapping.py): cold/warm start, incremental, model schimbat, hash pe lista
|
||||
filtrata, concurenta warmup/request sub lock (A9), echivalenta ranking cu
|
||||
toleranta float32 (A5).
|
||||
|
||||
Backend mock determinist (fara fastembed real). Toate testele seteaza explicit
|
||||
AUTOPASS_EMBEDDINGS_ENABLED=1 (A13d).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from app.embedding_cache import EMB_DIM
|
||||
|
||||
|
||||
def _det_vector(text: str, dim: int = EMB_DIM) -> list[float]:
|
||||
"""Vector determinist (384-dim) derivat din hash-ul textului. Suficient pentru
|
||||
ranking cosine in teste -- nu evaluam calitatea semantica, doar alinierea/cache-ul."""
|
||||
digest = hashlib.sha256(text.encode("utf-8")).digest()
|
||||
return [((digest[i % len(digest)] + i) % 256) / 255.0 for i in range(dim)]
|
||||
|
||||
|
||||
class CountingMockBackend:
|
||||
"""Backend determinist care numara textele primite la fiecare apel embed()."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def embed(self, texts):
|
||||
self.calls.append(list(texts))
|
||||
return [_det_vector(t) for t in texts]
|
||||
|
||||
@property
|
||||
def total_texts(self) -> int:
|
||||
return sum(len(c) for c in self.calls)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env(monkeypatch):
|
||||
tmp = tempfile.mkdtemp()
|
||||
monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "warmup.db"))
|
||||
monkeypatch.setenv("AUTOPASS_WEB_AUTH_REQUIRED", "false")
|
||||
monkeypatch.setenv("AUTOPASS_EMBEDDINGS_ENABLED", "true") # A13d: anti-vacuos
|
||||
from app.config import get_settings
|
||||
get_settings.cache_clear()
|
||||
from app.db import init_db
|
||||
init_db()
|
||||
yield monkeypatch
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def conn(env):
|
||||
from app.db import get_connection
|
||||
c = get_connection()
|
||||
yield c
|
||||
c.close()
|
||||
|
||||
|
||||
def _inject_engine(backend):
|
||||
import app.embeddings as emb
|
||||
from app.embeddings import EmbeddingEngine
|
||||
emb._engine = EmbeddingEngine(backend=backend)
|
||||
return emb
|
||||
|
||||
|
||||
def _seed_silver(conn, rows):
|
||||
"""rows = [(denumire_normalizata, cod, is_nul)]."""
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO mapping_suggestions "
|
||||
"(denumire_normalizata, cod_prestatie, is_nul, source, confidence) VALUES (?, ?, ?, 'llm_seed', 0.7)",
|
||||
rows,
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cold / warm / incremental start (US-005) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_cold_start_trimite_exact_n_texte_si_populeaza_cache(conn):
|
||||
backend = CountingMockBackend()
|
||||
emb = _inject_engine(backend)
|
||||
denumiri = ["SCHIMB ULEI MOTOR", "INLOCUIT PLACUTE FRANA", "VERIFICARE DIRECTIE"]
|
||||
_seed_silver(conn, [(d, "OE-1", 0) for d in denumiri])
|
||||
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend.total_texts == 3
|
||||
|
||||
from app.embedding_cache import load_cached_vectors, text_hash
|
||||
cached = load_cached_vectors(conn, emb.FASTEMBED_MODEL, [text_hash(d) for d in denumiri])
|
||||
assert len(cached) == 3
|
||||
|
||||
|
||||
def test_warm_start_al_doilea_proces_zero_texte_embed(conn):
|
||||
denumiri = ["SCHIMB ULEI MOTOR", "INLOCUIT PLACUTE FRANA"]
|
||||
_seed_silver(conn, [(d, "OE-1", 0) for d in denumiri])
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
|
||||
backend1 = CountingMockBackend()
|
||||
_inject_engine(backend1)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend1.total_texts == 2
|
||||
|
||||
# Simuleaza un al doilea proces: engine NOU, acelasi conn (cache-ul e in DB, nu in RAM).
|
||||
backend2 = CountingMockBackend()
|
||||
emb2 = _inject_engine(backend2)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend2.total_texts == 0
|
||||
assert emb2.has_corpus()
|
||||
|
||||
res = emb2.suggest_nearest("SCHIMB ULEI MOTOR", top_k=1)
|
||||
assert res and res[0]["cod"] == "OE-1"
|
||||
|
||||
|
||||
def test_incremental_un_rand_nou_trimite_exact_un_text(conn):
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
_seed_silver(conn, [("SCHIMB ULEI MOTOR", "OE-3", 0)])
|
||||
_inject_engine(CountingMockBackend())
|
||||
ensure_embeddings_corpus(conn)
|
||||
|
||||
_seed_silver(conn, [("INLOCUIT BATERIE", "OE-1", 0)])
|
||||
backend2 = CountingMockBackend()
|
||||
_inject_engine(backend2)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend2.total_texts == 1
|
||||
assert backend2.calls == [["INLOCUIT BATERIE"]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Model schimbat (US-004) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_model_schimbat_zero_hituri_cache_reindexare_completa_si_purjare(conn, monkeypatch):
|
||||
import app.embeddings as emb_module
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
|
||||
denumiri = ["SCHIMB ULEI MOTOR", "INLOCUIT PLACUTE FRANA"]
|
||||
_seed_silver(conn, [(d, "OE-1", 0) for d in denumiri])
|
||||
model_vechi = emb_module.FASTEMBED_MODEL
|
||||
|
||||
backend_old = CountingMockBackend()
|
||||
_inject_engine(backend_old)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend_old.total_texts == 2
|
||||
|
||||
monkeypatch.setattr(emb_module, "FASTEMBED_MODEL", "model-nou-v2")
|
||||
backend_new = CountingMockBackend()
|
||||
_inject_engine(backend_new)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend_new.total_texts == 2 # zero hit-uri sub noul model -> re-vectorizare integrala
|
||||
|
||||
from app.embedding_cache import load_cached_vectors, text_hash
|
||||
hashes = [text_hash(d) for d in denumiri]
|
||||
# Intrarile modelului vechi au fost purjate la reindexarea reusita sub noul model (A3).
|
||||
assert load_cached_vectors(conn, model_vechi, hashes) == {}
|
||||
assert len(load_cached_vectors(conn, "model-nou-v2", hashes)) == 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Hash pe lista FILTRATA (denumire goala exclusa din corpus) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_denumire_goala_nu_strica_alinierea_si_nu_produce_miss_permanent(conn):
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
_seed_silver(conn, [
|
||||
("", "OE-9", 0), # denumire_normalizata goala -- exclusa din corpus la filtrare
|
||||
("SCHIMB ULEI MOTOR", "OE-3", 0),
|
||||
("INLOCUIT PLACUTE FRANA", "OE-1", 0),
|
||||
])
|
||||
backend1 = CountingMockBackend()
|
||||
_inject_engine(backend1)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend1.total_texts == 2 # doar cele 2 randuri cu denumire nevida
|
||||
|
||||
backend2 = CountingMockBackend()
|
||||
emb2 = _inject_engine(backend2)
|
||||
ensure_embeddings_corpus(conn)
|
||||
assert backend2.total_texts == 0 # warm: fara miss permanent din cauza filtrarii
|
||||
assert emb2.suggest_nearest("SCHIMB ULEI MOTOR", top_k=1)[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Concurenta warmup/request sub lock de modul (A9) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_concurenta_doua_threaduri_fara_dubla_vectorizare(conn):
|
||||
class ReentrancyDetectingBackend:
|
||||
"""Detecteaza executie concurenta reala in embed(): daca lock-ul de modul
|
||||
NU serializeaza secventa hash->embed->save->purge, `max_active` ar depasi 1."""
|
||||
|
||||
def __init__(self):
|
||||
self.active = 0
|
||||
self.max_active = 0
|
||||
self.total_calls = 0
|
||||
self._guard = threading.Lock()
|
||||
|
||||
def embed(self, texts):
|
||||
with self._guard:
|
||||
self.active += 1
|
||||
self.max_active = max(self.max_active, self.active)
|
||||
self.total_calls += 1
|
||||
time.sleep(0.05) # largeste deliberat fereastra de suprapunere
|
||||
with self._guard:
|
||||
self.active -= 1
|
||||
return [_det_vector(t) for t in texts]
|
||||
|
||||
backend = ReentrancyDetectingBackend()
|
||||
emb = _inject_engine(backend)
|
||||
denumiri = ["SCHIMB ULEI MOTOR", "INLOCUIT PLACUTE FRANA"]
|
||||
_seed_silver(conn, [(d, "OE-1", 0) for d in denumiri])
|
||||
|
||||
from app.db import get_connection
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
|
||||
errors: list[Exception] = []
|
||||
|
||||
def _run():
|
||||
try:
|
||||
c = get_connection()
|
||||
try:
|
||||
ensure_embeddings_corpus(c, block=True)
|
||||
finally:
|
||||
c.close()
|
||||
except Exception as exc: # pragma: no cover - vizibil doar la regresie
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=_run) for _ in range(2)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(timeout=5)
|
||||
|
||||
assert not errors
|
||||
assert backend.max_active <= 1 # lock-ul de modul serializeaza cele doua treceri
|
||||
|
||||
from app.embedding_cache import load_cached_vectors, text_hash
|
||||
cached = load_cached_vectors(conn, emb.FASTEMBED_MODEL, [text_hash(d) for d in denumiri])
|
||||
assert len(cached) == 2 # niciun rand nou nu a fost purjat fals de trecerea concurenta
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Echivalenta ranking cu toleranta float32 (T6/A5) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_block_false_nu_asteapta_dupa_warmup_in_curs(conn):
|
||||
"""Calea de request (block=False) nu trebuie sa blocheze cat warmup-ul (block=True)
|
||||
tine lock-ul -- trebuie sa iasa imediat, nu sa astepte pana termina warmup-ul."""
|
||||
warmup_poate_continua = threading.Event()
|
||||
warmup_a_intrat_in_embed = threading.Event()
|
||||
|
||||
class SlowBackend:
|
||||
def embed(self, texts):
|
||||
warmup_a_intrat_in_embed.set()
|
||||
warmup_poate_continua.wait(timeout=5)
|
||||
return [_det_vector(t) for t in texts]
|
||||
|
||||
emb = _inject_engine(SlowBackend())
|
||||
_seed_silver(conn, [("SCHIMB ULEI MOTOR", "OE-1", 0)])
|
||||
|
||||
from app.db import get_connection
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
|
||||
warmup_thread = threading.Thread(
|
||||
target=lambda: ensure_embeddings_corpus(get_connection(), block=True)
|
||||
)
|
||||
warmup_thread.start()
|
||||
assert warmup_a_intrat_in_embed.wait(timeout=5), "warmup trebuia sa ajunga in embed()"
|
||||
|
||||
t0 = time.monotonic()
|
||||
ensure_embeddings_corpus(conn, block=False) # nu trebuie sa astepte lock-ul
|
||||
durata_request = time.monotonic() - t0
|
||||
assert durata_request < 1.0, "block=False nu are voie sa astepte warmup-ul in curs"
|
||||
assert not emb.has_corpus() # warmup-ul nu a terminat inca, request-ul a iesit fara sa faca nimic
|
||||
|
||||
warmup_poate_continua.set()
|
||||
warmup_thread.join(timeout=5)
|
||||
assert emb.has_corpus() # warmup-ul a terminat normal, neblocat de request
|
||||
|
||||
|
||||
def test_indexare_esuata_nu_purjeaza_cache_ul_vechi(conn):
|
||||
"""Cand embed-ul unui text NOU esueaza, indexarea nu se termina cu succes ->
|
||||
purge_stale nu trebuie sa ruleze, altfel randuri inca valide (disparute doar
|
||||
din setul CERUT, nu esecul lor) ar fi sterse fals din cache."""
|
||||
denumiri_initiale = ["SCHIMB ULEI MOTOR", "INLOCUIT PLACUTE FRANA"]
|
||||
_seed_silver(conn, [(d, "OE-1", 0) for d in denumiri_initiale])
|
||||
_inject_engine(CountingMockBackend())
|
||||
|
||||
from app.mapping import ensure_embeddings_corpus
|
||||
ensure_embeddings_corpus(conn)
|
||||
|
||||
from app.embedding_cache import load_cached_vectors, text_hash
|
||||
hashes_initiale = [text_hash(d) for d in denumiri_initiale]
|
||||
assert len(load_cached_vectors(conn, "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", hashes_initiale)) == 2
|
||||
|
||||
# Corpusul cerut se schimba: "INLOCUIT PLACUTE FRANA" dispare, apare un text nou
|
||||
# a carui vectorizare va esua -- indexarea intreaga trebuie sa rateze.
|
||||
conn.execute(
|
||||
"DELETE FROM mapping_suggestions WHERE denumire_normalizata=?",
|
||||
("INLOCUIT PLACUTE FRANA",),
|
||||
)
|
||||
conn.commit()
|
||||
_seed_silver(conn, [("TEXT NOU CARE ESUEAZA", "OE-2", 0)])
|
||||
|
||||
class BrokenOnNewText:
|
||||
def embed(self, texts):
|
||||
raise RuntimeError("model indisponibil pentru text nou")
|
||||
|
||||
emb = _inject_engine(BrokenOnNewText())
|
||||
ensure_embeddings_corpus(conn) # esueaza intern, prins de degradarea gratioasa
|
||||
|
||||
# Randul disparut din corpusul cerut RAMANE in cache -- purge nu a rulat.
|
||||
out = load_cached_vectors(conn, emb.FASTEMBED_MODEL, hashes_initiale)
|
||||
assert len(out) == 2
|
||||
|
||||
|
||||
def test_ranking_echivalent_index_direct_vs_din_cache_float32(conn):
|
||||
from app.embedding_cache import sync_corpus_vectors
|
||||
from app.embeddings import EmbeddingEngine
|
||||
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI MOTOR", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE CUTIE VITEZE", "cod": "OE-1"},
|
||||
{"denumire": "VERIFICARE DIRECTIE VOLAN", "cod": "OE-4"},
|
||||
{"denumire": "INLOCUIT PLACUTE FRANA", "cod": "OE-2"},
|
||||
]
|
||||
texts = [item["denumire"] for item in corpus]
|
||||
backend = CountingMockBackend()
|
||||
|
||||
engine_direct = EmbeddingEngine(backend=backend)
|
||||
engine_direct.index_corpus(corpus) # embed complet, vectori float64 in RAM
|
||||
|
||||
vectors_cache = sync_corpus_vectors(conn, "model-test-ranking", texts, backend.embed)
|
||||
engine_cache = EmbeddingEngine(backend=backend)
|
||||
engine_cache.index_corpus(corpus, vectors=vectors_cache) # round-trip float32 din cache
|
||||
|
||||
for query in ("SCHIMB ULEI", "VERIFICARE DIRECTIE"):
|
||||
r_direct = engine_direct.suggest_nearest(query, top_k=len(corpus))
|
||||
r_cache = engine_cache.suggest_nearest(query, top_k=len(corpus))
|
||||
assert [r["cod"] for r in r_direct] == [r["cod"] for r in r_cache]
|
||||
Reference in New Issue
Block a user