feat(embeddings): cache persistent de vectori in SQLite + warmup in fundal
Vectorii corpusului k-NN persista in tabela embedding_cache (PK model+text_hash,
blob float32 LE); la warmup se vectorizeaza doar textele lipsa din cache, deci
restartul cu corpus neschimbat nu mai plateste ~1-2 min de embed (embed=0).
- app/embedding_cache.py: serializare array('f'), load/save/purge chunk 500 cu
BEGIN/COMMIT explicit (conexiuni autocommit), validare dimensiune la scriere
si citire, orchestrare sync_corpus_vectors cu embed_fn injectat
- index_corpus(vectors=): vectori precalculati cu validare aliniere; mismatch
-> fallback embed complet
- ensure_embeddings_corpus: warmup in thread la startup (block=True), calea de
request ne-blocanta (acquire non-blocking pe lock; warmup in curs -> return
imediat); purjare orfane + modele vechi doar dupa indexare reusita
- log warmup: cache=N embed=M in Xs
- 31 teste noi (cold/warm/incremental, model schimbat, concurenta, ranking
exact, echivalenta float32); suita completa 1596 passed
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -166,6 +166,79 @@ def test_index_corpus_no_exception_on_backend_error():
|
||||
assert engine.suggest_nearest("CEVA") == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# index_corpus(vectors=) -- precalculati, aliniati cu items (A8) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_index_corpus_vectors_precalculati_nu_apeleaza_backend():
|
||||
"""Cand `vectors` e furnizat, backend-ul NU e apelat pentru corpus (embed=0)."""
|
||||
|
||||
class NoCallBackend:
|
||||
def embed(self, texts):
|
||||
raise AssertionError("backend.embed() NU trebuie apelat cand vectors e furnizat")
|
||||
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
vectors = [_vec("SCHIMB ULEI"), _vec("REPARATIE MOTOR")]
|
||||
engine = EmbeddingEngine(backend=NoCallBackend())
|
||||
engine.index_corpus(corpus, vectors=vectors)
|
||||
assert engine.has_corpus()
|
||||
|
||||
|
||||
def test_index_corpus_vectors_ranking_exact_warm_start():
|
||||
"""Vectori precalculati din 'cache' produc EXACT acelasi cod ca embed direct
|
||||
(nu doar non-empty) -- prinde o eventuala dezaliniere intre items si vectors."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI MOTOR", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE CUTIE VITEZE", "cod": "OE-1"},
|
||||
{"denumire": "VERIFICARE DIRECTIE VOLAN", "cod": "OE-4"},
|
||||
{"denumire": "INLOCUIT PLACUTE FRANA", "cod": "OE-2"},
|
||||
]
|
||||
vectors = [_vec(item["denumire"]) for item in corpus]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=vectors)
|
||||
|
||||
assert engine.suggest_nearest("SCHIMB ULEI MOTOR", top_k=1)[0]["cod"] == "OE-3"
|
||||
assert engine.suggest_nearest("VERIFICARE DIRECTIE VOLAN", top_k=1)[0]["cod"] == "OE-4"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_mismatch_lungime_fallback_embed_complet():
|
||||
"""len(vectors) != len(items) -> fallback pe embed complet (backend chemat), fara exceptie."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=[_vec("SCHIMB ULEI")]) # un singur vector pentru 2 itemi
|
||||
|
||||
assert engine.has_corpus()
|
||||
results = engine.suggest_nearest("SCHIMB ULEI", top_k=1)
|
||||
assert results and results[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_contine_none_fallback_embed_complet():
|
||||
"""Un `None` in `vectors` -> fallback pe embed complet, fara exceptie."""
|
||||
corpus = [
|
||||
{"denumire": "SCHIMB ULEI", "cod": "OE-3"},
|
||||
{"denumire": "REPARATIE MOTOR", "cod": "OE-1"},
|
||||
]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=[_vec("SCHIMB ULEI"), None])
|
||||
|
||||
assert engine.has_corpus()
|
||||
assert engine.suggest_nearest("SCHIMB ULEI", top_k=1)[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
def test_index_corpus_vectors_none_e_comportamentul_existent():
|
||||
"""`vectors=None` (default) -> embed complet prin backend, ca inainte."""
|
||||
corpus = [{"denumire": "SCHIMB ULEI", "cod": "OE-3"}]
|
||||
engine = EmbeddingEngine(backend=MockBackend())
|
||||
engine.index_corpus(corpus, vectors=None)
|
||||
assert engine.suggest_nearest("SCHIMB ULEI", top_k=1)[0]["cod"] == "OE-3"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# API la nivel de modul (singleton global) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
Reference in New Issue
Block a user