"""Ce intra in index (preferinta de format) si cum se taie XML-ul in chunk-uri.""" from __future__ import annotations import pytest import indexer import store XML = """ stocuri negative NIR Documentul nu poate fi sters: stocuri negative pentru BANDA HARTIE. Optiuni firma > VERIFICARESTOCNEGATIV = 0. Alta eroare. Alta rezolvare. """ # ------------------------------------------------------- preferinta de format def test_xml_umbreste_md_cu_acelasi_nume(write): write("d406.xml", "x") write("d406.md", "versiune veche") umbrite = {d["name"]: d["shadowed_by"] for d in store.list_documents()} assert umbrite["d406.md"] == "d406.xml" assert umbrite["d406.xml"] is None assert [d["name"] for d in store.documents_for_index()] == ["d406.xml"] def test_md_umbreste_txt(write): write("x.md", "md") write("x.txt", "txt") assert [d["name"] for d in store.documents_for_index()] == ["x.md"] def test_fisierele_fara_pereche_raman_toate(write): write("bilant.md", "a") write("roagest.xml", "b") write("note.txt", "c") assert len(store.documents_for_index()) == 3 def test_depozitul_e_vizibil_intreg_chiar_daca_nu_tot_se_indexeaza(write): write("d406.xml", "x") write("d406.md", "vechi") assert len(store.list_documents()) == 2 # dashboard le arata pe ambele assert len(store.documents_for_index()) == 1 # indexul ia doar una def test_extensiile_necunoscute_sunt_ignorate(write): write("chatflow.json", "{}") write("script.ps1", "echo") assert store.list_documents() == [] @pytest.mark.parametrize("name", ["a.xml", "a.md", "a.txt"]) def test_numele_acceptate(name): assert store.validate_name(name) == name @pytest.mark.parametrize("name", ["a.json", "a.pdf", "../a.md", "a b.md"]) def test_numele_respinse(name): with pytest.raises(ValueError): store.validate_name(name) # ----------------------------------------------------------- chunking XML def test_o_problema_ramane_un_singur_chunk(): """Miezul: mesajul de eroare si rezolvarea NU trebuie separate.""" chunks = indexer.chunk_document("x.xml", XML) assert len(chunks) == 2 assert "BANDA HARTIE" in chunks[0] and "VERIFICARESTOCNEGATIV" in chunks[0] assert "Alta eroare" in chunks[1] and "Alta rezolvare" in chunks[1] def test_etichetele_raman_ca_etichete_fara_paranteze(): chunk = indexer.chunk_document("x.xml", XML)[0] assert "mesaj eroare:" in chunk and "rezolvare:" in chunk assert "" not in chunk def test_entitatile_sunt_decodate(): assert ">" not in indexer.chunk_document("x.xml", XML)[0] def test_comentariile_nu_devin_chunk(): assert not any("comentariu" in c for c in indexer.chunk_document("x.xml", XML)) def test_xml_invalid_cade_pe_taierea_obisnuita(): chunks = indexer.chunk_document("stricat.xml", "") assert chunks == [""] def test_xml_fara_copii_e_tratat_ca_text(): assert indexer.chunk_document("x.xml", "doar text") == ["doar text"] def test_element_urias_e_taiat_mai_departe(): mare = "" + ("propozitie lunga. " * 500) + "" chunks = indexer.chunk_document("x.xml", mare) assert len(chunks) > 1 assert all(len(c) < indexer.MAX_XML_CHUNK * 2 for c in chunks) def test_md_foloseste_taierea_pe_paragrafe(): text = "primul paragraf\n\n" + "al doilea paragraf " * 20 assert len(indexer.chunk_document("x.md", text)) >= 1 assert indexer.chunk_document("x.md", text) == indexer.chunk_text(text) # --------------------------------------------- concurenta intre reindexari def test_lacatul_refuza_a_doua_reindexare(): """Timer-ul nu trebuie sa porneasca peste o sincronizare manuala.""" import config with config.exclusive(): with pytest.raises(config.Busy): with config.exclusive(): pass def test_lacatul_se_elibereaza_dupa_iesire(): import config with config.exclusive(): pass with config.exclusive(): # trebuie sa mearga din nou pass def test_lacatul_se_elibereaza_si_la_exceptie(): import config with pytest.raises(ValueError): with config.exclusive(): raise ValueError("ceva") with config.exclusive(): pass def test_indexul_se_scrie_atomic(write, monkeypatch): """Consumer-ul reciteste indexul la 30s; nu are voie sa prinda JSON pe jumatate.""" import config import indexer write("x.md", "un paragraf oarecare") monkeypatch.setattr(indexer, "embed", lambda text: [0.1, 0.2]) vazute = [] real_replace = indexer.os.replace def spion(src, dst): vazute.append((str(src), str(dst))) return real_replace(src, dst) monkeypatch.setattr(indexer.os, "replace", spion) indexer.build() assert vazute and vazute[0][1] == str(config.INDEX_FILE) assert not config.INDEX_FILE.with_suffix(".json.tmp").exists() def test_xml_invalid_e_raportat_ca_avertisment(write, monkeypatch): import indexer write("stricat.xml", "x") monkeypatch.setattr(indexer, "embed", lambda text: [0.0]) out = indexer.build() assert any("stricat.xml" in w for w in out.get("warnings", [])) def test_xml_valid_nu_produce_avertismente(write, monkeypatch): import indexer write("bun.xml", "x") monkeypatch.setattr(indexer, "embed", lambda text: [0.0]) assert "warnings" not in indexer.build() # --- refolosirea embeddings-urilor la reindexare ----------------------------- def test_vectorii_din_indexul_vechi_se_refolosesc(monkeypatch, write, tmp_path): """Un document nou nu are voie sa reforteze embeddings-urile celorlalte. Pe CPU un embedding costa ~7 secunde; fara refolosire, adaugarea unui document la un index de 170 de chunk-uri inseamna 20 de minute de asteptare. """ import json import config import indexer write("primul.md", "text vechi, deja indexat") calculate = [] def fals_embed(chunk): calculate.append(chunk) return [0.5, 0.5] monkeypatch.setattr(indexer, "embed", fals_embed) indexer.build() assert calculate == ["text vechi, deja indexat"] calculate.clear() write("al-doilea.md", "text nou") rezultat = indexer.build() # doar chunk-ul nou s-a calculat assert calculate == ["text nou"] assert rezultat["embeddings_noi"] == 1 assert rezultat["embeddings_refolosite"] == 1 index = json.loads(config.INDEX_FILE.read_text(encoding="utf-8")) assert {e["source"] for e in index} == {"primul.md", "al-doilea.md"} def test_textul_modificat_se_reembedeaza(monkeypatch, write): import indexer write("doc.md", "prima varianta") monkeypatch.setattr(indexer, "embed", lambda chunk: [1.0]) indexer.build() calculate = [] def fals_embed(chunk): calculate.append(chunk) return [2.0] monkeypatch.setattr(indexer, "embed", fals_embed) write("doc.md", "varianta modificata") rezultat = indexer.build() assert calculate == ["varianta modificata"] assert rezultat["embeddings_refolosite"] == 0