fix(memory): retry cu split la eroare de context Ollama in indexare KB

Chunk-urile mari (tabele dense cu IP-uri/cod) treceau de split-ul pe
paragrafe dar depaseau limita reala de tokeni a all-minilm, nu doar
un prag fix de caractere — 18/20 chunk-uri din infrastructure.md
esuau silentios la fiecare heartbeat. embed_chunk() acum injumatateste
textul recursiv la eroare 500 si retrimite pana reuseste.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-27 17:30:29 +00:00
parent b97edd5184
commit 164ff32031

View File

@@ -113,6 +113,28 @@ def get_embedding(text: str) -> list[float]:
raise ConnectionError(f"Ollama API error: {e.response.status_code}")
def embed_chunk(text: str, max_depth: int = 6) -> list[tuple[str, list[float]]]:
"""Embed text, halving on Ollama context-length errors until it fits.
The model's context limit is token-based, not character-based, so dense
content (tables, IPs, code) can overflow well under _CHUNK_MAX chars. This
self-corrects instead of guessing a safe static threshold.
"""
try:
return [(text, get_embedding(text))]
except ConnectionError:
if max_depth <= 0 or len(text) < _CHUNK_MIN:
raise
mid = len(text) // 2
split_at = text.rfind("\n", 0, mid)
if split_at <= 0:
split_at = mid
left, right = text[:split_at].strip(), text[split_at:].strip()
if not left or not right:
raise
return embed_chunk(left, max_depth - 1) + embed_chunk(right, max_depth - 1)
def serialize_embedding(embedding: list[float]) -> bytes:
"""Pack floats to bytes for SQLite storage."""
return struct.pack(f"{len(embedding)}f", *embedding)
@@ -194,15 +216,17 @@ def index_file(file_path: Path) -> int:
conn = get_db()
try:
conn.execute("DELETE FROM chunks WHERE file_path = ?", (rel_path,))
for i, chunk_text in enumerate(chunks):
embedding = get_embedding(chunk_text)
conn.execute(
"""INSERT INTO chunks (file_path, chunk_index, chunk_text, embedding, updated_at)
VALUES (?, ?, ?, ?, ?)""",
(rel_path, i, chunk_text, serialize_embedding(embedding), now),
)
i = 0
for chunk_text in chunks:
for sub_text, embedding in embed_chunk(chunk_text):
conn.execute(
"""INSERT INTO chunks (file_path, chunk_index, chunk_text, embedding, updated_at)
VALUES (?, ?, ?, ?, ?)""",
(rel_path, i, sub_text, serialize_embedding(embedding), now),
)
i += 1
conn.commit()
return len(chunks)
return i
finally:
conn.close()