fix(memory): retry cu split la eroare de context Ollama in indexare KB
Chunk-urile mari (tabele dense cu IP-uri/cod) treceau de split-ul pe paragrafe dar depaseau limita reala de tokeni a all-minilm, nu doar un prag fix de caractere — 18/20 chunk-uri din infrastructure.md esuau silentios la fiecare heartbeat. embed_chunk() acum injumatateste textul recursiv la eroare 500 si retrimite pana reuseste. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -113,6 +113,28 @@ def get_embedding(text: str) -> list[float]:
|
||||
raise ConnectionError(f"Ollama API error: {e.response.status_code}")
|
||||
|
||||
|
||||
def embed_chunk(text: str, max_depth: int = 6) -> list[tuple[str, list[float]]]:
|
||||
"""Embed text, halving on Ollama context-length errors until it fits.
|
||||
|
||||
The model's context limit is token-based, not character-based, so dense
|
||||
content (tables, IPs, code) can overflow well under _CHUNK_MAX chars. This
|
||||
self-corrects instead of guessing a safe static threshold.
|
||||
"""
|
||||
try:
|
||||
return [(text, get_embedding(text))]
|
||||
except ConnectionError:
|
||||
if max_depth <= 0 or len(text) < _CHUNK_MIN:
|
||||
raise
|
||||
mid = len(text) // 2
|
||||
split_at = text.rfind("\n", 0, mid)
|
||||
if split_at <= 0:
|
||||
split_at = mid
|
||||
left, right = text[:split_at].strip(), text[split_at:].strip()
|
||||
if not left or not right:
|
||||
raise
|
||||
return embed_chunk(left, max_depth - 1) + embed_chunk(right, max_depth - 1)
|
||||
|
||||
|
||||
def serialize_embedding(embedding: list[float]) -> bytes:
|
||||
"""Pack floats to bytes for SQLite storage."""
|
||||
return struct.pack(f"{len(embedding)}f", *embedding)
|
||||
@@ -194,15 +216,17 @@ def index_file(file_path: Path) -> int:
|
||||
conn = get_db()
|
||||
try:
|
||||
conn.execute("DELETE FROM chunks WHERE file_path = ?", (rel_path,))
|
||||
for i, chunk_text in enumerate(chunks):
|
||||
embedding = get_embedding(chunk_text)
|
||||
i = 0
|
||||
for chunk_text in chunks:
|
||||
for sub_text, embedding in embed_chunk(chunk_text):
|
||||
conn.execute(
|
||||
"""INSERT INTO chunks (file_path, chunk_index, chunk_text, embedding, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
(rel_path, i, chunk_text, serialize_embedding(embedding), now),
|
||||
(rel_path, i, sub_text, serialize_embedding(embedding), now),
|
||||
)
|
||||
i += 1
|
||||
conn.commit()
|
||||
return len(chunks)
|
||||
return i
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user