#!/usr/bin/env python3 """(Re)construieste rag_index.json din toate documentele din depozit (store.py), cu embeddings Ollama. Rulat manual sau declansat din dashboard (`/api/reindex`).""" from __future__ import annotations import json import re import sys import requests import config import store def chunk_text(text: str) -> list[str]: # imparte pe linii goale in paragrafe, uneste bucatile mici cu urmatoarea raw_parts = re.split(r"\n\s*\n", text.strip()) chunks: list[str] = [] buffer = "" for part in raw_parts: part = part.strip() if not part: continue buffer = f"{buffer}\n\n{part}" if buffer else part if len(buffer) >= 200: chunks.append(buffer) buffer = "" if buffer: chunks.append(buffer) return chunks def embed(text: str) -> list[float]: resp = requests.post( f"{config.get('OLLAMA_URL')}/api/embeddings", json={"model": config.get("EMBED_MODEL"), "prompt": text}, timeout=60, ) resp.raise_for_status() return resp.json()["embedding"] def build() -> dict: entries = [] docs = store.list_documents() for doc in docs: text = store.read_document(doc["name"]) for i, chunk in enumerate(chunk_text(text)): vec = embed(chunk) entries.append({"source": doc["name"], "chunk": i, "text": chunk, "embedding": vec}) config.STATE_DIR.mkdir(parents=True, exist_ok=True) config.INDEX_FILE.write_text(json.dumps(entries, ensure_ascii=False), encoding="utf-8") return {"documents": len(docs), "chunks": len(entries)} if __name__ == "__main__": result = build() print( f"[indexer] {result['documents']} documente, {result['chunks']} chunk-uri -> {config.INDEX_FILE}", file=sys.stderr, )