c514818ebe
Dictionarul de erori Oracle pus in `documents/` ar fi disparut in 10 minute: `rclone sync` sterge de acolo tot ce nu exista in Drive. Aceeasi problema o avea si butonul de adaugare document din dashboard — documentul traia pana la urmatorul tur de sincronizare, ceea ce README-ul mentiona ca pe o ciudatenie, nu ca pe un bug. Acum sunt doua directoare cu un singur spatiu de nume: `documents/` (oglinda Drive) si `documents-local/` (ce nu vine din Drive). La acelasi nume castiga Drive-ul — e sursa comuna a echipei, iar copia locala poate fi o versiune veche uitata acolo. Dashboard-ul scrie in cel local si marcheaza documentele „local". Recalibrat cu dictionarul indexat (169 chunk-uri): 19/19, cele patru intrebari Oracle noi ies la 0,73-0,75. Reindexarea a durat cat cele 29 de chunk-uri noi, nu cat toate 169 — refolosirea vectorilor isi face treaba (log: "29 embeddings noi, 140 refolosite"). Verificat pe canalul viu: o escaladare cu captura chiar pleaca pe WhatsApp, imagine + rezumat + referinta, si apare in dashboard. 73 pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Q4uzvgm7AyJch5WH8QHRhY
99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Sincronizeaza depozitul de documente cu un remote rclone (Google Drive) si
|
|
reconstruieste indexul RAG DOAR daca s-a schimbat efectiv ceva pe disc.
|
|
|
|
De ce prin rclone si nu direct cu Google Drive API: containerul e headless
|
|
(fara browser pentru OAuth interactiv) — rclone se configureaza o data cu un
|
|
cont de serviciu (`rclone config`, tip `drive`, `service_account_file=...`),
|
|
vezi README.md, sectiunea "Sincronizare cu Google Drive".
|
|
|
|
Config (`~/.maria-bridge/env`, vezi ops/env.example):
|
|
DRIVE_REMOTE - tinta rclone, ex: gdrive:romfast/document_store
|
|
(gol = sincronizare dezactivata, doar upload manual din dashboard)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import config
|
|
import indexer
|
|
import store
|
|
|
|
STATE_FILE_NAME = ".sync_state.json"
|
|
|
|
|
|
def _fingerprint() -> str:
|
|
"""Amprenta continutului depozitului (nume+mtime+marime), ca sa reindexam
|
|
doar cand s-a schimbat efectiv ceva, nu la fiecare tur de sincronizare."""
|
|
config.DOCS_DIR.mkdir(parents=True, exist_ok=True)
|
|
h = hashlib.sha256()
|
|
# amandoua directoarele: si un document local adaugat din dashboard trebuie sa
|
|
# declanseze reindexarea, nu doar ce vine din Drive
|
|
for f in store._all_files():
|
|
st = f.stat()
|
|
h.update(f"{f.name}:{st.st_mtime_ns}:{st.st_size}\n".encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def _sync_state_path():
|
|
return config.STATE_DIR / STATE_FILE_NAME
|
|
|
|
|
|
def read_sync_state() -> dict:
|
|
try:
|
|
return json.loads(_sync_state_path().read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def _write_state(fingerprint: str, extra: dict | None = None) -> None:
|
|
data = {"fingerprint": fingerprint, "synced_at": time.time()}
|
|
if extra:
|
|
data.update(extra)
|
|
config.STATE_DIR.mkdir(parents=True, exist_ok=True)
|
|
_sync_state_path().write_text(json.dumps(data), encoding="utf-8")
|
|
|
|
|
|
def pull_from_drive() -> dict:
|
|
"""`rclone sync <DRIVE_REMOTE> -> DOCS_DIR`. Fara remote configurat, e no-op."""
|
|
remote = config.get("DRIVE_REMOTE")
|
|
if not remote:
|
|
return {"ok": True, "skipped": "DRIVE_REMOTE nesetat in env"}
|
|
config.DOCS_DIR.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
r = subprocess.run(
|
|
["rclone", "sync", remote, str(config.DOCS_DIR)]
|
|
+ [arg for ext in store.DOC_EXTENSIONS for arg in ("--include", f"*{ext}")],
|
|
capture_output=True, text=True, timeout=300,
|
|
)
|
|
except FileNotFoundError:
|
|
return {"ok": False, "error": "rclone nu e instalat — vezi README.md"}
|
|
except subprocess.TimeoutExpired:
|
|
return {"ok": False, "error": "rclone a depasit timpul (300s)"}
|
|
return {"ok": r.returncode == 0, "stdout": r.stdout[-2000:], "stderr": r.stderr[-2000:]}
|
|
|
|
|
|
def sync_and_reindex(force: bool = False) -> dict:
|
|
pulled = pull_from_drive()
|
|
fp = _fingerprint()
|
|
last = read_sync_state().get("fingerprint")
|
|
if not force and fp == last:
|
|
return {"pulled": pulled, "reindexed": False, "reason": "fara schimbari"}
|
|
result = indexer.build()
|
|
_write_state(fp, {"last_build": result})
|
|
return {"pulled": pulled, "reindexed": True, "build": result}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
with config.exclusive():
|
|
out = sync_and_reindex(force="--force" in sys.argv[1:])
|
|
except config.Busy as exc:
|
|
out = {"skipped": str(exc)}
|
|
print(json.dumps(out, ensure_ascii=False), file=sys.stderr)
|