Files
rar-autopass/app/main.py
Claude Agent 705ad030fe feat(embeddings): cache persistent de vectori in SQLite + warmup in fundal
Vectorii corpusului k-NN persista in tabela embedding_cache (PK model+text_hash,
blob float32 LE); la warmup se vectorizeaza doar textele lipsa din cache, deci
restartul cu corpus neschimbat nu mai plateste ~1-2 min de embed (embed=0).

- app/embedding_cache.py: serializare array('f'), load/save/purge chunk 500 cu
  BEGIN/COMMIT explicit (conexiuni autocommit), validare dimensiune la scriere
  si citire, orchestrare sync_corpus_vectors cu embed_fn injectat
- index_corpus(vectors=): vectori precalculati cu validare aliniere; mismatch
  -> fallback embed complet
- ensure_embeddings_corpus: warmup in thread la startup (block=True), calea de
  request ne-blocanta (acquire non-blocking pe lock; warmup in curs -> return
  imediat); purjare orfane + modele vechi doar dupa indexare reusita
- log warmup: cache=N embed=M in Xs
- 31 teste noi (cold/warm/incremental, model schimbat, concurenta, ranking
  exact, echivalenta float32); suita completa 1596 passed

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-06 21:57:49 +00:00

230 lines
9.1 KiB
Python

"""Aplicatia FastAPI: API v1 + dashboard web + /healthz + /metrics.
Worker-ul ruleaza ca PROCES SEPARAT (python -m app.worker), NU ca task aici:
un worker mort nu trebuie sa lase containerul "sanatos".
"""
from __future__ import annotations
import secrets
import threading
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, PlainTextResponse
from fastapi.staticfiles import StaticFiles
from starlette.middleware.sessions import SessionMiddleware
from starlette.responses import RedirectResponse
import traceback
from . import __version__
from . import errors
from .api.v1.import_router import router as import_v1_router
from .api.v1.integrare_router import router as integrare_v1_router
from .api.v1.router import router as api_v1_router
from .config import get_settings, validate_prod_invariants
from .crypto import validate_creds_key
from .db import get_connection, init_db, queue_depth, read_heartbeat
from .observ import log_event, request_id_var
from .security import install_log_redaction, scrub_text
from .web.body_cap import BodyCapMiddleware
from .web.middleware import RequestIDMiddleware, SecurityHeadersMiddleware
from .web.routes import router as web_router
from .web.auth_routes import router as auth_router
from .web.admin_routes import router as admin_router
from .web.csrf import CsrfError
from .web.session import AdminRequired, LoginRequired
def _warmup_embeddings() -> None:
"""Incarca modelul de embeddings si indexeaza corpusul SILVER, in fundal.
Ruleaza intr-un thread daemon la startup: incarcarea modelului (~230MB) plus
vectorizarea corpusului dureaza zeci de secunde si NU are voie sa blocheze
primul request pe /mapari. Pana termina, sugestiile embeddings lipsesc
(degradare gratioasa); GOLD/SILVER/fuzzy functioneaza normal.
"""
from .mapping import ensure_embeddings_corpus
try:
conn = get_connection()
try:
ensure_embeddings_corpus(conn, block=True)
finally:
conn.close()
except Exception:
pass # best-effort: esecul warmup-ului nu opreste API-ul
@asynccontextmanager
async def lifespan(app: FastAPI):
install_log_redaction()
# Fail-fast: o cheie Fernet setata dar invalida opreste pornirea cu mesaj clar,
# in loc de 500 brut la primul POST /v1/prezentari.
validate_creds_key()
# Fail-fast (prod-only): elimina clasa "am uitat env var in Dokploy" pentru
# cheia API si secretul de sesiune, in loc de o instanta descoperita post-deploy.
validate_prod_invariants(get_settings())
init_db()
if get_settings().embeddings_enabled:
threading.Thread(target=_warmup_embeddings, name="emb-warmup", daemon=True).start()
yield
app = FastAPI(title="Gateway RAR AUTOPASS", version=__version__, lifespan=lifespan)
settings = get_settings()
_session_secret = settings.session_secret or secrets.token_hex(32)
app.add_middleware(
SessionMiddleware,
secret_key=_session_secret,
session_cookie="autopass_session",
https_only=settings.session_https_only,
same_site="strict",
)
# request_id pe fiecare cerere. Adaugat dupa SessionMiddleware -> ruleaza
# OUTERMOST (add_middleware prepend), deci `X-Request-ID` se pune pe TOATE raspunsurile,
# inclusiv 401/404/422/500 produse mai in interior.
app.add_middleware(RequestIDMiddleware)
# Headere de securitate (P1-4) pe TOATE raspunsurile.
app.add_middleware(SecurityHeadersMiddleware)
# Body-cap (P1-5): ADAUGAT ULTIMUL -> ruleaza CEL MAI OUTERMOST (add_middleware
# prepend), deci intercepteaza `receive` inaintea oricarui parser FastAPI/Starlette
# (multipart, JSON) — vezi docstring body_cap.py pentru motivul ASGI-pur.
app.add_middleware(BodyCapMiddleware)
@app.exception_handler(LoginRequired)
async def login_required_handler(request: Request, exc: LoginRequired) -> RedirectResponse:
return RedirectResponse("/login", status_code=303)
@app.exception_handler(AdminRequired)
async def admin_required_handler(request: Request, exc: AdminRequired) -> JSONResponse:
return JSONResponse(status_code=403, content={"detail": "acces interzis (necesita admin)"})
@app.exception_handler(CsrfError)
async def csrf_error_handler(request: Request, exc: CsrfError) -> JSONResponse:
return JSONResponse(status_code=403, content={"detail": "CSRF invalid"})
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
"""422 fara echo de credentiale.
Pydantic include implicit `input` (+ uneori `ctx`) in fiecare eroare — pe
/v1/prezentari asta ar reflecta inapoi `rar_credentials.password`. Pastram
type/loc/msg (clientul stie ce camp e gresit) si DROP-am input/ctx. Defense
in depth pe TOATE rutele, nu doar prezentari.
"""
cleaned = [{"type": e.get("type"), "loc": e.get("loc"), "msg": e.get("msg")} for e in exc.errors()]
return JSONResponse(status_code=422, content={"detail": cleaned})
@app.exception_handler(Exception)
async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Orice excepție neprinsa -> 500 STRUCTURAT din catalog in loc de 500 brut.
Body = envelope-ul standard din catalog (6 chei: field/cod/problema/cauza/fix/message)
+ `request_id` — fara traceback, fara mesaj de excepție brut, fara creds. Traceback-ul
complet + ruta + cont + request_id se scriu DOAR in jurnal (redactate prin scrub_text).
"""
request_id = getattr(request.state, "request_id", None) or request_id_var.get()
try:
account_id = request.session.get("account_id")
except (AssertionError, KeyError, AttributeError):
account_id = None
tb = scrub_text("".join(traceback.format_exception(type(exc), exc, exc.__traceback__)))
log_event(
"eroare_interna",
nivel="ERROR",
account_id=account_id,
cod="EROARE_INTERNA",
mesaj=f"{request.method} {request.url.path}: {type(exc).__name__}",
context={"path": request.url.path, "method": request.method, "traceback": tb},
request_id=request_id,
)
body = errors.eroare("EROARE_INTERNA")
body["request_id"] = request_id
return JSONResponse(status_code=500, content=body, headers={"X-Request-ID": request_id or ""})
# Assets servite local (htmx vendorizat), NU din CDN: gateway-ul ruleaza
# offline (LXC/VPS, Cloudflare Tunnel). Fara asta, dashboard-ul ramane static
# (zero polling banner/coada) cand unpkg e blocat/inaccesibil. Aceeasi decizie
# offline-first ca fontul UI (fara dependinta CDN).
_STATIC_DIR = Path(__file__).resolve().parent / "web" / "static"
app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static")
app.include_router(api_v1_router)
app.include_router(import_v1_router)
app.include_router(integrare_v1_router)
app.include_router(web_router)
app.include_router(auth_router)
app.include_router(admin_router)
@app.get("/healthz")
def healthz() -> dict:
"""Sanatate: worker viu + ultimul login RAR reusit + adancime coada.
Intoarce 200 mereu cu detalii; orchestratorul decide restartul pe campul
`worker_alive`.
"""
settings = get_settings()
conn = get_connection()
try:
hb = read_heartbeat(conn)
depth = queue_depth(conn)
finally:
conn.close()
worker_alive = False
last_beat = hb["last_beat"] if hb else None
if last_beat:
try:
age = (datetime.now(timezone.utc) - datetime.fromisoformat(last_beat)).total_seconds()
worker_alive = age <= settings.worker_heartbeat_stale_s
except ValueError:
worker_alive = False
return {
"ok": True,
"version": __version__,
"rar_env": settings.rar_env,
"worker_alive": worker_alive,
"last_beat": last_beat,
"last_rar_login_ok": hb["last_rar_login_ok"] if hb else None,
"queue_depth": depth,
}
@app.get("/metrics", response_class=PlainTextResponse)
def metrics() -> str:
"""Metrici text simplu (submissions pe status + backlog). Format Prometheus-lite."""
conn = get_connection()
try:
rows = conn.execute("SELECT status, COUNT(*) AS n FROM submissions GROUP BY status").fetchall()
# Coada tinuta: observabilitate ops pt. default Auto OFF.
# Gauge-uri DERIVATE (zero stare noua): total randuri tinute + varsta celui
# mai vechi (din created_at), scoped global. Semnaleaza esecul silentios
# cand prezentari raman nedeclarate tacit (risc L.142).
held = conn.execute(
"SELECT COUNT(*) AS n, "
"COALESCE(MAX(strftime('%s','now') - strftime('%s', created_at)), 0) AS oldest "
"FROM submissions WHERE status='queued' AND held=1"
).fetchone()
finally:
conn.close()
lines = ["# submissions pe status"]
for r in rows:
lines.append(f'autopass_submissions{{status="{r["status"]}"}} {r["n"]}')
# randuri tinute (queued AND held=1)
lines.append(f'autopass_held_submissions {held["n"]}')
lines.append(f'autopass_held_oldest_age_seconds {held["oldest"]}')
return "\n".join(lines) + "\n"