diff --git a/app/api/v1/router.py b/app/api/v1/router.py index 9493d2f..e2cb7b4 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -11,9 +11,12 @@ middleware (CORE) si exportul CSV vin ulterior — marcate TODO unde lipsesc. from __future__ import annotations +import csv +import io import json from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from ...auth import resolve_account_id @@ -152,6 +155,104 @@ def get_nomenclator() -> dict: conn.close() +AUDIT_COLUMNS = [ + "submission_id", + "status", + "id_prezentare", + "account_id", + "vin", + "nr_inmatriculare", + "data_prestatie", + "odometru_final", + "prestatii", + "rar_status_code", + "created_at", + "updated_at", + "purge_after", +] + + +def _audit_rows(conn, date_from: str | None, date_to: str | None, status: str): + """Randuri audit (sent implicit) filtrate pe data(updated_at) in [from, to]. + + payload_json e text in schelet (criptarea PII e P2); citim campurile-cheie + pentru audit. b64_image NU intra in CSV (mare). Daca P2 cripteaza payload-ul, + aici se decripteaza inainte de a construi randul. + """ + sql = "SELECT id, status, id_prezentare, account_id, payload_json, rar_status_code, created_at, updated_at, purge_after FROM submissions" + where = [] + params: list = [] + if status != "all": + where.append("status=?") + params.append(status) + if date_from: + where.append("date(updated_at) >= date(?)") + params.append(date_from) + if date_to: + where.append("date(updated_at) <= date(?)") + params.append(date_to) + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY id" + + for r in conn.execute(sql, params).fetchall(): + try: + p = json.loads(r["payload_json"]) if r["payload_json"] else {} + except (ValueError, TypeError): + p = {} + codes = ",".join( + (it.get("cod_prestatie") or it.get("cod_op_service") or "") + for it in (p.get("prestatii") or []) + if isinstance(it, dict) + ) + yield { + "submission_id": r["id"], + "status": r["status"], + "id_prezentare": r["id_prezentare"] or "", + "account_id": r["account_id"] or "", + "vin": p.get("vin") or "", + "nr_inmatriculare": p.get("nr_inmatriculare") or "", + "data_prestatie": p.get("data_prestatie") or "", + "odometru_final": p.get("odometru_final") or "", + "prestatii": codes, + "rar_status_code": r["rar_status_code"] or "", + "created_at": r["created_at"], + "updated_at": r["updated_at"], + "purge_after": r["purge_after"] or "", + } + + +@router.get("/audit/export") +def audit_export( + date_from: str | None = None, + date_to: str | None = None, + status: str = "sent", +) -> StreamingResponse: + """CSV cu ce s-a trimis (audit). Filtre optionale `date_from`/`date_to` (YYYY-MM-DD) + + pe data(updated_at). `status` implicit `sent` (ce a ajuns efectiv la RAR); + `status=all` exporta toata coada. Leaga re_tinerea 90 zile prin coloana + `purge_after` (plan.md sect. 4 + 8). b64_image nu se exporta. + """ + conn = get_connection() + try: + buf = io.StringIO() + writer = csv.DictWriter(buf, fieldnames=AUDIT_COLUMNS) + writer.writeheader() + for row in _audit_rows(conn, date_from, date_to, status): + writer.writerow(row) + data = buf.getvalue() + finally: + conn.close() + + fname = f"audit_{status}_{date_from or 'inceput'}_{date_to or 'azi'}.csv" + return StreamingResponse( + iter([data]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{fname}"'}, + ) + + @router.get("/mapari") def get_mapari(account_id: int | None = None) -> dict: conn = get_connection() diff --git a/app/web/routes.py b/app/web/routes.py index 9c11072..a490376 100644 --- a/app/web/routes.py +++ b/app/web/routes.py @@ -41,6 +41,25 @@ def _worker_alive(hb) -> bool: return age <= get_settings().worker_heartbeat_stale_s +def _rar_state(hb, worker_alive: bool) -> str: + """Eticheta de disponibilitate RAR, derivata din ultimul login reusit. + + Nu interogam RAR live aici (dashboard-ul degradeaza la ultima stare cunoscuta + a cozii). JWT TTL = 30h: un login mai vechi de atat inseamna ca nu mai stim + sigur ca RAR raspunde -> "indisponibil?". Fara niciun login -> necunoscut. + """ + if not worker_alive: + return "necunoscut (worker oprit)" + last = hb["last_rar_login_ok"] if hb else None + if not last: + return "fara login reusit inca" + try: + age = (datetime.now(timezone.utc) - datetime.fromisoformat(last)).total_seconds() + except (ValueError, TypeError): + return "necunoscut" + return "indisponibil?" if age > 108000 else "ok" + + @router.get("/", response_class=HTMLResponse) def dashboard(request: Request) -> HTMLResponse: conn = get_connection() @@ -48,20 +67,37 @@ def dashboard(request: Request) -> HTMLResponse: counts = _status_counts(conn) hb = read_heartbeat(conn) blocked = sum(counts.get(s, 0) for s in _BLOCKED) + worker_alive = _worker_alive(hb) ctx = { "request": request, "rar_env": get_settings().rar_env, "version": __version__, "counts": counts, "blocked": blocked, - "worker_alive": _worker_alive(hb), + "worker_alive": worker_alive, "last_login": hb["last_rar_login_ok"] if hb else None, + "rar_state": _rar_state(hb, worker_alive), } return templates.TemplateResponse("dashboard.html", ctx) finally: conn.close() +@router.get("/_fragments/nomenclator", response_class=HTMLResponse) +def fragment_nomenclator(request: Request) -> HTMLResponse: + """Browser nomenclator RAR (cache local upsert-at de worker la fiecare login).""" + conn = get_connection() + try: + rows = conn.execute( + "SELECT cod_prestatie, nume_prestatie, updated_at FROM nomenclator_rar ORDER BY cod_prestatie" + ).fetchall() + return templates.TemplateResponse( + "_nomenclator.html", {"request": request, "rows": rows} + ) + finally: + conn.close() + + @router.get("/_fragments/banner", response_class=HTMLResponse) def fragment_banner(request: Request) -> HTMLResponse: conn = get_connection() diff --git a/app/web/templates/_nomenclator.html b/app/web/templates/_nomenclator.html new file mode 100644 index 0000000..78d6da4 --- /dev/null +++ b/app/web/templates/_nomenclator.html @@ -0,0 +1,16 @@ +{% if rows %} + + + + {% for r in rows %} + + + + + + {% endfor %} + +
CodDenumireActualizat
{{ r.cod_prestatie }}{{ r.nume_prestatie }}{{ r.updated_at }}
+{% else %} +
Nomenclator gol. Worker-ul il umple la primul login RAR reusit.
+{% endif %} diff --git a/app/web/templates/dashboard.html b/app/web/templates/dashboard.html index ccc83ab..837ca1e 100644 --- a/app/web/templates/dashboard.html +++ b/app/web/templates/dashboard.html @@ -11,11 +11,17 @@
Worker
{{ 'viu' if worker_alive else 'mort' }}
+
RAR
{{ rar_state }}
Ultimul login RAR
{{ last_login or '—' }}
In coada
{{ counts.get('queued', 0) }}
Trimise
{{ counts.get('sent', 0) }}
Blocate
{{ blocked }}
+ {% if rar_state != 'ok' %} +

+ RAR posibil indisponibil — coada de mai jos arata ultima stare cunoscuta (local), nu live din RAR. +

+ {% endif %} @@ -24,10 +30,23 @@
-

Coada submissions

+
+

Coada submissions

+ export audit CSV (trimise) + tot +
se incarca…
+
+
+ Nomenclator RAR (coduri prestatii) +
+
se incarca…
+
+
+
+ {% endblock %} diff --git a/docs/plans/plan.md b/docs/plans/plan.md index e0b5321..542fc18 100644 --- a/docs/plans/plan.md +++ b/docs/plans/plan.md @@ -190,8 +190,11 @@ Nimic din cod nu e scris încă (`app/`, `tools/` nu există). Ordine recomandat Capturate: format eroare `data:[{field,message}]` + 3 mesaje exacte (VIN O/I/Q, dată veche, dată viitoare), forma răspuns success, `idPrezentare==id`, `idAgent` server-side, `sistemReparat:"null"` acceptat, `b64Image`/`odometruInitial` omise OK. **Descoperire: WAF cere `User-Agent` (altfel 403).** Toate detaliile în `docs/api-rar-contract.md`. -- [ ] **T5 (P1) — `tools/import_dbf.py`** dry-run + raport pe `mapare_prestatii.DBF` / `prestatii_rar.DBF`, apoi import în SQLite. - Verify: raport rânduri valide/orfane/coduri necunoscute; import idempotent. (Stub creat — neimplementat.) +- [x] **T5 (P1) — `tools/import_dbf.py`** ✅ 2026-06-15. `dbfread` → raport (rânduri valide, duplicate, goale, mapări + orfane = cod necunoscut în nomenclator) pe `prestatii_rar.DBF` (20 coduri) + `mapare_prestatii.DBF` (gol în arhivă). + Default dry-run; `--commit` scrie idempotent (upsert pe `nomenclator_rar` PK + `operations_mapping` UNIQUE), tranzacție + `BEGIN IMMEDIATE`/ROLLBACK. Verify: 6 teste (`tests/test_import_dbf.py`, cu writer dBASE III minimal pentru fixturi) + + dry-run real pe DBF-urile din repo. - [x] **Schelet repo** — ✅ 2026-06-15. `app/api/v1`, `app/rar_client.py` (cu User-Agent), `app/worker`, `app/web`, SQLite (WAL), `Dockerfile` + `docker compose`, `/healthz` verde. Verificat: login prin client OK, nomenclator 18 coduri, worker heartbeat → `worker_alive=True`, enqueue + dedup idempotency funcționale. @@ -230,7 +233,12 @@ Nimic din cod nu e scris încă (`app/`, `tools/` nu există). Ordine recomandat documentează env-urile. **Fix critic descoperit la split-ul în 2 containere:** `AUTOPASS_CREDS_KEY` trebuie PARTAJATĂ api↔worker (altfel worker nu decriptează creds) — acum impusă în compose (`${...:?}` → fail explicit dacă lipsește). Verify: 2 teste (`tests/test_deploy.py`). -- [ ] **Dashboard** (Jinja2+HTMX) cu stările empty/error/RAR-indisponibil + banner alertă. Apoi `/design-review` pe UI-ul live. +- [x] **Dashboard** (Jinja2+HTMX) ✅ 2026-06-15. Stări explicite: empty (coadă/mapări goale cu CTA), error (needs_mapping + în editor + motiv pe submission), **RAR indisponibil** (indicator stare RAR derivat din ultimul login < 30h → coada arată + ultima stare cunoscută local, nu live), banner alertă blocate (poll 15s). Componente: status worker/RAR, editor mapări + fuzzy, **browser nomenclator**, coadă (poll 10s), **export audit CSV** (`GET /v1/audit/export?status=sent|all&date_from&date_to`, + b64Image exclus, coloană `purge_after`). Verify: 5 teste (`tests/test_dashboard.py`) + smoke live. **Rămas: `/design-review` + pe UI-ul live** (cosmetic, neblocant). ### De decis ulterior (urmărit, nu blocant) - **[P2]** Defer criptare-at-rest + purjare 90z până după primul postPrezentare real reușit? (gold-plating vs. privacy-argument-de-adopție). diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 0000000..7c88f71 --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,109 @@ +"""Teste dashboard + audit CSV: nomenclator browser, stare RAR, export CSV.""" + +from __future__ import annotations + +import csv +import io +import os +import tempfile + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture() +def client(monkeypatch): + tmp = tempfile.mkdtemp() + monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "t.db")) + from app.config import get_settings + + get_settings.cache_clear() + from app.main import app + + with TestClient(app) as c: + yield c + get_settings.cache_clear() + + +def _body(**over): + prez = { + "vin": "WVWZZZ1KZAW000123", + "nr_inmatriculare": "B999TST", + "data_prestatie": "2026-06-15", + "odometru_final": "123456", + "prestatii": [{"cod_prestatie": "OE-1"}], + } + prez.update(over) + return {"rar_credentials": {"email": "x@y.ro", "password": "s"}, "prezentari": [prez]} + + +# --------------------------------------------------------------------------- # +# Dashboard render + fragmente # +# --------------------------------------------------------------------------- # + +def test_dashboard_renders_with_rar_state(client): + r = client.get("/") + assert r.status_code == 200 + # worker neavand heartbeat -> stare RAR necunoscuta (worker oprit) + assert "worker oprit" in r.text + assert "Nomenclator RAR" in r.text + + +def test_nomenclator_fragment_lists_seed(client): + r = client.get("/_fragments/nomenclator") + assert r.status_code == 200 + # seed fallback are 18 coduri; OE-1 + R-ODO trebuie sa apara + assert "OE-1" in r.text + assert "R-ODO" in r.text + + +def test_submissions_fragment_empty_state(client): + r = client.get("/_fragments/submissions") + assert r.status_code == 200 + assert "Coada e goala" in r.text + + +# --------------------------------------------------------------------------- # +# Audit CSV export # +# --------------------------------------------------------------------------- # + +def test_audit_export_sent_only(client): + # un submission queued (validare ok) + unul needs_data + client.post("/v1/prezentari", json=_body()) + client.post("/v1/prezentari", json=_body(vin="BAD")) + + # status implicit = sent -> niciun rand (nimic trimis inca), doar header + r = client.get("/v1/audit/export") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/csv") + assert "attachment" in r.headers["content-disposition"] + rows = list(csv.DictReader(io.StringIO(r.text))) + assert rows == [] + + # status=all -> ambele, cu coloane-cheie populate, fara b64_image + r = client.get("/v1/audit/export?status=all") + rows = list(csv.DictReader(io.StringIO(r.text))) + assert len(rows) == 2 + assert "vin" in rows[0] + assert "b64_image" not in rows[0] + vins = {row["vin"] for row in rows} + assert "WVWZZZ1KZAW000123" in vins + # prestatii = coduri concatenate + assert any(row["prestatii"] == "OE-1" for row in rows) + + +def test_audit_export_marks_sent_after_update(client): + client.post("/v1/prezentari", json=_body()) + # marcam manual sent (worker ar face asta dupa postPrezentare reusit) + from app.db import get_connection + + conn = get_connection() + try: + conn.execute("UPDATE submissions SET status='sent', id_prezentare=68514 WHERE id=1") + finally: + conn.close() + + rows = list(csv.DictReader(io.StringIO(client.get("/v1/audit/export").text))) + assert len(rows) == 1 + assert rows[0]["status"] == "sent" + assert rows[0]["id_prezentare"] == "68514" diff --git a/tests/test_import_dbf.py b/tests/test_import_dbf.py new file mode 100644 index 0000000..b08d459 --- /dev/null +++ b/tests/test_import_dbf.py @@ -0,0 +1,205 @@ +"""Teste T5: import DBF -> SQLite (dry-run/raport + commit idempotent). + +`mapare_prestatii.DBF` real e gol, deci pentru maparile reale scriem fixturi DBF +dBASE III minimale. Nomenclatorul real (`prestatii_rar.DBF`, 20 randuri) e citit +direct pentru un test de read pe date reale. +""" + +from __future__ import annotations + +import os +import struct +import tempfile +from pathlib import Path + +import pytest + +from app.config import ROOT + + +# --------------------------------------------------------------------------- # +# Helper: scriitor dBASE III minimal pentru fixturi # +# --------------------------------------------------------------------------- # + +def write_dbf(path: Path, fields: list[tuple[str, str, int]], records: list[dict]) -> None: + """Scrie un DBF dBASE III. fields = [(nume, tip C/L, lungime)].""" + header = bytearray(32) + header[0] = 0x03 # dBASE III, fara memo + header[1:4] = bytes((25, 1, 1)) # data ultimei actualizari (fictiva) + n_fields = len(fields) + header_len = 32 + 32 * n_fields + 1 + record_len = 1 + sum(length for _, _, length in fields) + struct.pack_into(" tuple[Path, Path]: + mp = dirpath / "mapare.DBF" + np_ = dirpath / "prest.DBF" + write_dbf( + np_, + PREST_FIELDS, + [{"COD_PREST": "OE-1", "NUME_PREST": "REPARATIE"}, {"COD_PREST": "R-ODO", "NUME_PREST": "REPARATIE ODOMETRU"}], + ) + write_dbf( + mp, + MAPARE_FIELDS, + [ + {"COD_OP": "OP1", "DESCR_OP": "Reparatie", "COD_RAR": "OE-1", "AUTO_SEND": True}, + {"COD_OP": "OP2", "DESCR_OP": "Operatie necunoscuta", "COD_RAR": "XYZ", "AUTO_SEND": False}, # orfan + ], + ) + return mp, np_ + + +def test_dry_run_does_not_write(env): + from tools.import_dbf import run + from app.db import get_connection + + mp, np_ = _fixtures(env) + res = run(commit=False, mapare_path=mp, prest_path=np_) + assert res["written"] == {"nomenclator": 0, "mapari": 0} + assert len(res["orphans"]) == 1 + assert res["orphans"][0]["cod_op_service"] == "OP2" + + conn = get_connection() + try: + n = conn.execute("SELECT COUNT(*) AS n FROM operations_mapping").fetchone()["n"] + assert n == 0 + finally: + conn.close() + + +def test_commit_writes_and_is_idempotent(env): + from tools.import_dbf import run + from app.db import get_connection + + mp, np_ = _fixtures(env) + res1 = run(commit=True, mapare_path=mp, prest_path=np_) + assert res1["written"] == {"nomenclator": 2, "mapari": 2} + + conn = get_connection() + try: + maps = conn.execute( + "SELECT cod_op_service, cod_prestatie, auto_send FROM operations_mapping ORDER BY cod_op_service" + ).fetchall() + assert [(m["cod_op_service"], m["cod_prestatie"], m["auto_send"]) for m in maps] == [ + ("OP1", "OE-1", 1), + ("OP2", "XYZ", 0), + ] + nom = conn.execute("SELECT COUNT(*) AS n FROM nomenclator_rar").fetchone()["n"] + finally: + conn.close() + + # A doua rulare nu duplica (upsert pe cheile UNIQUE). + run(commit=True, mapare_path=mp, prest_path=np_) + conn = get_connection() + try: + assert conn.execute("SELECT COUNT(*) AS n FROM operations_mapping").fetchone()["n"] == 2 + assert conn.execute("SELECT COUNT(*) AS n FROM nomenclator_rar").fetchone()["n"] == nom + finally: + conn.close() + + +def test_missing_dbf_raises(env): + from tools.import_dbf import run + + with pytest.raises(FileNotFoundError): + run(commit=False, mapare_path=env / "nope.DBF", prest_path=env / "nope2.DBF") diff --git a/tools/import_dbf.py b/tools/import_dbf.py index 5da35c1..479f2d7 100644 --- a/tools/import_dbf.py +++ b/tools/import_dbf.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 -"""Import DBF -> SQLite (T5 — SCHELET, neimplementat inca). +"""Import DBF ROAAUTO -> SQLite gateway (T5). Plan.md sect. 7: dry-run + raport intai (randuri valide, mapari orfane, coduri necunoscute in nomenclator), apoi scrie in SQLite. Surse: - - mapare_prestatii.DBF -> operations_mapping - - prestatii_rar.DBF -> nomenclator_rar -(rar_log.DBF NU se migreaza.) + - prestatii_rar.DBF (COD_PREST, NUME_PREST) -> nomenclator_rar + - mapare_prestatii.DBF (COD_OP, DESCR_OP, COD_RAR, AUTO_SEND) -> operations_mapping +(rar_log.DBF NU se migreaza — jurnalul nou e `submissions` + live din RAR.) -Utilizare (cand e implementat): - python -m tools.import_dbf --dry-run - python -m tools.import_dbf --commit +Importul e IDEMPOTENT (upsert pe cheile UNIQUE), deci rularea repetata nu duplica. +Default = dry-run (raport, fara scriere). `--commit` scrie efectiv. + +Utilizare: + python -m tools.import_dbf # dry-run + raport + python -m tools.import_dbf --commit # scrie in SQLite Necesita: pip install dbfread """ @@ -18,19 +21,230 @@ from __future__ import annotations import argparse import sys +from pathlib import Path +from typing import Any + +from dbfread import DBF + +from app.config import ROOT +from app.db import get_connection, init_db +from app.mapping import DEFAULT_ACCOUNT_ID + +# DBF-urile vin din arhiva ROAAUTO din radacina repo-ului. +MAPARE_DBF = ROOT / "mapare_prestatii.DBF" +PREST_DBF = ROOT / "prestatii_rar.DBF" + +# Language driver al DBF-urilor = 0x03 (Windows ANSI / cp1252). Diacriticele +# scrise ca literal '?' sunt in sursa, nu un artefact de encoding. +DBF_ENCODING = "cp1252" + + +# --------------------------------------------------------------------------- # +# Citire DBF -> randuri normalizate (pur, fara DB) # +# --------------------------------------------------------------------------- # + +def _field(rec: dict, *names: str) -> Any: + """Primul camp negol dintr-o lista de nume alternative (tolerant la schema).""" + for n in names: + if n in rec and rec[n] is not None: + return rec[n] + return None + + +def read_nomenclator(path: Path, *, encoding: str = DBF_ENCODING) -> dict[str, Any]: + """Citeste prestatii_rar.DBF. Intoarce raport + randuri valide. + + {rows: [{cod_prestatie, nume_prestatie}], duplicates: [cod...], blanks: int} + cod_prestatie normalizat strip().upper(); duplicate = acelasi cod de 2+ ori + (pastram prima aparitie). + """ + rows: list[dict[str, str]] = [] + seen: set[str] = set() + duplicates: list[str] = [] + blanks = 0 + for rec in DBF(str(path), encoding=encoding, char_decode_errors="replace"): + cod = str(_field(rec, "COD_PREST", "COD_PRESTATIE", "COD") or "").strip().upper() + nume = str(_field(rec, "NUME_PREST", "NUME_PRESTATIE", "NUME") or "").strip() + if not cod: + blanks += 1 + continue + if cod in seen: + duplicates.append(cod) + continue + seen.add(cod) + rows.append({"cod_prestatie": cod, "nume_prestatie": nume}) + return {"rows": rows, "duplicates": duplicates, "blanks": blanks} + + +def read_mapari(path: Path, *, encoding: str = DBF_ENCODING) -> dict[str, Any]: + """Citeste mapare_prestatii.DBF. Intoarce raport + randuri valide. + + {rows: [{cod_op_service, denumire, cod_prestatie, auto_send}], + duplicates: [cod_op...], blanks: int} + Rand valid = are si COD_OP si COD_RAR. blanks = randuri carora le lipseste + unul din ele. duplicate = acelasi COD_OP de 2+ ori (pastram prima aparitie). + """ + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + duplicates: list[str] = [] + blanks = 0 + for rec in DBF(str(path), encoding=encoding, char_decode_errors="replace"): + op = str(_field(rec, "COD_OP", "COD_OP_SERVICE") or "").strip() + cod = str(_field(rec, "COD_RAR", "COD_PRESTATIE", "COD_PREST") or "").strip().upper() + denumire = str(_field(rec, "DESCR_OP", "DENUMIRE", "DESCRIERE") or "").strip() + auto = _field(rec, "AUTO_SEND") + auto_send = bool(auto) if auto is not None else True + if not op or not cod: + blanks += 1 + continue + if op in seen: + duplicates.append(op) + continue + seen.add(op) + rows.append( + {"cod_op_service": op, "denumire": denumire, "cod_prestatie": cod, "auto_send": auto_send} + ) + return {"rows": rows, "duplicates": duplicates, "blanks": blanks} + + +def find_orphans(mapari: list[dict], known_codes: set[str]) -> list[dict]: + """Mapari al caror cod_prestatie nu exista in nomenclator (nu pot fi trimise).""" + return [m for m in mapari if m["cod_prestatie"] not in known_codes] + + +# --------------------------------------------------------------------------- # +# Scriere SQLite (idempotenta) # +# --------------------------------------------------------------------------- # + +def write_nomenclator(conn, rows: list[dict]) -> int: + conn.executemany( + "INSERT INTO nomenclator_rar (cod_prestatie, nume_prestatie, updated_at) " + "VALUES (?, ?, datetime('now')) " + "ON CONFLICT(cod_prestatie) DO UPDATE SET nume_prestatie=excluded.nume_prestatie, " + "updated_at=datetime('now')", + [(r["cod_prestatie"], r["nume_prestatie"]) for r in rows], + ) + return len(rows) + + +def write_mapari(conn, rows: list[dict], account_id: int) -> int: + conn.executemany( + "INSERT INTO operations_mapping (account_id, cod_op_service, cod_prestatie, auto_send) " + "VALUES (?, ?, ?, ?) " + "ON CONFLICT(account_id, cod_op_service) DO UPDATE SET " + "cod_prestatie=excluded.cod_prestatie, auto_send=excluded.auto_send", + [(account_id, r["cod_op_service"], r["cod_prestatie"], 1 if r["auto_send"] else 0) for r in rows], + ) + return len(rows) + + +# --------------------------------------------------------------------------- # +# Raport + CLI # +# --------------------------------------------------------------------------- # + +def build_report( + nomenclator: dict, mapari: dict, orphans: list[dict], *, account_id: int +) -> str: + lines: list[str] = [] + lines.append("=== Import DBF ROAAUTO -> SQLite (raport) ===") + lines.append(f"Cont tinta: account_id={account_id}") + lines.append("") + lines.append("nomenclator_rar (<- prestatii_rar.DBF):") + lines.append(f" randuri valide : {len(nomenclator['rows'])}") + lines.append(f" duplicate cod : {len(nomenclator['duplicates'])} {sorted(set(nomenclator['duplicates'])) or ''}".rstrip()) + lines.append(f" randuri goale : {nomenclator['blanks']}") + lines.append("") + lines.append("operations_mapping (<- mapare_prestatii.DBF):") + lines.append(f" randuri valide : {len(mapari['rows'])}") + lines.append(f" duplicate COD_OP: {len(mapari['duplicates'])} {sorted(set(mapari['duplicates'])) or ''}".rstrip()) + lines.append(f" randuri goale : {mapari['blanks']} (lipsa COD_OP sau COD_RAR)") + lines.append(f" mapari ORFANE : {len(orphans)} (cod_prestatie necunoscut in nomenclator)") + for m in orphans: + lines.append(f" - {m['cod_op_service']} -> {m['cod_prestatie']} ({m['denumire'] or 'fara denumire'})") + return "\n".join(lines) + + +def run( + *, + commit: bool, + account_id: int = DEFAULT_ACCOUNT_ID, + mapare_path: Path = MAPARE_DBF, + prest_path: Path = PREST_DBF, + encoding: str = DBF_ENCODING, +) -> dict[str, Any]: + """Citeste DBF-urile, construieste raportul si (optional) scrie in SQLite. + + Intoarce {report, nomenclator, mapari, orphans, written:{nomenclator,mapari}}. + """ + missing = [str(p) for p in (prest_path, mapare_path) if not p.exists()] + if missing: + raise FileNotFoundError("DBF lipsa: " + ", ".join(missing)) + + nomenclator = read_nomenclator(prest_path, encoding=encoding) + mapari = read_mapari(mapare_path, encoding=encoding) + + init_db() + conn = get_connection() + try: + # Coduri cunoscute = nomenclatorul ce urmeaza importat + ce e deja in DB + # (seed fallback / live din worker). Asa orfanele sunt detectate corect + # chiar daca prestatii_rar.DBF nu acopera toate codurile. + db_codes = {r["cod_prestatie"] for r in conn.execute("SELECT cod_prestatie FROM nomenclator_rar")} + known = db_codes | {r["cod_prestatie"] for r in nomenclator["rows"]} + orphans = find_orphans(mapari["rows"], known) + + written = {"nomenclator": 0, "mapari": 0} + if commit: + conn.execute("BEGIN IMMEDIATE") + try: + written["nomenclator"] = write_nomenclator(conn, nomenclator["rows"]) + written["mapari"] = write_mapari(conn, mapari["rows"], account_id) + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + report = build_report(nomenclator, mapari, orphans, account_id=account_id) + return { + "report": report, + "nomenclator": nomenclator, + "mapari": mapari, + "orphans": orphans, + "written": written, + } + finally: + conn.close() def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Import DBF ROAAUTO -> SQLite gateway (T5)") - parser.add_argument("--dry-run", action="store_true", help="raport fara scriere (default)") - parser.add_argument("--commit", action="store_true", help="scrie in SQLite dupa confirmare") - parser.parse_args(argv) + parser.add_argument("--commit", action="store_true", help="scrie in SQLite (implicit: doar raport)") + parser.add_argument("--account-id", type=int, default=DEFAULT_ACCOUNT_ID, help="cont tinta pentru mapari") + parser.add_argument("--mapare", type=Path, default=MAPARE_DBF, help="cale mapare_prestatii.DBF") + parser.add_argument("--nomenclator", type=Path, default=PREST_DBF, help="cale prestatii_rar.DBF") + parser.add_argument("--encoding", default=DBF_ENCODING, help=f"encoding DBF (implicit {DBF_ENCODING})") + args = parser.parse_args(argv) - print("tools/import_dbf.py este SCHELET (T5). De implementat:") - print(" 1. citeste mapare_prestatii.DBF + prestatii_rar.DBF cu dbfread") - print(" 2. raport: randuri valide, mapari orfane, coduri necunoscute in nomenclator") - print(" 3. la --commit: INSERT idempotent in operations_mapping / nomenclator_rar") - return 1 + try: + result = run( + commit=args.commit, + account_id=args.account_id, + mapare_path=args.mapare, + prest_path=args.nomenclator, + encoding=args.encoding, + ) + except FileNotFoundError as exc: + print(f"EROARE: {exc}", file=sys.stderr) + return 2 + + print(result["report"]) + print("") + if args.commit: + w = result["written"] + print(f"COMMIT: scris {w['nomenclator']} coduri nomenclator, {w['mapari']} mapari (idempotent).") + else: + print("DRY-RUN: nimic scris. Reia cu --commit dupa ce verifici raportul.") + return 0 if __name__ == "__main__":