feat(T5/dashboard): import DBF idempotent + nomenclator browser + audit CSV + stare RAR
T5 (tools/import_dbf.py): citire prestatii_rar.DBF / mapare_prestatii.DBF cu dbfread, raport dry-run (randuri valide/duplicate/goale, mapari orfane = cod necunoscut in nomenclator), --commit cu upsert idempotent in tranzactie. Dashboard: browser nomenclator, indicator stare RAR (indisponibil? derivat din ultimul login < 30h, coada arata ultima stare locala), export audit CSV (/v1/audit/export?status=sent|all&date_from&date_to, b64Image exclus, coloana purge_after pentru retentia 90z). Verify: 11 teste noi (test_import_dbf 6, test_dashboard 5), suita 111 pass, dry-run real pe DBF-urile din repo + smoke live dashboard/CSV. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -11,9 +11,12 @@ middleware (CORE) si exportul CSV vin ulterior — marcate TODO unde lipsesc.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ...auth import resolve_account_id
|
||||
@@ -152,6 +155,104 @@ def get_nomenclator() -> dict:
|
||||
conn.close()
|
||||
|
||||
|
||||
AUDIT_COLUMNS = [
|
||||
"submission_id",
|
||||
"status",
|
||||
"id_prezentare",
|
||||
"account_id",
|
||||
"vin",
|
||||
"nr_inmatriculare",
|
||||
"data_prestatie",
|
||||
"odometru_final",
|
||||
"prestatii",
|
||||
"rar_status_code",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"purge_after",
|
||||
]
|
||||
|
||||
|
||||
def _audit_rows(conn, date_from: str | None, date_to: str | None, status: str):
|
||||
"""Randuri audit (sent implicit) filtrate pe data(updated_at) in [from, to].
|
||||
|
||||
payload_json e text in schelet (criptarea PII e P2); citim campurile-cheie
|
||||
pentru audit. b64_image NU intra in CSV (mare). Daca P2 cripteaza payload-ul,
|
||||
aici se decripteaza inainte de a construi randul.
|
||||
"""
|
||||
sql = "SELECT id, status, id_prezentare, account_id, payload_json, rar_status_code, created_at, updated_at, purge_after FROM submissions"
|
||||
where = []
|
||||
params: list = []
|
||||
if status != "all":
|
||||
where.append("status=?")
|
||||
params.append(status)
|
||||
if date_from:
|
||||
where.append("date(updated_at) >= date(?)")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
where.append("date(updated_at) <= date(?)")
|
||||
params.append(date_to)
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY id"
|
||||
|
||||
for r in conn.execute(sql, params).fetchall():
|
||||
try:
|
||||
p = json.loads(r["payload_json"]) if r["payload_json"] else {}
|
||||
except (ValueError, TypeError):
|
||||
p = {}
|
||||
codes = ",".join(
|
||||
(it.get("cod_prestatie") or it.get("cod_op_service") or "")
|
||||
for it in (p.get("prestatii") or [])
|
||||
if isinstance(it, dict)
|
||||
)
|
||||
yield {
|
||||
"submission_id": r["id"],
|
||||
"status": r["status"],
|
||||
"id_prezentare": r["id_prezentare"] or "",
|
||||
"account_id": r["account_id"] or "",
|
||||
"vin": p.get("vin") or "",
|
||||
"nr_inmatriculare": p.get("nr_inmatriculare") or "",
|
||||
"data_prestatie": p.get("data_prestatie") or "",
|
||||
"odometru_final": p.get("odometru_final") or "",
|
||||
"prestatii": codes,
|
||||
"rar_status_code": r["rar_status_code"] or "",
|
||||
"created_at": r["created_at"],
|
||||
"updated_at": r["updated_at"],
|
||||
"purge_after": r["purge_after"] or "",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/audit/export")
|
||||
def audit_export(
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
status: str = "sent",
|
||||
) -> StreamingResponse:
|
||||
"""CSV cu ce s-a trimis (audit). Filtre optionale `date_from`/`date_to` (YYYY-MM-DD)
|
||||
|
||||
pe data(updated_at). `status` implicit `sent` (ce a ajuns efectiv la RAR);
|
||||
`status=all` exporta toata coada. Leaga re_tinerea 90 zile prin coloana
|
||||
`purge_after` (plan.md sect. 4 + 8). b64_image nu se exporta.
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(buf, fieldnames=AUDIT_COLUMNS)
|
||||
writer.writeheader()
|
||||
for row in _audit_rows(conn, date_from, date_to, status):
|
||||
writer.writerow(row)
|
||||
data = buf.getvalue()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
fname = f"audit_{status}_{date_from or 'inceput'}_{date_to or 'azi'}.csv"
|
||||
return StreamingResponse(
|
||||
iter([data]),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": f'attachment; filename="{fname}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/mapari")
|
||||
def get_mapari(account_id: int | None = None) -> dict:
|
||||
conn = get_connection()
|
||||
|
||||
@@ -41,6 +41,25 @@ def _worker_alive(hb) -> bool:
|
||||
return age <= get_settings().worker_heartbeat_stale_s
|
||||
|
||||
|
||||
def _rar_state(hb, worker_alive: bool) -> str:
|
||||
"""Eticheta de disponibilitate RAR, derivata din ultimul login reusit.
|
||||
|
||||
Nu interogam RAR live aici (dashboard-ul degradeaza la ultima stare cunoscuta
|
||||
a cozii). JWT TTL = 30h: un login mai vechi de atat inseamna ca nu mai stim
|
||||
sigur ca RAR raspunde -> "indisponibil?". Fara niciun login -> necunoscut.
|
||||
"""
|
||||
if not worker_alive:
|
||||
return "necunoscut (worker oprit)"
|
||||
last = hb["last_rar_login_ok"] if hb else None
|
||||
if not last:
|
||||
return "fara login reusit inca"
|
||||
try:
|
||||
age = (datetime.now(timezone.utc) - datetime.fromisoformat(last)).total_seconds()
|
||||
except (ValueError, TypeError):
|
||||
return "necunoscut"
|
||||
return "indisponibil?" if age > 108000 else "ok"
|
||||
|
||||
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
def dashboard(request: Request) -> HTMLResponse:
|
||||
conn = get_connection()
|
||||
@@ -48,20 +67,37 @@ def dashboard(request: Request) -> HTMLResponse:
|
||||
counts = _status_counts(conn)
|
||||
hb = read_heartbeat(conn)
|
||||
blocked = sum(counts.get(s, 0) for s in _BLOCKED)
|
||||
worker_alive = _worker_alive(hb)
|
||||
ctx = {
|
||||
"request": request,
|
||||
"rar_env": get_settings().rar_env,
|
||||
"version": __version__,
|
||||
"counts": counts,
|
||||
"blocked": blocked,
|
||||
"worker_alive": _worker_alive(hb),
|
||||
"worker_alive": worker_alive,
|
||||
"last_login": hb["last_rar_login_ok"] if hb else None,
|
||||
"rar_state": _rar_state(hb, worker_alive),
|
||||
}
|
||||
return templates.TemplateResponse("dashboard.html", ctx)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@router.get("/_fragments/nomenclator", response_class=HTMLResponse)
|
||||
def fragment_nomenclator(request: Request) -> HTMLResponse:
|
||||
"""Browser nomenclator RAR (cache local upsert-at de worker la fiecare login)."""
|
||||
conn = get_connection()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT cod_prestatie, nume_prestatie, updated_at FROM nomenclator_rar ORDER BY cod_prestatie"
|
||||
).fetchall()
|
||||
return templates.TemplateResponse(
|
||||
"_nomenclator.html", {"request": request, "rows": rows}
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@router.get("/_fragments/banner", response_class=HTMLResponse)
|
||||
def fragment_banner(request: Request) -> HTMLResponse:
|
||||
conn = get_connection()
|
||||
|
||||
16
app/web/templates/_nomenclator.html
Normal file
16
app/web/templates/_nomenclator.html
Normal file
@@ -0,0 +1,16 @@
|
||||
{% if rows %}
|
||||
<table>
|
||||
<thead><tr><th>Cod</th><th>Denumire</th><th>Actualizat</th></tr></thead>
|
||||
<tbody>
|
||||
{% for r in rows %}
|
||||
<tr>
|
||||
<td><span class="pill">{{ r.cod_prestatie }}</span></td>
|
||||
<td>{{ r.nume_prestatie }}</td>
|
||||
<td class="muted">{{ r.updated_at }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<div class="empty">Nomenclator gol. Worker-ul il umple la primul login RAR reusit.</div>
|
||||
{% endif %}
|
||||
@@ -11,11 +11,17 @@
|
||||
<div style="display:flex; gap:24px; flex-wrap:wrap;">
|
||||
<div><div class="muted">Worker</div><div class="{{ 's-sent' if worker_alive else 's-error' }}">
|
||||
{{ 'viu' if worker_alive else 'mort' }}</div></div>
|
||||
<div><div class="muted">RAR</div><div class="{{ 's-sent' if rar_state == 'ok' else 's-error' if 'indisponibil' in rar_state else 'muted' }}">{{ rar_state }}</div></div>
|
||||
<div><div class="muted">Ultimul login RAR</div><div>{{ last_login or '—' }}</div></div>
|
||||
<div><div class="muted">In coada</div><div>{{ counts.get('queued', 0) }}</div></div>
|
||||
<div><div class="muted">Trimise</div><div class="s-sent">{{ counts.get('sent', 0) }}</div></div>
|
||||
<div><div class="muted">Blocate</div><div class="{{ 's-error' if blocked else '' }}">{{ blocked }}</div></div>
|
||||
</div>
|
||||
{% if rar_state != 'ok' %}
|
||||
<p class="muted" style="margin:12px 0 0; font-size:12px;">
|
||||
RAR posibil indisponibil — coada de mai jos arata ultima stare cunoscuta (local), nu live din RAR.
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- incarcat o data; NU poll (sa nu stergem o selectie in curs). Se re-randeaza la salvare. -->
|
||||
@@ -24,10 +30,23 @@
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2 style="font-size:14px; margin:0 0 12px;">Coada submissions</h2>
|
||||
<div style="display:flex; align-items:center; gap:12px; margin:0 0 12px;">
|
||||
<h2 style="font-size:14px; margin:0;">Coada submissions</h2>
|
||||
<a href="/v1/audit/export?status=sent" style="margin-left:auto; font-size:13px;" download>export audit CSV (trimise)</a>
|
||||
<a href="/v1/audit/export?status=all" style="font-size:13px;" download>tot</a>
|
||||
</div>
|
||||
<div hx-get="/_fragments/submissions" hx-trigger="load, every 10s" hx-swap="innerHTML">
|
||||
<div class="empty">se incarca…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<details>
|
||||
<summary style="cursor:pointer; font-size:14px; font-weight:600;">Nomenclator RAR (coduri prestatii)</summary>
|
||||
<div style="margin-top:12px;" hx-get="/_fragments/nomenclator" hx-trigger="load" hx-swap="innerHTML">
|
||||
<div class="empty">se incarca…</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
Reference in New Issue
Block a user