From 3fc53534e2bcf2bea91dcc18c11024f6be68f65a Mon Sep 17 00:00:00 2001 From: Claude Agent Date: Sun, 28 Jun 2026 20:48:34 +0000 Subject: [PATCH] =?UTF-8?q?feat(5.15+5.14):=20CLOSE=20=E2=80=94=20fix-uri?= =?UTF-8?q?=20code-review=20+=20embeddings=20functional?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5.15 (propagare design + dashboard editare) si 5.14 (mapare LLM distilata) inchise dupa /code-review high. 8 buguri reparate TDD: - HIGH modal nu se deschidea pe randul slim (base.html: trimitere-slim) - HIGH /repune trunchia prestatii (declaratie incompleta la RAR) -> iterare peste existing, codes pozitional - HIGH embeddings incarca model ~230MB degeaba pe corpus gol -> poarta has_corpus() - HIGH picker chips gol pe re-render eroare -> conn/account_id pe toate ramurile - MED obs re-derivat dupa stergere explicita -> _merge_override pastreaza obs='' - MED mapare salvata fara denumire poluă GOLD -> _record_gold_validation guard - MED typo nome_prestatie -> nume_prestatie in select /repune - MED bucketare timp +3h gresita iarna -> SQLite localtime + TZ=Europe/Bucharest Embeddings WIRE-uit functional (PRD #15, decizie user): ensure_embeddings_corpus construieste corpus din nomenclator, gated pe AUTOPASS_EMBEDDINGS_ENABLED (default off). Marime model corectata ~50MB->~230MB (estimare PRD gresita). Cleanup: hoist load_* din bucla bulk-fix; import re la top. Regresie: 1256 passed, 1 deselected (live), 0 failed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 6 + CLAUDE.md | 2 +- DESIGN.md | 209 ++++- Dockerfile | 9 +- app/api/v1/import_router.py | 34 +- app/config.py | 7 + app/embeddings.py | 246 ++++++ app/mapping.py | 151 ++++ app/payload_view.py | 5 + app/schema.sql | 36 + app/shared_store.py | 139 +++ app/web/routes.py | 792 ++++++++++++++++-- app/web/templates/_chips_prestatii.html | 172 ++++ app/web/templates/_form_editare.html | 92 +- app/web/templates/_macros.html | 16 +- app/web/templates/_mapari.html | 5 +- app/web/templates/_status.html | 92 +- app/web/templates/_submissions.html | 150 ++-- app/web/templates/_trimitere_detaliu.html | 32 +- app/web/templates/base.html | 131 ++- docker-compose.yml | 5 + docs/ROADMAP.md | 4 +- docs/prd/prd-5.14-mapare-llm-distilata.md | 172 +++- ...5.15-propagare-design-dashboard-editare.md | 160 ++-- pytest.ini | 4 + requirements.txt | 5 + tests/conftest.py | 24 + tests/test_api_scope.py | 226 +++++ tests/test_dashboard.py | 8 +- tests/test_device_mix.py | 152 ++++ tests/test_embeddings.py | 233 ++++++ tests/test_heldout_eval.py | 527 ++++++++++++ tests/test_holdout.py | 286 +++++++ tests/test_mapare_integrare_l14.py | 578 +++++++++++++ tests/test_or_label.py | 491 +++++++++++ tests/test_shared_store.py | 290 +++++++ tests/test_tema.py | 184 ++++ tests/test_web_bulk_fix.py | 248 ++++++ tests/test_web_corectie.py | 7 +- tests/test_web_corectie_obs.py | 472 +++++++++++ tests/test_web_corectie_prestatii.py | 546 ++++++++++++ tests/test_web_form_editare_slim.py | 496 +++++++++++ tests/test_web_mapare_din_chip.py | 340 ++++++++ tests/test_web_modal_slim.py | 99 +++ tests/test_web_responsive.py | 238 ++++++ tests/test_web_scope.py | 231 +++++ tests/test_web_status.py | 258 +++++- tests/test_web_status_fragment.py | 17 +- tests/test_web_submissions.py | 159 +++- tests/test_web_submissions_layout.py | 68 +- tools/mapare-llm/heldout_eval.py | 567 +++++++++++++ tools/mapare-llm/holdout.py | 347 ++++++++ tools/mapare-llm/or_label.py | 300 +++++++ 53 files changed, 9684 insertions(+), 384 deletions(-) create mode 100644 app/embeddings.py create mode 100644 app/shared_store.py create mode 100644 app/web/templates/_chips_prestatii.html create mode 100644 pytest.ini create mode 100644 tests/test_api_scope.py create mode 100644 tests/test_device_mix.py create mode 100644 tests/test_embeddings.py create mode 100644 tests/test_heldout_eval.py create mode 100644 tests/test_holdout.py create mode 100644 tests/test_mapare_integrare_l14.py create mode 100644 tests/test_or_label.py create mode 100644 tests/test_shared_store.py create mode 100644 tests/test_web_bulk_fix.py create mode 100644 tests/test_web_corectie_obs.py create mode 100644 tests/test_web_corectie_prestatii.py create mode 100644 tests/test_web_form_editare_slim.py create mode 100644 tests/test_web_mapare_din_chip.py create mode 100644 tests/test_web_modal_slim.py create mode 100644 tests/test_web_scope.py create mode 100644 tools/mapare-llm/heldout_eval.py create mode 100644 tools/mapare-llm/holdout.py create mode 100644 tools/mapare-llm/or_label.py diff --git a/.env.example b/.env.example index 2252caf..0ab3247 100644 --- a/.env.example +++ b/.env.example @@ -19,3 +19,9 @@ AUTOPASS_WORKER_USE_TEST_CREDS=false # --- RAR --- # test | prod AUTOPASS_RAR_ENV=test + +# --- Embeddings (sugestie mapare, Stratul 2 PRD 5.14) --- +# false = dezactivat (default; /mapari instant, sugestii din GOLD/SILVER + fuzzy). +# true = sugestii semantice. Prima cerere /mapari lazy-load-eaza modelul fastembed/ONNX +# (~230MB pe disc) sincron -> hang la prima cerere. Doar API-ul il incarca. +AUTOPASS_EMBEDDINGS_ENABLED=false diff --git a/CLAUDE.md b/CLAUDE.md index 57f2626..f56cd93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,7 +67,7 @@ Flux: validare (`validation.py`) → mapare operatie→cod (`mapping.py`) → en - **Idempotenta = hash de continut canonic** server-side (`idempotency.py`), pentru ca RAR accepta duplicate si nu are nr. comanda. `build_key` normalizeaza INTOTDEAUNA `account_id` la `account_or_default` (None == 1) INAINTE de hash — altfel acelasi rand logic primeste chei diferite pe canalele API vs import (OV-2). `canonicalize_row` normeaza VIN/nr/odometru (strip ".0" din coercion Excel) inainte de validare si de cheie. - **`FINALIZATA` e terminal la RAR** — fara anulare/corectie prin API. De aceea reconcilierea anti-duplicat: pe eroare **ambigua** (timeout / TransportError / 502/503/504 / 429 / 408) sau rand `sending` orfan, worker-ul cauta in finalizate (match pe vin+dataPrestatie+odometruFinal) si marcheaza `sent` fara a re-trimite (`reconcile.py`). **EXCEPTIE: un RAR 500 cu mesaj** (`RarError.rar_message`, ex. `ORA-12899`) e un esec DEFINITIV (RAR a raspuns „am esuat", nu o pierdere de raspuns) → worker-ul NU reconciliaza si NU reincearca, marcheaza `error` cu mesajul RAR (`RAR_EROARE_SERVER`). Altfel ar marca fals `sent` pe un record PARTIAL pe care RAR (ne-tranzactional) il lasa la esec. - **Creds RAR per cont**: durabile in `accounts.rar_creds_enc` (canal web, fallback re-login) SAU efemere in `submissions.rar_creds_enc` (canal API, sterse dupa primul login reusit). Worker incearca submission-ul intai, apoi fallback la cont. Purjarea sterge DOAR `submissions.rar_creds_enc`, NU `accounts.rar_creds_enc`. -- **Auth API-key** (`auth.py`): identifica CONTUL ROAAUTO, separat de credentialele RAR. Stocam doar SHA-256 al cheii. Enforcement prin `AUTOPASS_REQUIRE_API_KEY`: `false` (dev) → fara cheie merge pe cont id=1, cheie invalida → 401; `true` (prod) → cheie obligatorie pe `/v1/*` protejat. POST-urile + rutele de import sunt account-scoped; GET-urile de listare sunt momentan globale + neprotejate (de remediat — vezi ROADMAP). +- **Auth API-key** (`auth.py`): identifica CONTUL ROAAUTO, separat de credentialele RAR. Stocam doar SHA-256 al cheii. Enforcement prin `AUTOPASS_REQUIRE_API_KEY`: `false` (dev) → fara cheie merge pe cont id=1, cheie invalida → 401; `true` (prod) → cheie obligatorie pe `/v1/*` protejat. POST-urile + rutele de import sunt account-scoped; GET-urile de listare sunt si ele account-scoped (5.15/US-011: fragmentele `_fragments/submissions|trimitere|mapari|status|jurnal|nomenclator|trimiteri-versiune` sub `require_login` + scope, 404-before-leak pe id strain; `GET /v1/prezentari(/{id})`/`/v1/mapari`/`/v1/audit/export` filtrate pe cont). `GET /v1/nomenclator` ramane public intentionat (coduri RAR publice, fara PII). - **Mapare coloane retinuta per `(account_id, signature_coloane)`** (`column_mappings`): la urmatorul fisier cu aceleasi coloane, pentru acelasi cont, maparea se reaplica automat. Un cont poate avea mai multe formate memorate simultan. - **Mapare operatie→cod**: prestatie poate veni cu `cod_prestatie` (cod RAR direct) sau `cod_op_service` (cod intern) + `denumire`. Nerezolvat → submission `needs_mapping` (nu se trimite), apare in editorul web cu sugestie fuzzy; la salvarea maparii se re-rezolva automat submission-urile blocate. - **`cod_prestatie` e VALIDAT fata de nomenclator la ingestie** (`resolve_prestatii(..., valid_codes)`): un cod direct NECUNOSCUT in nomenclator NU se mai trimite raw — e promovat la `cod_op_service` (denumire=cod) si tratat ca operatie de mapat. Motiv (confirmat live 2026-06-23): RAR accepta NUMAI coduri din nomenclator (coloana `COD_PRESTATIE` max 5 car.); un cod necunoscut da **HTTP 500** (`ORA-12899`), iar RAR **NU e tranzactional** → lasa un record PARTIAL `FINALIZATA` (terminal) chiar pe esec, pe care reconcilierea worker-ului l-ar marca fals `sent`. Comportamentul la cod necunoscut/nemapat: `on_unmapped_error` (camp boolean top-level pe `POST /v1/prezentari` + `/valideaza`) = `false` (intra in editor, `needs_mapping`) sau `true` (respinge fara enqueue → `submission_id=null` + `erori`). Default = `accounts.on_unmapped_error_default` (implicit `false`/`0`); precedenta cerere > cont > `false`. diff --git a/DESIGN.md b/DESIGN.md index bd34ec8..7547681 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -29,9 +29,11 @@ sistemul sa ramana discret. ``` --bg: #0f1218 fundal aplicatie --card: #181c24 suprafete (carduri, modal, inputuri pe fundal) +--card2: #0f1218 fundal input slim / carduri-contor (= --bg, nivelul cel mai adanc) --ink: #e6e9ef text principal --muted: #8b93a7 text secundar (label-uri, coduri, „by") --line: #262b36 borduri, separatoare +--line2: #1f2530 separator subtire lista slim (mai subtil decat --line) --accent:#2E74D6 azur ROMFAST — butoane primare, pill activ, linkuri, focus --ok: #2FBF8F sent / succes --warn: #E0A93B sending / atentie / Lipsa cod @@ -43,10 +45,12 @@ sistemul sa ramana discret. ``` --bg: #f5f7fa fundal (alb-rece ca romfast.ro) --card: #ffffff suprafete +--card2: #f5f7fa fundal input slim / carduri-contor (= --bg) --ink: #1a1d24 text principal --muted: #5c6473 text secundar --line: #e2e5ea borduri ---accent:#1F66C9 azur, variantă mai inchisa pentru contrast AA pe alb +--line2: #eaedf2 separator subtire lista slim (mai subtil decat --line) +--accent:#1F66C9 azur, varianta mai inchisa pentru contrast AA pe alb --ok: #15803d verde AA pe alb --warn: #b45309 chihlimbar AA pe alb --err: #dc2626 rosu AA pe alb @@ -60,15 +64,99 @@ Aceleasi neutre-calde inchise; doar accentul difera de azur. ``` --bg: #0e1416 fundal petrol-inchis --card: #161e20 suprafete +--card2: #0e1416 fundal input/contor (= --bg) --ink: #e6e9ef text principal --muted: #8b93a7 text secundar --line: #232c2e borduri +--line2: #1c2426 separator subtire (intre --bg si --line) --accent:#0E7C7B teal petrol — butoane, pill activ, linkuri, focus --ok: #2FBF8F sent --warn: #E0A93B atentie --err: #E05D5D eroare ``` +### Paleta — Grafit (`[data-theme="grafit"]`, tema selectabila — adaugata PRD 5.15) + +Similara cu dark, accent azur deschis (preluat din landing, `--infot`). Distinta de dark la cererea +userului (D2). Mapare landing->app: `--text->--ink`, `--sub->--muted`, `--okt->--ok`, +`--errt->--err`, `--infot->--accent`. + +``` +--bg: #0f1218 fundal (identic cu dark) +--card: #181c24 suprafete +--card2: #0f1218 fundal input/contor (= --bg) +--ink: #e6e9ef text principal +--muted: #8b93a7 text secundar +--line: #262b36 borduri +--line2: #1f2530 separator subtire +--accent:#6ea2ec azur deschis (landing --infot) — linkuri, focus, pill activ +--ok: #2FBF8F sent / succes +--warn: #E0A93B atentie +--err: #E05D5D eroare +``` + +### Paleta — Cobalt (`[data-theme="cobalt"]`, tema selectabila — adaugata PRD 5.15) + +Fundal bleumarin adanc, accent albastru viu. Atmosfera tehnica/corporatista rece. + +``` +--bg: #080d1c fundal bleumarin adanc +--card: #111a33 suprafete +--card2: #0b1226 fundal input/contor +--ink: #e9ecfb text principal (usor albastrat) +--muted: #8a93b8 text secundar +--line: #1d2747 borduri +--line2: #161f3a separator subtire +--accent:#8aa0ff albastru viu (landing --infot) +--ok: #2fd0a6 sent / succes (teal mai saturat) +--warn: #E0A93B atentie +--err: #f06a7a eroare (roz saturat pe bleumarin) +``` + +### Paleta — Cupru (`[data-theme="cupru"]`, tema selectabila — adaugata PRD 5.15) + +Fundal cald ciocolata, accent chihlimbar. Atmosfera artizanala/calda. + +``` +--bg: #15110b fundal maro inchis-cald +--card: #211a12 suprafete +--card2: #15110b fundal input/contor (= --bg) +--ink: #efe6d6 text principal (crem cald) +--muted: #a89a85 text secundar +--line: #36291c borduri +--line2: #281e14 separator subtire +--accent:#dfa45c chihlimbar cald (landing --infot) +--ok: #67b98c sent / succes (verde muted-cald) +--warn: #c97d2e atentie (chihlimbar mai inchis) +--err: #e2685a eroare (coral pe maro) +``` + +### Paleta — Hartie (`[data-theme="hartie"]`, tema selectabila — adaugata PRD 5.15) + +Fundal crem cald (hartie vintage), accent albastru clasic. Similara cu light, distinta la cererea +userului. Ambele teme luminoase (hartie + light) respecta contrast AA. + +``` +--bg: #f3efe6 fundal crem cald +--card: #fffdf7 suprafete (crem-alb) +--card2: #f3efe6 fundal input/contor (= --bg) +--ink: #1e1a13 text principal (maro-inchis, AA pe crem) +--muted: #6a6052 text secundar +--line: #e2dccc borduri +--line2: #ece6d9 separator subtire (mai deschis decat line) +--accent:#1F5FBF albastru clasic (landing --infot = --accent) — 6.5:1 pe --bg, AA +--ok: #1c7d5d sent / succes (verde AA pe crem) +--warn: #b45309 atentie (chihlimbar AA pe crem) +--err: #bd463c eroare (rosu AA pe crem) +``` + +### Tokeni noi adaugati la PRD 5.15 (in toate cele 7 teme) + +``` +--card2 fundal input slim si carduri-contor (US-001/002); pe dark = --bg (cel mai adanc nivel) +--line2 separator subtire intre randuri lista slim (US-001/002); mai subtil decat --line +``` + ### Culori de brand (doar wordmark, NU variabile de UI) ``` @@ -109,17 +197,27 @@ Inlocuieste comutatorul binar soare/luna cu un **buton ciclic** (pattern ca demo singur buton care roteste la fiecare click prin setul de teme, cu iconita + tooltip/`aria-label` care arata tema curenta („Tema: Light" etc.). -Ordinea ciclului: **Light → Dark → Petrol → Auto → (inapoi la Light)**. +Ordinea ciclului (PRD 5.15 — teme aditive D2): +**Light → Dark → Petrol → Grafit → Cobalt → Cupru → Hartie → Auto → (inapoi la Light)**. -- `Light` → `data-theme="light"` (azur pe alb) -- `Dark` → `data-theme="dark"` (azur pe inchis, comportamentul implicit actual) -- `Petrol` → `data-theme="petrol"` (teal pe petrol-inchis) -- `Auto` → urmeaza `prefers-color-scheme`; rezolva la Light azur sau Dark azur in functie de OS - (nu seteaza `data-theme` fix, ci il deriva la paint). +- `Light` → `data-theme="light"` (azur pe alb) — ☀ +- `Dark` → `data-theme="dark"` (azur pe inchis, comportamentul implicit actual) — ☾ +- `Petrol` → `data-theme="petrol"` (teal pe petrol-inchis) — ◐ +- `Grafit` → `data-theme="grafit"` (azur deschis pe negru-grafit, similar dark) — ◑ +- `Cobalt` → `data-theme="cobalt"` (albastru viu pe bleumarin adanc) — ◆ +- `Cupru` → `data-theme="cupru"` (chihlimbar pe maro cald) — ◇ +- `Hartie` → `data-theme="hartie"` (albastru clasic pe crem cald, similar light) — ○ +- `Auto` → urmeaza `prefers-color-scheme`; rezolva la `light` (OS light) sau `dark` (OS dark). — ◉ Persistenta: preferinta explicita (inclusiv „Auto") in `localStorage`, doar la click. Scriptul -anti-FOUC din `` trebuie sa rezolve „Auto"→light/dark inainte de primul paint (fara blink). -Iconite: ☀ Light, ☾ Dark, ◐ Petrol, ◉ Auto. Default la prima vizita = Auto (OS-aware), ca azi. +anti-FOUC din `` cunoaste toate cele 7+1 stari; valori vechi (light/dark/petrol) raman +valide fara migrare fortata; valoare lipsa/necunoscuta → auto (fallback sigur, fara blink). + +Implementare DRY (E2 PRD 5.15): configuratia temelor traieste intr-o singura structura JS +`var THEMES = [...]` (sursa de adevar), din care se DERIVA `CYCLE`/`VALID`/`ICONS`/`LABELS`/`NEXT`. +Adaugarea unei teme noi = O singura intrare in `THEMES`. + +Default la prima vizita = Auto (OS-aware), ca inainte. ## Componente — note de aplicare @@ -133,6 +231,99 @@ Iconite: ☀ Light, ☾ Dark, ◐ Petrol, ◉ Auto. Default la prima vizita = Au - **Suprafete de stare** (banner, flash, eroare-3n): raman pe `color-mix` peste `--err/--warn/--ok`, deci se adapteaza automat la noua paleta si la light/dark. +## Componente slim (PRD 5.15 US-002) + +Adaugate in `base.html` (sectiunea `SENTINEL-COMPONENTE-SLIM`). Toate culorile exclusiv prin +`var(--token)` — zero hex hardcodat. Consumate de US-003 (dashboard), US-004 (lista), US-007 (formular). + +### `.contor-card` + +Card cifra-contor compact: fundal `--card2`, bordura `--line`, `border-radius:8px`, padding 10-12px. + +```html +
+
847
+
Trimise (total)
+
luna 124 · azi 9
+
+``` + +Sub-elemente: +- `.contor-cifra` — `font-size:22px; font-weight:700`; culoare prin `.s-*` existente +- `.contor-label` — `font-size:11px; color:var(--muted)` +- `.contor-sub` — IBM Plex Mono, `font-size:10px; color:var(--muted)` + +### `.lista-trimiteri-slim` + `.trimitere-slim` + +Lista compacta cu separator `--line2`. Randul este clickabil (rol button), tinta `min-height:44px`. + +```html + +``` + +Sub-elemente: +- `.slim-vin` — IBM Plex Mono, `font-size:13px; font-weight:500; color:var(--ink)` +- `.slim-meta` — `font-size:11px; color:var(--muted)` (operatie + ora) + +### `.camp-slim` + macro `camp(slim=True)` + +Varianta compacta de camp formular: label 11px muted deasupra, input `height:30px`, fundal `--card2`. +Integrata in macro-ul `camp` din `_macros.html` prin flagul `slim=False` (default — randarea +actuala ramane neschimbata). + +```jinja2 +{{ camp('vin', 'VIN (serie sasiu)', vin, slim=True) }} +``` + +Pentru campuri mono (VIN, odometru, nr. inmatriculare): adauga clasa `camp-mono` pe input +(via `style=""` sau atribut `class=""` direct — macro-ul nu il pune automat, consumatorul decide). + +### `.chips` + `.chip` + `.chip-del` + +Prestatii multi-select: container `.chips` (fundal `--card2`), item `.chip` (accent 18%, IBM Plex +Mono 11px), buton de stergere `.chip-del` (accesibil cu `aria-label`). + +```html +
+ + + REV2 + + + + R-ODO + +
+``` + +Clase aditionale: +- `.chip-warn` — fundal `--warn` 22% (pentru coduri R-ODO/I-ODO care cer odometruInitial) + +### `.add-code` + `.op-row` (picker E4) + +Buton dashed pentru adaugare cod (`.add-code`) si randul operatie<->cod (`.op-row`, `.op-row-name`, +`.op-row-warn`). Folosite de picker-ul E4 din US-007 (formular editare). + +```html +
+ REVIZIE PERIODICA + REV2 + +
+
+ SCHIMBARE PLACUTE FRANA + +
+``` + ## Ce NU schimbam - Mecanismul light/dark existent (anti-FOUC, persistenta `localStorage`, comutator) — il pastram, diff --git a/Dockerfile b/Dockerfile index c6e933e..e6c64eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,17 @@ FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + PYTHONDONTWRITEBYTECODE=1 \ + # Fus orar RO: SQLite 'localtime' (bucketare contoare azi/luna, E7) depinde de TZ. + # tzdata ofera baza de fusuri; TZ alege Europe/Bucharest (DST-aware, UTC+2/+3). + TZ=Europe/Bucharest WORKDIR /app +# tzdata = necesar pentru ca 'localtime' din SQLite sa rezolve Europe/Bucharest. +RUN apt-get update && apt-get install -y --no-install-recommends tzdata \ + && rm -rf /var/lib/apt/lists/* + COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/app/api/v1/import_router.py b/app/api/v1/import_router.py index 0055502..b140339 100644 --- a/app/api/v1/import_router.py +++ b/app/api/v1/import_router.py @@ -186,6 +186,14 @@ def _resolve_row_for_preview( denumire = str(denumire_val).strip() if denumire_val not in (None, "") else str(operatie_val) mapped["prestatii"] = [{"cod_op_service": str(operatie_val), "denumire": denumire}] + # obs derive-on-empty (D7/E3 PRD 5.15): daca obs e gol si avem operatie, + # copiem denumirea operatiei in obs (nu o mutam — op_service ramane neatins). + # DERIVE-ON-EMPTY: doar cand obs e gol, ca sa fie idempotent la re-preview/re-editare. + obs_curent = str(mapped.get("obs") or "").strip() + if not obs_curent and operatie_val: + obs_denumire = str(denumire_val).strip() if denumire_val not in (None, "") else str(operatie_val) + mapped["obs"] = obs_denumire + # Canonicalizare: normalizeaza VIN/nr/odometru canon = canonicalize_row(mapped) mapped.update({ @@ -257,8 +265,9 @@ def _build_idempotency_key(account_id: int | None, resolved: dict[str, Any]) -> # Campuri de continut editabile in preview. Operatia/codul RAR NU se editeaza -# aici (raman in panoul de mapare). -EDIT_FIELDS = ("vin", "nr_inmatriculare", "data_prestatie", "odometru_initial", "odometru_final") +# aici (raman in panoul de mapare). obs = text liber, se trateaza ca non-canonic +# (doar .strip(), fara canonicalize_row) — urmeaza ramura `else` din _merge_override. +EDIT_FIELDS = ("vin", "nr_inmatriculare", "data_prestatie", "odometru_initial", "odometru_final", "obs") def _merge_override(current: dict[str, Any], fields: dict[str, str | None]) -> dict[str, Any]: @@ -279,7 +288,15 @@ def _merge_override(current: dict[str, Any], fields: dict[str, str | None]) -> d continue s = str(val).strip() if s == "": - out.pop(camp, None) # empty = clear + if camp == "obs": + # obs e camp DERIVAT (copiaza denumirea operatiei cand e gol). Empty = + # STERGERE EXPLICITA a userului -> pastram obs='' in override ca + # derive-on-empty sa NU il re-deriveze (override aplicat ULTIMUL + # suprascrie derivarea, in preview si la commit). Un pop ar fi pierdut + # semnalul "sters explicit" -> obs re-derivat silentios din denumire. + out["obs"] = "" + else: + out.pop(camp, None) # empty = clear (revine la valoarea din fisier) else: raw[camp] = s if raw: @@ -1078,6 +1095,13 @@ def commit_import( denumire = str(denumire_val).strip() if denumire_val not in (None, "") else str(operatie_val) mapped["prestatii"] = [{"cod_op_service": str(operatie_val), "denumire": denumire}] + # obs derive-on-empty (D7/E3 PRD 5.15): copiere denumire in obs daca obs e gol. + # Identic cu logica din _resolve_row_for_preview (override aplicat tot ultimul). + obs_curent = str(mapped.get("obs") or "").strip() + if not obs_curent and operatie_val: + obs_denumire = str(denumire_val).strip() if denumire_val not in (None, "") else str(operatie_val) + mapped["obs"] = obs_denumire + # Rezolva prestatii INAINTE de canonicalizare (altfel cheia difera de cea din preview) prestatii = mapped.get("prestatii") or [] resolved, _ = resolve_prestatii(prestatii, mapping, valid_codes, text_rules) @@ -1180,12 +1204,14 @@ def commit_import( class RandEditIn(BaseModel): """Campuri de continut editabile in preview. None = ne-trimis (neschimbat); - "" = sterge override-ul (revine la valoarea din fisier).""" + "" = sterge override-ul (revine la valoarea din fisier). + obs = text liber fara validare de continut (US-005 PRD 5.15).""" vin: str | None = None nr_inmatriculare: str | None = None data_prestatie: str | None = None odometru_initial: str | None = None odometru_final: str | None = None + obs: str | None = None @router.post("/{import_id}/rand/{row_index}/editeaza") diff --git a/app/config.py b/app/config.py index 1da059d..812ac11 100644 --- a/app/config.py +++ b/app/config.py @@ -104,6 +104,13 @@ class Settings(BaseSettings): worker_retry_max_s: int = 300 worker_max_retries: int = 8 # peste atat -> error + banner + # --- Embeddings (sugestie mapare, Stratul 2 PRD 5.14) --- + # DEZACTIVAT implicit: prima folosire lazy-load-eaza modelul fastembed/ONNX + # (~230MB pe disc) sincron in thread-ul de cerere -> hang la prima cerere /mapari. + # Activeaza explicit in productie (start.sh/Docker/.env) cand vrei sugestii semantice. + # OFF pastreaza suita de teste rapida si /mapari instant (cade pe GOLD/SILVER+fuzzy). + embeddings_enabled: bool = False + @property def rar_base_url(self) -> str: return self.rar_base_url_prod if self.rar_env == "prod" else self.rar_base_url_test diff --git a/app/embeddings.py b/app/embeddings.py new file mode 100644 index 0000000..1fd9e86 --- /dev/null +++ b/app/embeddings.py @@ -0,0 +1,246 @@ +"""Modul embedding in-proces pentru sugestie cod RAR -- L14-S4. + +Design (PRD 5.14, Decision #16/#16b): +- Model multilingv via fastembed/ONNX (~230MB pe disc, quantizat, fara torch) +- Lazy load la prima folosire, NU la import si NU pe /healthz +- Worker NU incarca modelul (API-only) +- Degradare gratioasa: daca modelul nu se incarca -> is_available()=False, + suggest_nearest() -> [] fara exceptie, ingestia NU e blocata +- Embeddings = DOAR sugestie (nu intra in lantul de enqueue/resolve_prestatii) +- NU apelat din resolve_prestatii/load_mapping (wiring vine in L14-S6 DUPA 5.15) + +API public (nivel modul): + index_corpus(items) -> None + suggest_nearest(text, top_k) -> [{cod, similaritate}] + is_available() -> bool + +Clase (pentru teste / injectare backend): + EmbeddingEngine(backend) -- motor testabil cu backend injectabil + FastEmbedBackend() -- backend real fastembed/ONNX +""" +from __future__ import annotations + +import logging +import math +from typing import Protocol, runtime_checkable + +log = logging.getLogger(__name__) + +# Modelul ales: paraphrase-multilingual-MiniLM-L12-v2 +# ~230MB pe disc (ONNX quantizat), 384 dim, multilingv (ro/en/etc.), suportat de +# fastembed, fara torch. (Estimarea initiala din PRD de ~50MB a fost gresita.) +FASTEMBED_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" + + +# --------------------------------------------------------------------------- # +# Protocol backend (mockabil in teste) # +# --------------------------------------------------------------------------- # + +@runtime_checkable +class EmbeddingBackend(Protocol): + """Interfata minimala pentru un backend de embedding.""" + + def embed(self, texts: list[str]) -> list[list[float]]: + """Intoarce o lista de vectori (cate unul per text).""" + ... + + +# --------------------------------------------------------------------------- # +# Backend real: fastembed/ONNX # +# --------------------------------------------------------------------------- # + +class FastEmbedBackend: + """Backend fastembed/ONNX. Lazy-load la constructie. + + Arunca ImportError daca fastembed nu e instalat, sau orice exceptie + de la TextEmbedding (download esuat, ONNX incompatibil etc.). + Apelantul (_load_engine) prinde aceste exceptii. + """ + + def __init__(self, model_name: str = FASTEMBED_MODEL): + from fastembed import TextEmbedding # import tardiv -- nu blocheaza la import modul + self._model = TextEmbedding(model_name=model_name) + + def embed(self, texts: list[str]) -> list[list[float]]: + # fastembed.embed() intoarce un generator de numpy arrays + return [vec.tolist() for vec in self._model.embed(texts)] + + +# --------------------------------------------------------------------------- # +# Motor de embedding (testabil, backend injectabil) # +# --------------------------------------------------------------------------- # + +def _cosine_similarity(a: list[float], b: list[float]) -> float: + """Similaritate cosine intre doi vectori. Returneaza 0.0 pe vectori nuli.""" + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(x * x for x in b)) + if na == 0.0 or nb == 0.0: + return 0.0 + return dot / (na * nb) + + +class EmbeddingEngine: + """Motor de embedding cu corpus indexat si cautare NN cosine. + + Parametri: + backend: instanta EmbeddingBackend (real sau mock). + None => degradare gratioasa (is_available=False). + """ + + def __init__(self, backend: EmbeddingBackend | None = None): + self._backend = backend + self._corpus_vecs: list[list[float]] = [] + self._corpus_items: list[dict] = [] + self._corpus_sig: str | None = None + + def is_available(self) -> bool: + """True daca backend-ul e disponibil si gata de folosire.""" + return self._backend is not None + + def has_corpus(self) -> bool: + """True daca un corpus a fost indexat (suggest_nearest poate produce ceva).""" + return bool(self._corpus_items) + + def corpus_signature(self) -> str | None: + """Semnatura corpusului indexat (None daca gol). Apelantul re-indexeaza + doar cand semnatura nomenclatorului s-a schimbat (evita re-embed inutil).""" + return self._corpus_sig + + def index_corpus(self, items: list[dict], signature: str | None = None) -> None: + """Vectorizeaza corpus [{denumire, cod}] si il pastreaza in memorie. + + Ignora silentios daca backend-ul lipseste, corpus-ul e gol sau apare + orice exceptie la vectorizare (degradare gratioasa). + """ + self._corpus_vecs = [] + self._corpus_items = [] + self._corpus_sig = None + + if not items or not self.is_available(): + return + + try: + texts = [str(item["denumire"]) for item in items] + vecs = self._backend.embed(texts) + self._corpus_vecs = vecs + self._corpus_items = list(items) + self._corpus_sig = signature + except Exception as exc: + log.warning("embeddings: index_corpus esuat: %s", exc) + # corpus ramane gol -- suggest_nearest va returna [] + + def suggest_nearest( + self, + denumire: str, + top_k: int = 3, + ) -> list[dict]: + """Returneaza top_k vecini cosine [{cod, similaritate}]. + + Returneaza [] daca backend-ul lipseste, corpus-ul e gol sau apare + orice exceptie (degradare gratioasa -- nu blocheaza ingestia). + """ + if not self.is_available() or not self._corpus_items: + return [] + + try: + query_vecs = self._backend.embed([str(denumire)]) + query_vec = query_vecs[0] + scored = [ + { + "cod": item["cod"], + "similaritate": _cosine_similarity(query_vec, vec), + } + for item, vec in zip(self._corpus_items, self._corpus_vecs) + ] + scored.sort(key=lambda r: r["similaritate"], reverse=True) + return scored[:top_k] + except Exception as exc: + log.warning("embeddings: suggest_nearest esuat: %s", exc) + return [] + + +# --------------------------------------------------------------------------- # +# Singleton global cu lazy load (API-only, NU worker) # +# --------------------------------------------------------------------------- # + +_engine: EmbeddingEngine | None = None + + +def _load_engine() -> EmbeddingEngine: + """Lazy load: construieste engine-ul la prima folosire. + + Captureaza ORICE exceptie la incarcare (import, download, ONNX init) + si returneaza un engine degradat (backend=None) -- ingestia continua + pe exact+fuzzy, embedding = sugestie dezactivata. + """ + try: + backend = FastEmbedBackend() + log.info("embeddings: backend fastembed incarcat (%s)", FASTEMBED_MODEL) + return EmbeddingEngine(backend=backend) + except ImportError: + log.warning( + "embeddings: fastembed nu e instalat -- sugestii NN dezactivate" + ) + except Exception as exc: + log.warning( + "embeddings: incarcare backend esuata (%s) -- sugestii NN dezactivate", + exc, + ) + return EmbeddingEngine(backend=None) + + +def _get_engine() -> EmbeddingEngine: + """Returneaza engine-ul global (lazy-init).""" + global _engine + if _engine is None: + _engine = _load_engine() + return _engine + + +# --------------------------------------------------------------------------- # +# API public la nivel de modul (wiring L14-S6) # +# --------------------------------------------------------------------------- # + +def is_available() -> bool: + """True daca modelul e incarcat si gata de folosire.""" + return _get_engine().is_available() + + +def has_corpus() -> bool: + """True daca un corpus a fost indexat in motorul global. + + NU forteaza incarcarea modelului: daca engine-ul nu a fost initializat inca + (`_engine is None`), corpus-ul e gol prin definitie -> False, fara cost. + Apelantii (ex. enrich_suggestions) folosesc asta ca poarta ieftina inainte de + a atinge calea scumpa (is_available/suggest_nearest, care lazy-load ~230MB). + """ + if _engine is None: + return False + return _engine.has_corpus() + + +def corpus_signature() -> str | None: + """Semnatura corpusului global indexat (None daca engine ne-initializat/gol). + + NU forteaza incarcarea modelului: `_engine is None` -> None fara cost. + """ + if _engine is None: + return None + return _engine.corpus_signature() + + +def index_corpus(items: list[dict], signature: str | None = None) -> None: + """Vectorizeaza corpus [{denumire, cod}] in motorul global. + + Silentios pe eroare (degradare gratioasa). + """ + _get_engine().index_corpus(items, signature=signature) + + +def suggest_nearest(denumire: str, top_k: int = 3) -> list[dict]: + """Returneaza top_k sugestii [{cod, similaritate}] sau [] la eroare. + + Sigur de apelat indiferent de starea backend-ului. + """ + return _get_engine().suggest_nearest(denumire, top_k=top_k) diff --git a/app/mapping.py b/app/mapping.py index 47249cd..2b41991 100644 --- a/app/mapping.py +++ b/app/mapping.py @@ -14,6 +14,7 @@ unit-testabile direct. Cele cu `conn` sunt helpere de persistenta. from __future__ import annotations +import hashlib import json import unicodedata from typing import Any @@ -483,10 +484,18 @@ def pending_unmapped(conn, account_id=None) -> list[dict]: entry["denumire"] = item.get("denumire") entry["_ids"].add(r["id"]) + # Indexeaza corpusul embeddings o data inainte de bucla (no-op cand flagul e off). + ensure_embeddings_corpus(conn, nomenclator) + out: list[dict] = [] for entry in agg.values(): entry["blocked"] = len(entry.pop("_ids")) entry["suggestions"] = suggest_codes(entry["denumire"], nomenclator, limit=5) + # L14-S6: imbogatire sugestii cu GOLD partajat > SILVER > embeddings (Eng-F2). + # SUGGESTION-ONLY: nu intra in resolve_prestatii/load_mapping (#13). + enriched = enrich_suggestions(conn, entry["denumire"]) + entry["sugestie_principala"] = enriched["sugestie_principala"] + entry["surse_sugestie"] = enriched["surse"] out.append(entry) out.sort(key=lambda e: (-e["blocked"], e["cod_op_service"])) return out @@ -561,6 +570,148 @@ def delete_text_rule(conn, account_id: int | None, pattern: str) -> None: ) +# Prag minim de similaritate cosine pentru sugestia din embeddings NN. +# Sub acest scor, sugestia NN e prea incerta si nu o afisam (previne recomandari +# irelevante cand corpus-ul e mic sau neindexat corect). +EMB_MIN_SIMILARITATE = 0.5 + + +def _corpus_signature(nomenclator: list[dict]) -> str: + """Semnatura stabila a nomenclatorului pentru cache-ul corpusului embeddings. + + Hash pe perechile (cod, denumire) sortate dupa cod -> se schimba la orice + add/remove/redenumire de cod, ramane stabila altfel (evita re-embed inutil). + """ + pairs = sorted( + (str(n.get("cod_prestatie") or ""), str(n.get("nume_prestatie") or "")) + for n in nomenclator + ) + blob = "".join(f"{c}{d}" for c, d in pairs) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def ensure_embeddings_corpus(conn, nomenclator: list[dict] | None = None) -> None: + """Construieste/actualizeaza corpusul embeddings din nomenclator (Stratul 2 PRD 5.14). + + Gated pe `AUTOPASS_EMBEDDINGS_ENABLED` (default OFF): cand e dezactivat, e un + no-op total (nu atinge modelul, nu interogheaza nomenclatorul) -> /mapari instant + + suita de teste rapida; sugestiile cad pe GOLD/SILVER + fuzzy. + + Cand e activat: indexeaza corpusul {denumire=nume_prestatie, cod=cod_prestatie} + o singura data (lazy-load modelul ~230MB la prima chemare), re-indexeaza doar + cand semnatura nomenclatorului s-a schimbat. Degradare gratioasa: orice eroare + (model absent, embed esuat) lasa corpusul gol -> enrich_suggestions cade pe restul. + + Apelat de apelantii care imbogatesc sugestii (pending_unmapped, + _nemapate_pentru_submission) INAINTE de bucla de enrich_suggestions, NU din + enrich_suggestions (care ramane o interogare ieftina cu garda has_corpus()). + """ + from .config import get_settings + if not get_settings().embeddings_enabled: + return + try: + from . import embeddings as _emb + nomen = nomenclator if nomenclator is not None else load_nomenclator(conn) + if not nomen: + return + sig = _corpus_signature(nomen) + if _emb.corpus_signature() == sig and _emb.has_corpus(): + return # deja indexat pe acelasi nomenclator -> nimic de facut + items = [ + {"denumire": str(n["nume_prestatie"]), "cod": str(n["cod_prestatie"])} + for n in nomen + if n.get("nume_prestatie") and n.get("cod_prestatie") + ] + _emb.index_corpus(items, signature=sig) + except Exception: + pass # degradare gratioasa (#16b): esecul indexarii nu blocheaza editorul + + +def enrich_suggestions( + conn, + denumire: str | None, + *, + include_embeddings: bool = True, +) -> dict: + """Imbogateste sugestiile cu GOLD partajat, SILVER LLM si embeddings NN. + + Precedenta Eng-F2 (pentru sugestie-only, nu auto-send): + shared GOLD > SILVER > embeddings + + (Account GOLD = operations_mapping propriu = deja rezolvat inainte de needs_mapping; + nu apare in needs_mapping, deci nu e in precedenta de sugestie.) + + Returneaza: + { + 'sugestie_principala': {'cod_prestatie': str, 'sursa': str} | None, + 'surse': {'gold_partajat': str|None, 'silver': str|None, 'embedding': str|None} + } + + INVARIANTE: + - Toate sursele = SUGGESTION-ONLY. NU intra in resolve_prestatii/load_mapping (#13). + - SILVER cu is_nul=1 (non-operatie/gunoi) NU produce sugestie (#4). + - Degradare gratioasa pe embeddings (#16b): daca motorul nu e disponibil sau arunca, + returneaza sugestia disponibila din celelalte surse, fara exceptie. + - Import local shared_store/embeddings: evita ciclu la import-time (shared_store + importa normalize_for_match din mapping). + """ + sugestie_principala: dict | None = None + surse: dict = {"gold_partajat": None, "silver": None, "embedding": None} + + if not denumire: + return {"sugestie_principala": sugestie_principala, "surse": surse} + + # Colecteaza TOATE sursele (fara short-circuit) in `surse`: editorul le poate afisa + # toate, independent de care castiga ca sugestie principala. + # Precedenta Eng-F2 se aplica DOAR la alegerea sugestiei_principale. + + # 1. GOLD partajat cross-account (validat de om, cel mai de incredere) + try: + from .shared_store import lookup_shared_gold + row_gold = lookup_shared_gold(conn, denumire) + if row_gold and row_gold["cod_prestatie"]: + surse["gold_partajat"] = str(row_gold["cod_prestatie"]) + except Exception: + pass # degradare gratioasa + + # 2. SILVER LLM (bootstrap, nevalidat de om; is_nul = supresie) + try: + from .shared_store import lookup_suggestion + row_silver = lookup_suggestion(conn, denumire) + if row_silver and not row_silver["is_nul"] and row_silver["cod_prestatie"]: + surse["silver"] = str(row_silver["cod_prestatie"]) + except Exception: + pass # degradare gratioasa + + # 3. Embeddings NN (similaritate semantica, degradare gratioasa #16b) + if include_embeddings: + try: + from . import embeddings as _emb + # Poarta IEFTINA: nu atinge is_available()/suggest_nearest cand corpus-ul + # e gol — `is_available()` lazy-load-eaza modelul de ~230MB (30-120s in + # thread-ul de cerere). Corpusul se construieste de apelant prin + # ensure_embeddings_corpus (gated pe AUTOPASS_EMBEDDINGS_ENABLED); cand + # flagul e off, has_corpus() ramane False si calea e un no-op real. + if _emb.has_corpus(): + nn = _emb.suggest_nearest(str(denumire), top_k=1) + # Prag minim: similaritate prea mica = sugestie inutila. + # Evita recomandari irelevante cand corpus-ul e mic/partial. + if nn and nn[0].get("similaritate", 0) >= EMB_MIN_SIMILARITATE: + surse["embedding"] = str(nn[0]["cod"]) + except Exception: + pass # degradare gratioasa (#16b): motorul absent nu blocheaza + + # Alege sugestia principala in ordinea de precedenta: GOLD > SILVER > embeddings + if surse["gold_partajat"]: + sugestie_principala = {"cod_prestatie": surse["gold_partajat"], "sursa": "gold_partajat"} + elif surse["silver"]: + sugestie_principala = {"cod_prestatie": surse["silver"], "sursa": "silver"} + elif surse["embedding"]: + sugestie_principala = {"cod_prestatie": surse["embedding"], "sursa": "embedding"} + + return {"sugestie_principala": sugestie_principala, "surse": surse} + + def _emite_text_rule_hits(conn, account_id: int, submission_id: int, resolved: list[dict] | None) -> None: """Emite `text_rule_hit` in app_events pentru fiecare item rezolvat prin regula text. diff --git a/app/payload_view.py b/app/payload_view.py index 6550e1f..b158b60 100644 --- a/app/payload_view.py +++ b/app/payload_view.py @@ -125,6 +125,10 @@ def prezentare_din_payload(payload: str | dict | None) -> dict[str, str]: # altfel ar expune denumirea RAR drept op. de service, ceea ce e semantic incorect. op_service_denumire = _clean_str(item.get("denumire")) if op_service_cod else "" + # obs: text liber observatii (camp RAR, optional). Conventie goala "" (nu EMPTY). + # US-005 PRD 5.15: obs traieste in payload_json (nu coloana separata). + obs = _clean_str(data.get("obs")) + return { "vehicul_nr": nr or EMPTY, "vin": vin or EMPTY, @@ -137,4 +141,5 @@ def prezentare_din_payload(payload: str | dict | None) -> dict[str, str]: # Chei cu conventie goala "" (nu EMPTY) — vezi comentariu de mai sus "op_service_cod": op_service_cod, "op_service_denumire": op_service_denumire, + "obs": obs, } diff --git a/app/schema.sql b/app/schema.sql index 2c9846e..94aa7de 100644 --- a/app/schema.sql +++ b/app/schema.sql @@ -200,6 +200,42 @@ CREATE TABLE IF NOT EXISTS operation_text_rules ( ); CREATE INDEX IF NOT EXISTS idx_text_rules_account ON operation_text_rules(account_id); +-- Sugestii de mapare (strat SILVER, L14-S3 PRD 5.14). +-- Etichete LLM/embedding — bootstrap; citita DOAR de suggest_codes/pending_unmapped, +-- NICIODATA de load_mapping/resolve_prestatii (separare structurala #13). +-- Cheia = denumire normalizata (fara diacritice, uppercase, spatii colapsate). +-- is_nul=1: non-operatie (ITP, discount, nr. inmatriculare) -> suprima (#4), cod NULL. +-- INSERT OR IGNORE la re-seed: nu suprascrie randuri existente (#2). +CREATE TABLE IF NOT EXISTS mapping_suggestions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + denumire_normalizata TEXT NOT NULL UNIQUE, + cod_prestatie TEXT, -- NULL cand is_nul=1 (supresie) + is_nul INTEGER NOT NULL DEFAULT 0 CHECK (is_nul IN (0, 1)), + source TEXT NOT NULL, -- 'llm', 'embedding', etc. (#5) + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); +CREATE INDEX IF NOT EXISTS idx_mapping_suggestions_cod + ON mapping_suggestions(cod_prestatie) WHERE cod_prestatie IS NOT NULL; + +-- Mapari validate de oameni (strat GOLD partajat cross-account, L14-S3 PRD 5.14). +-- Confirmarile umane din ORICE cont contribuie la acest store (#8). +-- cross-account = suggestion-only (pre-completeaza editorul, F-A/#11), NU auto-send. +-- Auto-send DOAR din operations_mapping (GOLD propriu per-cont, Eng-F2). +-- Cheia = denumire_normalizata (NU cod_op_service: spatii de chei diferite, #14). +CREATE TABLE IF NOT EXISTS shared_mappings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + denumire_normalizata TEXT NOT NULL UNIQUE, + cod_prestatie TEXT NOT NULL, -- cod RAR valid (GOLD = validat de om) + source TEXT NOT NULL DEFAULT 'human', -- 'human', 'human_import' (#5) + provenance TEXT, -- detalii: cont, email, batch (#5) + confidence REAL NOT NULL DEFAULT 1.0, + confirmations INTEGER NOT NULL DEFAULT 1, -- contor confirmari din orice cont + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + -- Heartbeat worker (un singur rand, id=1). /healthz citeste de aici. CREATE TABLE IF NOT EXISTS worker_heartbeat ( id INTEGER PRIMARY KEY CHECK (id = 1), diff --git a/app/shared_store.py b/app/shared_store.py new file mode 100644 index 0000000..ee0a13e --- /dev/null +++ b/app/shared_store.py @@ -0,0 +1,139 @@ +"""Store partajat pentru sugestii (SILVER) si mapari validate de oameni (GOLD cross-account). + +Straturi (L14-S3, PRD 5.14): +- mapping_suggestions (SILVER): sugestii LLM/embedding, citite DOAR de suggest/pending_unmapped, + NICIODATA de load_mapping/resolve_prestatii (separare structurala #13). +- shared_mappings (GOLD partajat): mapari validate de om din orice cont; pot pre-completa + editorul (suggestion-only cross-account, F-A/#11); auto-send DOAR GOLD propriu + (operations_mapping per-cont, Eng-F2). + +Invariante: +- INSERT OR IGNORE la seed: nu suprascrie randuri existente (#2). +- NUL = is_nul=1, cod_prestatie NULL; NU se promoveaza la cod RAR (#4). +- source/confidence pe fiecare rand (provenienta + rollback batch model prost, #5). +- Wiring in resolve_prestatii/load_mapping vine in L14-S6; modulul de fata e API pur. +""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + +from .mapping import normalize_for_match + + +def seed_suggestions( + conn: sqlite3.Connection, + items: list[dict[str, Any]], +) -> int: + """Insereaza sugestii in mapping_suggestions (SILVER). INSERT OR IGNORE. + + Nu suprascrie randuri deja existente (#2): re-rularea seeder-ului e sigura. + Fiecare item trebuie sa contina: + - 'denumire': str — text brut (se normalizeaza intern cu normalize_for_match) + - 'source': str — 'llm', 'embedding', etc. + Optional: + - 'cod_prestatie': str | None — ignorat cand is_nul=True + - 'is_nul': bool — True pt non-operatii (supresie, #4); cod_prestatie stocat NULL + - 'confidence': float — 0..1 (default 0.0) + + Returneaza numarul de randuri inserate efectiv (0 daca toate existau deja). + """ + inserted = 0 + for item in items: + den_norm = normalize_for_match(item.get("denumire") or "") + if not den_norm: + continue + is_nul = 1 if item.get("is_nul") else 0 + # NUL -> cod NULL obligatoriu (supresie stricta, #4) + cod = None if is_nul else ((item.get("cod_prestatie") or "") or None) + if cod: + cod = cod.strip().upper() + source = str(item.get("source") or "llm") + confidence = float(item.get("confidence") or 0.0) + cur = conn.execute( + """ + INSERT OR IGNORE INTO mapping_suggestions + (denumire_normalizata, cod_prestatie, is_nul, source, confidence) + VALUES (?, ?, ?, ?, ?) + """, + (den_norm, cod, is_nul, source, confidence), + ) + inserted += cur.rowcount + return inserted + + +def lookup_suggestion( + conn: sqlite3.Connection, + denumire: str, +) -> sqlite3.Row | None: + """Cauta sugestie SILVER dupa denumire normalizata. + + Returneaza randul din mapping_suggestions sau None daca nu exista. + NOTA: apelantul trebuie sa verifice is_nul inainte de a folosi cod_prestatie. + """ + den_norm = normalize_for_match(denumire) + if not den_norm: + return None + return conn.execute( + "SELECT * FROM mapping_suggestions WHERE denumire_normalizata = ?", + (den_norm,), + ).fetchone() + + +def lookup_shared_gold( + conn: sqlite3.Connection, + denumire: str, +) -> sqlite3.Row | None: + """Cauta mapare GOLD partajata dupa denumire normalizata. + + Returneaza randul din shared_mappings sau None daca nu exista. + NOTA (F-A/#11): acest GOLD partajat e suggestion-only cross-account; + auto-send vine DOAR din operations_mapping (GOLD propriu per-cont). + """ + den_norm = normalize_for_match(denumire) + if not den_norm: + return None + return conn.execute( + "SELECT * FROM shared_mappings WHERE denumire_normalizata = ?", + (den_norm,), + ).fetchone() + + +def record_human_validation( + conn: sqlite3.Connection, + denumire: str, + cod_prestatie: str, + *, + source: str = "human", + provenance: str | None = None, + confidence: float = 1.0, +) -> None: + """Inregistreaza o mapare validata de om in GOLD partajat (shared_mappings). + + Daca denumirea exista deja: incrementeaza confirmations + actualizeaza updated_at. + Daca nu exista: insert nou cu confirmations=1. + + Apelat la confirmarea umana a unei mapari (din editorul needs_mapping). + Wiring efectiv vine in L14-S6 (dupa 5.15); aceasta functie e API-ul store. + + NOTA: NU intra in operations_mapping (GOLD per-cont) — acela e gestionat + separat de editorul existent. Ambele pot coexista. + """ + den_norm = normalize_for_match(denumire) + if not den_norm: + return + cod = (cod_prestatie or "").strip().upper() + if not cod: + return + conn.execute( + """ + INSERT INTO shared_mappings + (denumire_normalizata, cod_prestatie, source, provenance, confidence, confirmations) + VALUES (?, ?, ?, ?, ?, 1) + ON CONFLICT(denumire_normalizata) DO UPDATE SET + confirmations = confirmations + 1, + updated_at = datetime('now') + """, + (den_norm, cod, source, provenance, confidence), + ) diff --git a/app/web/routes.py b/app/web/routes.py index 052ebbc..8120117 100644 --- a/app/web/routes.py +++ b/app/web/routes.py @@ -13,6 +13,7 @@ from __future__ import annotations import hashlib import json import math +import re as _re import sqlite3 from datetime import datetime, timezone from pathlib import Path @@ -53,6 +54,7 @@ from ..api.v1.import_router import ( from ..config import get_settings from ..crypto import decrypt_creds, encrypt_creds from ..db import get_connection, read_app_events, read_heartbeat +from ..observ import log_event from ..idempotency import build_key, canonicalize_row from ..validation import validate_prezentare from ..import_parse import FileTooLarge, HeaderError, MultipleSheets, parse_date_value, parse_file @@ -69,6 +71,8 @@ from ..mapping import ( account_or_default, account_scope_clause, delete_text_rule, + enrich_suggestions, + ensure_embeddings_corpus, has_no_auto_send, load_mapping_meta, load_nomenclator, @@ -83,6 +87,7 @@ from ..mapping import ( suggest_codes, text_rules_overlap, ) +from ..shared_store import record_human_validation # Campuri canonice cu eticheta umana pentru dropdown mapare coloane _CANONICAL_FIELDS = [(k, v[0]) for k, v in _CANONICAL_SYNONYMS.items()] @@ -95,6 +100,76 @@ templates.env.globals["parse_erori"] = parse_erori _BLOCKED = ("error", "needs_data", "needs_mapping") +def _record_gold_validation(conn, denumire: str | None, cod_op_service: str, + cod_prestatie: str, provenance: str) -> None: + """Scrie GOLD partajat (shared_mappings) DOAR cand denumirea umana e reala. + + shared_mappings e cheiat pe `denumire_normalizata` (text uman din prezentari). + `cod_op_service` e codul INTERN al operatiei, NU denumirea — a-l scrie ca si cheie + polueaza GOLD cu intrari pe care `lookup_shared_gold` (cauta pe denumirea umana) nu + le potriveste niciodata. Sarim scrierea cand denumirea lipseste sau == cod_op_service. + Best-effort: confirmarea GOLD nu blocheaza fluxul principal. + """ + den = (denumire or "").strip() + if not den or den == (cod_op_service or "").strip(): + return + try: + record_human_validation(conn, den, cod_prestatie, provenance=provenance) + except Exception: + pass + +# --------------------------------------------------------------------------- # +# Analytics device-mix (US-012, PRD 5.15) # +# --------------------------------------------------------------------------- # + +_UA_MOBIL = _re.compile( + r"Mobile|Android|iPhone|iPad|iPod|BlackBerry|Windows Phone|webOS", + _re.IGNORECASE, +) + + +def _clasificare_device(user_agent: str) -> str: + """Clasifica grosier un User-Agent in 'mobil' sau 'desktop'. + + Regex pe markeri standard (Mobile/Android/iPhone/iPad/iPod/BlackBerry/ + Windows Phone/webOS) — suficient pentru a valida premisa de utilizare mobil. + Nicio librarie externa noua. + """ + if _UA_MOBIL.search(user_agent or ""): + return "mobil" + return "desktop" + + +def _log_device_mix(request: Request, account_id: int | None) -> None: + """Inregistreaza semnalul agregat de device-mix in app_events. + + Stocheaza DOAR eticheta grosiera ('desktop'/'mobil') in campul `cod`. + NU stocheaza UA brut, IP sau alte PII suplimentare. + + Citire raport agregat (SQL): + SELECT cod, COUNT(*) AS n + FROM app_events + WHERE tip='device_mix' + GROUP BY cod; + + Sau cu evolutie zilnica: + SELECT date(ts) AS zi, cod, COUNT(*) AS n + FROM app_events + WHERE tip='device_mix' + GROUP BY zi, cod + ORDER BY zi DESC; + """ + ua = request.headers.get("user-agent", "") + clasificare = _clasificare_device(ua) + log_event( + "device_mix", + nivel="INFO", + account_id=account_id, + cod=clasificare, + mesaj=clasificare, # doar eticheta — nu UA brut + ) + + def _ctx(request: Request, **extra) -> dict: """Context de baza pentru template-uri cu formulare: include mereu csrf_token. @@ -111,7 +186,30 @@ def _status_counts(conn, account_id: int) -> dict[str, int]: "GROUP BY status", (account_id, account_id), ).fetchall() - return {r["status"]: int(r["n"]) for r in rows} + counts = {r["status"]: int(r["n"]) for r in rows} + + # sent_today si sent_month — bucketare in TIMP LOCAL RO (E7 CRITIC). + # updated_at e stocat ca datetime('now') UTC; date(updated_at) pur ar bucketiza + # trimiterile dintre miezul noptii local (21:xx-24:xx UTC) pe ziua gresita. + # Folosim modificatorul SQLite 'localtime' (DST-aware) in loc de offset fix '+3 hours': + # RO e UTC+2 (EET) iarna si UTC+3 (EEST) vara; un '+3 hours' fix gresea cu 1h iarna + # (ex. 21:30 UTC iarna = 23:30 RO azi, dar +3h = 00:30 maine -> ziua gresita). + # Presupune TZ=Europe/Bucharest in mediul procesului (docker-compose/Dockerfile). + row = conn.execute( + "SELECT " + " COUNT(CASE WHEN date(updated_at, 'localtime') = date('now', 'localtime') THEN 1 END) AS sent_today, " + " COUNT(CASE WHEN strftime('%Y-%m', updated_at, 'localtime') = strftime('%Y-%m', 'now', 'localtime') THEN 1 END) AS sent_month " + "FROM submissions " + "WHERE status = 'sent' AND (account_id = ? OR (? = 1 AND account_id IS NULL))", + (account_id, account_id), + ).fetchone() + if row: + counts["sent_today"] = int(row["sent_today"] or 0) + counts["sent_month"] = int(row["sent_month"] or 0) + else: + counts["sent_today"] = 0 + counts["sent_month"] = 0 + return counts def _trimiteri_versiune(conn, account_id: int) -> str: @@ -457,6 +555,8 @@ def dashboard(request: Request, tab: str = "acasa", status: str | None = None) - "csrf_token": get_csrf_token(request), }, ) + # US-012: semnal agregat de device-mix (fara PII) + _log_device_mix(request, account_id) active_tab = tab if tab in _TABS_VALIDE else "acasa" conn = get_connection() try: @@ -518,7 +618,12 @@ def fragment_coada(request: Request) -> HTMLResponse: @router.get("/_fragments/nomenclator", response_class=HTMLResponse) def fragment_nomenclator(request: Request) -> HTMLResponse: - """Browser nomenclator RAR (cache local upsert-at de worker la fiecare login).""" + """Browser nomenclator RAR (cache local upsert-at de worker la fiecare login). + + Necesita autentificare: este un fragment al dashboard-ului intern, nu un + endpoint public. Fara sesiune -> redirect /login (via require_login). + """ + require_login(request) conn = get_connection() try: rows = conn.execute( @@ -634,16 +739,30 @@ def _build_status_ctx(request: Request, conn, account_id: int, *, oob: bool = Fa rar_ok = rar_state == "ok" rar_lbl = eticheta_rar("ok" if rar_ok else rar_state) blocate_total = sum(counts.get(s, 0) for s in _BLOCKED) + + # D6 (strip sanatate mereu-vizibil): text compus + stare verde/rosu + sanatate_ok = worker_alive and rar_ok + if not worker_alive: + sanatate_text = "Blocat: worker oprit — declaratiile NU pleaca" + elif not rar_ok: + sanatate_text = "Blocat: RAR inaccesibil — declaratiile NU pleaca" + else: + sanatate_text = "Declaratiile curg normal" + return { "request": request, "worker_lbl": worker_lbl, "rar_lbl": rar_lbl, "worker_ok": worker_alive, "rar_ok": rar_ok, + "sanatate_ok": sanatate_ok, + "sanatate_text": sanatate_text, "eticheta_ultima_auth": ETICHETA_ULTIMA_AUTENTIFICARE_RAR, "last_login": format_data_rar(hb["last_rar_login_ok"] if hb else None), "counts_queued": counts.get("queued", 0), "counts_sent": counts.get("sent", 0), + "sent_today": counts.get("sent_today", 0), + "sent_month": counts.get("sent_month", 0), "blocate_total": blocate_total, "blocate_defalcat": _blocate_defalcat(counts), "pills_categorii": _pills_categorii(counts), @@ -876,8 +995,13 @@ _EDITABILE_OP = ("needs_data", "needs_mapping", "error") _GESTIONABILE_WEB = ("error", "needs_data", "needs_mapping") -def _render_submissions(request: Request, conn, account_id: int) -> HTMLResponse: - """Re-randeaza lista Trimiteri (fara filtre) — folosit dupa actiuni bulk.""" +def _render_submissions(request: Request, conn, account_id: int, + *, message: str | None = None) -> HTMLResponse: + """Re-randeaza lista Trimiteri (fara filtre) — folosit dupa actiuni bulk. + + `message`: sumar optional (ex. "2 reusite, 0 esuate") afisat ca banner la + inceputul fragmentului — folosit de bulk-fix (US-010, PRD 5.15). + """ scope_sql, scope_params = account_scope_clause(account_id) rows = conn.execute( "SELECT id, status, id_prezentare, rar_status_code, rar_error, retry_count, " @@ -893,6 +1017,7 @@ def _render_submissions(request: Request, conn, account_id: int) -> HTMLResponse "pills_categorii": _pills_categorii(_status_counts(conn, account_id)), "status_filtru": "", "versiune_trimiteri": _trimiteri_versiune(conn, account_id), + "bulk_message": message, # US-010 (PRD 5.15): sumar bulk-fix }) @@ -913,14 +1038,46 @@ def _payload_form_values(payload_json) -> dict: } -def _nemapate_pentru_submission(row, nomenclator: list[dict]) -> list[dict]: - """Operatiile nemapate ale UNUI submission needs_mapping, cu sugestii fuzzy. +def _prestatii_chips_from_payload(payload_json) -> list[dict]: + """Extrage lista de chips prestatii din payload_json pentru _form_editare.html (US-007). + + Returneaza lista de dicts {cod_prestatie, cod_op_service, denumire}. + Itemele fara cod_prestatie (operatii nemapate) sunt incluse cu cod_prestatie=''. + """ + try: + data = json.loads(payload_json) if payload_json else {} + if not isinstance(data, dict): + data = {} + except (ValueError, TypeError): + data = {} + chips = [] + for item in (data.get("prestatii") or []): + if not isinstance(item, dict): + continue + chips.append({ + "cod_prestatie": (item.get("cod_prestatie") or "").strip().upper(), + "cod_op_service": (item.get("cod_op_service") or "").strip(), + "denumire": (item.get("denumire") or "").strip(), + }) + return chips + + +def _has_r_odo_chips(prestatii_chips: list[dict]) -> bool: + """True daca orice chip are cod R-ODO sau I-ODO (trigger pentru reveal odo initial).""" + return any(c.get("cod_prestatie") in ("R-ODO", "I-ODO") for c in prestatii_chips) + + +def _nemapate_pentru_submission(row, nomenclator: list[dict], conn=None) -> list[dict]: + """Operatiile nemapate ale UNUI submission needs_mapping, cu sugestii fuzzy + enriched. Echivalentul `pending_unmapped` restrans la un singur rand: parseaza payload_json, aduna prestatiile fara cod_prestatie (cu cod_op_service) si ataseaza sugestii din `nomenclator` (pasat de apelant — evita un SELECT redundant in _detaliu_ctx). Goala daca randul nu e needs_mapping sau nu are operatii nemapate reale (ex. needs_mapping din auto_send=0 — codul exista deja, doar trimiterea e oprita). + + L14-S6: cand `conn` e dat, adauga `sugestie_principala` (GOLD partajat > SILVER > + embeddings) si `surse_sugestie` din `enrich_suggestions`. SUGGESTION-ONLY (#13). """ if row["status"] != "needs_mapping": return [] @@ -930,6 +1087,9 @@ def _nemapate_pentru_submission(row, nomenclator: list[dict]) -> list[dict]: content = {} except (ValueError, TypeError): content = {} + # Indexeaza corpusul embeddings o data inainte de bucla (no-op cand flagul e off). + if conn is not None: + ensure_embeddings_corpus(conn, nomenclator) seen: set[str] = set() out: list[dict] = [] for item in content.get("prestatii") or []: @@ -939,11 +1099,19 @@ def _nemapate_pentru_submission(row, nomenclator: list[dict]) -> list[dict]: if not op or op in seen: continue seen.add(op) - out.append({ + entry: dict = { "cod_op_service": op, "denumire": item.get("denumire"), "suggestions": suggest_codes(item.get("denumire"), nomenclator, limit=5), - }) + "sugestie_principala": None, + "surse_sugestie": {"gold_partajat": None, "silver": None, "embedding": None}, + } + # L14-S6: imbogatire cu GOLD partajat > SILVER > embeddings (SUGGESTION-ONLY, #13) + if conn is not None: + enriched = enrich_suggestions(conn, item.get("denumire")) + entry["sugestie_principala"] = enriched["sugestie_principala"] + entry["surse_sugestie"] = enriched["surse"] + out.append(entry) return out @@ -963,7 +1131,8 @@ def _detaliu_ctx(request: Request, row, *, message: str | None = None, if conn is not None and row["status"] == "needs_mapping": # Un singur SELECT pe nomenclator: il refolosim si pentru sugestii si pentru dropdown. _nomenclator_complet = load_nomenclator(conn) - nemapate_inline = _nemapate_pentru_submission(row, _nomenclator_complet) + # L14-S6: pasam conn pt enrich_suggestions (GOLD/SILVER/embeddings, suggestion-only) + nemapate_inline = _nemapate_pentru_submission(row, _nomenclator_complet, conn=conn) nomenclator = _nomenclator_complet if nemapate_inline else [] # Nomenclator pentru selectul cod_prestatie — needs_data/needs_mapping (in formularul @@ -1014,7 +1183,27 @@ def _detaliu_ctx(request: Request, row, *, message: str | None = None, "corectie_msg": message, "corectie_error": error, "corectie_errors": corectie_errors or [], + # US-007 (PRD 5.15): chips prestatii + obs pentru formularul slim. + # prestatii_chips: lista {cod_prestatie, cod_op_service, denumire} pentru _chips_prestatii.html. + # has_r_odo: True daca chips contin R-ODO/I-ODO (trigger reveal odo initial, D10c). + # obs_val: text liber observatii (campul obs din payload_json). + # form_chips_url: endpoint HTMX pentru add/remove chip (E6 server-driven). } + # Chips context (US-007): derivat din payload_json + _pjson = row["payload_json"] + prestatii_chips = _prestatii_chips_from_payload(_pjson) + ctx["prestatii_chips"] = prestatii_chips + ctx["has_r_odo"] = _has_r_odo_chips(prestatii_chips) + ctx["form_chips_url"] = "/form-chips" + # US-009: submission_id pentru butonul "salveaza ca regula" din _chips_prestatii.html. + # Cand chips sunt rerandate via /form-chips (stateless), chips_submission_id lipseste + # → butonul nu apare (corect: /form-chips nu are scop de submission). + ctx["chips_submission_id"] = row["id"] + try: + _pdata = json.loads(_pjson or "{}") + ctx["obs_val"] = (str(_pdata.get("obs") or "") if isinstance(_pdata, dict) else "").strip() + except (ValueError, TypeError): + ctx["obs_val"] = "" ctx.update(_payload_form_values(row["payload_json"])) return ctx @@ -1089,6 +1278,21 @@ async def post_mapeaza_inline(request: Request, submission_id: int) -> HTMLRespo message=f"Cod necunoscut in nomenclator: {cod_prestatie}."), ) save_mapping(conn, account_id, cod_op_service, cod_prestatie, auto_send) + # L14-S6: inregistreaza confirmare umana in GOLD partajat (shared_mappings). + # Gaseste denumirea operatiei din payload (cheia partajata e denumirea, nu cod_op_service). + try: + _pj = json.loads(row["payload_json"]) if row["payload_json"] else {} + _den_gold = None + for _pit in (_pj.get("prestatii") or []): + if isinstance(_pit, dict) and _pit.get("cod_op_service") == cod_op_service: + _den_gold = _pit.get("denumire") + break + _record_gold_validation( + conn, _den_gold, cod_op_service, cod_prestatie, + provenance=f"account_{account_or_default(account_id)}/mapeaza_inline", + ) + except Exception: + pass # best-effort: confirmare GOLD nu blocheaza fluxul principal # Re-rezolva scoped pe canalul randului: batch_id None (API) sau batch import. reresolve_account(conn, account_id, batch_id=row["batch_id"]) row2 = _fetch_submission_scoped(conn, account_id, submission_id) @@ -1140,29 +1344,67 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR if isinstance(val, str) and val.strip() != "": content[camp] = val.strip() - # Injectare cod_prestatie din form INAINTE de resolve_prestatii. Oglindeste - # validarea din post_mapeaza_inline (nomenclator check). Codul nou e injectat in - # prima prestatie (index 0); build_key il include in hash. - _cod_raw = form.get("cod_prestatie") - cod_prestatie_form = (_cod_raw.strip().upper() if isinstance(_cod_raw, str) else "") - if cod_prestatie_form: - exists_nom = conn.execute( - "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod_prestatie_form,) - ).fetchone() - if not exists_nom: - return templates.TemplateResponse( - "_trimitere_detaliu.html", - _detaliu_ctx( - request, row, conn=conn, account_id=account_id, error=True, - message=f"Cod RAR necunoscut in nomenclator: {cod_prestatie_form}. " - "Alege un cod valid din lista.", - ), - ) - prestatii_form = content.get("prestatii") - if isinstance(prestatii_form, list) and prestatii_form: - p0 = dict(prestatii_form[0]) - p0["cod_prestatie"] = cod_prestatie_form - content["prestatii"] = [p0] + list(prestatii_form[1:]) + # obs: text liber optional (US-005 PRD 5.15). Permite si string gol (sterge obs). + # None = absent din form (neschimbat); "" = curatare explicita. + obs_val = form.get("obs") + if isinstance(obs_val, str): + content["obs"] = obs_val.strip() + + # Injectare coduri_prestatie din form (lista multi-select) INAINTE de resolve_prestatii. + # US-006 (PRD 5.15): form.getlist permite N coduri; fiecare se ataseaza itemului + # corespondent din prestatii (by index), pastrand cod_op_service/denumire (D7/E1). + # US-007 (PRD 5.15): form-ul slim trimite TOATE itemele (inclusiv "" pentru nemapate), + # permitand 1-1 aliniere by-index chiar cand un item de mijloc ramane nemapat. + # Cod necunoscut in nomenclator -> respins imediat (invariant ORA-12899). + codes_raw = form.getlist("cod_prestatie") + # Acceptam lista cu "" pentru pozitii nemapate (US-007); filtrare doar pt detectia + # "fara niciun cod trimis" (cazul in care form-ul nu a inclus deloc cod_prestatie). + codes_positional = [ + c.strip().upper() if isinstance(c, str) else "" + for c in codes_raw + ] + # Verifica daca cel putin un cod non-gol a fost trimis + codes_nonempty = [c for c in codes_positional if c] + if codes_nonempty: + # Valideaza FIECARE cod non-gol fata de nomenclator (ORA-12899: RAR accepta NUMAI coduri valide) + for cod in codes_nonempty: + exists_nom = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod,) + ).fetchone() + if not exists_nom: + return templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx( + request, row, conn=conn, account_id=account_id, error=True, + message=f"Cod RAR necunoscut in nomenclator: {cod}. " + "Alege un cod valid din lista.", + ), + ) + # Pereche operatie<->cod (E4): fiecare cod se ataseaza itemului by index. + # Itemii existenti cu cod_op_service/denumire sunt PASTRATI (D7, E1 IRON RULE). + # Coduri "" (pozitii nemapate) lasa itemul fara cod_prestatie -> needs_mapping. + existing = content.get("prestatii") or [] + new_prestatii = [] + for i, cod in enumerate(codes_positional): + if i >= len(existing) and not cod: + continue # extra pozitii goale fara item corespondent — sarite + item = dict(existing[i]) if i < len(existing) else {} + if cod: + item["cod_prestatie"] = cod + # E1: cod_op_service/denumire NU se sterg; perechea op<->cod ramane intacta + new_prestatii.append(item) + # Dedup per-item (E4): (cod_op_service, cod_prestatie) identice -> pastreaza primul. + # Doua operatii DIFERITE cu acelasi cod RAR sunt legitime si NU se dedupeaza. + seen_pairs: set = set() + deduped: list = [] + for item in new_prestatii: + pair = (item.get("cod_op_service"), item.get("cod_prestatie")) + if pair not in seen_pairs: + seen_pairs.add(pair) + deduped.append(item) + content["prestatii"] = deduped + # else: fara coduri trimise -> content["prestatii"] neatins; resolve_prestatii + # detecteaza operatii nemapate si randul ramane needs_mapping. # Re-rezolva prestatiile cu maparea curenta (ca reresolve_account): NU re-pune # niciodata in coada un cod nemapat (codPrestatie null) — FINALIZATA e ireversibil @@ -1212,7 +1454,8 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR row2 = _fetch_submission_scoped(conn, account_id, submission_id) return templates.TemplateResponse( "_trimitere_detaliu.html", - _detaliu_ctx(request, row2, message="Mai sunt campuri invalide — vezi mai jos.", + _detaliu_ctx(request, row2, conn=conn, account_id=account_id, + message="Mai sunt campuri invalide — vezi mai jos.", error=True, corectie_errors=errors), ) @@ -1228,7 +1471,7 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR return templates.TemplateResponse( "_trimitere_detaliu.html", _detaliu_ctx( - request, row2, + request, row2, conn=conn, account_id=account_id, message=f"Exista deja o trimitere identica (rand #{dup['id']}). Corectia a fost oprita.", error=True, ), @@ -1247,13 +1490,14 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR row2 = _fetch_submission_scoped(conn, account_id, submission_id) return templates.TemplateResponse( "_trimitere_detaliu.html", - _detaliu_ctx(request, row2, error=True, + _detaliu_ctx(request, row2, conn=conn, account_id=account_id, error=True, message="Exista deja o trimitere identica. Corectia a fost oprita."), ) row2 = _fetch_submission_scoped(conn, account_id, submission_id) resp = templates.TemplateResponse( "_trimitere_detaliu.html", - _detaliu_ctx(request, row2, message="Corectat — randul a fost re-pus in coada."), + _detaliu_ctx(request, row2, conn=conn, account_id=account_id, + message="Corectat — randul a fost re-pus in coada."), ) # Pe succes, lista se reincarca (trimiteriChanged) si modalul se inchide # (inchideModal). After-settle ca inchiderea sa urmeze swap-ul fragmentului. @@ -1283,12 +1527,13 @@ async def post_repune_trimitere(request: Request, submission_id: int) -> HTMLRes verify_csrf(request, str(form.get("csrf_token") or "")) conn = get_connection() try: - # Prelucrare cod_prestatie pentru starea error (inaintea requeue_submission - # standard, care nu actualizeaza cheia de idempotency). - _cod_raw = form.get("cod_prestatie") - cod_prestatie_form = (_cod_raw.strip().upper() if isinstance(_cod_raw, str) else "") + # Prelucrare coduri_prestatie (lista multi-select) pentru starea error, inaintea + # requeue_submission standard care nu actualizeaza cheia de idempotency. + # US-006 (PRD 5.15): form.getlist; cod_op_service/denumire RAMAN pe item (E1 IRON RULE). + codes_raw = form.getlist("cod_prestatie") + codes = [c.strip().upper() for c in codes_raw if isinstance(c, str) and c.strip()] - if cod_prestatie_form: + if codes: row = _fetch_submission_scoped(conn, account_id, submission_id) if not row: raise HTTPException(status_code=404, detail="trimitere inexistenta") @@ -1298,33 +1543,51 @@ async def post_repune_trimitere(request: Request, submission_id: int) -> HTMLRes status_code=409, detail="modificarea cod_prestatie prin repune e valida doar pentru starea error", ) - # Valideaza cod-ul fata de nomenclator - exists_nom = conn.execute( - "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod_prestatie_form,) - ).fetchone() - if not exists_nom: - return templates.TemplateResponse( - "_trimitere_detaliu.html", - _detaliu_ctx( - request, row, conn=conn, account_id=account_id, - error=True, - message=f"Cod RAR necunoscut: {cod_prestatie_form}. Alege un cod valid.", - ), - ) - # Parseaza payload si injecteaza cod_prestatie + # Valideaza FIECARE cod fata de nomenclator (ORA-12899) + for cod in codes: + exists_nom = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod,) + ).fetchone() + if not exists_nom: + return templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx( + request, row, conn=conn, account_id=account_id, + error=True, + message=f"Cod RAR necunoscut: {cod}. Alege un cod valid.", + ), + ) + # Parseaza payload try: content = json.loads(row["payload_json"]) if row["payload_json"] else {} if not isinstance(content, dict): content = {} except (ValueError, TypeError): content = {} - prestatii = content.get("prestatii") or [] - if isinstance(prestatii, list) and prestatii: - p0 = dict(prestatii[0]) - p0["cod_prestatie"] = cod_prestatie_form - # sterge cod_op_service/denumire daca exista (codul direct preia prioritate) - p0.pop("cod_op_service", None) - content["prestatii"] = [p0] + list(prestatii[1:]) + # Pereche operatie<->cod (E4): fiecare cod se ataseaza itemului by index. + # E1 IRON RULE: cod_op_service/denumire RAMAN pe item (pop-ul vechi ELIMINAT). + # ITERAM peste `existing`, NU peste `codes`: formularul /repune trimite un + # SINGUR select cod_prestatie, deci pentru un rand cu 2+ prestatii o iterare + # pe codes ar fi pastrat doar len(codes) itemi -> prestatii[1:] PIERDUTE -> + # declaratie incompleta la RAR (FINALIZATA ireversibil). Aplicam codes pozitional + # si pastram intacte toate prestatiile existente fara cod nou. + existing = content.get("prestatii") or [] + new_prestatii = [] + for i in range(max(len(existing), len(codes))): + item = dict(existing[i]) if i < len(existing) else {} + if i < len(codes): + item["cod_prestatie"] = codes[i] + # E1: cod_op_service/denumire NU se sterg — perechea op<->cod ramane intacta + new_prestatii.append(item) + # Dedup per-item (E4): (cod_op_service, cod_prestatie) identice -> pastreaza primul + seen_pairs: set = set() + deduped: list = [] + for item in new_prestatii: + pair = (item.get("cod_op_service"), item.get("cod_prestatie")) + if pair not in seen_pairs: + seen_pairs.add(pair) + deduped.append(item) + content["prestatii"] = deduped # Re-rezolva prestatii cu noul cod mapping_meta = load_mapping_meta(conn, account_id) mapping = {op: m["cod_prestatie"] for op, m in mapping_meta.items()} @@ -1451,6 +1714,353 @@ async def post_sterge_bulk(request: Request) -> HTMLResponse: conn.close() +# =========================================================================== # +# US-010 (PRD 5.15): Bulk-fix — aplica un cod RAR la selectia de randuri # +# blocate. Reuse form #bulk-trimiteri + validare cod din post_corectie. # +# Regiune izolata in routes.py (fara conflict cu alte endpoints). # +# =========================================================================== # + + +@router.post("/trimiteri/bulk-fix", response_class=HTMLResponse) +async def post_bulk_fix(request: Request) -> HTMLResponse: + """Aplica un cod RAR la TOATE randurile blocate selectate (US-010, PRD 5.15). + + Reuse form #bulk-trimiteri (checkbox-uri pe gestionabil). Fiecare rand e + re-validat + idempotenta recalculata individual — un rand invalid nu pica lotul. + Sumar 'N reusite, M esuate'. Scoped pe cont (cross-account = silentios sarite). + Randurile sent/sending/queued sarite silentios (gard gestionabil, ca sterge-bulk). + """ + account_id = require_login(request) + form = await request.form() + verify_csrf(request, str(form.get("csrf_token") or "")) + ids = form.getlist("submission_id") + cod_prestatie = str(form.get("cod_prestatie") or "").strip().upper() + + conn = get_connection() + try: + # Validare cod fata de nomenclator INAINTE de procesarea randurilor + # (aceeasi regula ORA-12899 ca la corectie individuala: RAR accepta NUMAI coduri valide) + if not cod_prestatie: + resp = _render_submissions( + request, conn, account_id, + message="Cod RAR lipsa — introdu un cod inainte de aplicare.", + ) + return resp + + exists_nom = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod_prestatie,) + ).fetchone() + if not exists_nom: + resp = _render_submissions( + request, conn, account_id, + message=f"Cod RAR necunoscut in nomenclator: {cod_prestatie}. " + "Alege un cod valid din lista.", + ) + return resp + + reusite = 0 + esuate = 0 + + # Maparea contului + nomenclator + reguli text NU depind de rand -> incarcate + # O DATA inainte de bucla (evita 3xN query-uri redundante pe bulk-fix). + mapping_meta = load_mapping_meta(conn, account_id) + mapping = {op: m["cod_prestatie"] for op, m in mapping_meta.items()} + valid_codes = load_nomenclator_codes(conn) or None + text_rules = load_text_rules(conn, account_id) + + for raw in ids: + try: + sid = int(str(raw)) + except (ValueError, TypeError): + continue + + # 404-before-409: rand inexistent SAU al altui cont -> sarit silentios + row = _fetch_submission_scoped(conn, account_id, sid) + if not row: + continue + + # Doar randuri gestionabile (blocate); sent/sending/queued sarite silentios + if row["status"] not in _GESTIONABILE_WEB: + continue + + # Aplica cod_prestatie la itemele prestatii nemapate (fara cod_prestatie) + # Itemele cu cod existent sunt pastrate (E1: cod_op_service/denumire intacte) + try: + content = json.loads(row["payload_json"]) if row["payload_json"] else {} + if not isinstance(content, dict): + content = {} + except (ValueError, TypeError): + content = {} + + prestatii = content.get("prestatii") or [] + new_prestatii = [] + for item in prestatii: + if not isinstance(item, dict): + continue + it = dict(item) + # Aplica cod DOAR la itemii fara cod_prestatie (nemapati) + if not it.get("cod_prestatie"): + it["cod_prestatie"] = cod_prestatie + new_prestatii.append(it) + # Lista goala (rand fara prestatii) -> adauga un item cu cod direct + if not new_prestatii: + new_prestatii = [{"cod_prestatie": cod_prestatie}] + content["prestatii"] = new_prestatii + + # Re-resolve cu maparea curenta a contului (ca reresolve_account; + # mapping/valid_codes/text_rules hoistate inainte de bucla) + resolved, unmapped = resolve_prestatii( + content.get("prestatii"), mapping, valid_codes, text_rules + ) + content["prestatii"] = resolved + + if unmapped: + # Inca nemapat (ex. alte operatii fara cod) -> persista needs_mapping + payload_json = json.dumps(content, ensure_ascii=False) + conn.execute( + "UPDATE submissions SET status='needs_mapping', payload_json=?, rar_error=?, " + "updated_at=datetime('now') WHERE id=?", + (payload_json, json.dumps({"unmapped": unmapped}, ensure_ascii=False), sid), + ) + esuate += 1 + continue + + # Canonicalizare (strip ".0", VIN/nr upper) INAINTE de validare si cheie + canon = canonicalize_row(content) + content.update({ + "vin": canon["vin"], + "nr_inmatriculare": canon["nr_inmatriculare"], + "odometru_final": canon["odometru_final"], + }) + payload_json = json.dumps(content, ensure_ascii=False) + + # Validare individuala — un rand invalid nu pica lotul (AC US-010) + errors = validate_prezentare(content) + if errors: + conn.execute( + "UPDATE submissions SET status='needs_data', payload_json=?, rar_error=?, " + "updated_at=datetime('now') WHERE id=?", + (payload_json, json.dumps(errors, ensure_ascii=False), sid), + ) + esuate += 1 + continue + + # Recalcul idempotenta — coliziune detectata INAINTE de UPDATE (fara 500/duplicat) + new_key = build_key(account_id, canon) + if new_key != row["idempotency_key"]: + dup = conn.execute( + "SELECT id FROM submissions WHERE idempotency_key=? AND id<>?", + (new_key, sid), + ).fetchone() + if dup: + esuate += 1 + continue + + try: + conn.execute( + "UPDATE submissions SET idempotency_key=?, status='queued', payload_json=?, " + "rar_error=NULL, retry_count=0, next_attempt_at=datetime('now'), " + "updated_at=datetime('now') WHERE id=?", + (new_key, payload_json, sid), + ) + reusite += 1 + except sqlite3.IntegrityError: + # Plasa de siguranta TOCTOU pe UNIQUE(idempotency_key) + esuate += 1 + + # Compune mesajul sumar "N reusite, M esuate" (AC US-010) + parts: list[str] = [] + if reusite: + suffix_r = "a" if reusite == 1 else "e" + parts.append(f"{reusite} reusit{suffix_r}") + if esuate: + suffix_e = "a" if esuate == 1 else "e" + parts.append(f"{esuate} esuat{suffix_e}") + message: str | None = (", ".join(parts) + ".") if parts else None + + resp = _render_submissions(request, conn, account_id, message=message) + resp.headers["HX-Trigger"] = "trimiteriChanged" + return resp + finally: + conn.close() + + +# =========================================================================== # +# US-007 (PRD 5.15): Endpoint /form-chips — re-randare chips prestatii (E6). # +# Preia starea curenta din form + actiunea (add/remove) si re-randeaza # +# _chips_prestatii.html. Fara persistenta mid-edit (stare in input-uri form). # +# Minim si izolat (regiune noua, fara conflict cu post_corecteaza). # +# =========================================================================== # + + +@router.post("/form-chips", response_class=HTMLResponse) +async def post_form_chips(request: Request) -> HTMLResponse: + """Re-randeaza sectiunea chips prestatii (HTMX server-driven, E6, US-007). + + Primeste starea curenta a chip-urilor (3 liste paralele: cod_prestatie, + chip_op_service, chip_denumire) + actiunea (add/remove) si returneaza + _chips_prestatii.html actualizat. Fara scriere in DB (stateless mid-edit). + Auth: sesiune activa; CSRF verificat. + """ + account_id = require_login(request) + form = await request.form() + verify_csrf(request, str(form.get("csrf_token") or "")) + + # Reconstruct current chips state from parallel hidden inputs (emise de _chips_prestatii.html). + # Toate cele 3 liste sunt aceeasi lungime (emise index-by-index in template). + cod_list = [c.strip().upper() if isinstance(c, str) else "" for c in form.getlist("cod_prestatie")] + op_list = [o.strip() if isinstance(o, str) else "" for o in form.getlist("chip_op_service")] + den_list = [d.strip() if isinstance(d, str) else "" for d in form.getlist("chip_denumire")] + + # Aliniaza listele la lungimea maxima (defensive) + n = max(len(cod_list), len(op_list), len(den_list)) if (cod_list or op_list or den_list) else 0 + chips: list[dict] = [] + for i in range(n): + chips.append({ + "cod_prestatie": cod_list[i] if i < len(cod_list) else "", + "cod_op_service": op_list[i] if i < len(op_list) else "", + "denumire": den_list[i] if i < len(den_list) else "", + }) + + action = str(form.get("chips_action") or "").strip() + + conn = get_connection() + try: + if action == "add": + # Adauga cod la operatia specificata prin chips_add_op_index + try: + op_idx = int(str(form.get("chips_add_op_index") or 0)) + except (ValueError, TypeError): + op_idx = 0 + add_cod = str(form.get(f"chips_add_cod_{op_idx}") or "").strip().upper() + if add_cod and 0 <= op_idx < len(chips): + exists = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (add_cod,) + ).fetchone() + if exists: + chips[op_idx]["cod_prestatie"] = add_cod + + elif action == "add_flat": + # Adauga cod nou in lista plata (fara op_service) + add_cod_flat = str(form.get("chips_add_cod_flat") or "").strip().upper() + if add_cod_flat: + exists = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (add_cod_flat,) + ).fetchone() + if exists: + chips.append({"cod_prestatie": add_cod_flat, "cod_op_service": "", "denumire": ""}) + + elif action == "remove": + # Sterge codul de la indexul dat (lasa op_service intact -> operatie ramane nemapata) + try: + remove_idx = int(str(form.get("chips_remove_index") or 0)) + except (ValueError, TypeError): + remove_idx = 0 + if 0 <= remove_idx < len(chips): + chips[remove_idx]["cod_prestatie"] = "" + + elif action == "remove_flat": + # Sterge un chip plat dupa cod (in mod fara op_service) + remove_cod = str(form.get("chips_remove_cod") or "").strip().upper() + chips = [ + c for c in chips + if not (not c.get("cod_op_service") and c.get("cod_prestatie") == remove_cod) + ] + + # Compute has_r_odo dupa actiune + has_r_odo = _has_r_odo_chips(chips) + + # Incarca nomenclatorul pentru picker + nomenclator_rar = load_nomenclator(conn) + finally: + conn.close() + + return templates.TemplateResponse("_chips_prestatii.html", { + "request": request, + "csrf_token": get_csrf_token(request), + "prestatii_chips": chips, + "nomenclator_rar": nomenclator_rar, + "has_r_odo": has_r_odo, + "form_chips_url": "/form-chips", + "chips_section_id": "chips-section", + }) + + +# =========================================================================== # +# US-009 (PRD 5.15): Salvare mapare din chip. # +# Reuse EXACT save_mapping + reresolve_account (ca maparea inline 5.7). # +# Scoped pe sesiune (404 cross-account), CSRF obligatoriu. # +# =========================================================================== # + + +@router.post("/trimitere/{submission_id}/salveaza-regula-chip", response_class=HTMLResponse) +async def post_salveaza_regula_din_chip(request: Request, submission_id: int) -> HTMLResponse: + """Salveaza regula op->cod din chip (US-009, PRD 5.15). + + Reuse EXACT save_mapping + reresolve_account (acelasi mecanism ca maparea inline 5.7). + Scoped pe sesiune (404 cross-account/inexistent), CSRF obligatoriu. + Re-rezolva deblocand si submission-urile frate cu aceeasi operatie (pe batch_id). + auto_send implicit False (conservator — userul poate activa din tab-ul Mapari). + """ + account_id = require_login(request) + form = await request.form() + verify_csrf(request, str(form.get("csrf_token") or "")) + cod_op_service = str(form.get("salveaza_op") or "").strip() + cod_prestatie = str(form.get("salveaza_cod") or "").strip().upper() + conn = get_connection() + try: + row = _fetch_submission_scoped(conn, account_id, submission_id) + if not row: + raise HTTPException(status_code=404, detail="trimitere inexistenta") + if not cod_op_service or not cod_prestatie: + return templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx(request, row, conn=conn, account_id=account_id, error=True, + message="Operatia sau codul RAR lipsesc pentru salvarea regulii."), + ) + # Valideaza codul in nomenclator (invariant ORA-12899) + exists = conn.execute( + "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod_prestatie,) + ).fetchone() + if not exists: + return templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx(request, row, conn=conn, account_id=account_id, error=True, + message=f"Cod necunoscut in nomenclator: {cod_prestatie}."), + ) + # Reuse EXACT save_mapping + reresolve_account (ca post_mapeaza_inline 5.7) + save_mapping(conn, account_id, cod_op_service, cod_prestatie, False) + # L14-S6: inregistreaza confirmare umana in GOLD partajat (shared_mappings). + try: + _pj2 = json.loads(row["payload_json"]) if row["payload_json"] else {} + _den_chip = None + for _pit2 in (_pj2.get("prestatii") or []): + if isinstance(_pit2, dict) and _pit2.get("cod_op_service") == cod_op_service: + _den_chip = _pit2.get("denumire") + break + _record_gold_validation( + conn, _den_chip, cod_op_service, cod_prestatie, + provenance=f"account_{account_or_default(account_id)}/salveaza_chip", + ) + except Exception: + pass # best-effort + # Re-rezolva scoped pe canalul randului: batch_id None (API) sau batch import. + # Deblocheaza si submission-urile frate cu aceeasi operatie. + reresolve_account(conn, account_id, batch_id=row["batch_id"]) + row2 = _fetch_submission_scoped(conn, account_id, submission_id) + eticheta = eticheta_stare(row2["status"]) + resp = templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx(request, row2, conn=conn, account_id=account_id, + message=f"Regula salvata: {cod_op_service} -> {cod_prestatie}. " + f"Stare noua: {eticheta[0]}."), + ) + resp.headers["HX-Trigger"] = "trimiteriChanged" + return resp + finally: + conn.close() + + def _load_saved_op_mappings(conn, account_id: int) -> list[dict]: """Mapari operatie->cod salvate (operations_mapping) ale contului, cu numele prestatiei jonctionat din nomenclator. Scoped pe cont (NOT NULL → simplu).""" @@ -1541,8 +2151,14 @@ def post_mapare( cod_prestatie: str = Form(...), csrf_token: str | None = Form(None), auto_send: bool = Form(False), + denumire: str | None = Form(None), ) -> HTMLResponse: - """Salveaza maparea aleasa de user, re-rezolva submission-urile blocate, re-randeaza editorul.""" + """Salveaza maparea aleasa de user, re-rezolva submission-urile blocate, re-randeaza editorul. + + L14-S6: parametrul optional `denumire` permite inregistrarea in GOLD partajat + (shared_mappings) cu cheia corecta (denumire normalizata, nu cod_op_service). + Formularul din _mapari.html include un input hidden `denumire` per operatie. + """ account_id = require_login(request) verify_csrf(request, csrf_token) conn = get_connection() @@ -1552,6 +2168,11 @@ def post_mapare( if not exists: return _render_mapari(request, conn, account_id, message=f"Cod necunoscut: {cod}") save_mapping(conn, account_id, cod_op_service, cod, auto_send) + # L14-S6: inregistreaza confirmare umana in GOLD partajat (doar cu denumire reala). + _record_gold_validation( + conn, denumire, cod_op_service, cod, + provenance=f"account_{account_or_default(account_id)}/mapare_tab", + ) stats = reresolve_account(conn, account_id) msg = ( f"Mapat {cod_op_service.strip()} -> {cod}. " @@ -1575,11 +2196,14 @@ def post_editeaza_mapare_salvata( cod_prestatie: str = Form(...), csrf_token: str | None = Form(None), auto_send: bool = Form(False), + denumire: str | None = Form(None), ) -> HTMLResponse: """Editeaza o mapare op->cod salvata (cod RAR / auto-send) + re-rezolva blocatele. Scoped pe contul sesiunii (save_mapping foloseste account_or_default(sesiune) — cross-account imposibil). Respinge cod inexistent in nomenclator. + + L14-S6: `denumire` optional actualizeaza si GOLD partajat (shared_mappings). """ account_id = require_login(request) verify_csrf(request, csrf_token) @@ -1592,6 +2216,13 @@ def post_editeaza_mapare_salvata( if not exists: return _render_mapari(request, conn, account_id, message=f"Cod necunoscut: {cod}") save_mapping(conn, account_id, cod_op_service, cod, auto_send) + # L14-S6: actualizeaza GOLD partajat la editarea maparii salvate. Formularul + # din _mapari.html NU trimite denumire (maparea salvata nu retine textul uman + # original), deci de regula nu se scrie GOLD aici -> evita poluarea cu cod_op_service. + _record_gold_validation( + conn, denumire, cod_op_service, cod, + provenance=f"account_{account_or_default(account_id)}/editeaza_mapare", + ) stats = reresolve_account(conn, account_id) msg = ( f"Mapare actualizata: {cod_op_service.strip()} -> {cod}. " @@ -2459,6 +3090,18 @@ def web_rand_editare_modal(request: Request, import_id: int, row_index: int) -> err_map[e["field"]] = e.get("message") or e.get("msg") or "" if e.get("fix"): fix_map[e["field"]] = e["fix"] + # US-007: chips prestatii + obs pentru formularul slim + _preview_chips = [ + { + "cod_prestatie": (p.get("cod_prestatie") or "").strip().upper(), + "cod_op_service": (p.get("cod_op_service") or "").strip(), + "denumire": (p.get("denumire") or "").strip(), + } + for p in (res.get("prestatii") or []) + if isinstance(p, dict) + ] + _preview_has_r_odo = _has_r_odo_chips(_preview_chips) + _preview_nomenclator = load_nomenclator(conn) return templates.TemplateResponse("_editare_preview_modal.html", { "request": request, "import_id": import_id, @@ -2479,6 +3122,12 @@ def web_rand_editare_modal(request: Request, import_id: int, row_index: int) -> "message": None, # T2 (US-007): butonul 'Confirma valorile' apare DOAR pe randurile needs_review. "is_needs_review": row.get("resolved_status") == "needs_review", + # US-007: chips slim + "prestatii_chips": _preview_chips, + "has_r_odo": _preview_has_r_odo, + "obs_val": (res.get("obs") or "").strip(), + "nomenclator_rar": _preview_nomenclator, + "form_chips_url": "/form-chips", }) finally: conn.close() @@ -2557,6 +3206,17 @@ async def web_editeaza_rand(request: Request, import_id: int, row_index: int) -> err_map[e["field"]] = e.get("message") or e.get("msg") or "" if e.get("fix"): fix_map[e["field"]] = e["fix"] + # US-007: chips context pentru re-randare modal cu erori + _err_chips = [ + { + "cod_prestatie": (p.get("cod_prestatie") or "").strip().upper(), + "cod_op_service": (p.get("cod_op_service") or "").strip(), + "denumire": (p.get("denumire") or "").strip(), + } + for p in (res.get("prestatii") or []) + if isinstance(p, dict) + ] + _err_nomenclator = load_nomenclator(conn) return templates.TemplateResponse("_editare_preview_modal.html", { "request": request, "import_id": import_id, @@ -2576,6 +3236,12 @@ async def web_editeaza_rand(request: Request, import_id: int, row_index: int) -> "vin_context": res.get("vin") or "", "btn_label": "Salveaza", "message": "Mai sunt valori invalide — corecteaza campurile marcate.", + # US-007: chips slim + "prestatii_chips": _err_chips, + "has_r_odo": _has_r_odo_chips(_err_chips), + "obs_val": str(form.get("obs") or res.get("obs") or "").strip(), + "nomenclator_rar": _err_nomenclator, + "form_chips_url": "/form-chips", }) # Succes: reincarca preview-ul complet + toast + inchide modal (vezi helper). diff --git a/app/web/templates/_chips_prestatii.html b/app/web/templates/_chips_prestatii.html new file mode 100644 index 0000000..cc6f81a --- /dev/null +++ b/app/web/templates/_chips_prestatii.html @@ -0,0 +1,172 @@ +{# + _chips_prestatii.html — sectiunea de prestatii chips (E4, server-driven via /form-chips). + + Re-randata de endpoint-ul /form-chips la fiecare add/remove de chip. + Inclusa si din _form_editare.html pentru randarea initiala. + + Starea chip-urilor traieste in input-uri hidden din form (NU in DB mid-edit). + Fiecare operatie are un picker propriu cand e nemapata (E4 binding op<->cod). + Reveal odometru initial semnalat prin data-has-r-odo="true" si chip-warn pe R-ODO/I-ODO. + + Context vars (toate cu defaults): + prestatii_chips — list of {cod_prestatie, cod_op_service, denumire} + nomenclator_rar — list of {cod_prestatie, nume_prestatie} pentru picker + has_r_odo — True daca orice chip e R-ODO sau I-ODO (server-computed) + form_chips_url — URL pentru HTMX; default '/form-chips' + chips_section_id — ID div (default 'chips-section') + csrf_token — CSRF (trecut prin hx-include din form parinte) +#} +{% set _chips_url = form_chips_url or '/form-chips' %} +{% set _sec_id = chips_section_id or 'chips-section' %} +{% set _chips = prestatii_chips or [] %} +{% set _has_ops = _chips | selectattr('cod_op_service') | list | length > 0 %} +{# US-009: chips_submission_id e setat din _detaliu_ctx cand chips sunt randate in modalul de detaliu. + Lipseste cand _chips_prestatii.html e rerandat via /form-chips (stateless, fara submission). #} +{% set _sub_id = chips_submission_id if chips_submission_id is defined else none %} + +
+ + {# ===== Input-uri hidden pentru starea curenta a chip-urilor ===== + TOATE itemele emit 3 hidden inputs (cod poate fi "" pentru unmapped). + Paralele index-by-index: cod_prestatie[i], chip_op_service[i], chip_denumire[i]. + Filtrate la submit de post_corectie_trimitere (coduri goale = neschimbate). #} + {% for chip in _chips %} + + + + {% endfor %} + +
+ + + {% if _has_ops %} + {# ===== Mod operatii: UN picker PE operatie (E4 binding) ===== #} + {% for chip in _chips %} + {% if chip.cod_op_service %} + {% set _is_warn = chip.cod_prestatie in ('R-ODO', 'I-ODO') %} + {% set _nemapat = not chip.cod_prestatie %} +
+ + {{ chip.cod_op_service }} + {% if chip.denumire and chip.denumire != chip.cod_op_service %} + — {{ chip.denumire }} + {% endif %} + {% if _nemapat %} + · lipsa cod + {% endif %} + + + {% if chip.cod_prestatie %} + {# ===== Operatie mapata: chip cu × ===== #} + + {{ chip.cod_prestatie }} + + + {# US-009: "salveaza ca regula op->cod" — apare doar cand submission_id e cunoscut + (in modalul de detaliu, nu la re-randarea stateless via /form-chips). + Reuse EXACT save_mapping + reresolve_account via endpoint dedicat. + hx-include="closest form" propaga csrf_token din form-ul parinte. #} + + {% if _sub_id and chip.cod_op_service and chip.cod_prestatie %} + + {% endif %} + + {% else %} + {# ===== Operatie nemapata: picker galben cu "alege cod RAR" ===== #} + + + {% endif %} + +
+ {% endif %} + {% endfor %} + + {% else %} + {# ===== Mod plat: lista de coduri libere (corectie pura, fara op_service) ===== #} +
+ {% for chip in _chips %} + {% if chip.cod_prestatie %} + {% set _is_warn_flat = chip.cod_prestatie in ('R-ODO', 'I-ODO') %} + + {{ chip.cod_prestatie }} + + + {% endif %} + {% endfor %} + {# Picker adaugare cod nou in mod plat #} + {% if nomenclator_rar %} + + + + + {% endif %} +
+ {% endif %} + + {# Hint discret fara chips (debut) #} + {% if not _chips %} +
+ Niciun cod RAR inca — alege din picker (sus) sau adauga prin mapare. +
+ {% endif %} +
+
diff --git a/app/web/templates/_form_editare.html b/app/web/templates/_form_editare.html index f10b527..50a3ee6 100644 --- a/app/web/templates/_form_editare.html +++ b/app/web/templates/_form_editare.html @@ -1,17 +1,22 @@ -{# _form_editare.html — partial partajat: campurile vehicul/data/odometru. - US-005 (PRD 5.12): extras DRY din _trimitere_detaliu.html; refolosit si de - _preview_rand.html (US-006) pentru editarea randurilor de import in modal. +{# _form_editare.html — partial partajat slim: campurile vehicul/data/odo + obs + chips prestatii. + US-007 (PRD 5.15): redesign slim cu VIN unic, Observatii textarea, chips prestatii (E4), + si reveal dinamic odometru initial cand chips contin R-ODO/I-ODO (D10c, E6 server-driven). Inclus cu {% include "_form_editare.html" %} INSIDE un
element al template-ului parinte. Acel parinte pune form-ul, CSRF-ul si orice campuri - suplimentare (ex. select cod_prestatie din _trimitere_detaliu.html). + suplimentare. - Necesita din context (setate de parinte inainte de include): + Variabile necesare din context (setate de parinte inainte de include): form_nr — valoare curenta nr_inmatriculare form_vin — valoare curenta vin form_data — valoare curenta data_prestatie (YYYY-MM-DD sau brut) form_odo_final — valoare curenta odometru_final form_odo_initial — valoare curenta odometru_initial + obs_val — valoare curenta obs (Observatii), text liber (default '') + prestatii_chips — list of {cod_prestatie, cod_op_service, denumire} (default []) + nomenclator_rar — list of {cod_prestatie, nume_prestatie} pentru picker (default []) + has_r_odo — True daca chips contin R-ODO/I-ODO (server-computed, default False) + form_chips_url — URL pentru HTMX chip endpoint (default '/form-chips') err_map — dict {field_name: mesaj_eroare} (poate fi {}) fix_map — dict {field_name: hint_fix} (poate fi {}) vin_context — string VIN pentru aria-label (poate fi '') @@ -19,23 +24,78 @@ #} {% from "_macros.html" import camp, icon %} -{# Nr. inmatriculare pe rand propriu, VIN dedesubt — ambele latime plina. #} -{{ camp('nr_inmatriculare', 'Numar inmatriculare', form_nr, - err_map=err_map, fix_map=fix_map, vin_context=vin_context) }} -{{ camp('vin', 'VIN (serie sasiu)', form_vin, +{# === 1. VIN — camp unic (fara "Confirma VIN"; contractul RAR cere un singur VIN) === #} +{{ camp('vin', 'VIN (serie sasiu)', form_vin, slim=True, mono=True, err_map=err_map, fix_map=fix_map, vin_context=vin_context) }} -{# Restul campurilor in grila responsiva existenta. #} -
- {{ camp('data_prestatie', 'Data prestatie', form_data, tip='date', +{# === 2. Data prestatie + Nr. inmatriculare — grila 2 coloane === #} +
+ {{ camp('data_prestatie', 'Data prestatiei', form_data, tip='date', slim=True, err_map=err_map, fix_map=fix_map, vin_context=vin_context) }} - {{ camp('odometru_final', 'Odometru final', form_odo_final, - err_map=err_map, fix_map=fix_map, vin_context=vin_context) }} - {{ camp('odometru_initial', 'Odometru initial (daca e cerut)', form_odo_initial, + {{ camp('nr_inmatriculare', 'Numar inmatriculare', form_nr, slim=True, mono=True, err_map=err_map, fix_map=fix_map, vin_context=vin_context) }}
-{# Buton primar parametrizat. +{# === 3. Observatii (obs) — textarea liber, US-005 === #} +
+ + +
+ +{# === 4. Prestatii chips (E4 server-driven, US-007) === #} +{% set form_chips_url = form_chips_url or '/form-chips' %} +{% set chips_section_id = 'chips-section' %} +{% include "_chips_prestatii.html" %} + +{# === 5. Odometru final — intotdeauna vizibil === #} +{{ camp('odometru_final', 'Odometru final (km)', form_odo_final, slim=True, mono=True, + err_map=err_map, fix_map=fix_map, vin_context=vin_context) }} + +{# === 6. Odometru initial — reveal dinamic server cand chips contin R-ODO/I-ODO (D10c) === + has_r_odo=True (server-computed din lista de chips): sectiune vizibila cu marcaj warn. + has_r_odo=False: hint discret, campul optional si vizual neutru. #} +{% if has_r_odo %} +
+
+ + + {% if err_map.get('odometru_initial') %} +
{{ err_map.get('odometru_initial') }}
+ {% endif %} +
+
+{% else %} +{# Hint discret cand nu e necesar #} +
+ + + {% if err_map.get('odometru_initial') %} +
{{ err_map.get('odometru_initial') }}
+ {% endif %} + + Odometru initial se cere doar pentru coduri R-ODO / I-ODO. + +
+{% endif %} + +{# === 7. Buton primar parametrizat === with_cancel=True (modal editare preview): Salveaza + Anuleaza pe ACELASI rand, sistemul .act (desktop = text alaturat; mobil = doua iconite Lucide 44px alaturate). Implicit (ex. _trimitere_detaliu): un singur buton text, neschimbat. #} diff --git a/app/web/templates/_macros.html b/app/web/templates/_macros.html index e756118..03d8b05 100644 --- a/app/web/templates/_macros.html +++ b/app/web/templates/_macros.html @@ -18,9 +18,13 @@ vin_context — string VIN pentru aria-label cu context (default '') id_prefix — prefix pentru id="" al input-ului (default 'c'; preview poate folosi 'e-N') #} -{% macro camp(nome, eticheta, valoare, tip='text', err_map={}, fix_map={}, vin_context='', id_prefix='c') %} -
- +{% macro camp(nome, eticheta, valoare, tip='text', err_map={}, fix_map={}, vin_context='', id_prefix='c', slim=False, mono=False) %} +{# slim=False: randare clasica (neschimbata). slim=True: varianta compacta (.camp-slim) din US-002 PRD 5.15: + label 11px muted deasupra, input ~30px, fundal --card2. + mono=True (valid numai cu slim=True): adauga clasa 'camp-mono' pe input pentru campuri + VIN/odometru/nr (IBM Plex Mono, prin .camp-slim .camp-mono din base.html). #} +
+ {% if tip == 'date' %} {# D#10/R3: degradare grijulie pentru valori ne-YYYY-MM-DD. Daca valoarea nu e in formatul corect, inputul ramane gol + hint + hidden cu valoarea bruta @@ -28,7 +32,8 @@ {%- set _dp_ok = (valoare and valoare|length == 10 and valoare[4:5] == '-' and valoare[7:8] == '-') -%} {% if not _dp_ok and valoare %} @@ -38,7 +43,8 @@ {% else %} {% endif %} diff --git a/app/web/templates/_mapari.html b/app/web/templates/_mapari.html index 7d7264c..b6da289 100644 --- a/app/web/templates/_mapari.html +++ b/app/web/templates/_mapari.html @@ -37,7 +37,8 @@ {% for e in pending %} {% set top = e.suggestions[0] if e.suggestions else None %} - {% set preselect = top.cod_prestatie if (top and top.score >= 60) else '' %} + {# L14-S6: pre-selectare din sugestie_principala (GOLD/SILVER/embedding) > fuzzy #} + {% set preselect = e.sugestie_principala.cod_prestatie if e.sugestie_principala else (top.cod_prestatie if (top and top.score >= 60) else '') %} {# data-dt-row = haystack de cautare (randul contine un + {# L14-S6: denumire pt record_human_validation in GOLD partajat #} +
{{ e.cod_op_service }} {{ e.blocked }} blocate
diff --git a/app/web/templates/_status.html b/app/web/templates/_status.html index 6a44735..9d0377a 100644 --- a/app/web/templates/_status.html +++ b/app/web/templates/_status.html @@ -4,7 +4,7 @@ hx-swap="outerHTML" {% if oob %}hx-swap-oob="outerHTML"{% endif %}> - + {# Banner cont in asteptare de activare (mereu vizibil cand contul e inactiv) #} {% if not account_active %}
@@ -14,50 +14,68 @@
{% endif %} - -
- - {# Bifa: glifa (✓/✗) + culoare + text — accesibil (nu doar culoare, design review) #} - {% macro bifa(ok, text, tip) %} - - {% if ok %} - - {{ text }} - {% else %} - - {{ text }} - {% endif %} - - {% endmacro %} - - {{ bifa(worker_ok, worker_lbl[0], worker_lbl[1]) }} - {{ bifa(rar_ok, rar_lbl[0], rar_lbl[1]) }} - - - {{ eticheta_ultima_auth }}: - {{ last_login }} + {# === D6: Strip sanatate mereu-vizibil DEASUPRA contoarelor === + Verde: worker viu + RAR ok → "Declaratiile curg normal" + Rosu: worker oprit SAU RAR inaccesibil → "Blocat: ... — declaratiile NU pleaca" + Glife accesibile ✓/✗ (nu doar culoare). Layout: glifa+text stanga, ultima auth dreapta. + #} +
+
+ {% if sanatate_ok %} + + {% else %} + + {% endif %} + {{ sanatate_text }} +
+ + {{ eticheta_ultima_auth }}: {{ last_login }}
- -
- In asteptare: {{ counts_queued }} - Declarate la RAR: {{ counts_sent }} - Blocate: - {{ blocate_total }} - + {# === D4: 3 carduri-contor (mockup exact: Trimise / In coada / De corectat) === + Responsive: flex-wrap => 3 pe rand desktop, 2/stivuite pe mobil (min-width:120px). + Trimise: all-time (cifra mare) + sub-linie "luna N · azi N" (D4 + E7). + De corectat: rosu cand >0 (s-error), muted cand 0. + #} +
+ + {# Trimise (all-time principal, luna/azi secundar) #} +
+
{{ counts_sent }}
+
Trimise (total)
+
luna {{ sent_month }} · azi {{ sent_today }}
+
+ + {# In coada (accent/albastru) #} +
+
{{ counts_queued }}
+
In coada
+
+ + {# De corectat (rosu daca >0, muted la 0; link catre lista) #} + +
{{ blocate_total }}
+
De corectat
+ +
- {# Pill-urile de stare s-au mutat in bara de filtre din sectiunea Trimiteri (_coada.html). #} - - {# === Rand 3: navigatie rapida sub contoare (US-005) === - Linkurile Trimiteri + Mapari apar pe FIECARE pagina sub status-bar. - Marcajul activ vine din variabila de context tab_activ (transmisa de dashboard via ?tab= - sau default 'acasa'). Badge-ul Mapari = mapari_badge (aceeasi sursa: counts.needs_mapping). + {# === Navigatie rapida: Trimiteri + Mapari cu badge needs_mapping === + Pastrata exact ca inainte (US-005): tab_activ determina marcajul activ. #} {% set _tab = tab_activ | default('acasa') %}