diff --git a/CLAUDE.md b/CLAUDE.md index f42da03..bf64c34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,6 +63,7 @@ Flux: validare (`validation.py`) → mapare operatie→cod (`mapping.py`) → en - **Admin bootstrap**: primul user din TOATA baza la `/signup` (`count_admins()==0`, in tranzactie) devine `is_admin=1`; restul nu. `is_admin` da acces la `/admin` (panou global) si e independent de contul id=1 (acela e doar fallback dev pentru API neautentificat). Fix manual: `python3 -m tools.account set-admin --account N`. - **Mapare coloane memorata per `(account_id, signature_coloane)`** (`column_mappings`): reaplicata automat la fisiere cu aceleasi coloane; un cont poate avea mai multe formate. - **Mapare operatie→cod**: prestatia vine cu `cod_prestatie` (cod RAR) sau `cod_op_service` + `denumire`. Nerezolvat → `needs_mapping` (nu se trimite), editor web cu sugestie fuzzy; salvarea maparii re-rezolva automat submission-urile blocate. +- **Excludere de la declarare** (`operations_mapping.exclus=1`; optiunea "Nu se declara la RAR" = sentinel `__NEDECLARAT__` in select-urile de mapare): operatia NU pleaca la RAR. Precedenta in `resolve_prestatii`: cod explicit valid pe rand > exclus > mapare exacta > reguli text. Itemii adnotati `exclus` sunt scosi din payload/cheie cu `split_prestatii_excluse` inainte de enqueue; rand de import cu toate operatiile excluse → stare `excluded` ("Nedeclarat", nu se comite); submission cu toate excluse → `needs_data` cu motiv explicit. `save_mapping` peste o regula exclusa reseteaza `exclus=0`. - **`cod_prestatie` VALIDAT fata de nomenclator la ingestie** (`resolve_prestatii(..., valid_codes)`): cod necunoscut NU se trimite raw — e promovat la `cod_op_service` (denumire=cod) si intra la mapat. Motiv (confirmat live): RAR accepta doar coduri din nomenclator (max 5 car.); cod necunoscut → HTTP 500 `ORA-12899`, iar RAR ne-tranzactional lasa record PARTIAL `FINALIZATA` pe care reconcilierea l-ar marca fals `sent`. La cod nemapat: `on_unmapped_error` (boolean top-level pe `POST /v1/prezentari` + `/valideaza`) = `false` → editor/`needs_mapping`; `true` → respins fara enqueue (`submission_id=null` + `erori`). Precedenta: cerere > `accounts.on_unmapped_error_default` > `false`. - **WAF RAR da 403 fara User-Agent de browser** — httpx trimite mereu `User-Agent: Mozilla/5.0` (`config.py`). - **422 fara echo de credentiale**: handler-ul din `main.py` pastreaza type/loc/msg dar DROP-a `input`/`ctx` (altfel reflecta `rar_credentials.password`). diff --git a/app/api/v1/import_router.py b/app/api/v1/import_router.py index 42a6f4e..95dfe30 100644 --- a/app/api/v1/import_router.py +++ b/app/api/v1/import_router.py @@ -49,11 +49,13 @@ from ...mapping import ( _emite_text_rule_hits, account_or_default, has_no_auto_send, + load_excluded_ops, load_mapping_meta, load_nomenclator_codes, load_text_rules, normalize_for_match, resolve_prestatii, + split_prestatii_excluse, ) from ...validation import validate_prezentare from ...rar_env import MediuIndisponibil, rar_env_efectiv_cont, rezolva_rar_env @@ -135,11 +137,12 @@ def _resolve_row_for_preview( valid_codes: set[str] | None = None, text_rules: list[dict] | None = None, reviewed: bool = False, + excluded_ops: set[str] | None = None, ) -> dict[str, Any]: """Rezolva un rand din import pentru preview: aplica mapare coloane + validare. Intoarce un dict cu: - resolved_status: ok/needs_mapping/needs_data/needs_review + resolved_status: ok/needs_mapping/needs_data/needs_review/excluded resolved: valorile finale rezolvate (VIN, data, km, prestatii) errors: lista erori validare flags: motive needs_review @@ -218,7 +221,7 @@ def _resolve_row_for_preview( # Rezolvare prestatii prestatii = mapped.get("prestatii") or [] - resolved, unmapped = resolve_prestatii(prestatii, mapping, valid_codes, text_rules) + resolved, unmapped = resolve_prestatii(prestatii, mapping, valid_codes, text_rules, excluded_ops) mapped["prestatii"] = resolved # Determinare stare @@ -230,6 +233,19 @@ def _resolve_row_for_preview( "flags": all_flags, } + # Operatii excluse de la declarare: ies din prestatiile trimise (si din cheia + # de idempotenta). Toate excluse -> rand 'excluded' (nu se trimite, nu blocheaza). + declarabile, excluse = split_prestatii_excluse(resolved) + if excluse and not declarabile: + return { + "resolved_status": "excluded", + "resolved": mapped, + "errors": [], + "flags": all_flags, + } + if excluse: + mapped["prestatii"] = declarabile + # Validare continut errors = validate_prezentare(mapped) @@ -770,9 +786,10 @@ def preview_import( # Incarca maparea de operatii o singura data mapping_meta = load_mapping_meta(conn, acct) mapping = {op: meta["cod_prestatie"] for op, meta in mapping_meta.items()} - # Validare nomenclator + reguli text incarcate O DATA, inainte de bucla pe randuri. + # Validare nomenclator + reguli text + excluderi incarcate O DATA, inainte de bucla. valid_codes = load_nomenclator_codes(conn) or None text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) # Mediul RAR efectiv al contului — folosit la calculul cheii de idempotenta # la preview (trebuie sa coincida cu ce va folosi commit-ul fara rar_env explicit). @@ -828,6 +845,7 @@ def preview_import( override=overrides[i] or None, valid_codes=valid_codes, text_rules=text_rules, + excluded_ops=excluded_ops, ) # Calculeaza cheia de idempotenta pentru randurile ok/needs_review @@ -1118,9 +1136,10 @@ def commit_import( # Incarca maparea de operatii mapping_meta = load_mapping_meta(conn, acct) mapping = {op: meta["cod_prestatie"] for op, meta in mapping_meta.items()} - # Validare nomenclator + reguli text incarcate O DATA, inainte de bucla pe randuri. + # Validare nomenclator + reguli text + excluderi incarcate O DATA, inainte de bucla. valid_codes = load_nomenclator_codes(conn) or None text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) # Construieste payload-urile submissions enqueued: list[dict] = [] @@ -1184,7 +1203,12 @@ def commit_import( # Rezolva prestatii INAINTE de canonicalizare (altfel cheia difera de cea din preview) prestatii = mapped.get("prestatii") or [] - resolved, _ = resolve_prestatii(prestatii, mapping, valid_codes, text_rules) + resolved, _ = resolve_prestatii(prestatii, mapping, valid_codes, text_rules, excluded_ops) + # Prestatiile excluse de la declarare NU pleaca la RAR (ca in preview); + # rand fara nicio prestatie declarabila = 'excluded' in preview -> defensiv skip. + resolved, _excluse = split_prestatii_excluse(resolved) + if not resolved: + continue mapped["prestatii"] = resolved # Canonicalizare (dupa rezolvare prestatii -> cod_prestatie inclus in cheie; diff --git a/app/api/v1/router.py b/app/api/v1/router.py index f23ddd7..a8be7e1 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -31,6 +31,7 @@ from ...mapping import ( account_or_default, account_scope_clause, classify_prezentare, + load_excluded_ops, load_mapping_meta, load_nomenclator_codes, load_text_rules, @@ -69,13 +70,13 @@ def _effective_on_unmapped_error(conn, acct: int, req_value: bool | None) -> boo return bool(row["on_unmapped_error_default"]) if row else False -def _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules=None) -> dict: +def _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules=None, excluded_ops=None) -> dict: """classify_prezentare + aplicarea modului on_unmapped_error. Cand exista coduri nemapate si error_mode=True, marcheaza outcome-ul ca respingere (blocked_error=True): rutele NU mai fac enqueue, ci intorc o eroare per-element. """ - cl = classify_prezentare(content, mapping, mapping_meta, valid_codes, text_rules) + cl = classify_prezentare(content, mapping, mapping_meta, valid_codes, text_rules, excluded_ops) cl["blocked_error"] = bool(cl["unmapped"]) and error_mode return cl @@ -186,6 +187,7 @@ def create_prezentari( valid_codes = load_nomenclator_codes(conn) or None # Reguli text incarcate o data per cerere (seam partajat cu dry-run). text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) error_mode = _effective_on_unmapped_error(conn, acct, req.on_unmapped_error) # Rezolva mediul RAR tinta (cerut > default cont > ancora globala). @@ -285,7 +287,7 @@ def create_prezentari( # retrimiterea aceluiasi continut. Il RE-ACTIVAM (re-clasificam + actualizam # creds + reset), printr-un UPDATE compare-and-swap pe status='error'. if existing["status"] == "error": - cl = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules) + cl = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules, excluded_ops) if cl["blocked_error"]: # on_unmapped_error=True: nu reactivam; randul ramane 'error'. results.append(_rezultat_respins(existing["id"], cl, rar_env=env)) @@ -337,7 +339,7 @@ def create_prezentari( # Helper pur partajat cu dry-run: reproduce EXACT clasificarea # (canonicalize + mapare op->cod + validare; fara gate auto_send). - cl = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules) + cl = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules, excluded_ops) if cl["blocked_error"]: # on_unmapped_error=True: respinge fara enqueue (cod necunoscut/nemapat). results.append(_rezultat_respins(None, cl, rar_env=env)) @@ -397,6 +399,7 @@ def valideaza_prezentari( valid_codes = load_nomenclator_codes(conn) or None # Acelasi seam ca trimiterea reala: dry-run trebuie sa vada aceleasi reguli text. text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) error_mode = _effective_on_unmapped_error(conn, acct, req.on_unmapped_error) # Rezolva env identic ca trimiterea reala si ecou-ieste in raspuns. @@ -429,7 +432,7 @@ def valideaza_prezentari( for i, prez in enumerate(req.prezentari): content = prez.model_dump() - res = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules) + res = _classify_modal(content, mapping, mapping_meta, valid_codes, error_mode, text_rules, excluded_ops) if res["blocked_error"]: res = {**res, "status": "error"} # Imbogatim fiecare element nemapat cu 3 niveluri COD_NEMAPAT diff --git a/app/db.py b/app/db.py index d3fbe0f..08f854b 100644 --- a/app/db.py +++ b/app/db.py @@ -178,6 +178,26 @@ def _migrate(conn: sqlite3.Connection) -> None: conn.execute( "ALTER TABLE import_rows ADD COLUMN reviewed INTEGER NOT NULL DEFAULT 0" ) + # CHECK-ul resolved_status trebuie sa accepte 'excluded' (operatii excluse de la + # declarare). SQLite nu poate altera CHECK -> rebuild o singura data (detectat + # pe textul CREATE din sqlite_master). + create_sql = str(conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='import_rows'" + ).fetchone()["sql"] or "") + if "'excluded'" not in create_sql: + _rebuild_import_rows_cu_excluded(conn) + + # Coloana operations_mapping.exclus: regula "nu se declara la RAR" per operatie. + om_tbl = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='operations_mapping'" + ).fetchone() + if om_tbl: + om_cols = {r["name"] for r in conn.execute("PRAGMA table_info(operations_mapping)").fetchall()} + if "exclus" not in om_cols: + conn.execute( + "ALTER TABLE operations_mapping ADD COLUMN exclus INTEGER NOT NULL DEFAULT 0 " + "CHECK (exclus IN (0, 1))" + ) # Index batch_id pe submissions (poate lipsi pe DB veche) existing_idx = {r["name"] for r in conn.execute( @@ -201,6 +221,45 @@ def _migrate(conn: sqlite3.Connection) -> None: ) +def _rebuild_import_rows_cu_excluded(conn: sqlite3.Connection) -> None: + """Rebuild import_rows cu 'excluded' in CHECK-ul resolved_status. + + Ruleaza O SINGURA DATA pe DB-uri create inainte de starea 'excluded' (gardat de + apelant pe textul CREATE). Copiaza toate randurile 1:1 si recreeaza indexul. + """ + conn.execute("BEGIN IMMEDIATE") + try: + conn.execute( + "CREATE TABLE import_rows_new (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " batch_id INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE," + " row_index INTEGER NOT NULL," + " raw_json TEXT NOT NULL," + " override_json TEXT," + " reviewed INTEGER NOT NULL DEFAULT 0," + " resolved_status TEXT NOT NULL DEFAULT 'pending'" + " CHECK (resolved_status IN (" + " 'pending','ok','needs_mapping','needs_data'," + " 'needs_review','already_sent','duplicate_in_file','excluded'" + " ))," + " error TEXT" + ")" + ) + conn.execute( + "INSERT INTO import_rows_new " + " (id, batch_id, row_index, raw_json, override_json, reviewed, resolved_status, error) " + "SELECT id, batch_id, row_index, raw_json, override_json, reviewed, resolved_status, error " + "FROM import_rows" + ) + conn.execute("DROP TABLE import_rows") + conn.execute("ALTER TABLE import_rows_new RENAME TO import_rows") + conn.execute("CREATE INDEX IF NOT EXISTS idx_import_rows_batch ON import_rows(batch_id)") + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + def _migrate_accounts_medii(conn: sqlite3.Connection, acc_cols: set[str]) -> None: """Coloane medii RAR per cont + backfill din ancora globala. diff --git a/app/embeddings.py b/app/embeddings.py index 0445a8a..a0b8195 100644 --- a/app/embeddings.py +++ b/app/embeddings.py @@ -226,7 +226,9 @@ class EmbeddingEngine: { "cod": self._corpus_items[i]["cod"], "is_nul": bool(self._corpus_items[i].get("is_nul", False)), - "similaritate": float(sims[i]), + # clamp la [-1, 1]: eroarea de rotunjire float32 poate da + # 1.0000001 pe text identic (ar afisa >100% in UI). + "similaritate": min(1.0, max(-1.0, float(sims[i]))), } for i in idx ] diff --git a/app/mapping.py b/app/mapping.py index fe78ed9..7c478b0 100644 --- a/app/mapping.py +++ b/app/mapping.py @@ -39,6 +39,10 @@ DEFAULT_ACCOUNT_ID = 1 # Sub acest scor (0..100) nu preselectam nicio sugestie — userul alege manual. SUGGEST_MIN_SCORE = 60 +# Sentinel pentru optiunea "nu se declara la RAR" din select-urile web de mapare. +# NU e cod RAR (codurile au max 5 caractere) — nu poate coliziona cu nomenclatorul. +EXCLUDE_SENTINEL = "__NEDECLARAT__" + # --------------------------------------------------------------------------- # # Pur: normalizare + fuzzy + rezolvare # @@ -243,11 +247,15 @@ def resolve_prestatii( mapping: dict[str, str], valid_codes: set[str] | None = None, text_rules: list[dict] | None = None, + excluded_ops: set[str] | None = None, ) -> tuple[list[dict], list[dict]]: """Rezolva fiecare item: umple `cod_prestatie` din maparea op->cod unde lipseste. Reguli (hibrid): - item cu `cod_prestatie` valid (in nomenclator) -> pastrat ca atare. + - item fara cod, cu `cod_op_service` in `excluded_ops` -> adnotat `exclus=True` + (NU e nemapat; nu se declara la RAR — apelantii il scot din payload cu + `split_prestatii_excluse` inainte de enqueue). - item fara cod, cu `cod_op_service` in `mapping` -> umplem cod_prestatie. - item fara cod, nemapat exact, dar al carui text da match pe o regula text (substring) -> umplem cod_prestatie din prima regula care potriveste. @@ -258,9 +266,11 @@ def resolve_prestatii( COD_PRESTATIE max 5 car.); un cod necunoscut da HTTP 500 si RECORD PARTIAL la RAR (terminal) -> nu-l trimitem niciodata raw. - Precedenta (stricta): `cod_prestatie` direct valid > mapare exacta `cod_op_service` - in `mapping` > reguli text > nemapat. Regulile text se incearca DOAR cand nu exista - cod valid SI op nu e in `mapping`. + Precedenta (stricta): `cod_prestatie` direct valid > exclus de la declarare > + mapare exacta `cod_op_service` in `mapping` > reguli text > nemapat. Un cod ales + explicit pe rand (editor) bate regula de excludere; regula de excludere bate + regulile text. Regulile text se incearca DOAR cand nu exista cod valid SI op + nu e in `mapping`. `valid_codes` = setul de coduri RAR valide (uppercase) din nomenclator. Cand e None, validarea e dezactivata (compat: comportamentul vechi „cod_prestatie trece @@ -285,6 +295,7 @@ def resolve_prestatii( # un cod_sursa/flag stale din payload -> telemetrie falsa + hold gresit. it.pop("cod_sursa", None) it.pop("regula_fara_autosend", None) + it.pop("exclus", None) cod = (it.get("cod_prestatie") or "").strip().upper() op = (it.get("cod_op_service") or "").strip() cod_valid = bool(cod) and (valid_codes is None or cod in valid_codes) @@ -299,7 +310,11 @@ def resolve_prestatii( it["cod_op_service"] = op if not it.get("denumire"): it["denumire"] = cod - if op and op in mapping: + if op and excluded_ops and op in excluded_ops: + # Exclus de la declarare: nu e nemapat, nu se trimite la RAR. + it["cod_prestatie"] = None + it["exclus"] = True + elif op and op in mapping: it["cod_prestatie"] = mapping[op] elif op: # Mapare exacta absenta -> incearca regulile text (substring). @@ -322,6 +337,17 @@ def resolve_prestatii( return resolved, unmapped +def split_prestatii_excluse(prestatii: list[dict] | None) -> tuple[list[dict], list[dict]]: + """Separa prestatiile declarabile de cele adnotate `exclus` de resolve_prestatii. + + Payload-ul trimis la RAR (si cheia de idempotenta) se construieste NUMAI din + declarabile; cele excluse nu parasesc niciodata sistemul. + """ + declarabile = [p for p in (prestatii or []) if not p.get("exclus")] + excluse = [p for p in (prestatii or []) if p.get("exclus")] + return declarabile, excluse + + # --------------------------------------------------------------------------- # # Persistenta (conn) # # --------------------------------------------------------------------------- # @@ -404,10 +430,11 @@ def load_nomenclator_codes(conn) -> set[str]: def load_mapping(conn, account_id: int | None) -> dict[str, str]: - """{cod_op_service -> cod_prestatie} pentru un cont.""" + """{cod_op_service -> cod_prestatie} pentru un cont. Fara regulile de excludere.""" acct = account_or_default(account_id) rows = conn.execute( - "SELECT cod_op_service, cod_prestatie FROM operations_mapping WHERE account_id=?", + "SELECT cod_op_service, cod_prestatie FROM operations_mapping " + "WHERE account_id=? AND exclus=0", (acct,), ).fetchall() return {r["cod_op_service"]: r["cod_prestatie"] for r in rows} @@ -417,10 +444,13 @@ def load_mapping_meta(conn, account_id: int | None) -> dict[str, dict]: """{cod_op_service -> {cod_prestatie, auto_send}} pentru un cont. Varianta extinsa care include si flagul auto_send per operatie. + Fara regulile de excludere (cod_prestatie gol) — consumatorii construiesc din ea + dict-uri de mapare op->cod; excluderile se incarca separat (load_excluded_ops). """ acct = account_or_default(account_id) rows = conn.execute( - "SELECT cod_op_service, cod_prestatie, auto_send FROM operations_mapping WHERE account_id=?", + "SELECT cod_op_service, cod_prestatie, auto_send FROM operations_mapping " + "WHERE account_id=? AND exclus=0", (acct,), ).fetchall() return { @@ -429,12 +459,23 @@ def load_mapping_meta(conn, account_id: int | None) -> dict[str, dict]: } +def load_excluded_ops(conn, account_id: int | None) -> set[str]: + """Setul de operatii excluse de la declarare pentru un cont.""" + acct = account_or_default(account_id) + rows = conn.execute( + "SELECT cod_op_service FROM operations_mapping WHERE account_id=? AND exclus=1", + (acct,), + ).fetchall() + return {r["cod_op_service"] for r in rows} + + def classify_prezentare( content: dict, mapping: dict[str, str], mapping_meta: dict[str, dict], valid_codes: set[str] | None = None, text_rules: list[dict] | None = None, + excluded_ops: set[str] | None = None, ) -> dict: """Helper pur de clasificare: reproduce EXACT logica create_prezentari fara DB/efecte. @@ -454,7 +495,7 @@ def classify_prezentare( "odometru_final": canon["odometru_final"], }) - resolved, unmapped = resolve_prestatii(c.get("prestatii"), mapping, valid_codes, text_rules) + resolved, unmapped = resolve_prestatii(c.get("prestatii"), mapping, valid_codes, text_rules, excluded_ops) c["prestatii"] = resolved if unmapped: @@ -466,7 +507,20 @@ def classify_prezentare( ) errors: list[dict] = [] else: - errors = validate_prezentare(c) + # Prestatiile excluse de la declarare NU intra in payload-ul trimis (nici in + # cheia de idempotenta). Toate excluse -> needs_data cu motiv explicit; + # payload-ul pastreaza itemii adnotati `exclus` ca detaliul sa arate operatiile. + declarabile, excluse = split_prestatii_excluse(resolved) + if not declarabile and excluse: + ops_excluse = ", ".join((p.get("cod_op_service") or "") for p in excluse) + errors = [{ + "field": "prestatii", + "message": f"Toate operatiile sunt excluse de la declarare ({ops_excluse}) — randul nu se trimite la RAR.", + }] + else: + if excluse: + c["prestatii"] = declarabile + errors = validate_prezentare(c) if errors: status = "needs_data" rar_error = json.dumps(errors, ensure_ascii=False) @@ -522,6 +576,7 @@ def pending_unmapped(conn, account_id=None) -> list[dict]: ).fetchall() agg: dict[tuple[int, str], dict[str, Any]] = {} + excluded_by_acct: dict[int, set[str]] = {} for r in rows: acct = r["account_id"] if r["account_id"] is not None else DEFAULT_ACCOUNT_ID try: @@ -536,6 +591,11 @@ def pending_unmapped(conn, account_id=None) -> list[dict]: op = (item.get("cod_op_service") or "").strip() if not op: continue + # Operatiile excluse de la declarare nu sunt "de mapat" — nu apar in editor. + if acct not in excluded_by_acct: + excluded_by_acct[acct] = load_excluded_ops(conn, acct) + if op in excluded_by_acct[acct]: + continue key = (acct, op) entry = agg.setdefault( key, @@ -563,21 +623,44 @@ def pending_unmapped(conn, account_id=None) -> list[dict]: def save_mapping(conn, account_id: int | None, cod_op_service: str, cod_prestatie: str, auto_send: bool) -> None: - """Upsert o mapare op->cod (UNIQUE pe account_id+cod_op_service).""" + """Upsert o mapare op->cod (UNIQUE pe account_id+cod_op_service). + + Reseteaza `excluded=0`: maparea unei operatii excluse anterior o readuce + in fluxul de declarare (aceeasi cheie UNIQUE, o operatie = o regula). + """ acct = account_or_default(account_id) op = (cod_op_service or "").strip() cod = (cod_prestatie or "").strip().upper() if not op or not cod: raise ValueError("cod_op_service si cod_prestatie sunt obligatorii") conn.execute( - "INSERT INTO operations_mapping (account_id, cod_op_service, cod_prestatie, auto_send) " - "VALUES (?, ?, ?, ?) " + "INSERT INTO operations_mapping (account_id, cod_op_service, cod_prestatie, auto_send, exclus) " + "VALUES (?, ?, ?, ?, 0) " "ON CONFLICT(account_id, cod_op_service) DO UPDATE SET " - "cod_prestatie=excluded.cod_prestatie, auto_send=excluded.auto_send", + "cod_prestatie=excluded.cod_prestatie, auto_send=excluded.auto_send, exclus=0", (acct, op, cod, 1 if auto_send else 0), ) +def save_exclusion(conn, account_id: int | None, cod_op_service: str) -> None: + """Upsert o regula 'nu se declara' pentru o operatie (UNIQUE pe account_id+op). + + cod_prestatie ramane gol — regula nu mapeaza, doar exclude de la declarare. + Suprascrie o mapare existenta pe aceeasi operatie (o operatie = o regula). + """ + acct = account_or_default(account_id) + op = (cod_op_service or "").strip() + if not op: + raise ValueError("cod_op_service este obligatoriu") + conn.execute( + "INSERT INTO operations_mapping (account_id, cod_op_service, cod_prestatie, auto_send, exclus) " + "VALUES (?, ?, '', 0, 1) " + "ON CONFLICT(account_id, cod_op_service) DO UPDATE SET " + "cod_prestatie='', auto_send=0, exclus=1", + (acct, op), + ) + + def load_text_rules(conn, account_id: int | None) -> list[dict]: """Returneaza regulile text ale unui cont, ordonate priority ASC, id ASC. @@ -929,6 +1012,7 @@ def reresolve_account(conn, account_id: int | None, batch_id: int | None = None) valid_codes = load_nomenclator_codes(conn) or None # Incarca regulile text O DATA, inainte de bucla pe randuri. text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) if batch_id is not None: # Scope la batch-ul specificat (import commit explicit). @@ -954,7 +1038,7 @@ def reresolve_account(conn, account_id: int | None, batch_id: int | None = None) content = json.loads(r["payload_json"]) except (ValueError, TypeError): continue - resolved, unmapped = resolve_prestatii(content.get("prestatii"), mapping, valid_codes, text_rules) + resolved, unmapped = resolve_prestatii(content.get("prestatii"), mapping, valid_codes, text_rules, excluded_ops) content["prestatii"] = resolved payload_json = json.dumps(content, ensure_ascii=False) @@ -969,6 +1053,28 @@ def reresolve_account(conn, account_id: int | None, batch_id: int | None = None) stats["still_blocked"] += 1 continue + # Prestatiile excluse ies din payload-ul trimis; toate excluse -> needs_data + # cu motiv explicit (randul nu se mai trimite; operatorul il poate sterge). + # Payload-ul pastreaza itemii adnotati `exclus` DOAR in ramura blocata, ca + # detaliul sa arate operatiile; la queued pleaca numai declarabilele. + declarabile, excluse = split_prestatii_excluse(resolved) + if not declarabile and excluse: + ops_excluse = ", ".join((p.get("cod_op_service") or "") for p in excluse) + motiv = [{ + "field": "prestatii", + "message": f"Toate operatiile sunt excluse de la declarare ({ops_excluse}) — randul nu se trimite la RAR.", + }] + conn.execute( + "UPDATE submissions SET status='needs_data', payload_json=?, rar_error=?, " + "updated_at=datetime('now') WHERE id=?", + (payload_json, json.dumps(motiv, ensure_ascii=False), r["id"]), + ) + stats["needs_data"] += 1 + continue + if excluse: + content["prestatii"] = declarabile + payload_json = json.dumps(content, ensure_ascii=False) + # Ramura auto_send eliminata din reresolve. # Un cod rezolvat -> queued direct (review_manual ramane 0). diff --git a/app/schema.sql b/app/schema.sql index 2b60113..948f8ea 100644 --- a/app/schema.sql +++ b/app/schema.sql @@ -82,8 +82,9 @@ CREATE TABLE IF NOT EXISTS operations_mapping ( id INTEGER PRIMARY KEY AUTOINCREMENT, account_id INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, cod_op_service TEXT NOT NULL, - cod_prestatie TEXT NOT NULL, + cod_prestatie TEXT NOT NULL, -- gol ('') cand exclus=1 (regula nu mapeaza) auto_send INTEGER NOT NULL DEFAULT 1, + exclus INTEGER NOT NULL DEFAULT 0, -- 1 = operatia NU se declara la RAR created_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE (account_id, cod_op_service) ); @@ -175,7 +176,8 @@ CREATE TABLE IF NOT EXISTS import_rows ( resolved_status TEXT NOT NULL DEFAULT 'pending' CHECK (resolved_status IN ( 'pending','ok','needs_mapping','needs_data', - 'needs_review','already_sent','duplicate_in_file' + 'needs_review','already_sent','duplicate_in_file', + 'excluded' )), error TEXT ); diff --git a/app/submissions_admin.py b/app/submissions_admin.py index 921fc2f..722421a 100644 --- a/app/submissions_admin.py +++ b/app/submissions_admin.py @@ -25,6 +25,7 @@ from .mapping import ( account_or_default, account_scope_clause, classify_prezentare, + load_excluded_ops, load_mapping_meta, load_nomenclator_codes, ) @@ -102,7 +103,8 @@ def requeue_submission(conn, account_id: int, sid: int) -> dict: mapping_meta = load_mapping_meta(conn, account_id) mapping = {op: m["cod_prestatie"] for op, m in mapping_meta.items()} valid_codes = load_nomenclator_codes(conn) or None - cl = classify_prezentare(content, mapping, mapping_meta, valid_codes) + excluded_ops = load_excluded_ops(conn, account_id) + cl = classify_prezentare(content, mapping, mapping_meta, valid_codes, excluded_ops=excluded_ops) # Re-snapshot `held` din comutatorul contului la re-punere in coada # (paritate cu create_prezentari/reresolve_account). Fara asta un rand repus pastra diff --git a/app/web/labels.py b/app/web/labels.py index 2045e44..ddd75cd 100644 --- a/app/web/labels.py +++ b/app/web/labels.py @@ -400,6 +400,7 @@ STARI_PREVIEW: dict[str, tuple[str, str]] = { "needs_data": ("Date incomplete", "s-needs_data"), "already_sent": ("Deja trimis", "s-already_sent"), "duplicate_in_file": ("Duplicat in fisier", "s-duplicate_in_file"), + "excluded": ("Nedeclarat", "s-excluded"), } @@ -422,6 +423,8 @@ def nota_umana_preview(status: str, errors: list, flags: list) -> str: """ if status in ("already_sent", "duplicate_in_file"): return "" + if status == "excluded": + return "Operatie exclusa de la declarare — randul nu se trimite la RAR." # needs_mapping: codul RAR lipseste — prioritizeaza 'unmapped' inaintea flags, # altfel un rand cu si un flag (ex. VIN numeric) ar afisa textul flag-ului # si ascunde motivul real (cod lipsa). diff --git a/app/web/routes.py b/app/web/routes.py index caa06b5..bb10381 100644 --- a/app/web/routes.py +++ b/app/web/routes.py @@ -72,12 +72,14 @@ from ..submissions_admin import ( from ..mapping import ( DEFAULT_ACCOUNT_ID, _emite_text_rule_hits, + EXCLUDE_SENTINEL, account_or_default, account_scope_clause, delete_text_rule, enrich_suggestions, ensure_embeddings_corpus, has_no_auto_send, + load_excluded_ops, load_mapping_meta, load_nomenclator, load_nomenclator_codes, @@ -86,8 +88,10 @@ from ..mapping import ( pending_unmapped, reresolve_account, resolve_prestatii, + save_exclusion, save_mapping, save_text_rule, + split_prestatii_excluse, suggest_codes, text_rules_overlap, ) @@ -1727,12 +1731,19 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR mapping = {op: m["cod_prestatie"] for op, m in mapping_meta.items()} valid_codes = load_nomenclator_codes(conn) or None text_rules = load_text_rules(conn, account_id) - resolved, unmapped = resolve_prestatii(content.get("prestatii"), mapping, valid_codes, text_rules) + excluded_ops = load_excluded_ops(conn, account_id) + resolved, unmapped = resolve_prestatii(content.get("prestatii"), mapping, valid_codes, text_rules, excluded_ops) content["prestatii"] = resolved # telemetrie pentru itemii rezolvati prin regula text (calea corectie web). _emite_text_rule_hits(conn, account_id, row["id"], resolved) + # Prestatiile excluse de la declarare ies din payload-ul trimis (ca in + # reresolve_account); toate excluse -> needs_data cu motiv explicit mai jos. + declarabile, excluse = split_prestatii_excluse(resolved) + if not unmapped and excluse and declarabile: + content["prestatii"] = declarabile + # Canonicalizare (strip ".0" odometru, VIN/nr upper) INAINTE de validare si cheie. canon = canonicalize_row(content) content.update({ @@ -1755,6 +1766,24 @@ async def post_corectie_trimitere(request: Request, submission_id: int) -> HTMLR message="Lipseste inca un cod RAR — alege-l mai jos sau in tab-ul Mapari."), ) + if excluse and not declarabile: + ops_excluse = ", ".join((p.get("cod_op_service") or "") for p in excluse) + motiv = [{ + "field": "prestatii", + "message": f"Toate operatiile sunt excluse de la declarare ({ops_excluse}) — randul nu se trimite la RAR.", + }] + conn.execute( + "UPDATE submissions SET status='needs_data', payload_json=?, rar_error=?, " + "updated_at=datetime('now') WHERE id=?", + (payload_json, json.dumps(motiv, ensure_ascii=False), row["id"]), + ) + row2 = _fetch_submission_scoped(conn, account_id, submission_id) + return templates.TemplateResponse( + "_trimitere_detaliu.html", + _detaliu_ctx(request, row2, conn=conn, account_id=account_id, error=True, + message="Toate operatiile randului sunt excluse de la declarare — nu se trimite la RAR."), + ) + errors = validate_prezentare(content) if errors: # Inca invalid: persista valorile introduse, ramane needs_data, arata motivul pe camp. @@ -2571,7 +2600,7 @@ def _load_saved_op_mappings(conn, account_id: int) -> list[dict]: prestatiei jonctionat din nomenclator. Scoped pe cont (NOT NULL → simplu).""" acct = account_or_default(account_id) rows = conn.execute( - "SELECT o.id, o.cod_op_service, o.cod_prestatie, o.auto_send, n.nume_prestatie " + "SELECT o.id, o.cod_op_service, o.cod_prestatie, o.auto_send, o.exclus, n.nume_prestatie " "FROM operations_mapping o " "LEFT JOIN nomenclator_rar n ON n.cod_prestatie = o.cod_prestatie " "WHERE o.account_id=? ORDER BY o.cod_op_service", @@ -2583,6 +2612,7 @@ def _load_saved_op_mappings(conn, account_id: int) -> list[dict]: "cod_op_service": r["cod_op_service"], "cod_prestatie": r["cod_prestatie"], "auto_send": bool(r["auto_send"]), + "exclus": bool(r["exclus"]), "nume_prestatie": r["nume_prestatie"], } for r in rows @@ -2669,6 +2699,17 @@ def post_mapare( conn = get_connection() try: cod = cod_prestatie.strip().upper() + if cod == EXCLUDE_SENTINEL: + # Regula "nu se declara la RAR": exclude + re-rezolva blocatele + # (randurile cu toate operatiile excluse trec pe needs_data cu motiv). + save_exclusion(conn, account_id, cod_op_service) + stats = reresolve_account(conn, account_id) + msg = ( + f"{cod_op_service.strip()} exclus de la declarare. " + f"Deblocate: {stats['requeued']} in coada, {stats['needs_data']} oprite/date lipsa, " + f"{stats['still_blocked']} inca nemapate." + ) + return _render_mapari(request, conn, account_id, message=msg) exists = conn.execute("SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod,)).fetchone() if not exists: return _render_mapari(request, conn, account_id, message=f"Cod necunoscut: {cod}") @@ -2715,6 +2756,16 @@ def post_editeaza_mapare_salvata( conn = get_connection() try: cod = cod_prestatie.strip().upper() + if cod == EXCLUDE_SENTINEL: + # Trecerea unei mapari salvate pe "nu se declara" — exclude + re-rezolva. + save_exclusion(conn, account_id, cod_op_service) + stats = reresolve_account(conn, account_id) + msg = ( + f"{cod_op_service.strip()} exclus de la declarare. " + f"Deblocate: {stats['requeued']} in coada, {stats['needs_data']} oprite/date lipsa, " + f"{stats['still_blocked']} inca nemapate." + ) + return _render_mapari(request, conn, account_id, message=msg) exists = conn.execute( "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod,) ).fetchone() @@ -3131,6 +3182,7 @@ def _web_compute_preview( # needs_mapping si exclus din commit. Incarcate o data. valid_codes = load_nomenclator_codes(conn) or None text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) # Detectie coercion flags din valorile stocate (VIN numeric) coercion_flags_map: dict[int, list[str]] = {} @@ -3177,6 +3229,7 @@ def _web_compute_preview( valid_codes=valid_codes, text_rules=text_rules, reviewed=reviewed_flags[i], + excluded_ops=excluded_ops, ) key: str | None = None @@ -4162,6 +4215,7 @@ async def web_mapare_operatii( codes_list = form.getlist("cod_prestatie") salvate: list[str] = [] + excluse_ops: list[str] = [] sarite_invalide: list[str] = [] for cod_op_service, cod_prestatie in zip(ops_list, codes_list): @@ -4172,6 +4226,12 @@ async def web_mapare_operatii( if not cod_op_service or not cod_prestatie: continue + # Regula "nu se declara la RAR": exclude operatia de la declarare. + if cod_prestatie == EXCLUDE_SENTINEL: + save_exclusion(conn, account_id, cod_op_service) + excluse_ops.append(cod_op_service) + continue + # Validare per-item (D#12): cod invalid -> skip + sumar, nu all-or-nothing exists = conn.execute( "SELECT 1 FROM nomenclator_rar WHERE cod_prestatie=?", (cod_prestatie,) @@ -4187,10 +4247,12 @@ async def web_mapare_operatii( parts: list[str] = [] if salvate: parts.append(f"Salvate: {', '.join(salvate)}.") + if excluse_ops: + parts.append(f"Excluse de la declarare: {', '.join(excluse_ops)}.") if sarite_invalide: parts.append(f"Coduri necunoscute ignorate: {', '.join(sarite_invalide)}.") message = " ".join(parts) if parts else None - error = bool(sarite_invalide) and not salvate + error = bool(sarite_invalide) and not salvate and not excluse_ops result = _web_compute_preview(conn, import_id, account_id) if isinstance(result, str): @@ -4404,9 +4466,10 @@ async def web_confirma_import( # Mapare operatii mapping_meta = load_mapping_meta(conn, acct) mapping_ops = {op: meta["cod_prestatie"] for op, meta in mapping_meta.items()} - # validare nomenclator + reguli text incarcate O DATA, inainte de bucla pe randuri. + # validare nomenclator + reguli text + excluderi incarcate O DATA, inainte de bucla. valid_codes = load_nomenclator_codes(conn) or None text_rules = load_text_rules(conn, acct) + excluded_ops = load_excluded_ops(conn, acct) # Rezolva mediul RAR tinta al lotului: form > default cont > ancora globala. try: @@ -4468,7 +4531,12 @@ async def web_confirma_import( # Rezolva prestatii prestatii = mapped.get("prestatii") or [] - resolved_p, _ = resolve_prestatii(prestatii, mapping_ops, valid_codes, text_rules) + resolved_p, _ = resolve_prestatii(prestatii, mapping_ops, valid_codes, text_rules, excluded_ops) + # Prestatiile excluse NU pleaca la RAR (ca in preview); rand fara nicio + # prestatie declarabila = 'excluded' in preview -> defensiv skip. + resolved_p, _excluse_p = split_prestatii_excluse(resolved_p) + if not resolved_p: + continue mapped["prestatii"] = resolved_p # Canonicalizare (cheia identica cu preview: canonicalize_row + build_key) diff --git a/app/web/templates/_mapari.html b/app/web/templates/_mapari.html index c970efc..8e5f46e 100644 --- a/app/web/templates/_mapari.html +++ b/app/web/templates/_mapari.html @@ -82,6 +82,7 @@ cu tot nomenclatorul). #} - +
@@ -142,14 +143,16 @@
{{ m.cod_op_service }}
- acum: {{ m.cod_prestatie }}{% if m.nume_prestatie %} — {{ m.nume_prestatie }}{% endif %} + {% if m.exclus %}acum: Nu se declara la RAR + {% else %}acum: {{ m.cod_prestatie }}{% if m.nume_prestatie %} — {{ m.nume_prestatie }}{% endif %}{% endif %}
+ {% for n in nomenclator %}