diff --git a/app/db.py b/app/db.py index 5b0e6ed..8b367fd 100644 --- a/app/db.py +++ b/app/db.py @@ -43,11 +43,33 @@ def init_db() -> None: def _migrate(conn: sqlite3.Connection) -> None: """Migrari aditive pentru DB create inainte de o coloana noua (CREATE IF NOT EXISTS nu altereaza).""" - cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()} - if "next_attempt_at" not in cols: + # Coloane submissions + sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()} + if "next_attempt_at" not in sub_cols: conn.execute("ALTER TABLE submissions ADD COLUMN next_attempt_at TEXT") - if "rar_creds_enc" not in cols: + if "rar_creds_enc" not in sub_cols: conn.execute("ALTER TABLE submissions ADD COLUMN rar_creds_enc TEXT") + if "purge_after" not in sub_cols: + conn.execute("ALTER TABLE submissions ADD COLUMN purge_after TEXT") + if "batch_id" not in sub_cols: + conn.execute("ALTER TABLE submissions ADD COLUMN batch_id INTEGER") + if "row_index" not in sub_cols: + conn.execute("ALTER TABLE submissions ADD COLUMN row_index INTEGER") + + # Coloane accounts + acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()} + if "rar_creds_enc" not in acc_cols: + conn.execute("ALTER TABLE accounts ADD COLUMN rar_creds_enc TEXT") + + # Index batch_id pe submissions (poate lipsi pe DB veche) + existing_idx = {r["name"] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='submissions'" + ).fetchall()} + if "idx_submissions_batch" not in existing_idx: + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_submissions_batch ON submissions(batch_id) " + "WHERE batch_id IS NOT NULL" + ) def _now_iso() -> str: diff --git a/app/schema.sql b/app/schema.sql index 107e8e9..947e10e 100644 --- a/app/schema.sql +++ b/app/schema.sql @@ -1,18 +1,17 @@ -- Schema SQLite (WAL) pentru gateway RAR AUTOPASS. --- Vezi plan.md sect. 5. NICIUN camp pentru parole RAR. --- Validarea completa (T3) si criptarea PII (P2) vin ulterior; in schelet --- payload-ul e stocat ca JSON text (camp payload_json), de inlocuit cu BLOB --- criptat + purge_after cand se face T7/criptare. +-- Vezi plan.md sect. 5 + plan-treapta2.md sect. 4. +-- Treapta 2: adauga conturi cu creds RAR durabile, tabele import, atestari. PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; -- Conturi ROAAUTO (clientii care folosesc gateway-ul). CREATE TABLE IF NOT EXISTS accounts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - cui TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')) + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + cui TEXT, + rar_creds_enc TEXT, -- creds RAR criptate (Fernet) durabile per-cont (D4/Eng#1) + created_at TEXT NOT NULL DEFAULT (datetime('now')) ); -- Cont implicit (id=1): auth API-key (CORE) inca neimplementat, deci ingestiile vin -- cu account_id NULL. Le atribuim contului default ca FK + UNIQUE(account_id,...) din @@ -54,20 +53,79 @@ CREATE TABLE IF NOT EXISTS submissions ( account_id INTEGER REFERENCES accounts(id) ON DELETE SET NULL, status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','sending','sent','needs_mapping','needs_data','error')), - payload_json TEXT NOT NULL, -- TODO(P2): inlocuit cu BLOB criptat - rar_creds_enc TEXT, -- creds RAR criptate (Fernet), sterse dupa primul login reusit (plan sect.5) + payload_json TEXT NOT NULL, + rar_creds_enc TEXT, -- creds RAR criptate (Fernet), sterse dupa primul login reusit rar_status_code INTEGER, rar_error TEXT, id_prezentare INTEGER, -- data.id intors de RAR la succes retry_count INTEGER NOT NULL DEFAULT 0, next_attempt_at TEXT, -- backoff: randul nu se ia inainte de acest moment (T2) sending_since TEXT, -- pentru lease/timeout pe randuri 'sending' orfane (T2) - purge_after TEXT, -- sent + 90z (P2) + purge_after TEXT, -- sent + 90z (T16) + batch_id INTEGER, -- import batch (T7; NULL = canal API) + row_index INTEGER, -- rand in batch (T7; NULL = canal API) created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_submissions_status ON submissions(status); +-- Nota: idx_submissions_batch se creeaza in _migrate (dupa ALTER care adauga batch_id pe DB veche). + +-- Mapare coloane fisier -> campuri canonice (retinuta per cont, semnatura coloane). +CREATE TABLE IF NOT EXISTS column_mappings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + account_id INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, + signature_coloane TEXT NOT NULL, -- hash/lista sortata a coloanelor fisierului + json_mapare TEXT NOT NULL, -- {col_fisier: camp_canonic, ...} JSON + format_data TEXT, -- ex. "DD.MM.YYYY" + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (account_id, signature_coloane) +); + +-- Loturi de import (fisiere incarcate). +CREATE TABLE IF NOT EXISTS import_batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + account_id INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'staging' + CHECK (status IN ('staging','committed','error')), + total INTEGER NOT NULL DEFAULT 0, + ok INTEGER NOT NULL DEFAULT 0, + needs_mapping INTEGER NOT NULL DEFAULT 0, + needs_data INTEGER NOT NULL DEFAULT 0, + needs_review INTEGER NOT NULL DEFAULT 0, + already_sent INTEGER NOT NULL DEFAULT 0, + duplicate_in_file INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + purge_after TEXT -- created_at + 90z (T16) +); + +-- Randuri din lot de import (PII criptat cu Fernet). +CREATE TABLE IF NOT EXISTS import_rows ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch_id INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE, + row_index INTEGER NOT NULL, + raw_json TEXT NOT NULL, -- PII criptat (Fernet, ca submissions) + resolved_status TEXT NOT NULL DEFAULT 'pending' + CHECK (resolved_status IN ( + 'pending','ok','needs_mapping','needs_data', + 'needs_review','already_sent','duplicate_in_file' + )), + error TEXT +); + +CREATE INDEX IF NOT EXISTS idx_import_rows_batch ON import_rows(batch_id); + +-- Log atestare legala (confirmare import batch, L.142/2023). +CREATE TABLE IF NOT EXISTS import_attestations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch_id INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE, + account_id INTEGER NOT NULL, + confirmed_by TEXT, -- email/identifier utilizator + ts TEXT NOT NULL DEFAULT (datetime('now')), + rows_hash TEXT NOT NULL, -- sha256 peste valorile rezolvate confirmate + n_confirmed INTEGER NOT NULL +); -- Heartbeat worker (un singur rand, id=1). /healthz citeste de aici. CREATE TABLE IF NOT EXISTS worker_heartbeat ( diff --git a/requirements.txt b/requirements.txt index e6cdc33..cc009b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,8 @@ rapidfuzz==3.14.5 # Criptare creds RAR efemere in submissions (app/crypto.py, Fernet). Zero-storage at rest. cryptography==46.0.5 +# Parsare xlsx/xls pentru import fisiere (Treapta 2, Issue 4 — PINNED). +openpyxl==3.1.5 + # Migrare DBF (tools/import_dbf.py). Necesar doar pentru import optional, nu pentru runtime. dbfread==2.0.7 diff --git a/tests/test_foundation.py b/tests/test_foundation.py new file mode 100644 index 0000000..6682cc1 --- /dev/null +++ b/tests/test_foundation.py @@ -0,0 +1,173 @@ +"""Teste FOUNDATION (Task #1): schema + migrari idempotente + openpyxl disponibil.""" + +from __future__ import annotations + +import os +import tempfile + +import pytest + + +@pytest.fixture() +def db_conn(monkeypatch): + tmp = tempfile.mkdtemp() + monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "test_foundation.db")) + from app.config import get_settings + get_settings.cache_clear() + from app.db import get_connection, init_db + init_db() + conn = get_connection() + yield conn + conn.close() + get_settings.cache_clear() + + +def _table_cols(conn, table: str) -> set[str]: + return {r["name"] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()} + + +def _tables(conn) -> set[str]: + return {r["name"] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall()} + + +# --- Coloane noi --- + +def test_accounts_rar_creds_enc(db_conn): + cols = _table_cols(db_conn, "accounts") + assert "rar_creds_enc" in cols + + +def test_submissions_batch_id(db_conn): + cols = _table_cols(db_conn, "submissions") + assert "batch_id" in cols + + +def test_submissions_row_index(db_conn): + cols = _table_cols(db_conn, "submissions") + assert "row_index" in cols + + +def test_submissions_purge_after(db_conn): + cols = _table_cols(db_conn, "submissions") + assert "purge_after" in cols + + +# --- Tabele noi --- + +def test_column_mappings_table(db_conn): + assert "column_mappings" in _tables(db_conn) + + +def test_import_batches_table(db_conn): + assert "import_batches" in _tables(db_conn) + + +def test_import_rows_table(db_conn): + assert "import_rows" in _tables(db_conn) + + +def test_import_attestations_table(db_conn): + assert "import_attestations" in _tables(db_conn) + + +def test_import_rows_cols(db_conn): + cols = _table_cols(db_conn, "import_rows") + for c in ("id", "batch_id", "row_index", "raw_json", "resolved_status", "error"): + assert c in cols, f"coloana lipsa: {c}" + + +def test_import_batches_cols(db_conn): + cols = _table_cols(db_conn, "import_batches") + for c in ("id", "account_id", "filename", "status", "total", "ok", + "needs_mapping", "needs_data", "needs_review", "already_sent", + "duplicate_in_file", "created_at", "purge_after"): + assert c in cols, f"coloana lipsa: {c}" + + +def test_import_attestations_cols(db_conn): + cols = _table_cols(db_conn, "import_attestations") + for c in ("id", "batch_id", "account_id", "confirmed_by", "ts", "rows_hash", "n_confirmed"): + assert c in cols, f"coloana lipsa: {c}" + + +# --- Idempotenta init_db --- + +def test_init_db_idempotent(monkeypatch): + """init_db() ruleaza de doua ori fara eroare.""" + tmp = tempfile.mkdtemp() + monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "idem.db")) + from app.config import get_settings + get_settings.cache_clear() + from app.db import init_db + init_db() + init_db() # a doua oara trebuie sa fie silentioasa + get_settings.cache_clear() + + +def test_migrate_on_existing_db(monkeypatch): + """Migrarea functioneaza pe o DB veche (fara coloane noi).""" + import sqlite3 + tmp = tempfile.mkdtemp() + db_path = os.path.join(tmp, "old.db") + # Creeaza schema minima fara coloanele noi + old_conn = sqlite3.connect(db_path) + old_conn.executescript(""" + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, cui TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))); + INSERT OR IGNORE INTO accounts (id, name) VALUES (1, 'default'); + CREATE TABLE IF NOT EXISTS submissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + idempotency_key TEXT NOT NULL UNIQUE, + account_id INTEGER, + status TEXT NOT NULL DEFAULT 'queued', + payload_json TEXT NOT NULL, + rar_status_code INTEGER, + rar_error TEXT, + id_prezentare INTEGER, + retry_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS worker_heartbeat ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_beat TEXT, last_rar_login_ok TEXT, detail TEXT + ); + INSERT OR IGNORE INTO worker_heartbeat (id, detail) VALUES (1, 'never started'); + CREATE TABLE IF NOT EXISTS api_keys (id INTEGER PRIMARY KEY, account_id INTEGER, key_hash TEXT UNIQUE, active INTEGER DEFAULT 1, created_at TEXT); + CREATE TABLE IF NOT EXISTS operations_mapping (id INTEGER PRIMARY KEY, account_id INTEGER, cod_op_service TEXT, cod_prestatie TEXT, auto_send INTEGER DEFAULT 1, created_at TEXT, UNIQUE(account_id, cod_op_service)); + CREATE TABLE IF NOT EXISTS nomenclator_rar (cod_prestatie TEXT PRIMARY KEY, nume_prestatie TEXT, updated_at TEXT); + """) + old_conn.close() + + monkeypatch.setenv("AUTOPASS_DB_PATH", db_path) + from app.config import get_settings + get_settings.cache_clear() + from app.db import init_db, get_connection + init_db() + + conn = get_connection() + sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()} + acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()} + assert "batch_id" in sub_cols + assert "row_index" in sub_cols + assert "purge_after" in sub_cols + assert "rar_creds_enc" in acc_cols + conn.close() + get_settings.cache_clear() + + +# --- openpyxl disponibil --- + +def test_openpyxl_importabil(): + import openpyxl + assert openpyxl.__version__.startswith("3.1") + + +def test_openpyxl_create_workbook(): + from openpyxl import Workbook + wb = Workbook() + ws = wb.active + ws["A1"] = "test" + assert ws["A1"].value == "test"