feat(foundation): schema Treapta 2 + migrari aditive + openpyxl pinned (#1)

- accounts.rar_creds_enc TEXT (creds RAR durabile per-cont, D4)
- submissions.batch_id, row_index (T7 scoping R1)
- submissions.purge_after (T16 GDPR)
- Tabele noi: column_mappings, import_batches, import_rows, import_attestations
- _migrate idempotent pe DB veche (ALTER aditiv, pattern existent)
- openpyxl==3.1.5 adaugat in requirements.txt (Issue 4, PINNED)
- 15 teste noi: coloane, tabele, idempotenta, migrare DB veche, openpyxl

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app/db.py

@@ -43,11 +43,33 @@ def init_db() -> None:
 
 def _migrate(conn: sqlite3.Connection) -> None:
     """Migrari aditive pentru DB create inainte de o coloana noua (CREATE IF NOT EXISTS nu altereaza)."""
-    cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
+    # Coloane submissions
-    if "next_attempt_at" not in cols:
+    sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
+    if "next_attempt_at" not in sub_cols:
         conn.execute("ALTER TABLE submissions ADD COLUMN next_attempt_at TEXT")
-    if "rar_creds_enc" not in cols:
+    if "rar_creds_enc" not in sub_cols:
         conn.execute("ALTER TABLE submissions ADD COLUMN rar_creds_enc TEXT")
+    if "purge_after" not in sub_cols:
+        conn.execute("ALTER TABLE submissions ADD COLUMN purge_after TEXT")
+    if "batch_id" not in sub_cols:
+        conn.execute("ALTER TABLE submissions ADD COLUMN batch_id INTEGER")
+    if "row_index" not in sub_cols:
+        conn.execute("ALTER TABLE submissions ADD COLUMN row_index INTEGER")
+
+    # Coloane accounts
+    acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()}
+    if "rar_creds_enc" not in acc_cols:
+        conn.execute("ALTER TABLE accounts ADD COLUMN rar_creds_enc TEXT")
+
+    # Index batch_id pe submissions (poate lipsi pe DB veche)
+    existing_idx = {r["name"] for r in conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='submissions'"
+    ).fetchall()}
+    if "idx_submissions_batch" not in existing_idx:
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_submissions_batch ON submissions(batch_id) "
+            "WHERE batch_id IS NOT NULL"
+        )
 
 
 def _now_iso() -> str:

app/schema.sql

@@ -1,8 +1,6 @@
 -- Schema SQLite (WAL) pentru gateway RAR AUTOPASS.
--- Vezi plan.md sect. 5. NICIUN camp pentru parole RAR.
+-- Vezi plan.md sect. 5 + plan-treapta2.md sect. 4.
--- Validarea completa (T3) si criptarea PII (P2) vin ulterior; in schelet
+-- Treapta 2: adauga conturi cu creds RAR durabile, tabele import, atestari.
--- payload-ul e stocat ca JSON text (camp payload_json), de inlocuit cu BLOB
--- criptat + purge_after cand se face T7/criptare.
 
 PRAGMA journal_mode = WAL;
 PRAGMA foreign_keys = ON;
@@ -12,6 +10,7 @@ CREATE TABLE IF NOT EXISTS accounts (
     id              INTEGER PRIMARY KEY AUTOINCREMENT,
     name            TEXT NOT NULL,
     cui             TEXT,
+    rar_creds_enc   TEXT,       -- creds RAR criptate (Fernet) durabile per-cont (D4/Eng#1)
     created_at      TEXT NOT NULL DEFAULT (datetime('now'))
 );
 -- Cont implicit (id=1): auth API-key (CORE) inca neimplementat, deci ingestiile vin
@@ -54,20 +53,79 @@ CREATE TABLE IF NOT EXISTS submissions (
     account_id       INTEGER REFERENCES accounts(id) ON DELETE SET NULL,
     status           TEXT NOT NULL DEFAULT 'queued'
                        CHECK (status IN ('queued','sending','sent','needs_mapping','needs_data','error')),
-    payload_json     TEXT NOT NULL,          -- TODO(P2): inlocuit cu BLOB criptat
+    payload_json     TEXT NOT NULL,
-    rar_creds_enc    TEXT,                   -- creds RAR criptate (Fernet), sterse dupa primul login reusit (plan sect.5)
+    rar_creds_enc    TEXT,                   -- creds RAR criptate (Fernet), sterse dupa primul login reusit
     rar_status_code  INTEGER,
     rar_error        TEXT,
     id_prezentare    INTEGER,                -- data.id intors de RAR la succes
     retry_count      INTEGER NOT NULL DEFAULT 0,
     next_attempt_at  TEXT,                   -- backoff: randul nu se ia inainte de acest moment (T2)
     sending_since    TEXT,                   -- pentru lease/timeout pe randuri 'sending' orfane (T2)
-    purge_after      TEXT,                   -- sent + 90z (P2)
+    purge_after      TEXT,                   -- sent + 90z (T16)
+    batch_id         INTEGER,               -- import batch (T7; NULL = canal API)
+    row_index        INTEGER,               -- rand in batch (T7; NULL = canal API)
     created_at       TEXT NOT NULL DEFAULT (datetime('now')),
     updated_at       TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
 CREATE INDEX IF NOT EXISTS idx_submissions_status ON submissions(status);
+-- Nota: idx_submissions_batch se creeaza in _migrate (dupa ALTER care adauga batch_id pe DB veche).
+
+-- Mapare coloane fisier -> campuri canonice (retinuta per cont, semnatura coloane).
+CREATE TABLE IF NOT EXISTS column_mappings (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    account_id          INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+    signature_coloane   TEXT NOT NULL,      -- hash/lista sortata a coloanelor fisierului
+    json_mapare         TEXT NOT NULL,      -- {col_fisier: camp_canonic, ...} JSON
+    format_data         TEXT,               -- ex. "DD.MM.YYYY"
+    created_at          TEXT NOT NULL DEFAULT (datetime('now')),
+    UNIQUE (account_id, signature_coloane)
+);
+
+-- Loturi de import (fisiere incarcate).
+CREATE TABLE IF NOT EXISTS import_batches (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    account_id          INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
+    filename            TEXT NOT NULL,
+    status              TEXT NOT NULL DEFAULT 'staging'
+                            CHECK (status IN ('staging','committed','error')),
+    total               INTEGER NOT NULL DEFAULT 0,
+    ok                  INTEGER NOT NULL DEFAULT 0,
+    needs_mapping       INTEGER NOT NULL DEFAULT 0,
+    needs_data          INTEGER NOT NULL DEFAULT 0,
+    needs_review        INTEGER NOT NULL DEFAULT 0,
+    already_sent        INTEGER NOT NULL DEFAULT 0,
+    duplicate_in_file   INTEGER NOT NULL DEFAULT 0,
+    created_at          TEXT NOT NULL DEFAULT (datetime('now')),
+    purge_after         TEXT                -- created_at + 90z (T16)
+);
+
+-- Randuri din lot de import (PII criptat cu Fernet).
+CREATE TABLE IF NOT EXISTS import_rows (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    batch_id        INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE,
+    row_index       INTEGER NOT NULL,
+    raw_json        TEXT NOT NULL,          -- PII criptat (Fernet, ca submissions)
+    resolved_status TEXT NOT NULL DEFAULT 'pending'
+                        CHECK (resolved_status IN (
+                            'pending','ok','needs_mapping','needs_data',
+                            'needs_review','already_sent','duplicate_in_file'
+                        )),
+    error           TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_import_rows_batch ON import_rows(batch_id);
+
+-- Log atestare legala (confirmare import batch, L.142/2023).
+CREATE TABLE IF NOT EXISTS import_attestations (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    batch_id        INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE,
+    account_id      INTEGER NOT NULL,
+    confirmed_by    TEXT,                   -- email/identifier utilizator
+    ts              TEXT NOT NULL DEFAULT (datetime('now')),
+    rows_hash       TEXT NOT NULL,          -- sha256 peste valorile rezolvate confirmate
+    n_confirmed     INTEGER NOT NULL
+);
 
 -- Heartbeat worker (un singur rand, id=1). /healthz citeste de aici.
 CREATE TABLE IF NOT EXISTS worker_heartbeat (

requirements.txt

@@ -12,5 +12,8 @@ rapidfuzz==3.14.5
 # Criptare creds RAR efemere in submissions (app/crypto.py, Fernet). Zero-storage at rest.
 cryptography==46.0.5
 
+# Parsare xlsx/xls pentru import fisiere (Treapta 2, Issue 4 — PINNED).
+openpyxl==3.1.5
+
 # Migrare DBF (tools/import_dbf.py). Necesar doar pentru import optional, nu pentru runtime.
 dbfread==2.0.7

tests/test_foundation.py

@@ -0,0 +1,173 @@
+"""Teste FOUNDATION (Task #1): schema + migrari idempotente + openpyxl disponibil."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+
+import pytest
+
+
+@pytest.fixture()
+def db_conn(monkeypatch):
+    tmp = tempfile.mkdtemp()
+    monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "test_foundation.db"))
+    from app.config import get_settings
+    get_settings.cache_clear()
+    from app.db import get_connection, init_db
+    init_db()
+    conn = get_connection()
+    yield conn
+    conn.close()
+    get_settings.cache_clear()
+
+
+def _table_cols(conn, table: str) -> set[str]:
+    return {r["name"] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
+
+
+def _tables(conn) -> set[str]:
+    return {r["name"] for r in conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table'"
+    ).fetchall()}
+
+
+# --- Coloane noi ---
+
+def test_accounts_rar_creds_enc(db_conn):
+    cols = _table_cols(db_conn, "accounts")
+    assert "rar_creds_enc" in cols
+
+
+def test_submissions_batch_id(db_conn):
+    cols = _table_cols(db_conn, "submissions")
+    assert "batch_id" in cols
+
+
+def test_submissions_row_index(db_conn):
+    cols = _table_cols(db_conn, "submissions")
+    assert "row_index" in cols
+
+
+def test_submissions_purge_after(db_conn):
+    cols = _table_cols(db_conn, "submissions")
+    assert "purge_after" in cols
+
+
+# --- Tabele noi ---
+
+def test_column_mappings_table(db_conn):
+    assert "column_mappings" in _tables(db_conn)
+
+
+def test_import_batches_table(db_conn):
+    assert "import_batches" in _tables(db_conn)
+
+
+def test_import_rows_table(db_conn):
+    assert "import_rows" in _tables(db_conn)
+
+
+def test_import_attestations_table(db_conn):
+    assert "import_attestations" in _tables(db_conn)
+
+
+def test_import_rows_cols(db_conn):
+    cols = _table_cols(db_conn, "import_rows")
+    for c in ("id", "batch_id", "row_index", "raw_json", "resolved_status", "error"):
+        assert c in cols, f"coloana lipsa: {c}"
+
+
+def test_import_batches_cols(db_conn):
+    cols = _table_cols(db_conn, "import_batches")
+    for c in ("id", "account_id", "filename", "status", "total", "ok",
+              "needs_mapping", "needs_data", "needs_review", "already_sent",
+              "duplicate_in_file", "created_at", "purge_after"):
+        assert c in cols, f"coloana lipsa: {c}"
+
+
+def test_import_attestations_cols(db_conn):
+    cols = _table_cols(db_conn, "import_attestations")
+    for c in ("id", "batch_id", "account_id", "confirmed_by", "ts", "rows_hash", "n_confirmed"):
+        assert c in cols, f"coloana lipsa: {c}"
+
+
+# --- Idempotenta init_db ---
+
+def test_init_db_idempotent(monkeypatch):
+    """init_db() ruleaza de doua ori fara eroare."""
+    tmp = tempfile.mkdtemp()
+    monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "idem.db"))
+    from app.config import get_settings
+    get_settings.cache_clear()
+    from app.db import init_db
+    init_db()
+    init_db()  # a doua oara trebuie sa fie silentioasa
+    get_settings.cache_clear()
+
+
+def test_migrate_on_existing_db(monkeypatch):
+    """Migrarea functioneaza pe o DB veche (fara coloane noi)."""
+    import sqlite3
+    tmp = tempfile.mkdtemp()
+    db_path = os.path.join(tmp, "old.db")
+    # Creeaza schema minima fara coloanele noi
+    old_conn = sqlite3.connect(db_path)
+    old_conn.executescript("""
+        PRAGMA journal_mode = WAL;
+        CREATE TABLE IF NOT EXISTS accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, cui TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')));
+        INSERT OR IGNORE INTO accounts (id, name) VALUES (1, 'default');
+        CREATE TABLE IF NOT EXISTS submissions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            idempotency_key TEXT NOT NULL UNIQUE,
+            account_id INTEGER,
+            status TEXT NOT NULL DEFAULT 'queued',
+            payload_json TEXT NOT NULL,
+            rar_status_code INTEGER,
+            rar_error TEXT,
+            id_prezentare INTEGER,
+            retry_count INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT NOT NULL DEFAULT (datetime('now')),
+            updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+        );
+        CREATE TABLE IF NOT EXISTS worker_heartbeat (
+            id INTEGER PRIMARY KEY CHECK (id = 1),
+            last_beat TEXT, last_rar_login_ok TEXT, detail TEXT
+        );
+        INSERT OR IGNORE INTO worker_heartbeat (id, detail) VALUES (1, 'never started');
+        CREATE TABLE IF NOT EXISTS api_keys (id INTEGER PRIMARY KEY, account_id INTEGER, key_hash TEXT UNIQUE, active INTEGER DEFAULT 1, created_at TEXT);
+        CREATE TABLE IF NOT EXISTS operations_mapping (id INTEGER PRIMARY KEY, account_id INTEGER, cod_op_service TEXT, cod_prestatie TEXT, auto_send INTEGER DEFAULT 1, created_at TEXT, UNIQUE(account_id, cod_op_service));
+        CREATE TABLE IF NOT EXISTS nomenclator_rar (cod_prestatie TEXT PRIMARY KEY, nume_prestatie TEXT, updated_at TEXT);
+    """)
+    old_conn.close()
+
+    monkeypatch.setenv("AUTOPASS_DB_PATH", db_path)
+    from app.config import get_settings
+    get_settings.cache_clear()
+    from app.db import init_db, get_connection
+    init_db()
+
+    conn = get_connection()
+    sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
+    acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()}
+    assert "batch_id" in sub_cols
+    assert "row_index" in sub_cols
+    assert "purge_after" in sub_cols
+    assert "rar_creds_enc" in acc_cols
+    conn.close()
+    get_settings.cache_clear()
+
+
+# --- openpyxl disponibil ---
+
+def test_openpyxl_importabil():
+    import openpyxl
+    assert openpyxl.__version__.startswith("3.1")
+
+
+def test_openpyxl_create_workbook():
+    from openpyxl import Workbook
+    wb = Workbook()
+    ws = wb.active
+    ws["A1"] = "test"
+    assert ws["A1"].value == "test"