feat(foundation): schema Treapta 2 + migrari aditive + openpyxl pinned (#1)
- accounts.rar_creds_enc TEXT (creds RAR durabile per-cont, D4) - submissions.batch_id, row_index (T7 scoping R1) - submissions.purge_after (T16 GDPR) - Tabele noi: column_mappings, import_batches, import_rows, import_attestations - _migrate idempotent pe DB veche (ALTER aditiv, pattern existent) - openpyxl==3.1.5 adaugat in requirements.txt (Issue 4, PINNED) - 15 teste noi: coloane, tabele, idempotenta, migrare DB veche, openpyxl Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Agent
28
app/db.py
28
app/db.py
@@ -43,11 +43,33 @@ def init_db() -> None:
|
||||
|
||||
def _migrate(conn: sqlite3.Connection) -> None:
|
||||
"""Migrari aditive pentru DB create inainte de o coloana noua (CREATE IF NOT EXISTS nu altereaza)."""
|
||||
cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
|
||||
if "next_attempt_at" not in cols:
|
||||
# Coloane submissions
|
||||
sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
|
||||
if "next_attempt_at" not in sub_cols:
|
||||
conn.execute("ALTER TABLE submissions ADD COLUMN next_attempt_at TEXT")
|
||||
if "rar_creds_enc" not in cols:
|
||||
if "rar_creds_enc" not in sub_cols:
|
||||
conn.execute("ALTER TABLE submissions ADD COLUMN rar_creds_enc TEXT")
|
||||
if "purge_after" not in sub_cols:
|
||||
conn.execute("ALTER TABLE submissions ADD COLUMN purge_after TEXT")
|
||||
if "batch_id" not in sub_cols:
|
||||
conn.execute("ALTER TABLE submissions ADD COLUMN batch_id INTEGER")
|
||||
if "row_index" not in sub_cols:
|
||||
conn.execute("ALTER TABLE submissions ADD COLUMN row_index INTEGER")
|
||||
|
||||
# Coloane accounts
|
||||
acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()}
|
||||
if "rar_creds_enc" not in acc_cols:
|
||||
conn.execute("ALTER TABLE accounts ADD COLUMN rar_creds_enc TEXT")
|
||||
|
||||
# Index batch_id pe submissions (poate lipsi pe DB veche)
|
||||
existing_idx = {r["name"] for r in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='submissions'"
|
||||
).fetchall()}
|
||||
if "idx_submissions_batch" not in existing_idx:
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_submissions_batch ON submissions(batch_id) "
|
||||
"WHERE batch_id IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
-- Schema SQLite (WAL) pentru gateway RAR AUTOPASS.
|
||||
-- Vezi plan.md sect. 5. NICIUN camp pentru parole RAR.
|
||||
-- Validarea completa (T3) si criptarea PII (P2) vin ulterior; in schelet
|
||||
-- payload-ul e stocat ca JSON text (camp payload_json), de inlocuit cu BLOB
|
||||
-- criptat + purge_after cand se face T7/criptare.
|
||||
-- Vezi plan.md sect. 5 + plan-treapta2.md sect. 4.
|
||||
-- Treapta 2: adauga conturi cu creds RAR durabile, tabele import, atestari.
|
||||
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
@@ -12,6 +10,7 @@ CREATE TABLE IF NOT EXISTS accounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
cui TEXT,
|
||||
rar_creds_enc TEXT, -- creds RAR criptate (Fernet) durabile per-cont (D4/Eng#1)
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
-- Cont implicit (id=1): auth API-key (CORE) inca neimplementat, deci ingestiile vin
|
||||
@@ -54,20 +53,79 @@ CREATE TABLE IF NOT EXISTS submissions (
|
||||
account_id INTEGER REFERENCES accounts(id) ON DELETE SET NULL,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued','sending','sent','needs_mapping','needs_data','error')),
|
||||
payload_json TEXT NOT NULL, -- TODO(P2): inlocuit cu BLOB criptat
|
||||
rar_creds_enc TEXT, -- creds RAR criptate (Fernet), sterse dupa primul login reusit (plan sect.5)
|
||||
payload_json TEXT NOT NULL,
|
||||
rar_creds_enc TEXT, -- creds RAR criptate (Fernet), sterse dupa primul login reusit
|
||||
rar_status_code INTEGER,
|
||||
rar_error TEXT,
|
||||
id_prezentare INTEGER, -- data.id intors de RAR la succes
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT, -- backoff: randul nu se ia inainte de acest moment (T2)
|
||||
sending_since TEXT, -- pentru lease/timeout pe randuri 'sending' orfane (T2)
|
||||
purge_after TEXT, -- sent + 90z (P2)
|
||||
purge_after TEXT, -- sent + 90z (T16)
|
||||
batch_id INTEGER, -- import batch (T7; NULL = canal API)
|
||||
row_index INTEGER, -- rand in batch (T7; NULL = canal API)
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_submissions_status ON submissions(status);
|
||||
-- Nota: idx_submissions_batch se creeaza in _migrate (dupa ALTER care adauga batch_id pe DB veche).
|
||||
|
||||
-- Mapare coloane fisier -> campuri canonice (retinuta per cont, semnatura coloane).
|
||||
CREATE TABLE IF NOT EXISTS column_mappings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
|
||||
signature_coloane TEXT NOT NULL, -- hash/lista sortata a coloanelor fisierului
|
||||
json_mapare TEXT NOT NULL, -- {col_fisier: camp_canonic, ...} JSON
|
||||
format_data TEXT, -- ex. "DD.MM.YYYY"
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (account_id, signature_coloane)
|
||||
);
|
||||
|
||||
-- Loturi de import (fisiere incarcate).
|
||||
CREATE TABLE IF NOT EXISTS import_batches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
|
||||
filename TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'staging'
|
||||
CHECK (status IN ('staging','committed','error')),
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
ok INTEGER NOT NULL DEFAULT 0,
|
||||
needs_mapping INTEGER NOT NULL DEFAULT 0,
|
||||
needs_data INTEGER NOT NULL DEFAULT 0,
|
||||
needs_review INTEGER NOT NULL DEFAULT 0,
|
||||
already_sent INTEGER NOT NULL DEFAULT 0,
|
||||
duplicate_in_file INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
purge_after TEXT -- created_at + 90z (T16)
|
||||
);
|
||||
|
||||
-- Randuri din lot de import (PII criptat cu Fernet).
|
||||
CREATE TABLE IF NOT EXISTS import_rows (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE,
|
||||
row_index INTEGER NOT NULL,
|
||||
raw_json TEXT NOT NULL, -- PII criptat (Fernet, ca submissions)
|
||||
resolved_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (resolved_status IN (
|
||||
'pending','ok','needs_mapping','needs_data',
|
||||
'needs_review','already_sent','duplicate_in_file'
|
||||
)),
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_import_rows_batch ON import_rows(batch_id);
|
||||
|
||||
-- Log atestare legala (confirmare import batch, L.142/2023).
|
||||
CREATE TABLE IF NOT EXISTS import_attestations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id INTEGER NOT NULL REFERENCES import_batches(id) ON DELETE CASCADE,
|
||||
account_id INTEGER NOT NULL,
|
||||
confirmed_by TEXT, -- email/identifier utilizator
|
||||
ts TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
rows_hash TEXT NOT NULL, -- sha256 peste valorile rezolvate confirmate
|
||||
n_confirmed INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Heartbeat worker (un singur rand, id=1). /healthz citeste de aici.
|
||||
CREATE TABLE IF NOT EXISTS worker_heartbeat (
|
||||
|
||||
@@ -12,5 +12,8 @@ rapidfuzz==3.14.5
|
||||
# Criptare creds RAR efemere in submissions (app/crypto.py, Fernet). Zero-storage at rest.
|
||||
cryptography==46.0.5
|
||||
|
||||
# Parsare xlsx/xls pentru import fisiere (Treapta 2, Issue 4 — PINNED).
|
||||
openpyxl==3.1.5
|
||||
|
||||
# Migrare DBF (tools/import_dbf.py). Necesar doar pentru import optional, nu pentru runtime.
|
||||
dbfread==2.0.7
|
||||
|
||||
173
tests/test_foundation.py
Normal file
173
tests/test_foundation.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""Teste FOUNDATION (Task #1): schema + migrari idempotente + openpyxl disponibil."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db_conn(monkeypatch):
|
||||
tmp = tempfile.mkdtemp()
|
||||
monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "test_foundation.db"))
|
||||
from app.config import get_settings
|
||||
get_settings.cache_clear()
|
||||
from app.db import get_connection, init_db
|
||||
init_db()
|
||||
conn = get_connection()
|
||||
yield conn
|
||||
conn.close()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def _table_cols(conn, table: str) -> set[str]:
|
||||
return {r["name"] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
|
||||
|
||||
|
||||
def _tables(conn) -> set[str]:
|
||||
return {r["name"] for r in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()}
|
||||
|
||||
|
||||
# --- Coloane noi ---
|
||||
|
||||
def test_accounts_rar_creds_enc(db_conn):
|
||||
cols = _table_cols(db_conn, "accounts")
|
||||
assert "rar_creds_enc" in cols
|
||||
|
||||
|
||||
def test_submissions_batch_id(db_conn):
|
||||
cols = _table_cols(db_conn, "submissions")
|
||||
assert "batch_id" in cols
|
||||
|
||||
|
||||
def test_submissions_row_index(db_conn):
|
||||
cols = _table_cols(db_conn, "submissions")
|
||||
assert "row_index" in cols
|
||||
|
||||
|
||||
def test_submissions_purge_after(db_conn):
|
||||
cols = _table_cols(db_conn, "submissions")
|
||||
assert "purge_after" in cols
|
||||
|
||||
|
||||
# --- Tabele noi ---
|
||||
|
||||
def test_column_mappings_table(db_conn):
|
||||
assert "column_mappings" in _tables(db_conn)
|
||||
|
||||
|
||||
def test_import_batches_table(db_conn):
|
||||
assert "import_batches" in _tables(db_conn)
|
||||
|
||||
|
||||
def test_import_rows_table(db_conn):
|
||||
assert "import_rows" in _tables(db_conn)
|
||||
|
||||
|
||||
def test_import_attestations_table(db_conn):
|
||||
assert "import_attestations" in _tables(db_conn)
|
||||
|
||||
|
||||
def test_import_rows_cols(db_conn):
|
||||
cols = _table_cols(db_conn, "import_rows")
|
||||
for c in ("id", "batch_id", "row_index", "raw_json", "resolved_status", "error"):
|
||||
assert c in cols, f"coloana lipsa: {c}"
|
||||
|
||||
|
||||
def test_import_batches_cols(db_conn):
|
||||
cols = _table_cols(db_conn, "import_batches")
|
||||
for c in ("id", "account_id", "filename", "status", "total", "ok",
|
||||
"needs_mapping", "needs_data", "needs_review", "already_sent",
|
||||
"duplicate_in_file", "created_at", "purge_after"):
|
||||
assert c in cols, f"coloana lipsa: {c}"
|
||||
|
||||
|
||||
def test_import_attestations_cols(db_conn):
|
||||
cols = _table_cols(db_conn, "import_attestations")
|
||||
for c in ("id", "batch_id", "account_id", "confirmed_by", "ts", "rows_hash", "n_confirmed"):
|
||||
assert c in cols, f"coloana lipsa: {c}"
|
||||
|
||||
|
||||
# --- Idempotenta init_db ---
|
||||
|
||||
def test_init_db_idempotent(monkeypatch):
|
||||
"""init_db() ruleaza de doua ori fara eroare."""
|
||||
tmp = tempfile.mkdtemp()
|
||||
monkeypatch.setenv("AUTOPASS_DB_PATH", os.path.join(tmp, "idem.db"))
|
||||
from app.config import get_settings
|
||||
get_settings.cache_clear()
|
||||
from app.db import init_db
|
||||
init_db()
|
||||
init_db() # a doua oara trebuie sa fie silentioasa
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_migrate_on_existing_db(monkeypatch):
|
||||
"""Migrarea functioneaza pe o DB veche (fara coloane noi)."""
|
||||
import sqlite3
|
||||
tmp = tempfile.mkdtemp()
|
||||
db_path = os.path.join(tmp, "old.db")
|
||||
# Creeaza schema minima fara coloanele noi
|
||||
old_conn = sqlite3.connect(db_path)
|
||||
old_conn.executescript("""
|
||||
PRAGMA journal_mode = WAL;
|
||||
CREATE TABLE IF NOT EXISTS accounts (id INTEGER PRIMARY KEY, name TEXT NOT NULL, cui TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')));
|
||||
INSERT OR IGNORE INTO accounts (id, name) VALUES (1, 'default');
|
||||
CREATE TABLE IF NOT EXISTS submissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
idempotency_key TEXT NOT NULL UNIQUE,
|
||||
account_id INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
payload_json TEXT NOT NULL,
|
||||
rar_status_code INTEGER,
|
||||
rar_error TEXT,
|
||||
id_prezentare INTEGER,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS worker_heartbeat (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
last_beat TEXT, last_rar_login_ok TEXT, detail TEXT
|
||||
);
|
||||
INSERT OR IGNORE INTO worker_heartbeat (id, detail) VALUES (1, 'never started');
|
||||
CREATE TABLE IF NOT EXISTS api_keys (id INTEGER PRIMARY KEY, account_id INTEGER, key_hash TEXT UNIQUE, active INTEGER DEFAULT 1, created_at TEXT);
|
||||
CREATE TABLE IF NOT EXISTS operations_mapping (id INTEGER PRIMARY KEY, account_id INTEGER, cod_op_service TEXT, cod_prestatie TEXT, auto_send INTEGER DEFAULT 1, created_at TEXT, UNIQUE(account_id, cod_op_service));
|
||||
CREATE TABLE IF NOT EXISTS nomenclator_rar (cod_prestatie TEXT PRIMARY KEY, nume_prestatie TEXT, updated_at TEXT);
|
||||
""")
|
||||
old_conn.close()
|
||||
|
||||
monkeypatch.setenv("AUTOPASS_DB_PATH", db_path)
|
||||
from app.config import get_settings
|
||||
get_settings.cache_clear()
|
||||
from app.db import init_db, get_connection
|
||||
init_db()
|
||||
|
||||
conn = get_connection()
|
||||
sub_cols = {r["name"] for r in conn.execute("PRAGMA table_info(submissions)").fetchall()}
|
||||
acc_cols = {r["name"] for r in conn.execute("PRAGMA table_info(accounts)").fetchall()}
|
||||
assert "batch_id" in sub_cols
|
||||
assert "row_index" in sub_cols
|
||||
assert "purge_after" in sub_cols
|
||||
assert "rar_creds_enc" in acc_cols
|
||||
conn.close()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
# --- openpyxl disponibil ---
|
||||
|
||||
def test_openpyxl_importabil():
|
||||
import openpyxl
|
||||
assert openpyxl.__version__.startswith("3.1")
|
||||
|
||||
|
||||
def test_openpyxl_create_workbook():
|
||||
from openpyxl import Workbook
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws["A1"] = "test"
|
||||
assert ws["A1"].value == "test"
|
||||
Reference in New Issue
Block a user