feat(sqlite): refactor orders schema + dashboard period filter

Replace import_orders (insert-per-run) with orders table (one row per
order, upsert on conflict). Eliminates dedup CTE on every dashboard
query and prevents unbounded row growth at 4-500 orders/sync.

Key changes:
- orders table: PK order_number, upsert via ON CONFLICT DO UPDATE;
  COALESCE preserves id_comanda once set; times_skipped auto-increments
- sync_run_orders: lightweight junction (sync_run_id, order_number)
  replaces sync_run_id column on orders
- order_items: PK changed to (order_number, sku), INSERT OR IGNORE
- Auto-migration in init_sqlite(): import_orders → orders on first boot,
  old table renamed to import_orders_bak
- /api/dashboard/orders: period_days param (3/7/30/0=all, default 7)
- Dashboard: period selector buttons in orders card header
- start.sh: stop existing process on port 5003 before restart;
  remove --reload (broken on WSL2 /mnt/e/)
- Add invoice_service, E2E Playwright tests, Oracle package updates

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-13 16:18:57 +02:00
parent 650e98539e
commit 82196b9dc0
32 changed files with 4164 additions and 1192 deletions

View File

@@ -15,10 +15,11 @@ def search_articles(query: str, limit: int = 20):
with database.pool.acquire() as conn:
with conn.cursor() as cur:
cur.execute("""
SELECT id_articol, codmat, denumire
SELECT id_articol, codmat, denumire, um
FROM nom_articole
WHERE (UPPER(codmat) LIKE UPPER(:q) || '%'
OR UPPER(denumire) LIKE '%' || UPPER(:q) || '%')
AND sters = 0 AND inactiv = 0
AND ROWNUM <= :lim
ORDER BY CASE WHEN UPPER(codmat) LIKE UPPER(:q) || '%' THEN 0 ELSE 1 END, codmat
""", {"q": query, "lim": limit})

View File

@@ -44,9 +44,12 @@ def convert_web_date(date_str: str) -> datetime:
if not date_str:
return datetime.now()
try:
return datetime.strptime(date_str[:10], '%Y-%m-%d')
return datetime.strptime(date_str.strip(), '%Y-%m-%d %H:%M:%S')
except ValueError:
return datetime.now()
try:
return datetime.strptime(date_str.strip()[:10], '%Y-%m-%d')
except ValueError:
return datetime.now()
def format_address_for_oracle(address: str, city: str, region: str) -> str:
@@ -78,18 +81,26 @@ def import_single_order(order, id_pol: int = None, id_sectie: int = None) -> dic
success: bool
id_comanda: int or None
id_partener: int or None
id_adresa_facturare: int or None
id_adresa_livrare: int or None
error: str or None
"""
result = {
"success": False,
"id_comanda": None,
"id_partener": None,
"id_adresa_facturare": None,
"id_adresa_livrare": None,
"error": None
}
try:
order_number = clean_web_text(order.number)
order_date = convert_web_date(order.date)
logger.info(
f"Order {order.number}: raw date={order.date!r}"
f"parsed={order_date.strftime('%Y-%m-%d %H:%M:%S')}"
)
if database.pool is None:
raise RuntimeError("Oracle pool not initialized")
@@ -99,14 +110,14 @@ def import_single_order(order, id_pol: int = None, id_sectie: int = None) -> dic
id_partener = cur.var(oracledb.DB_TYPE_NUMBER)
if order.billing.is_company:
denumire = clean_web_text(order.billing.company_name)
denumire = clean_web_text(order.billing.company_name).upper()
cod_fiscal = clean_web_text(order.billing.company_code) or None
registru = clean_web_text(order.billing.company_reg) or None
is_pj = 1
else:
denumire = clean_web_text(
f"{order.billing.firstname} {order.billing.lastname}"
)
f"{order.billing.lastname} {order.billing.firstname}"
).upper()
cod_fiscal = None
registru = None
is_pj = 0
@@ -151,6 +162,11 @@ def import_single_order(order, id_pol: int = None, id_sectie: int = None) -> dic
])
addr_livr_id = id_adresa_livr.getvalue()
if addr_fact_id is not None:
result["id_adresa_facturare"] = int(addr_fact_id)
if addr_livr_id is not None:
result["id_adresa_livrare"] = int(addr_livr_id)
# Step 4: Build articles JSON and import order
articles_json = build_articles_json(order.items)

View File

@@ -0,0 +1,43 @@
import logging
from .. import database
logger = logging.getLogger(__name__)
def check_invoices_for_orders(id_comanda_list: list) -> dict:
"""Check which orders have been invoiced in Oracle (vanzari table).
Returns {id_comanda: {facturat, numar_act, serie_act, total_fara_tva, total_tva, total_cu_tva}}
"""
if not id_comanda_list or database.pool is None:
return {}
result = {}
conn = database.get_oracle_connection()
try:
with conn.cursor() as cur:
for i in range(0, len(id_comanda_list), 500):
batch = id_comanda_list[i:i+500]
placeholders = ",".join([f":c{j}" for j in range(len(batch))])
params = {f"c{j}": cid for j, cid in enumerate(batch)}
cur.execute(f"""
SELECT id_comanda, numar_act, serie_act,
total_fara_tva, total_tva, total_cu_tva
FROM vanzari
WHERE id_comanda IN ({placeholders}) AND sters = 0
""", params)
for row in cur:
result[row[0]] = {
"facturat": True,
"numar_act": row[1],
"serie_act": row[2],
"total_fara_tva": float(row[3]) if row[3] else 0,
"total_tva": float(row[4]) if row[4] else 0,
"total_cu_tva": float(row[5]) if row[5] else 0,
}
except Exception as e:
logger.warning(f"Invoice check failed (table may not exist): {e}")
finally:
database.pool.release(conn)
return result

View File

@@ -7,23 +7,47 @@ from .. import database
logger = logging.getLogger(__name__)
def get_mappings(search: str = "", page: int = 1, per_page: int = 50):
"""Get paginated mappings with optional search."""
def get_mappings(search: str = "", page: int = 1, per_page: int = 50,
sort_by: str = "sku", sort_dir: str = "asc",
show_deleted: bool = False):
"""Get paginated mappings with optional search and sorting."""
if database.pool is None:
raise HTTPException(status_code=503, detail="Oracle unavailable")
offset = (page - 1) * per_page
# Validate and resolve sort parameters
allowed_sort = {
"sku": "at.sku",
"codmat": "at.codmat",
"denumire": "na.denumire",
"um": "na.um",
"cantitate_roa": "at.cantitate_roa",
"procent_pret": "at.procent_pret",
"activ": "at.activ",
}
sort_col = allowed_sort.get(sort_by, "at.sku")
if sort_dir.lower() not in ("asc", "desc"):
sort_dir = "asc"
order_clause = f"{sort_col} {sort_dir}"
# Always add secondary sort to keep groups together
if sort_col not in ("at.sku",):
order_clause += ", at.sku"
order_clause += ", at.codmat"
with database.pool.acquire() as conn:
with conn.cursor() as cur:
# Build WHERE clause
where = ""
where_clauses = []
params = {}
if not show_deleted:
where_clauses.append("at.sters = 0")
if search:
where = """WHERE (UPPER(at.sku) LIKE '%' || UPPER(:search) || '%'
where_clauses.append("""(UPPER(at.sku) LIKE '%' || UPPER(:search) || '%'
OR UPPER(at.codmat) LIKE '%' || UPPER(:search) || '%'
OR UPPER(na.denumire) LIKE '%' || UPPER(:search) || '%')"""
OR UPPER(na.denumire) LIKE '%' || UPPER(:search) || '%')""")
params["search"] = search
where = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
# Count total
count_sql = f"""
@@ -36,13 +60,13 @@ def get_mappings(search: str = "", page: int = 1, per_page: int = 50):
# Get page
data_sql = f"""
SELECT at.sku, at.codmat, na.denumire, at.cantitate_roa,
at.procent_pret, at.activ,
SELECT at.sku, at.codmat, na.denumire, na.um, at.cantitate_roa,
at.procent_pret, at.activ, at.sters,
TO_CHAR(at.data_creare, 'YYYY-MM-DD HH24:MI') as data_creare
FROM ARTICOLE_TERTI at
LEFT JOIN nom_articole na ON na.codmat = at.codmat
{where}
ORDER BY at.sku, at.codmat
ORDER BY {order_clause}
OFFSET :offset ROWS FETCH NEXT :per_page ROWS ONLY
"""
params["offset"] = offset
@@ -68,8 +92,8 @@ def create_mapping(sku: str, codmat: str, cantitate_roa: float = 1, procent_pret
with database.pool.acquire() as conn:
with conn.cursor() as cur:
cur.execute("""
INSERT INTO ARTICOLE_TERTI (sku, codmat, cantitate_roa, procent_pret, activ, data_creare, id_util_creare)
VALUES (:sku, :codmat, :cantitate_roa, :procent_pret, 1, SYSDATE, -3)
INSERT INTO ARTICOLE_TERTI (sku, codmat, cantitate_roa, procent_pret, activ, sters, data_creare, id_util_creare)
VALUES (:sku, :codmat, :cantitate_roa, :procent_pret, 1, 0, SYSDATE, -3)
""", {"sku": sku, "codmat": codmat, "cantitate_roa": cantitate_roa, "procent_pret": procent_pret})
conn.commit()
return {"sku": sku, "codmat": codmat}
@@ -108,8 +132,68 @@ def update_mapping(sku: str, codmat: str, cantitate_roa: float = None, procent_p
return cur.rowcount > 0
def delete_mapping(sku: str, codmat: str):
"""Soft delete (set activ=0)."""
return update_mapping(sku, codmat, activ=0)
"""Soft delete (set sters=1)."""
if database.pool is None:
raise HTTPException(status_code=503, detail="Oracle unavailable")
with database.pool.acquire() as conn:
with conn.cursor() as cur:
cur.execute("""
UPDATE ARTICOLE_TERTI SET sters = 1, data_modif = SYSDATE
WHERE sku = :sku AND codmat = :codmat
""", {"sku": sku, "codmat": codmat})
conn.commit()
return cur.rowcount > 0
def edit_mapping(old_sku: str, old_codmat: str, new_sku: str, new_codmat: str,
cantitate_roa: float = 1, procent_pret: float = 100):
"""Edit a mapping. If PK changed, soft-delete old and insert new."""
if database.pool is None:
raise HTTPException(status_code=503, detail="Oracle unavailable")
if old_sku == new_sku and old_codmat == new_codmat:
# Simple update - only cantitate/procent changed
return update_mapping(new_sku, new_codmat, cantitate_roa, procent_pret)
else:
# PK changed: soft-delete old, upsert new (MERGE handles existing soft-deleted target)
with database.pool.acquire() as conn:
with conn.cursor() as cur:
# Mark old record as deleted
cur.execute("""
UPDATE ARTICOLE_TERTI SET sters = 1, data_modif = SYSDATE
WHERE sku = :sku AND codmat = :codmat
""", {"sku": old_sku, "codmat": old_codmat})
# Upsert new record (MERGE in case target PK exists as soft-deleted)
cur.execute("""
MERGE INTO ARTICOLE_TERTI t
USING (SELECT :sku AS sku, :codmat AS codmat FROM DUAL) s
ON (t.sku = s.sku AND t.codmat = s.codmat)
WHEN MATCHED THEN UPDATE SET
cantitate_roa = :cantitate_roa,
procent_pret = :procent_pret,
activ = 1, sters = 0,
data_modif = SYSDATE
WHEN NOT MATCHED THEN INSERT
(sku, codmat, cantitate_roa, procent_pret, activ, sters, data_creare, id_util_creare)
VALUES (:sku, :codmat, :cantitate_roa, :procent_pret, 1, 0, SYSDATE, -3)
""", {"sku": new_sku, "codmat": new_codmat,
"cantitate_roa": cantitate_roa, "procent_pret": procent_pret})
conn.commit()
return True
def restore_mapping(sku: str, codmat: str):
"""Restore a soft-deleted mapping (set sters=0)."""
if database.pool is None:
raise HTTPException(status_code=503, detail="Oracle unavailable")
with database.pool.acquire() as conn:
with conn.cursor() as cur:
cur.execute("""
UPDATE ARTICOLE_TERTI SET sters = 0, data_modif = SYSDATE
WHERE sku = :sku AND codmat = :codmat
""", {"sku": sku, "codmat": codmat})
conn.commit()
return cur.rowcount > 0
def import_csv(file_content: str):
"""Import mappings from CSV content. Returns summary."""
@@ -143,10 +227,11 @@ def import_csv(file_content: str):
cantitate_roa = :cantitate_roa,
procent_pret = :procent_pret,
activ = 1,
sters = 0,
data_modif = SYSDATE
WHEN NOT MATCHED THEN INSERT
(sku, codmat, cantitate_roa, procent_pret, activ, data_creare, id_util_creare)
VALUES (:sku, :codmat, :cantitate_roa, :procent_pret, 1, SYSDATE, -3)
(sku, codmat, cantitate_roa, procent_pret, activ, sters, data_creare, id_util_creare)
VALUES (:sku, :codmat, :cantitate_roa, :procent_pret, 1, 0, SYSDATE, -3)
""", {"sku": sku, "codmat": codmat, "cantitate_roa": cantitate, "procent_pret": procent})
# Check if it was insert or update by rowcount
@@ -172,7 +257,7 @@ def export_csv():
with conn.cursor() as cur:
cur.execute("""
SELECT sku, codmat, cantitate_roa, procent_pret, activ
FROM ARTICOLE_TERTI ORDER BY sku, codmat
FROM ARTICOLE_TERTI WHERE sters = 0 ORDER BY sku, codmat
""")
for row in cur:
writer.writerow(row)

View File

@@ -41,21 +41,48 @@ async def update_sync_run(run_id: str, status: str, total_orders: int = 0,
await db.close()
async def add_import_order(sync_run_id: str, order_number: str, order_date: str,
customer_name: str, status: str, id_comanda: int = None,
id_partener: int = None, error_message: str = None,
missing_skus: list = None, items_count: int = 0):
"""Record an individual order import result."""
async def upsert_order(sync_run_id: str, order_number: str, order_date: str,
customer_name: str, status: str, id_comanda: int = None,
id_partener: int = None, error_message: str = None,
missing_skus: list = None, items_count: int = 0):
"""Upsert a single order — one row per order_number, status updated in place."""
db = await get_sqlite()
try:
await db.execute("""
INSERT INTO import_orders
(sync_run_id, order_number, order_date, customer_name, status,
id_comanda, id_partener, error_message, missing_skus, items_count)
INSERT INTO orders
(order_number, order_date, customer_name, status,
id_comanda, id_partener, error_message, missing_skus, items_count,
last_sync_run_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (sync_run_id, order_number, order_date, customer_name, status,
ON CONFLICT(order_number) DO UPDATE SET
status = excluded.status,
error_message = excluded.error_message,
missing_skus = excluded.missing_skus,
items_count = excluded.items_count,
id_comanda = COALESCE(excluded.id_comanda, orders.id_comanda),
id_partener = COALESCE(excluded.id_partener, orders.id_partener),
times_skipped = CASE WHEN excluded.status = 'SKIPPED'
THEN orders.times_skipped + 1
ELSE orders.times_skipped END,
last_sync_run_id = excluded.last_sync_run_id,
updated_at = datetime('now')
""", (order_number, order_date, customer_name, status,
id_comanda, id_partener, error_message,
json.dumps(missing_skus) if missing_skus else None, items_count))
json.dumps(missing_skus) if missing_skus else None,
items_count, sync_run_id))
await db.commit()
finally:
await db.close()
async def add_sync_run_order(sync_run_id: str, order_number: str, status_at_run: str):
"""Record that this run processed this order (junction table)."""
db = await get_sqlite()
try:
await db.execute("""
INSERT OR IGNORE INTO sync_run_orders (sync_run_id, order_number, status_at_run)
VALUES (?, ?, ?)
""", (sync_run_id, order_number, status_at_run))
await db.commit()
finally:
await db.close()
@@ -71,7 +98,6 @@ async def track_missing_sku(sku: str, product_name: str = "",
INSERT OR IGNORE INTO missing_skus (sku, product_name)
VALUES (?, ?)
""", (sku, product_name))
# Update context columns (always update with latest data)
if order_count or order_numbers or customers:
await db.execute("""
UPDATE missing_skus SET
@@ -99,24 +125,35 @@ async def resolve_missing_sku(sku: str):
async def get_missing_skus_paginated(page: int = 1, per_page: int = 20, resolved: int = 0):
"""Get paginated missing SKUs."""
"""Get paginated missing SKUs. resolved=-1 means show all."""
db = await get_sqlite()
try:
offset = (page - 1) * per_page
cursor = await db.execute(
"SELECT COUNT(*) FROM missing_skus WHERE resolved = ?", (resolved,)
)
total = (await cursor.fetchone())[0]
if resolved == -1:
cursor = await db.execute("SELECT COUNT(*) FROM missing_skus")
total = (await cursor.fetchone())[0]
cursor = await db.execute("""
SELECT sku, product_name, first_seen, resolved, resolved_at,
order_count, order_numbers, customers
FROM missing_skus
ORDER BY resolved ASC, order_count DESC, first_seen DESC
LIMIT ? OFFSET ?
""", (per_page, offset))
else:
cursor = await db.execute(
"SELECT COUNT(*) FROM missing_skus WHERE resolved = ?", (resolved,)
)
total = (await cursor.fetchone())[0]
cursor = await db.execute("""
SELECT sku, product_name, first_seen, resolved, resolved_at,
order_count, order_numbers, customers
FROM missing_skus
WHERE resolved = ?
ORDER BY order_count DESC, first_seen DESC
LIMIT ? OFFSET ?
""", (resolved, per_page, offset))
cursor = await db.execute("""
SELECT sku, product_name, first_seen, resolved, resolved_at,
order_count, order_numbers, customers
FROM missing_skus
WHERE resolved = ?
ORDER BY order_count DESC, first_seen DESC
LIMIT ? OFFSET ?
""", (resolved, per_page, offset))
rows = await cursor.fetchall()
return {
@@ -157,7 +194,7 @@ async def get_sync_runs(page: int = 1, per_page: int = 20):
async def get_sync_run_detail(run_id: str):
"""Get details for a specific sync run including its orders."""
"""Get details for a specific sync run including its orders via sync_run_orders."""
db = await get_sqlite()
try:
cursor = await db.execute(
@@ -168,9 +205,10 @@ async def get_sync_run_detail(run_id: str):
return None
cursor = await db.execute("""
SELECT * FROM import_orders
WHERE sync_run_id = ?
ORDER BY created_at
SELECT o.* FROM orders o
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
WHERE sro.sync_run_id = ?
ORDER BY o.order_date
""", (run_id,))
orders = await cursor.fetchall()
@@ -186,42 +224,34 @@ async def get_dashboard_stats():
"""Get stats for the dashboard."""
db = await get_sqlite()
try:
# Total imported
cursor = await db.execute(
"SELECT COUNT(*) FROM import_orders WHERE status = 'IMPORTED'"
"SELECT COUNT(*) FROM orders WHERE status = 'IMPORTED'"
)
imported = (await cursor.fetchone())[0]
# Total skipped
cursor = await db.execute(
"SELECT COUNT(*) FROM import_orders WHERE status = 'SKIPPED'"
"SELECT COUNT(*) FROM orders WHERE status = 'SKIPPED'"
)
skipped = (await cursor.fetchone())[0]
# Total errors
cursor = await db.execute(
"SELECT COUNT(*) FROM import_orders WHERE status = 'ERROR'"
"SELECT COUNT(*) FROM orders WHERE status = 'ERROR'"
)
errors = (await cursor.fetchone())[0]
# Missing SKUs (unresolved)
cursor = await db.execute(
"SELECT COUNT(*) FROM missing_skus WHERE resolved = 0"
)
missing = (await cursor.fetchone())[0]
# Article stats from last sync
cursor = await db.execute("""
SELECT COUNT(DISTINCT sku) FROM missing_skus
""")
cursor = await db.execute("SELECT COUNT(DISTINCT sku) FROM missing_skus")
total_missing_skus = (await cursor.fetchone())[0]
cursor = await db.execute("""
SELECT COUNT(DISTINCT sku) FROM missing_skus WHERE resolved = 0
""")
cursor = await db.execute(
"SELECT COUNT(DISTINCT sku) FROM missing_skus WHERE resolved = 0"
)
unresolved_skus = (await cursor.fetchone())[0]
# Last sync run
cursor = await db.execute("""
SELECT * FROM sync_runs ORDER BY started_at DESC LIMIT 1
""")
@@ -262,3 +292,266 @@ async def set_scheduler_config(key: str, value: str):
await db.commit()
finally:
await db.close()
# ── web_products ─────────────────────────────────
async def upsert_web_product(sku: str, product_name: str):
"""Insert or update a web product, incrementing order_count."""
db = await get_sqlite()
try:
await db.execute("""
INSERT INTO web_products (sku, product_name, order_count)
VALUES (?, ?, 1)
ON CONFLICT(sku) DO UPDATE SET
product_name = COALESCE(NULLIF(excluded.product_name, ''), web_products.product_name),
last_seen = datetime('now'),
order_count = web_products.order_count + 1
""", (sku, product_name))
await db.commit()
finally:
await db.close()
async def get_web_product_name(sku: str) -> str:
"""Lookup product name by SKU."""
db = await get_sqlite()
try:
cursor = await db.execute(
"SELECT product_name FROM web_products WHERE sku = ?", (sku,)
)
row = await cursor.fetchone()
return row["product_name"] if row else ""
finally:
await db.close()
async def get_web_products_batch(skus: list) -> dict:
"""Batch lookup product names by SKU list. Returns {sku: product_name}."""
if not skus:
return {}
db = await get_sqlite()
try:
placeholders = ",".join("?" for _ in skus)
cursor = await db.execute(
f"SELECT sku, product_name FROM web_products WHERE sku IN ({placeholders})",
list(skus)
)
rows = await cursor.fetchall()
return {row["sku"]: row["product_name"] for row in rows}
finally:
await db.close()
# ── order_items ──────────────────────────────────
async def add_order_items(order_number: str, items: list):
"""Bulk insert order items. Uses INSERT OR IGNORE — PK is (order_number, sku)."""
if not items:
return
db = await get_sqlite()
try:
await db.executemany("""
INSERT OR IGNORE INTO order_items
(order_number, sku, product_name, quantity, price, vat,
mapping_status, codmat, id_articol, cantitate_roa)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", [
(order_number,
item.get("sku"), item.get("product_name"),
item.get("quantity"), item.get("price"), item.get("vat"),
item.get("mapping_status"), item.get("codmat"),
item.get("id_articol"), item.get("cantitate_roa"))
for item in items
])
await db.commit()
finally:
await db.close()
async def get_order_items(order_number: str) -> list:
"""Fetch items for one order."""
db = await get_sqlite()
try:
cursor = await db.execute("""
SELECT * FROM order_items
WHERE order_number = ?
ORDER BY sku
""", (order_number,))
rows = await cursor.fetchall()
return [dict(row) for row in rows]
finally:
await db.close()
async def get_order_detail(order_number: str) -> dict:
"""Get full order detail: order metadata + items."""
db = await get_sqlite()
try:
cursor = await db.execute("""
SELECT * FROM orders WHERE order_number = ?
""", (order_number,))
order = await cursor.fetchone()
if not order:
return None
cursor = await db.execute("""
SELECT * FROM order_items WHERE order_number = ?
ORDER BY sku
""", (order_number,))
items = await cursor.fetchall()
return {
"order": dict(order),
"items": [dict(i) for i in items]
}
finally:
await db.close()
async def get_run_orders_filtered(run_id: str, status_filter: str = "all",
page: int = 1, per_page: int = 50,
sort_by: str = "order_date", sort_dir: str = "asc"):
"""Get paginated orders for a run via sync_run_orders junction table."""
db = await get_sqlite()
try:
where = "WHERE sro.sync_run_id = ?"
params = [run_id]
if status_filter and status_filter != "all":
where += " AND UPPER(o.status) = ?"
params.append(status_filter.upper())
allowed_sort = {"order_date", "order_number", "customer_name", "items_count",
"status", "first_seen_at", "updated_at"}
if sort_by not in allowed_sort:
sort_by = "order_date"
if sort_dir.lower() not in ("asc", "desc"):
sort_dir = "asc"
cursor = await db.execute(
f"SELECT COUNT(*) FROM orders o INNER JOIN sync_run_orders sro "
f"ON sro.order_number = o.order_number {where}", params
)
total = (await cursor.fetchone())[0]
offset = (page - 1) * per_page
cursor = await db.execute(f"""
SELECT o.* FROM orders o
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
{where}
ORDER BY o.{sort_by} {sort_dir}
LIMIT ? OFFSET ?
""", params + [per_page, offset])
rows = await cursor.fetchall()
cursor = await db.execute("""
SELECT o.status, COUNT(*) as cnt
FROM orders o
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
WHERE sro.sync_run_id = ?
GROUP BY o.status
""", (run_id,))
status_counts = {row["status"]: row["cnt"] for row in await cursor.fetchall()}
return {
"orders": [dict(r) for r in rows],
"total": total,
"page": page,
"per_page": per_page,
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
"counts": {
"imported": status_counts.get("IMPORTED", 0),
"skipped": status_counts.get("SKIPPED", 0),
"error": status_counts.get("ERROR", 0),
"total": sum(status_counts.values())
}
}
finally:
await db.close()
async def get_orders(page: int = 1, per_page: int = 50,
search: str = "", status_filter: str = "all",
sort_by: str = "order_date", sort_dir: str = "desc",
period_days: int = 7):
"""Get orders with filters, sorting, and period. period_days=0 means all time."""
db = await get_sqlite()
try:
where_clauses = []
params = []
if period_days and period_days > 0:
where_clauses.append("order_date >= date('now', ?)")
params.append(f"-{period_days} days")
if search:
where_clauses.append("(order_number LIKE ? OR customer_name LIKE ?)")
params.extend([f"%{search}%", f"%{search}%"])
if status_filter and status_filter not in ("all", "UNINVOICED"):
where_clauses.append("UPPER(status) = ?")
params.append(status_filter.upper())
where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else ""
allowed_sort = {"order_date", "order_number", "customer_name", "items_count",
"status", "first_seen_at", "updated_at"}
if sort_by not in allowed_sort:
sort_by = "order_date"
if sort_dir.lower() not in ("asc", "desc"):
sort_dir = "desc"
cursor = await db.execute(f"SELECT COUNT(*) FROM orders {where}", params)
total = (await cursor.fetchone())[0]
offset = (page - 1) * per_page
cursor = await db.execute(f"""
SELECT * FROM orders
{where}
ORDER BY {sort_by} {sort_dir}
LIMIT ? OFFSET ?
""", params + [per_page, offset])
rows = await cursor.fetchall()
# Counts by status (on full period, not just this page)
cursor = await db.execute(f"""
SELECT status, COUNT(*) as cnt FROM orders
{where}
GROUP BY status
""", params)
status_counts = {row["status"]: row["cnt"] for row in await cursor.fetchall()}
return {
"orders": [dict(r) for r in rows],
"total": total,
"page": page,
"per_page": per_page,
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
"counts": {
"imported": status_counts.get("IMPORTED", 0),
"skipped": status_counts.get("SKIPPED", 0),
"error": status_counts.get("ERROR", 0),
"total": sum(status_counts.values())
}
}
finally:
await db.close()
async def update_import_order_addresses(order_number: str,
id_adresa_facturare: int = None,
id_adresa_livrare: int = None):
"""Update ROA address IDs on an order record."""
db = await get_sqlite()
try:
await db.execute("""
UPDATE orders SET
id_adresa_facturare = ?,
id_adresa_livrare = ?,
updated_at = datetime('now')
WHERE order_number = ?
""", (id_adresa_facturare, id_adresa_livrare, order_number))
await db.commit()
finally:
await db.close()

View File

@@ -16,6 +16,9 @@ _current_sync = None # dict with run_id, status, progress info
# SSE subscriber system
_subscribers: list[asyncio.Queue] = []
# In-memory text log buffer per run
_run_logs: dict[str, list[str]] = {}
def subscribe() -> asyncio.Queue:
"""Subscribe to sync events. Returns a queue that will receive event dicts."""
@@ -32,6 +35,22 @@ def unsubscribe(q: asyncio.Queue):
pass
def _log_line(run_id: str, message: str):
"""Append a timestamped line to the in-memory log buffer."""
if run_id not in _run_logs:
_run_logs[run_id] = []
ts = datetime.now().strftime("%H:%M:%S")
_run_logs[run_id].append(f"[{ts}] {message}")
def get_run_text_log(run_id: str) -> str | None:
"""Return the accumulated text log for a run, or None if not found."""
lines = _run_logs.get(run_id)
if lines is None:
return None
return "\n".join(lines)
async def _emit(event: dict):
"""Push an event to all subscriber queues."""
for q in _subscribers:
@@ -87,13 +106,30 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
_current_sync["progress"] = "Reading JSON files..."
await _emit({"type": "phase", "run_id": run_id, "message": "Reading JSON files..."})
started_dt = datetime.now()
_run_logs[run_id] = [
f"=== Sync Run {run_id} ===",
f"Inceput: {started_dt.strftime('%d.%m.%Y %H:%M:%S')}",
""
]
_log_line(run_id, "Citire fisiere JSON...")
try:
# Step 1: Read orders
# Step 1: Read orders and sort chronologically (oldest first - R3)
orders, json_count = order_reader.read_json_orders()
orders.sort(key=lambda o: o.date or '')
await sqlite_service.create_sync_run(run_id, json_count)
await _emit({"type": "phase", "run_id": run_id, "message": f"Found {len(orders)} orders in {json_count} files"})
_log_line(run_id, f"Gasite {len(orders)} comenzi in {json_count} fisiere")
# Populate web_products catalog from all orders (R4)
for order in orders:
for item in order.items:
if item.sku and item.name:
await sqlite_service.upsert_web_product(item.sku, item.name)
if not orders:
_log_line(run_id, "Nicio comanda gasita.")
await sqlite_service.update_sync_run(run_id, "completed", 0, 0, 0, 0)
summary = {"run_id": run_id, "status": "completed", "message": "No orders found", "json_files": json_count}
await _emit({"type": "completed", "run_id": run_id, "summary": summary})
@@ -114,6 +150,7 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
importable, skipped = validation_service.classify_orders(orders, validation)
await _emit({"type": "phase", "run_id": run_id, "message": f"{len(importable)} importable, {len(skipped)} skipped (missing SKUs)"})
_log_line(run_id, f"Validare SKU-uri: {len(importable)} importabile, {len(skipped)} nemapate")
# Step 2c: Build SKU context from skipped orders
sku_context = {} # {sku: {"orders": [], "customers": []}}
@@ -148,9 +185,13 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
# Step 2d: Pre-validate prices for importable articles
id_pol = id_pol or settings.ID_POL
id_sectie = id_sectie or settings.ID_SECTIE
logger.info(f"Sync params: ID_POL={id_pol}, ID_SECTIE={id_sectie}")
_log_line(run_id, f"Parametri import: ID_POL={id_pol}, ID_SECTIE={id_sectie}")
if id_pol and importable:
_current_sync["progress"] = "Validating prices..."
await _emit({"type": "phase", "run_id": run_id, "message": "Validating prices..."})
_log_line(run_id, "Validare preturi...")
# Gather all CODMATs from importable orders
all_codmats = set()
for order in importable:
@@ -175,11 +216,11 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
price_result["missing_price"], id_pol
)
# Step 3: Record skipped orders + emit events
# Step 3: Record skipped orders + emit events + store items
for order, missing_skus in skipped:
customer = order.billing.company_name or \
f"{order.billing.firstname} {order.billing.lastname}"
await sqlite_service.add_import_order(
await sqlite_service.upsert_order(
sync_run_id=run_id,
order_number=order.number,
order_date=order.date,
@@ -188,9 +229,24 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
missing_skus=missing_skus,
items_count=len(order.items)
)
await sqlite_service.add_sync_run_order(run_id, order.number, "SKIPPED")
# Store order items with mapping status (R9)
order_items_data = []
for item in order.items:
ms = "missing" if item.sku in validation["missing"] else \
"mapped" if item.sku in validation["mapped"] else "direct"
order_items_data.append({
"sku": item.sku, "product_name": item.name,
"quantity": item.quantity, "price": item.price, "vat": item.vat,
"mapping_status": ms, "codmat": None, "id_articol": None,
"cantitate_roa": None
})
await sqlite_service.add_order_items(order.number, order_items_data)
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → OMIS (lipsa: {', '.join(missing_skus)})")
await _emit({
"type": "order_result", "run_id": run_id,
"order_number": order.number, "customer_name": customer,
"order_date": order.date,
"status": "SKIPPED", "missing_skus": missing_skus,
"items_count": len(order.items), "progress": f"0/{len(importable)}"
})
@@ -210,9 +266,20 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
customer = order.billing.company_name or \
f"{order.billing.firstname} {order.billing.lastname}"
# Build order items data for storage (R9)
order_items_data = []
for item in order.items:
ms = "mapped" if item.sku in validation["mapped"] else "direct"
order_items_data.append({
"sku": item.sku, "product_name": item.name,
"quantity": item.quantity, "price": item.price, "vat": item.vat,
"mapping_status": ms, "codmat": None, "id_articol": None,
"cantitate_roa": None
})
if result["success"]:
imported_count += 1
await sqlite_service.add_import_order(
await sqlite_service.upsert_order(
sync_run_id=run_id,
order_number=order.number,
order_date=order.date,
@@ -222,15 +289,25 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
id_partener=result["id_partener"],
items_count=len(order.items)
)
await sqlite_service.add_sync_run_order(run_id, order.number, "IMPORTED")
# Store ROA address IDs (R9)
await sqlite_service.update_import_order_addresses(
order.number,
id_adresa_facturare=result.get("id_adresa_facturare"),
id_adresa_livrare=result.get("id_adresa_livrare")
)
await sqlite_service.add_order_items(order.number, order_items_data)
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → IMPORTAT (ID: {result['id_comanda']})")
await _emit({
"type": "order_result", "run_id": run_id,
"order_number": order.number, "customer_name": customer,
"order_date": order.date,
"status": "IMPORTED", "items_count": len(order.items),
"id_comanda": result["id_comanda"], "progress": progress_str
})
else:
error_count += 1
await sqlite_service.add_import_order(
await sqlite_service.upsert_order(
sync_run_id=run_id,
order_number=order.number,
order_date=order.date,
@@ -240,9 +317,13 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
error_message=result["error"],
items_count=len(order.items)
)
await sqlite_service.add_sync_run_order(run_id, order.number, "ERROR")
await sqlite_service.add_order_items(order.number, order_items_data)
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → EROARE: {result['error']}")
await _emit({
"type": "order_result", "run_id": run_id,
"order_number": order.number, "customer_name": customer,
"order_date": order.date,
"status": "ERROR", "error_message": result["error"],
"items_count": len(order.items), "progress": progress_str
})
@@ -275,10 +356,16 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
f"{len(skipped)} skipped, {error_count} errors"
)
await _emit({"type": "completed", "run_id": run_id, "summary": summary})
duration = (datetime.now() - started_dt).total_seconds()
_log_line(run_id, "")
_run_logs[run_id].append(f"Finalizat: {imported_count} importate, {len(skipped)} nemapate, {error_count} erori din {len(orders)} comenzi | Durata: {int(duration)}s")
return summary
except Exception as e:
logger.error(f"Sync {run_id} failed: {e}")
_log_line(run_id, f"EROARE FATALA: {e}")
await sqlite_service.update_sync_run(run_id, "failed", 0, 0, 0, 1, error_message=str(e))
_current_sync["error"] = str(e)
await _emit({"type": "failed", "run_id": run_id, "error": str(e)})
@@ -291,6 +378,11 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
_current_sync = None
asyncio.ensure_future(_clear_current_sync())
async def _clear_run_logs():
await asyncio.sleep(300) # 5 minutes
_run_logs.pop(run_id, None)
asyncio.ensure_future(_clear_run_logs())
def stop_sync():
"""Signal sync to stop. Currently sync runs to completion."""

View File

@@ -29,7 +29,7 @@ def validate_skus(skus: set[str]) -> dict:
# Check ARTICOLE_TERTI
cur.execute(f"""
SELECT DISTINCT sku FROM ARTICOLE_TERTI
WHERE sku IN ({placeholders}) AND activ = 1
WHERE sku IN ({placeholders}) AND activ = 1 AND sters = 0
""", params)
for row in cur:
mapped.add(row[0])
@@ -41,7 +41,7 @@ def validate_skus(skus: set[str]) -> dict:
params2 = {f"n{j}": sku for j, sku in enumerate(remaining)}
cur.execute(f"""
SELECT DISTINCT codmat FROM NOM_ARTICOLE
WHERE codmat IN ({placeholders2})
WHERE codmat IN ({placeholders2}) AND sters = 0 AND inactiv = 0
""", params2)
for row in cur:
direct.add(row[0])