feat(sqlite): refactor orders schema + dashboard period filter
Replace import_orders (insert-per-run) with orders table (one row per order, upsert on conflict). Eliminates dedup CTE on every dashboard query and prevents unbounded row growth at 4-500 orders/sync. Key changes: - orders table: PK order_number, upsert via ON CONFLICT DO UPDATE; COALESCE preserves id_comanda once set; times_skipped auto-increments - sync_run_orders: lightweight junction (sync_run_id, order_number) replaces sync_run_id column on orders - order_items: PK changed to (order_number, sku), INSERT OR IGNORE - Auto-migration in init_sqlite(): import_orders → orders on first boot, old table renamed to import_orders_bak - /api/dashboard/orders: period_days param (3/7/30/0=all, default 7) - Dashboard: period selector buttons in orders card header - start.sh: stop existing process on port 5003 before restart; remove --reload (broken on WSL2 /mnt/e/) - Add invoice_service, E2E Playwright tests, Oracle package updates Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -41,21 +41,48 @@ async def update_sync_run(run_id: str, status: str, total_orders: int = 0,
|
||||
await db.close()
|
||||
|
||||
|
||||
async def add_import_order(sync_run_id: str, order_number: str, order_date: str,
|
||||
customer_name: str, status: str, id_comanda: int = None,
|
||||
id_partener: int = None, error_message: str = None,
|
||||
missing_skus: list = None, items_count: int = 0):
|
||||
"""Record an individual order import result."""
|
||||
async def upsert_order(sync_run_id: str, order_number: str, order_date: str,
|
||||
customer_name: str, status: str, id_comanda: int = None,
|
||||
id_partener: int = None, error_message: str = None,
|
||||
missing_skus: list = None, items_count: int = 0):
|
||||
"""Upsert a single order — one row per order_number, status updated in place."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
await db.execute("""
|
||||
INSERT INTO import_orders
|
||||
(sync_run_id, order_number, order_date, customer_name, status,
|
||||
id_comanda, id_partener, error_message, missing_skus, items_count)
|
||||
INSERT INTO orders
|
||||
(order_number, order_date, customer_name, status,
|
||||
id_comanda, id_partener, error_message, missing_skus, items_count,
|
||||
last_sync_run_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (sync_run_id, order_number, order_date, customer_name, status,
|
||||
ON CONFLICT(order_number) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
error_message = excluded.error_message,
|
||||
missing_skus = excluded.missing_skus,
|
||||
items_count = excluded.items_count,
|
||||
id_comanda = COALESCE(excluded.id_comanda, orders.id_comanda),
|
||||
id_partener = COALESCE(excluded.id_partener, orders.id_partener),
|
||||
times_skipped = CASE WHEN excluded.status = 'SKIPPED'
|
||||
THEN orders.times_skipped + 1
|
||||
ELSE orders.times_skipped END,
|
||||
last_sync_run_id = excluded.last_sync_run_id,
|
||||
updated_at = datetime('now')
|
||||
""", (order_number, order_date, customer_name, status,
|
||||
id_comanda, id_partener, error_message,
|
||||
json.dumps(missing_skus) if missing_skus else None, items_count))
|
||||
json.dumps(missing_skus) if missing_skus else None,
|
||||
items_count, sync_run_id))
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def add_sync_run_order(sync_run_id: str, order_number: str, status_at_run: str):
|
||||
"""Record that this run processed this order (junction table)."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
await db.execute("""
|
||||
INSERT OR IGNORE INTO sync_run_orders (sync_run_id, order_number, status_at_run)
|
||||
VALUES (?, ?, ?)
|
||||
""", (sync_run_id, order_number, status_at_run))
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
@@ -71,7 +98,6 @@ async def track_missing_sku(sku: str, product_name: str = "",
|
||||
INSERT OR IGNORE INTO missing_skus (sku, product_name)
|
||||
VALUES (?, ?)
|
||||
""", (sku, product_name))
|
||||
# Update context columns (always update with latest data)
|
||||
if order_count or order_numbers or customers:
|
||||
await db.execute("""
|
||||
UPDATE missing_skus SET
|
||||
@@ -99,24 +125,35 @@ async def resolve_missing_sku(sku: str):
|
||||
|
||||
|
||||
async def get_missing_skus_paginated(page: int = 1, per_page: int = 20, resolved: int = 0):
|
||||
"""Get paginated missing SKUs."""
|
||||
"""Get paginated missing SKUs. resolved=-1 means show all."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
offset = (page - 1) * per_page
|
||||
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM missing_skus WHERE resolved = ?", (resolved,)
|
||||
)
|
||||
total = (await cursor.fetchone())[0]
|
||||
if resolved == -1:
|
||||
cursor = await db.execute("SELECT COUNT(*) FROM missing_skus")
|
||||
total = (await cursor.fetchone())[0]
|
||||
cursor = await db.execute("""
|
||||
SELECT sku, product_name, first_seen, resolved, resolved_at,
|
||||
order_count, order_numbers, customers
|
||||
FROM missing_skus
|
||||
ORDER BY resolved ASC, order_count DESC, first_seen DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", (per_page, offset))
|
||||
else:
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM missing_skus WHERE resolved = ?", (resolved,)
|
||||
)
|
||||
total = (await cursor.fetchone())[0]
|
||||
cursor = await db.execute("""
|
||||
SELECT sku, product_name, first_seen, resolved, resolved_at,
|
||||
order_count, order_numbers, customers
|
||||
FROM missing_skus
|
||||
WHERE resolved = ?
|
||||
ORDER BY order_count DESC, first_seen DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", (resolved, per_page, offset))
|
||||
|
||||
cursor = await db.execute("""
|
||||
SELECT sku, product_name, first_seen, resolved, resolved_at,
|
||||
order_count, order_numbers, customers
|
||||
FROM missing_skus
|
||||
WHERE resolved = ?
|
||||
ORDER BY order_count DESC, first_seen DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", (resolved, per_page, offset))
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
return {
|
||||
@@ -157,7 +194,7 @@ async def get_sync_runs(page: int = 1, per_page: int = 20):
|
||||
|
||||
|
||||
async def get_sync_run_detail(run_id: str):
|
||||
"""Get details for a specific sync run including its orders."""
|
||||
"""Get details for a specific sync run including its orders via sync_run_orders."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
cursor = await db.execute(
|
||||
@@ -168,9 +205,10 @@ async def get_sync_run_detail(run_id: str):
|
||||
return None
|
||||
|
||||
cursor = await db.execute("""
|
||||
SELECT * FROM import_orders
|
||||
WHERE sync_run_id = ?
|
||||
ORDER BY created_at
|
||||
SELECT o.* FROM orders o
|
||||
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
|
||||
WHERE sro.sync_run_id = ?
|
||||
ORDER BY o.order_date
|
||||
""", (run_id,))
|
||||
orders = await cursor.fetchall()
|
||||
|
||||
@@ -186,42 +224,34 @@ async def get_dashboard_stats():
|
||||
"""Get stats for the dashboard."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
# Total imported
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM import_orders WHERE status = 'IMPORTED'"
|
||||
"SELECT COUNT(*) FROM orders WHERE status = 'IMPORTED'"
|
||||
)
|
||||
imported = (await cursor.fetchone())[0]
|
||||
|
||||
# Total skipped
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM import_orders WHERE status = 'SKIPPED'"
|
||||
"SELECT COUNT(*) FROM orders WHERE status = 'SKIPPED'"
|
||||
)
|
||||
skipped = (await cursor.fetchone())[0]
|
||||
|
||||
# Total errors
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM import_orders WHERE status = 'ERROR'"
|
||||
"SELECT COUNT(*) FROM orders WHERE status = 'ERROR'"
|
||||
)
|
||||
errors = (await cursor.fetchone())[0]
|
||||
|
||||
# Missing SKUs (unresolved)
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(*) FROM missing_skus WHERE resolved = 0"
|
||||
)
|
||||
missing = (await cursor.fetchone())[0]
|
||||
|
||||
# Article stats from last sync
|
||||
cursor = await db.execute("""
|
||||
SELECT COUNT(DISTINCT sku) FROM missing_skus
|
||||
""")
|
||||
cursor = await db.execute("SELECT COUNT(DISTINCT sku) FROM missing_skus")
|
||||
total_missing_skus = (await cursor.fetchone())[0]
|
||||
|
||||
cursor = await db.execute("""
|
||||
SELECT COUNT(DISTINCT sku) FROM missing_skus WHERE resolved = 0
|
||||
""")
|
||||
cursor = await db.execute(
|
||||
"SELECT COUNT(DISTINCT sku) FROM missing_skus WHERE resolved = 0"
|
||||
)
|
||||
unresolved_skus = (await cursor.fetchone())[0]
|
||||
|
||||
# Last sync run
|
||||
cursor = await db.execute("""
|
||||
SELECT * FROM sync_runs ORDER BY started_at DESC LIMIT 1
|
||||
""")
|
||||
@@ -262,3 +292,266 @@ async def set_scheduler_config(key: str, value: str):
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
# ── web_products ─────────────────────────────────
|
||||
|
||||
async def upsert_web_product(sku: str, product_name: str):
|
||||
"""Insert or update a web product, incrementing order_count."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
await db.execute("""
|
||||
INSERT INTO web_products (sku, product_name, order_count)
|
||||
VALUES (?, ?, 1)
|
||||
ON CONFLICT(sku) DO UPDATE SET
|
||||
product_name = COALESCE(NULLIF(excluded.product_name, ''), web_products.product_name),
|
||||
last_seen = datetime('now'),
|
||||
order_count = web_products.order_count + 1
|
||||
""", (sku, product_name))
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_web_product_name(sku: str) -> str:
|
||||
"""Lookup product name by SKU."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
cursor = await db.execute(
|
||||
"SELECT product_name FROM web_products WHERE sku = ?", (sku,)
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return row["product_name"] if row else ""
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_web_products_batch(skus: list) -> dict:
|
||||
"""Batch lookup product names by SKU list. Returns {sku: product_name}."""
|
||||
if not skus:
|
||||
return {}
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
placeholders = ",".join("?" for _ in skus)
|
||||
cursor = await db.execute(
|
||||
f"SELECT sku, product_name FROM web_products WHERE sku IN ({placeholders})",
|
||||
list(skus)
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
return {row["sku"]: row["product_name"] for row in rows}
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
# ── order_items ──────────────────────────────────
|
||||
|
||||
async def add_order_items(order_number: str, items: list):
|
||||
"""Bulk insert order items. Uses INSERT OR IGNORE — PK is (order_number, sku)."""
|
||||
if not items:
|
||||
return
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
await db.executemany("""
|
||||
INSERT OR IGNORE INTO order_items
|
||||
(order_number, sku, product_name, quantity, price, vat,
|
||||
mapping_status, codmat, id_articol, cantitate_roa)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", [
|
||||
(order_number,
|
||||
item.get("sku"), item.get("product_name"),
|
||||
item.get("quantity"), item.get("price"), item.get("vat"),
|
||||
item.get("mapping_status"), item.get("codmat"),
|
||||
item.get("id_articol"), item.get("cantitate_roa"))
|
||||
for item in items
|
||||
])
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_order_items(order_number: str) -> list:
|
||||
"""Fetch items for one order."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
cursor = await db.execute("""
|
||||
SELECT * FROM order_items
|
||||
WHERE order_number = ?
|
||||
ORDER BY sku
|
||||
""", (order_number,))
|
||||
rows = await cursor.fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_order_detail(order_number: str) -> dict:
|
||||
"""Get full order detail: order metadata + items."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
cursor = await db.execute("""
|
||||
SELECT * FROM orders WHERE order_number = ?
|
||||
""", (order_number,))
|
||||
order = await cursor.fetchone()
|
||||
if not order:
|
||||
return None
|
||||
|
||||
cursor = await db.execute("""
|
||||
SELECT * FROM order_items WHERE order_number = ?
|
||||
ORDER BY sku
|
||||
""", (order_number,))
|
||||
items = await cursor.fetchall()
|
||||
|
||||
return {
|
||||
"order": dict(order),
|
||||
"items": [dict(i) for i in items]
|
||||
}
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_run_orders_filtered(run_id: str, status_filter: str = "all",
|
||||
page: int = 1, per_page: int = 50,
|
||||
sort_by: str = "order_date", sort_dir: str = "asc"):
|
||||
"""Get paginated orders for a run via sync_run_orders junction table."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
where = "WHERE sro.sync_run_id = ?"
|
||||
params = [run_id]
|
||||
|
||||
if status_filter and status_filter != "all":
|
||||
where += " AND UPPER(o.status) = ?"
|
||||
params.append(status_filter.upper())
|
||||
|
||||
allowed_sort = {"order_date", "order_number", "customer_name", "items_count",
|
||||
"status", "first_seen_at", "updated_at"}
|
||||
if sort_by not in allowed_sort:
|
||||
sort_by = "order_date"
|
||||
if sort_dir.lower() not in ("asc", "desc"):
|
||||
sort_dir = "asc"
|
||||
|
||||
cursor = await db.execute(
|
||||
f"SELECT COUNT(*) FROM orders o INNER JOIN sync_run_orders sro "
|
||||
f"ON sro.order_number = o.order_number {where}", params
|
||||
)
|
||||
total = (await cursor.fetchone())[0]
|
||||
|
||||
offset = (page - 1) * per_page
|
||||
cursor = await db.execute(f"""
|
||||
SELECT o.* FROM orders o
|
||||
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
|
||||
{where}
|
||||
ORDER BY o.{sort_by} {sort_dir}
|
||||
LIMIT ? OFFSET ?
|
||||
""", params + [per_page, offset])
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
cursor = await db.execute("""
|
||||
SELECT o.status, COUNT(*) as cnt
|
||||
FROM orders o
|
||||
INNER JOIN sync_run_orders sro ON sro.order_number = o.order_number
|
||||
WHERE sro.sync_run_id = ?
|
||||
GROUP BY o.status
|
||||
""", (run_id,))
|
||||
status_counts = {row["status"]: row["cnt"] for row in await cursor.fetchall()}
|
||||
|
||||
return {
|
||||
"orders": [dict(r) for r in rows],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
|
||||
"counts": {
|
||||
"imported": status_counts.get("IMPORTED", 0),
|
||||
"skipped": status_counts.get("SKIPPED", 0),
|
||||
"error": status_counts.get("ERROR", 0),
|
||||
"total": sum(status_counts.values())
|
||||
}
|
||||
}
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def get_orders(page: int = 1, per_page: int = 50,
|
||||
search: str = "", status_filter: str = "all",
|
||||
sort_by: str = "order_date", sort_dir: str = "desc",
|
||||
period_days: int = 7):
|
||||
"""Get orders with filters, sorting, and period. period_days=0 means all time."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
where_clauses = []
|
||||
params = []
|
||||
|
||||
if period_days and period_days > 0:
|
||||
where_clauses.append("order_date >= date('now', ?)")
|
||||
params.append(f"-{period_days} days")
|
||||
|
||||
if search:
|
||||
where_clauses.append("(order_number LIKE ? OR customer_name LIKE ?)")
|
||||
params.extend([f"%{search}%", f"%{search}%"])
|
||||
|
||||
if status_filter and status_filter not in ("all", "UNINVOICED"):
|
||||
where_clauses.append("UPPER(status) = ?")
|
||||
params.append(status_filter.upper())
|
||||
|
||||
where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else ""
|
||||
|
||||
allowed_sort = {"order_date", "order_number", "customer_name", "items_count",
|
||||
"status", "first_seen_at", "updated_at"}
|
||||
if sort_by not in allowed_sort:
|
||||
sort_by = "order_date"
|
||||
if sort_dir.lower() not in ("asc", "desc"):
|
||||
sort_dir = "desc"
|
||||
|
||||
cursor = await db.execute(f"SELECT COUNT(*) FROM orders {where}", params)
|
||||
total = (await cursor.fetchone())[0]
|
||||
|
||||
offset = (page - 1) * per_page
|
||||
cursor = await db.execute(f"""
|
||||
SELECT * FROM orders
|
||||
{where}
|
||||
ORDER BY {sort_by} {sort_dir}
|
||||
LIMIT ? OFFSET ?
|
||||
""", params + [per_page, offset])
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
# Counts by status (on full period, not just this page)
|
||||
cursor = await db.execute(f"""
|
||||
SELECT status, COUNT(*) as cnt FROM orders
|
||||
{where}
|
||||
GROUP BY status
|
||||
""", params)
|
||||
status_counts = {row["status"]: row["cnt"] for row in await cursor.fetchall()}
|
||||
|
||||
return {
|
||||
"orders": [dict(r) for r in rows],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
|
||||
"counts": {
|
||||
"imported": status_counts.get("IMPORTED", 0),
|
||||
"skipped": status_counts.get("SKIPPED", 0),
|
||||
"error": status_counts.get("ERROR", 0),
|
||||
"total": sum(status_counts.values())
|
||||
}
|
||||
}
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def update_import_order_addresses(order_number: str,
|
||||
id_adresa_facturare: int = None,
|
||||
id_adresa_livrare: int = None):
|
||||
"""Update ROA address IDs on an order record."""
|
||||
db = await get_sqlite()
|
||||
try:
|
||||
await db.execute("""
|
||||
UPDATE orders SET
|
||||
id_adresa_facturare = ?,
|
||||
id_adresa_livrare = ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE order_number = ?
|
||||
""", (id_adresa_facturare, id_adresa_livrare, order_number))
|
||||
await db.commit()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
Reference in New Issue
Block a user