feat(sqlite): refactor orders schema + dashboard period filter
Replace import_orders (insert-per-run) with orders table (one row per order, upsert on conflict). Eliminates dedup CTE on every dashboard query and prevents unbounded row growth at 4-500 orders/sync. Key changes: - orders table: PK order_number, upsert via ON CONFLICT DO UPDATE; COALESCE preserves id_comanda once set; times_skipped auto-increments - sync_run_orders: lightweight junction (sync_run_id, order_number) replaces sync_run_id column on orders - order_items: PK changed to (order_number, sku), INSERT OR IGNORE - Auto-migration in init_sqlite(): import_orders → orders on first boot, old table renamed to import_orders_bak - /api/dashboard/orders: period_days param (3/7/30/0=all, default 7) - Dashboard: period selector buttons in orders card header - start.sh: stop existing process on port 5003 before restart; remove --reload (broken on WSL2 /mnt/e/) - Add invoice_service, E2E Playwright tests, Oracle package updates Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,9 @@ _current_sync = None # dict with run_id, status, progress info
|
||||
# SSE subscriber system
|
||||
_subscribers: list[asyncio.Queue] = []
|
||||
|
||||
# In-memory text log buffer per run
|
||||
_run_logs: dict[str, list[str]] = {}
|
||||
|
||||
|
||||
def subscribe() -> asyncio.Queue:
|
||||
"""Subscribe to sync events. Returns a queue that will receive event dicts."""
|
||||
@@ -32,6 +35,22 @@ def unsubscribe(q: asyncio.Queue):
|
||||
pass
|
||||
|
||||
|
||||
def _log_line(run_id: str, message: str):
|
||||
"""Append a timestamped line to the in-memory log buffer."""
|
||||
if run_id not in _run_logs:
|
||||
_run_logs[run_id] = []
|
||||
ts = datetime.now().strftime("%H:%M:%S")
|
||||
_run_logs[run_id].append(f"[{ts}] {message}")
|
||||
|
||||
|
||||
def get_run_text_log(run_id: str) -> str | None:
|
||||
"""Return the accumulated text log for a run, or None if not found."""
|
||||
lines = _run_logs.get(run_id)
|
||||
if lines is None:
|
||||
return None
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def _emit(event: dict):
|
||||
"""Push an event to all subscriber queues."""
|
||||
for q in _subscribers:
|
||||
@@ -87,13 +106,30 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
_current_sync["progress"] = "Reading JSON files..."
|
||||
await _emit({"type": "phase", "run_id": run_id, "message": "Reading JSON files..."})
|
||||
|
||||
started_dt = datetime.now()
|
||||
_run_logs[run_id] = [
|
||||
f"=== Sync Run {run_id} ===",
|
||||
f"Inceput: {started_dt.strftime('%d.%m.%Y %H:%M:%S')}",
|
||||
""
|
||||
]
|
||||
_log_line(run_id, "Citire fisiere JSON...")
|
||||
|
||||
try:
|
||||
# Step 1: Read orders
|
||||
# Step 1: Read orders and sort chronologically (oldest first - R3)
|
||||
orders, json_count = order_reader.read_json_orders()
|
||||
orders.sort(key=lambda o: o.date or '')
|
||||
await sqlite_service.create_sync_run(run_id, json_count)
|
||||
await _emit({"type": "phase", "run_id": run_id, "message": f"Found {len(orders)} orders in {json_count} files"})
|
||||
_log_line(run_id, f"Gasite {len(orders)} comenzi in {json_count} fisiere")
|
||||
|
||||
# Populate web_products catalog from all orders (R4)
|
||||
for order in orders:
|
||||
for item in order.items:
|
||||
if item.sku and item.name:
|
||||
await sqlite_service.upsert_web_product(item.sku, item.name)
|
||||
|
||||
if not orders:
|
||||
_log_line(run_id, "Nicio comanda gasita.")
|
||||
await sqlite_service.update_sync_run(run_id, "completed", 0, 0, 0, 0)
|
||||
summary = {"run_id": run_id, "status": "completed", "message": "No orders found", "json_files": json_count}
|
||||
await _emit({"type": "completed", "run_id": run_id, "summary": summary})
|
||||
@@ -114,6 +150,7 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
importable, skipped = validation_service.classify_orders(orders, validation)
|
||||
|
||||
await _emit({"type": "phase", "run_id": run_id, "message": f"{len(importable)} importable, {len(skipped)} skipped (missing SKUs)"})
|
||||
_log_line(run_id, f"Validare SKU-uri: {len(importable)} importabile, {len(skipped)} nemapate")
|
||||
|
||||
# Step 2c: Build SKU context from skipped orders
|
||||
sku_context = {} # {sku: {"orders": [], "customers": []}}
|
||||
@@ -148,9 +185,13 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
|
||||
# Step 2d: Pre-validate prices for importable articles
|
||||
id_pol = id_pol or settings.ID_POL
|
||||
id_sectie = id_sectie or settings.ID_SECTIE
|
||||
logger.info(f"Sync params: ID_POL={id_pol}, ID_SECTIE={id_sectie}")
|
||||
_log_line(run_id, f"Parametri import: ID_POL={id_pol}, ID_SECTIE={id_sectie}")
|
||||
if id_pol and importable:
|
||||
_current_sync["progress"] = "Validating prices..."
|
||||
await _emit({"type": "phase", "run_id": run_id, "message": "Validating prices..."})
|
||||
_log_line(run_id, "Validare preturi...")
|
||||
# Gather all CODMATs from importable orders
|
||||
all_codmats = set()
|
||||
for order in importable:
|
||||
@@ -175,11 +216,11 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
price_result["missing_price"], id_pol
|
||||
)
|
||||
|
||||
# Step 3: Record skipped orders + emit events
|
||||
# Step 3: Record skipped orders + emit events + store items
|
||||
for order, missing_skus in skipped:
|
||||
customer = order.billing.company_name or \
|
||||
f"{order.billing.firstname} {order.billing.lastname}"
|
||||
await sqlite_service.add_import_order(
|
||||
await sqlite_service.upsert_order(
|
||||
sync_run_id=run_id,
|
||||
order_number=order.number,
|
||||
order_date=order.date,
|
||||
@@ -188,9 +229,24 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
missing_skus=missing_skus,
|
||||
items_count=len(order.items)
|
||||
)
|
||||
await sqlite_service.add_sync_run_order(run_id, order.number, "SKIPPED")
|
||||
# Store order items with mapping status (R9)
|
||||
order_items_data = []
|
||||
for item in order.items:
|
||||
ms = "missing" if item.sku in validation["missing"] else \
|
||||
"mapped" if item.sku in validation["mapped"] else "direct"
|
||||
order_items_data.append({
|
||||
"sku": item.sku, "product_name": item.name,
|
||||
"quantity": item.quantity, "price": item.price, "vat": item.vat,
|
||||
"mapping_status": ms, "codmat": None, "id_articol": None,
|
||||
"cantitate_roa": None
|
||||
})
|
||||
await sqlite_service.add_order_items(order.number, order_items_data)
|
||||
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → OMIS (lipsa: {', '.join(missing_skus)})")
|
||||
await _emit({
|
||||
"type": "order_result", "run_id": run_id,
|
||||
"order_number": order.number, "customer_name": customer,
|
||||
"order_date": order.date,
|
||||
"status": "SKIPPED", "missing_skus": missing_skus,
|
||||
"items_count": len(order.items), "progress": f"0/{len(importable)}"
|
||||
})
|
||||
@@ -210,9 +266,20 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
customer = order.billing.company_name or \
|
||||
f"{order.billing.firstname} {order.billing.lastname}"
|
||||
|
||||
# Build order items data for storage (R9)
|
||||
order_items_data = []
|
||||
for item in order.items:
|
||||
ms = "mapped" if item.sku in validation["mapped"] else "direct"
|
||||
order_items_data.append({
|
||||
"sku": item.sku, "product_name": item.name,
|
||||
"quantity": item.quantity, "price": item.price, "vat": item.vat,
|
||||
"mapping_status": ms, "codmat": None, "id_articol": None,
|
||||
"cantitate_roa": None
|
||||
})
|
||||
|
||||
if result["success"]:
|
||||
imported_count += 1
|
||||
await sqlite_service.add_import_order(
|
||||
await sqlite_service.upsert_order(
|
||||
sync_run_id=run_id,
|
||||
order_number=order.number,
|
||||
order_date=order.date,
|
||||
@@ -222,15 +289,25 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
id_partener=result["id_partener"],
|
||||
items_count=len(order.items)
|
||||
)
|
||||
await sqlite_service.add_sync_run_order(run_id, order.number, "IMPORTED")
|
||||
# Store ROA address IDs (R9)
|
||||
await sqlite_service.update_import_order_addresses(
|
||||
order.number,
|
||||
id_adresa_facturare=result.get("id_adresa_facturare"),
|
||||
id_adresa_livrare=result.get("id_adresa_livrare")
|
||||
)
|
||||
await sqlite_service.add_order_items(order.number, order_items_data)
|
||||
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → IMPORTAT (ID: {result['id_comanda']})")
|
||||
await _emit({
|
||||
"type": "order_result", "run_id": run_id,
|
||||
"order_number": order.number, "customer_name": customer,
|
||||
"order_date": order.date,
|
||||
"status": "IMPORTED", "items_count": len(order.items),
|
||||
"id_comanda": result["id_comanda"], "progress": progress_str
|
||||
})
|
||||
else:
|
||||
error_count += 1
|
||||
await sqlite_service.add_import_order(
|
||||
await sqlite_service.upsert_order(
|
||||
sync_run_id=run_id,
|
||||
order_number=order.number,
|
||||
order_date=order.date,
|
||||
@@ -240,9 +317,13 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
error_message=result["error"],
|
||||
items_count=len(order.items)
|
||||
)
|
||||
await sqlite_service.add_sync_run_order(run_id, order.number, "ERROR")
|
||||
await sqlite_service.add_order_items(order.number, order_items_data)
|
||||
_log_line(run_id, f"#{order.number} [{order.date or '?'}] {customer} → EROARE: {result['error']}")
|
||||
await _emit({
|
||||
"type": "order_result", "run_id": run_id,
|
||||
"order_number": order.number, "customer_name": customer,
|
||||
"order_date": order.date,
|
||||
"status": "ERROR", "error_message": result["error"],
|
||||
"items_count": len(order.items), "progress": progress_str
|
||||
})
|
||||
@@ -275,10 +356,16 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
f"{len(skipped)} skipped, {error_count} errors"
|
||||
)
|
||||
await _emit({"type": "completed", "run_id": run_id, "summary": summary})
|
||||
|
||||
duration = (datetime.now() - started_dt).total_seconds()
|
||||
_log_line(run_id, "")
|
||||
_run_logs[run_id].append(f"Finalizat: {imported_count} importate, {len(skipped)} nemapate, {error_count} erori din {len(orders)} comenzi | Durata: {int(duration)}s")
|
||||
|
||||
return summary
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Sync {run_id} failed: {e}")
|
||||
_log_line(run_id, f"EROARE FATALA: {e}")
|
||||
await sqlite_service.update_sync_run(run_id, "failed", 0, 0, 0, 1, error_message=str(e))
|
||||
_current_sync["error"] = str(e)
|
||||
await _emit({"type": "failed", "run_id": run_id, "error": str(e)})
|
||||
@@ -291,6 +378,11 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
_current_sync = None
|
||||
asyncio.ensure_future(_clear_current_sync())
|
||||
|
||||
async def _clear_run_logs():
|
||||
await asyncio.sleep(300) # 5 minutes
|
||||
_run_logs.pop(run_id, None)
|
||||
asyncio.ensure_future(_clear_run_logs())
|
||||
|
||||
|
||||
def stop_sync():
|
||||
"""Signal sync to stop. Currently sync runs to completion."""
|
||||
|
||||
Reference in New Issue
Block a user