diff --git a/.gitignore b/.gitignore index 50c646a..97684f3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,16 @@ vfp/output/ vfp/*.json *.~pck .claude/HANDOFF.md + +# Virtual environments +venv/ +.venv/ + +# SQLite databases +*.db + +# Generated/duplicate directories +api/api/ + +# Logs directory +logs/ diff --git a/api/app/config.py b/api/app/config.py index cba1372..0323fb6 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -30,6 +30,11 @@ class Settings(BaseSettings): API_USERNAME: str = "" API_PASSWORD: str = "" + # ROA Import Settings + ID_POL: int = 0 + ID_GESTIUNE: int = 0 + ID_SECTIE: int = 0 + model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} settings = Settings() diff --git a/api/app/database.py b/api/app/database.py index d108631..fbce776 100644 --- a/api/app/database.py +++ b/api/app/database.py @@ -91,7 +91,10 @@ CREATE TABLE IF NOT EXISTS missing_skus ( product_name TEXT, first_seen TEXT DEFAULT (datetime('now')), resolved INTEGER DEFAULT 0, - resolved_at TEXT + resolved_at TEXT, + order_count INTEGER DEFAULT 0, + order_numbers TEXT, + customers TEXT ); CREATE TABLE IF NOT EXISTS scheduler_config ( @@ -115,6 +118,21 @@ def init_sqlite(): # Create tables synchronously conn = sqlite3.connect(_sqlite_db_path) conn.executescript(SQLITE_SCHEMA) + + # Migrate: add columns if missing (for existing databases) + try: + cursor = conn.execute("PRAGMA table_info(missing_skus)") + cols = {row[1] for row in cursor.fetchall()} + for col, typedef in [("order_count", "INTEGER DEFAULT 0"), + ("order_numbers", "TEXT"), + ("customers", "TEXT")]: + if col not in cols: + conn.execute(f"ALTER TABLE missing_skus ADD COLUMN {col} {typedef}") + logger.info(f"Migrated missing_skus: added column {col}") + conn.commit() + except Exception as e: + logger.warning(f"Migration check failed: {e}") + conn.close() logger.info(f"SQLite initialized: {_sqlite_db_path}") diff --git a/api/app/routers/sync.py b/api/app/routers/sync.py index 5cad174..a4827f9 100644 --- a/api/app/routers/sync.py +++ b/api/app/routers/sync.py @@ -60,6 +60,11 @@ async def sync_history(page: int = 1, per_page: int = 20): return await sqlite_service.get_sync_runs(page, per_page) +@router.get("/logs", response_class=HTMLResponse) +async def logs_page(request: Request): + return templates.TemplateResponse("logs.html", {"request": request}) + + @router.get("/api/sync/run/{run_id}") async def sync_run_detail(run_id: str): """Get details for a specific sync run.""" @@ -69,6 +74,30 @@ async def sync_run_detail(run_id: str): return detail +@router.get("/api/sync/run/{run_id}/log") +async def sync_run_log(run_id: str): + """Get detailed log per order for a sync run.""" + detail = await sqlite_service.get_sync_run_detail(run_id) + if not detail: + return {"error": "Run not found", "status_code": 404} + orders = detail.get("orders", []) + return { + "run_id": run_id, + "run": detail.get("run", {}), + "orders": [ + { + "order_number": o.get("order_number"), + "customer_name": o.get("customer_name"), + "items_count": o.get("items_count"), + "status": o.get("status"), + "error_message": o.get("error_message"), + "missing_skus": o.get("missing_skus"), + } + for o in orders + ] + } + + @router.put("/api/sync/schedule") async def update_schedule(config: ScheduleConfig): """Update scheduler configuration.""" diff --git a/api/app/routers/validation.py b/api/app/routers/validation.py index a81a70a..5d491f8 100644 --- a/api/app/routers/validation.py +++ b/api/app/routers/validation.py @@ -1,9 +1,11 @@ +import asyncio import csv import io -from fastapi import APIRouter +import json +from fastapi import APIRouter, Query from fastapi.responses import StreamingResponse -from ..services import order_reader, validation_service +from ..services import order_reader, validation_service, sqlite_service from ..database import get_sqlite router = APIRouter(prefix="/api/validate", tags=["validation"]) @@ -20,27 +22,40 @@ async def scan_and_validate(): result = validation_service.validate_skus(all_skus) importable, skipped = validation_service.classify_orders(orders, result) - # Track missing SKUs in SQLite - db = await get_sqlite() - try: - for sku in result["missing"]: - # Find product name from orders - product_name = "" - for order in orders: - for item in order.items: - if item.sku == sku: - product_name = item.name - break - if product_name: - break + # Find new orders (not yet in Oracle) + all_order_numbers = [o.number for o in orders] + new_orders = await asyncio.to_thread(validation_service.find_new_orders, all_order_numbers) - await db.execute(""" - INSERT OR IGNORE INTO missing_skus (sku, product_name) - VALUES (?, ?) - """, (sku, product_name)) - await db.commit() - finally: - await db.close() + # Build SKU context from skipped orders and track missing SKUs + sku_context = {} # sku -> {order_numbers: [], customers: []} + for order, missing_list in skipped: + customer = order.billing.company_name or f"{order.billing.firstname} {order.billing.lastname}" + for sku in missing_list: + if sku not in sku_context: + sku_context[sku] = {"order_numbers": [], "customers": []} + sku_context[sku]["order_numbers"].append(order.number) + if customer not in sku_context[sku]["customers"]: + sku_context[sku]["customers"].append(customer) + + for sku in result["missing"]: + # Find product name from orders + product_name = "" + for order in orders: + for item in order.items: + if item.sku == sku: + product_name = item.name + break + if product_name: + break + + ctx = sku_context.get(sku, {}) + await sqlite_service.track_missing_sku( + sku=sku, + product_name=product_name, + order_count=len(ctx.get("order_numbers", [])), + order_numbers=json.dumps(ctx.get("order_numbers", [])), + customers=json.dumps(ctx.get("customers", [])) + ) return { "json_files": json_count, @@ -48,11 +63,15 @@ async def scan_and_validate(): "total_skus": len(all_skus), "importable": len(importable), "skipped": len(skipped), + "new_orders": len(new_orders), "skus": { "mapped": len(result["mapped"]), "direct": len(result["direct"]), "missing": len(result["missing"]), - "missing_list": sorted(result["missing"]) + "missing_list": sorted(result["missing"]), + "total_skus": len(all_skus), + "mapped_skus": len(result["mapped"]), + "direct_skus": len(result["direct"]) }, "skipped_orders": [ { @@ -66,23 +85,24 @@ async def scan_and_validate(): } @router.get("/missing-skus") -async def get_missing_skus(): - """Get all tracked missing SKUs.""" +async def get_missing_skus( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + resolved: int = Query(0, ge=0, le=1) +): + """Get paginated missing SKUs.""" + result = await sqlite_service.get_missing_skus_paginated(page, per_page, resolved) + # Backward compat: also include 'unresolved' count db = await get_sqlite() try: - cursor = await db.execute(""" - SELECT sku, product_name, first_seen, resolved, resolved_at - FROM missing_skus - ORDER BY resolved ASC, first_seen DESC - """) - rows = await cursor.fetchall() - return { - "missing_skus": [dict(row) for row in rows], - "total": len(rows), - "unresolved": sum(1 for r in rows if not r["resolved"]) - } + cursor = await db.execute( + "SELECT COUNT(*) FROM missing_skus WHERE resolved = 0" + ) + unresolved = (await cursor.fetchone())[0] finally: await db.close() + result["unresolved"] = unresolved + return result @router.get("/missing-skus-csv") async def export_missing_skus_csv(): diff --git a/api/app/services/sqlite_service.py b/api/app/services/sqlite_service.py index 6177581..895c5b2 100644 --- a/api/app/services/sqlite_service.py +++ b/api/app/services/sqlite_service.py @@ -59,14 +59,25 @@ async def add_import_order(sync_run_id: str, order_number: str, order_date: str, await db.close() -async def track_missing_sku(sku: str, product_name: str = ""): - """Track a missing SKU.""" +async def track_missing_sku(sku: str, product_name: str = "", + order_count: int = 0, order_numbers: str = None, + customers: str = None): + """Track a missing SKU with order context.""" db = await get_sqlite() try: await db.execute(""" INSERT OR IGNORE INTO missing_skus (sku, product_name) VALUES (?, ?) """, (sku, product_name)) + # Update context columns (always update with latest data) + if order_count or order_numbers or customers: + await db.execute(""" + UPDATE missing_skus SET + order_count = ?, + order_numbers = ?, + customers = ? + WHERE sku = ? + """, (order_count, order_numbers, customers, sku)) await db.commit() finally: await db.close() @@ -85,6 +96,38 @@ async def resolve_missing_sku(sku: str): await db.close() +async def get_missing_skus_paginated(page: int = 1, per_page: int = 20, resolved: int = 0): + """Get paginated missing SKUs.""" + db = await get_sqlite() + try: + offset = (page - 1) * per_page + + cursor = await db.execute( + "SELECT COUNT(*) FROM missing_skus WHERE resolved = ?", (resolved,) + ) + total = (await cursor.fetchone())[0] + + cursor = await db.execute(""" + SELECT sku, product_name, first_seen, resolved, resolved_at, + order_count, order_numbers, customers + FROM missing_skus + WHERE resolved = ? + ORDER BY order_count DESC, first_seen DESC + LIMIT ? OFFSET ? + """, (resolved, per_page, offset)) + rows = await cursor.fetchall() + + return { + "missing_skus": [dict(row) for row in rows], + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if total > 0 else 0 + } + finally: + await db.close() + + async def get_sync_runs(page: int = 1, per_page: int = 20): """Get paginated sync run history.""" db = await get_sqlite() @@ -165,6 +208,17 @@ async def get_dashboard_stats(): ) missing = (await cursor.fetchone())[0] + # Article stats from last sync + cursor = await db.execute(""" + SELECT COUNT(DISTINCT sku) FROM missing_skus + """) + total_missing_skus = (await cursor.fetchone())[0] + + cursor = await db.execute(""" + SELECT COUNT(DISTINCT sku) FROM missing_skus WHERE resolved = 0 + """) + unresolved_skus = (await cursor.fetchone())[0] + # Last sync run cursor = await db.execute(""" SELECT * FROM sync_runs ORDER BY started_at DESC LIMIT 1 @@ -176,6 +230,8 @@ async def get_dashboard_stats(): "skipped": skipped, "errors": errors, "missing_skus": missing, + "total_tracked_skus": total_missing_skus, + "unresolved_skus": unresolved_skus, "last_run": dict(last_run) if last_run else None } finally: diff --git a/api/app/services/sync_service.py b/api/app/services/sync_service.py index 0842f72..ca8a2ec 100644 --- a/api/app/services/sync_service.py +++ b/api/app/services/sync_service.py @@ -1,9 +1,11 @@ import asyncio +import json import logging import uuid from datetime import datetime from . import order_reader, validation_service, import_service, sqlite_service +from ..config import settings logger = logging.getLogger(__name__) @@ -52,12 +54,31 @@ async def run_sync(id_pol: int = None, id_sectie: int = None) -> dict: _current_sync["progress"] = f"Validating {len(orders)} orders..." - # Step 2: Validate SKUs (blocking Oracle call -> run in thread) + # Step 2a: Find new orders (not yet in Oracle) + all_order_numbers = [o.number for o in orders] + new_orders = await asyncio.to_thread( + validation_service.find_new_orders, all_order_numbers + ) + + # Step 2b: Validate SKUs (blocking Oracle call -> run in thread) all_skus = order_reader.get_all_skus(orders) validation = await asyncio.to_thread(validation_service.validate_skus, all_skus) importable, skipped = validation_service.classify_orders(orders, validation) - # Track missing SKUs + # Step 2c: Build SKU context from skipped orders + sku_context = {} # {sku: {"orders": [], "customers": []}} + for order, missing_skus_list in skipped: + customer = order.billing.company_name or \ + f"{order.billing.firstname} {order.billing.lastname}" + for sku in missing_skus_list: + if sku not in sku_context: + sku_context[sku] = {"orders": [], "customers": []} + if order.number not in sku_context[sku]["orders"]: + sku_context[sku]["orders"].append(order.number) + if customer not in sku_context[sku]["customers"]: + sku_context[sku]["customers"].append(customer) + + # Track missing SKUs with context for sku in validation["missing"]: product_name = "" for order in orders: @@ -67,7 +88,41 @@ async def run_sync(id_pol: int = None, id_sectie: int = None) -> dict: break if product_name: break - await sqlite_service.track_missing_sku(sku, product_name) + ctx = sku_context.get(sku, {}) + await sqlite_service.track_missing_sku( + sku, product_name, + order_count=len(ctx.get("orders", [])), + order_numbers=json.dumps(ctx.get("orders", [])) if ctx.get("orders") else None, + customers=json.dumps(ctx.get("customers", [])) if ctx.get("customers") else None, + ) + + # Step 2d: Pre-validate prices for importable articles + id_pol = id_pol or settings.ID_POL + if id_pol and importable: + _current_sync["progress"] = "Validating prices..." + # Gather all CODMATs from importable orders + all_codmats = set() + for order in importable: + for item in order.items: + if item.sku in validation["mapped"]: + # Mapped SKUs resolve to codmat via ARTICOLE_TERTI (handled by import) + pass + elif item.sku in validation["direct"]: + all_codmats.add(item.sku) + # For mapped SKUs, we'd need the ARTICOLE_TERTI lookup - direct SKUs = codmat + if all_codmats: + price_result = await asyncio.to_thread( + validation_service.validate_prices, all_codmats, id_pol + ) + if price_result["missing_price"]: + logger.info( + f"Auto-adding price 0 for {len(price_result['missing_price'])} " + f"direct articles in policy {id_pol}" + ) + await asyncio.to_thread( + validation_service.ensure_prices, + price_result["missing_price"], id_pol + ) # Step 3: Record skipped orders for order, missing_skus in skipped: @@ -138,6 +193,7 @@ async def run_sync(id_pol: int = None, id_sectie: int = None) -> dict: "status": status, "json_files": json_count, "total_orders": len(orders), + "new_orders": len(new_orders), "imported": imported_count, "skipped": len(skipped), "errors": error_count, diff --git a/api/app/services/validation_service.py b/api/app/services/validation_service.py index 753d072..63cfcaa 100644 --- a/api/app/services/validation_service.py +++ b/api/app/services/validation_service.py @@ -69,3 +69,123 @@ def classify_orders(orders, validation_result): importable.append(order) return importable, skipped + +def find_new_orders(order_numbers: list[str]) -> set[str]: + """Check which order numbers do NOT already exist in Oracle COMENZI. + Returns: set of order numbers that are truly new (not yet imported). + """ + if not order_numbers: + return set() + + existing = set() + num_list = list(order_numbers) + + with database.pool.acquire() as conn: + with conn.cursor() as cur: + for i in range(0, len(num_list), 500): + batch = num_list[i:i+500] + placeholders = ",".join([f":o{j}" for j in range(len(batch))]) + params = {f"o{j}": num for j, num in enumerate(batch)} + + cur.execute(f""" + SELECT DISTINCT comanda_externa FROM COMENZI + WHERE comanda_externa IN ({placeholders}) AND sters = 0 + """, params) + for row in cur: + existing.add(row[0]) + + new_orders = set(order_numbers) - existing + logger.info(f"Order check: {len(new_orders)} new, {len(existing)} already exist out of {len(order_numbers)} total") + return new_orders + +def validate_prices(codmats: set[str], id_pol: int) -> dict: + """Check which CODMATs have a price entry in CRM_POLITICI_PRET_ART for the given policy. + Returns: {"has_price": set_of_codmats, "missing_price": set_of_codmats} + """ + if not codmats: + return {"has_price": set(), "missing_price": set()} + + codmat_to_id = {} + ids_with_price = set() + codmat_list = list(codmats) + + with database.pool.acquire() as conn: + with conn.cursor() as cur: + # Step 1: Get ID_ARTICOL for each CODMAT + for i in range(0, len(codmat_list), 500): + batch = codmat_list[i:i+500] + placeholders = ",".join([f":c{j}" for j in range(len(batch))]) + params = {f"c{j}": cm for j, cm in enumerate(batch)} + + cur.execute(f""" + SELECT id_articol, codmat FROM NOM_ARTICOLE + WHERE codmat IN ({placeholders}) + """, params) + for row in cur: + codmat_to_id[row[1]] = row[0] + + # Step 2: Check which ID_ARTICOLs have a price in the policy + id_list = list(codmat_to_id.values()) + for i in range(0, len(id_list), 500): + batch = id_list[i:i+500] + placeholders = ",".join([f":a{j}" for j in range(len(batch))]) + params = {f"a{j}": aid for j, aid in enumerate(batch)} + params["id_pol"] = id_pol + + cur.execute(f""" + SELECT DISTINCT pa.ID_ARTICOL FROM CRM_POLITICI_PRET_ART pa + WHERE pa.ID_POL = :id_pol AND pa.ID_ARTICOL IN ({placeholders}) + """, params) + for row in cur: + ids_with_price.add(row[0]) + + # Map back to CODMATs + has_price = {cm for cm, aid in codmat_to_id.items() if aid in ids_with_price} + missing_price = codmats - has_price + + logger.info(f"Price validation (policy {id_pol}): {len(has_price)} have price, {len(missing_price)} missing price") + return {"has_price": has_price, "missing_price": missing_price} + +def ensure_prices(codmats: set[str], id_pol: int): + """Insert price 0 entries for CODMATs missing from the given price policy.""" + if not codmats: + return + + with database.pool.acquire() as conn: + with conn.cursor() as cur: + # Get ID_VALUTA for this policy + cur.execute(""" + SELECT ID_VALUTA FROM CRM_POLITICI_PRETURI WHERE ID_POL = :id_pol + """, {"id_pol": id_pol}) + row = cur.fetchone() + if not row: + logger.error(f"Price policy {id_pol} not found in CRM_POLITICI_PRETURI") + return + id_valuta = row[0] + + for codmat in codmats: + # Get ID_ARTICOL + cur.execute(""" + SELECT id_articol FROM NOM_ARTICOLE WHERE codmat = :codmat + """, {"codmat": codmat}) + row = cur.fetchone() + if not row: + logger.warning(f"CODMAT {codmat} not found in NOM_ARTICOLE, skipping price insert") + continue + id_articol = row[0] + + cur.execute(""" + INSERT INTO CRM_POLITICI_PRET_ART + (ID_POL_ART, ID_POL, ID_ARTICOL, PRET, ID_COMANDA, ID_VALUTA, + ID_UTIL, DATAORA, PROC_TVAV, ID_PARTR, ID_PARTZ, + PRETFTVA, PRETCTVA, CANTITATE, ID_UM, PRET_MIN, PRET_MIN_TVA) + VALUES + (SEQ_CRM_POLITICI_PRET_ART.NEXTVAL, :id_pol, :id_articol, 0, NULL, :id_valuta, + -3, SYSDATE, 1.19, NULL, NULL, + 0, 0, 0, NULL, 0, 0) + """, {"id_pol": id_pol, "id_articol": id_articol, "id_valuta": id_valuta}) + logger.info(f"Pret 0 adaugat pentru CODMAT {codmat} in politica {id_pol}") + + conn.commit() + + logger.info(f"Ensure prices done: {len(codmats)} CODMATs processed for policy {id_pol}") diff --git a/api/app/static/js/dashboard.js b/api/app/static/js/dashboard.js index efa1787..c5cd722 100644 --- a/api/app/static/js/dashboard.js +++ b/api/app/static/js/dashboard.js @@ -1,9 +1,22 @@ let refreshInterval = null; +let currentMapSku = ''; +let acTimeout = null; document.addEventListener('DOMContentLoaded', () => { loadDashboard(); // Auto-refresh every 10 seconds refreshInterval = setInterval(loadDashboard, 10000); + + const input = document.getElementById('mapCodmat'); + if (input) { + input.addEventListener('input', () => { + clearTimeout(acTimeout); + acTimeout = setTimeout(() => autocompleteMap(input.value), 250); + }); + input.addEventListener('blur', () => { + setTimeout(() => document.getElementById('mapAutocomplete').classList.add('d-none'), 200); + }); + } }); async function loadDashboard() { @@ -20,11 +33,36 @@ async function loadSyncStatus() { const res = await fetch('/api/sync/status'); const data = await res.json(); - // Update stats const stats = data.stats || {}; - document.getElementById('stat-imported').textContent = stats.imported || 0; - document.getElementById('stat-skipped').textContent = stats.skipped || 0; - document.getElementById('stat-missing').textContent = stats.missing_skus || 0; + + // Order-level stat cards from sync status + document.getElementById('stat-imported').textContent = stats.imported != null ? stats.imported : 0; + document.getElementById('stat-skipped').textContent = stats.skipped != null ? stats.skipped : 0; + document.getElementById('stat-errors').textContent = stats.errors != null ? stats.errors : 0; + + // Article-level stats from sync status + if (stats.total_tracked_skus != null) { + document.getElementById('stat-total-skus').textContent = stats.total_tracked_skus; + } + if (stats.unresolved_skus != null) { + document.getElementById('stat-missing-skus').textContent = stats.unresolved_skus; + const total = stats.total_tracked_skus || 0; + const unresolved = stats.unresolved_skus || 0; + document.getElementById('stat-mapped-skus').textContent = total - unresolved; + } + + // Restore scan-derived stats from sessionStorage (preserved across auto-refresh) + const scanData = getScanData(); + if (scanData) { + document.getElementById('stat-new').textContent = scanData.new_orders != null ? scanData.new_orders : (scanData.total_orders || '-'); + document.getElementById('stat-ready').textContent = scanData.importable != null ? scanData.importable : '-'; + if (scanData.skus) { + document.getElementById('stat-total-skus').textContent = scanData.skus.total_skus || stats.total_tracked_skus || '-'; + document.getElementById('stat-missing-skus').textContent = scanData.skus.missing || stats.unresolved_skus || 0; + const mapped = (scanData.skus.total_skus || 0) - (scanData.skus.missing || 0); + document.getElementById('stat-mapped-skus').textContent = mapped >= 0 ? mapped : '-'; + } + } // Update sync status badge const badge = document.getElementById('syncStatusBadge'); @@ -46,7 +84,7 @@ async function loadSyncStatus() { const lr = stats.last_run; const started = lr.started_at ? new Date(lr.started_at).toLocaleString('ro-RO') : ''; document.getElementById('syncProgressText').textContent = - `Ultimul: ${started} | ${lr.imported || 0} ok, ${lr.skipped || 0} skip, ${lr.errors || 0} err`; + `Ultimul: ${started} | ${lr.imported || 0} ok, ${lr.skipped || 0} fara mapare, ${lr.errors || 0} erori`; } else { document.getElementById('syncProgressText').textContent = ''; } @@ -93,32 +131,42 @@ async function loadSyncHistory() { async function loadMissingSkus() { try { - const res = await fetch('/api/validate/missing-skus'); + const res = await fetch('/api/validate/missing-skus?page=1&per_page=10'); const data = await res.json(); const tbody = document.getElementById('missingSkusBody'); - // Update stat card - document.getElementById('stat-missing').textContent = data.unresolved || 0; + // Update article-level stat card (unresolved count) + if (data.total != null) { + document.getElementById('stat-missing-skus').textContent = data.total; + } const unresolved = (data.missing_skus || []).filter(s => !s.resolved); if (unresolved.length === 0) { - tbody.innerHTML = '
${esc(s.sku)}${esc(s)}`).join('') +
+ '${esc(order.order_number || '-')}