romfast/gomag-vending
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): already_imported tracking, invoice cache, path fixes, remove vfp

Browse Source 
- Track already_imported/new_imported counts separately in sync_runs
  and surface them in status API + dashboard last-run card
- Cache invoice data in SQLite orders table (factura_* columns);
  dashboard falls back to Oracle only for uncached imported orders
- Resolve JSON_OUTPUT_DIR and SQLITE_DB_PATH relative to known
  anchored roots in config.py, independent of CWD (fixes WSL2 start)
- Use single Oracle connection for entire validation phase (perf)
- Batch upsert web_products instead of one-by-one
- Remove stale VFP scripts (replaced by gomag-vending.prg workflow)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Marius Mutu
2026-03-14 00:15:37 +02:00
parent 8681a92eec
commit 2e65855fe2
24 changed files with 485 additions and 3458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
api/app/services/validation_service.py
View File

@@ -1,23 +1,53 @@
import logging
from datetime import datetime, timedelta
from .. import database
logger = logging.getLogger(__name__)
def validate_skus(skus: set[str]) -> dict:
def check_orders_in_roa(min_date, conn) -> dict:
"""Check which orders already exist in Oracle COMENZI by date range.
Returns: {comanda_externa: id_comanda} for all existing orders.
Much faster than IN-clause batching — single query using date index.
"""
if conn is None:
return {}
existing = {}
try:
with conn.cursor() as cur:
cur.execute("""
SELECT comanda_externa, id_comanda FROM COMENZI
WHERE data_comanda >= :min_date
AND comanda_externa IS NOT NULL AND sters = 0
""", {"min_date": min_date})
for row in cur:
existing[str(row[0])] = row[1]
except Exception as e:
logger.error(f"check_orders_in_roa failed: {e}")
logger.info(f"ROA order check (since {min_date}): {len(existing)} existing orders found")
return existing
def validate_skus(skus: set[str], conn=None) -> dict:
"""Validate a set of SKUs against Oracle.
Returns: {mapped: set, direct: set, missing: set}
Returns: {mapped: set, direct: set, missing: set, direct_id_map: {codmat: id_articol}}
- mapped: found in ARTICOLE_TERTI (active)
- direct: found in NOM_ARTICOLE by codmat (not in ARTICOLE_TERTI)
- missing: not found anywhere
- direct_id_map: {codmat: id_articol} for direct SKUs (saves a round-trip in validate_prices)
"""
if not skus:
return {"mapped": set(), "direct": set(), "missing": set()}
return {"mapped": set(), "direct": set(), "missing": set(), "direct_id_map": {}}
mapped = set()
direct = set()
direct_id_map = {}
sku_list = list(skus)
conn = database.get_oracle_connection()
own_conn = conn is None
if own_conn:
conn = database.get_oracle_connection()
try:
with conn.cursor() as cur:
# Check in batches of 500
@@ -34,24 +64,26 @@ def validate_skus(skus: set[str]) -> dict:
for row in cur:
mapped.add(row[0])
# Check NOM_ARTICOLE for remaining
# Check NOM_ARTICOLE for remaining — also fetch id_articol
remaining = [s for s in batch if s not in mapped]
if remaining:
placeholders2 = ",".join([f":n{j}" for j in range(len(remaining))])
params2 = {f"n{j}": sku for j, sku in enumerate(remaining)}
cur.execute(f"""
SELECT DISTINCT codmat FROM NOM_ARTICOLE
SELECT codmat, id_articol FROM NOM_ARTICOLE
WHERE codmat IN ({placeholders2}) AND sters = 0 AND inactiv = 0
""", params2)
for row in cur:
direct.add(row[0])
direct_id_map[row[0]] = row[1]
finally:
database.pool.release(conn)
if own_conn:
database.pool.release(conn)
missing = skus - mapped - direct
logger.info(f"SKU validation: {len(mapped)} mapped, {len(direct)} direct, {len(missing)} missing")
return {"mapped": mapped, "direct": direct, "missing": missing}
return {"mapped": mapped, "direct": direct, "missing": missing, "direct_id_map": direct_id_map}
def classify_orders(orders, validation_result):
"""Classify orders as importable or skipped based on SKU validation.
@@ -73,39 +105,9 @@ def classify_orders(orders, validation_result):
return importable, skipped
def find_new_orders(order_numbers: list[str]) -> set[str]:
"""Check which order numbers do NOT already exist in Oracle COMENZI.
Returns: set of order numbers that are truly new (not yet imported).
"""
if not order_numbers:
return set()
existing = set()
num_list = list(order_numbers)
conn = database.get_oracle_connection()
try:
with conn.cursor() as cur:
for i in range(0, len(num_list), 500):
batch = num_list[i:i+500]
placeholders = ",".join([f":o{j}" for j in range(len(batch))])
params = {f"o{j}": num for j, num in enumerate(batch)}
cur.execute(f"""
SELECT DISTINCT comanda_externa FROM COMENZI
WHERE comanda_externa IN ({placeholders}) AND sters = 0
""", params)
for row in cur:
existing.add(row[0])
finally:
database.pool.release(conn)
new_orders = set(order_numbers) - existing
logger.info(f"Order check: {len(new_orders)} new, {len(existing)} already exist out of {len(order_numbers)} total")
return new_orders
def validate_prices(codmats: set[str], id_pol: int) -> dict:
def validate_prices(codmats: set[str], id_pol: int, conn=None, direct_id_map: dict=None) -> dict:
"""Check which CODMATs have a price entry in CRM_POLITICI_PRET_ART for the given policy.
If direct_id_map is provided, skips the NOM_ARTICOLE lookup for those CODMATs.
Returns: {"has_price": set_of_codmats, "missing_price": set_of_codmats}
"""
if not codmats:
@@ -115,21 +117,31 @@ def validate_prices(codmats: set[str], id_pol: int) -> dict:
ids_with_price = set()
codmat_list = list(codmats)
conn = database.get_oracle_connection()
# Pre-populate from direct_id_map if available
if direct_id_map:
for cm in codmat_list:
if cm in direct_id_map:
codmat_to_id[cm] = direct_id_map[cm]
own_conn = conn is None
if own_conn:
conn = database.get_oracle_connection()
try:
with conn.cursor() as cur:
# Step 1: Get ID_ARTICOL for each CODMAT
for i in range(0, len(codmat_list), 500):
batch = codmat_list[i:i+500]
placeholders = ",".join([f":c{j}" for j in range(len(batch))])
params = {f"c{j}": cm for j, cm in enumerate(batch)}
# Step 1: Get ID_ARTICOL for CODMATs not already in direct_id_map
remaining = [cm for cm in codmat_list if cm not in codmat_to_id]
if remaining:
for i in range(0, len(remaining), 500):
batch = remaining[i:i+500]
placeholders = ",".join([f":c{j}" for j in range(len(batch))])
params = {f"c{j}": cm for j, cm in enumerate(batch)}
cur.execute(f"""
SELECT id_articol, codmat FROM NOM_ARTICOLE
WHERE codmat IN ({placeholders})
""", params)
for row in cur:
codmat_to_id[row[1]] = row[0]
cur.execute(f"""
SELECT id_articol, codmat FROM NOM_ARTICOLE
WHERE codmat IN ({placeholders})
""", params)
for row in cur:
codmat_to_id[row[1]] = row[0]
# Step 2: Check which ID_ARTICOLs have a price in the policy
id_list = list(codmat_to_id.values())
@@ -146,7 +158,8 @@ def validate_prices(codmats: set[str], id_pol: int) -> dict:
for row in cur:
ids_with_price.add(row[0])
finally:
database.pool.release(conn)
if own_conn:
database.pool.release(conn)
# Map back to CODMATs
has_price = {cm for cm, aid in codmat_to_id.items() if aid in ids_with_price}
@@ -155,12 +168,17 @@ def validate_prices(codmats: set[str], id_pol: int) -> dict:
logger.info(f"Price validation (policy {id_pol}): {len(has_price)} have price, {len(missing_price)} missing price")
return {"has_price": has_price, "missing_price": missing_price}
def ensure_prices(codmats: set[str], id_pol: int):
"""Insert price 0 entries for CODMATs missing from the given price policy."""
def ensure_prices(codmats: set[str], id_pol: int, conn=None, direct_id_map: dict=None):
"""Insert price 0 entries for CODMATs missing from the given price policy.
Uses batch executemany instead of individual INSERTs.
Relies on TRG_CRM_POLITICI_PRET_ART trigger for ID_POL_ART sequence.
"""
if not codmats:
return
conn = database.get_oracle_connection()
own_conn = conn is None
if own_conn:
conn = database.get_oracle_connection()
try:
with conn.cursor() as cur:
# Get ID_VALUTA for this policy
@@ -173,31 +191,53 @@ def ensure_prices(codmats: set[str], id_pol: int):
return
id_valuta = row[0]
# Build batch params using direct_id_map where available
batch_params = []
need_lookup = []
codmat_id_map = dict(direct_id_map) if direct_id_map else {}
for codmat in codmats:
# Get ID_ARTICOL
cur.execute("""
SELECT id_articol FROM NOM_ARTICOLE WHERE codmat = :codmat
""", {"codmat": codmat})
row = cur.fetchone()
if not row:
if codmat not in codmat_id_map:
need_lookup.append(codmat)
# Batch lookup remaining CODMATs
if need_lookup:
for i in range(0, len(need_lookup), 500):
batch = need_lookup[i:i+500]
placeholders = ",".join([f":c{j}" for j in range(len(batch))])
params = {f"c{j}": cm for j, cm in enumerate(batch)}
cur.execute(f"""
SELECT codmat, id_articol FROM NOM_ARTICOLE
WHERE codmat IN ({placeholders}) AND sters = 0 AND inactiv = 0
""", params)
for r in cur:
codmat_id_map[r[0]] = r[1]
for codmat in codmats:
id_articol = codmat_id_map.get(codmat)
if not id_articol:
logger.warning(f"CODMAT {codmat} not found in NOM_ARTICOLE, skipping price insert")
continue
id_articol = row[0]
batch_params.append({
"id_pol": id_pol,
"id_articol": id_articol,
"id_valuta": id_valuta
})
cur.execute("""
if batch_params:
cur.executemany("""
INSERT INTO CRM_POLITICI_PRET_ART
(ID_POL_ART, ID_POL, ID_ARTICOL, PRET, ID_VALUTA,
ID_UTIL, DATAORA, PROC_TVAV,
PRETFTVA, PRETCTVA)
(ID_POL, ID_ARTICOL, PRET, ID_VALUTA,
ID_UTIL, DATAORA, PROC_TVAV, PRETFTVA, PRETCTVA)
VALUES
(SEQ_CRM_POLITICI_PRET_ART.NEXTVAL, :id_pol, :id_articol, 0, :id_valuta,
-3, SYSDATE, 1.19,
0, 0)
""", {"id_pol": id_pol, "id_articol": id_articol, "id_valuta": id_valuta})
logger.info(f"Pret 0 adaugat pentru CODMAT {codmat} in politica {id_pol}")
(:id_pol, :id_articol, 0, :id_valuta,
-3, SYSDATE, 1.19, 0, 0)
""", batch_params)
logger.info(f"Batch inserted {len(batch_params)} price entries for policy {id_pol}")
conn.commit()
finally:
database.pool.release(conn)
if own_conn:
database.pool.release(conn)
logger.info(f"Ensure prices done: {len(codmats)} CODMATs processed for policy {id_pol}")