fix(sync): guard against mass DELETED_IN_ROA when ROA is recovering
After a power loss + reboot, COMENZI was queryable but not yet recovered; phase 4b-3 read it as empty and sticky-marked 3794 live orders DELETED_IN_ROA (nulling id_comanda). check_orders_exist also swallowed Oracle errors and returned a partial set, which callers misread as deletions. - check_orders_exist now re-raises on Oracle error instead of returning partial - new invoice_service.deletions_or_guard() raises MassDeletionGuard when the would-delete fraction is implausibly high (>30% of >=25 imported orders) - both deletion sites (auto sync + manual refresh) skip + log on guard trip Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -846,10 +846,14 @@ async def refresh_invoices():
|
|||||||
existing_ids = await asyncio.to_thread(
|
existing_ids = await asyncio.to_thread(
|
||||||
invoice_service.check_orders_exist, id_comanda_list
|
invoice_service.check_orders_exist, id_comanda_list
|
||||||
)
|
)
|
||||||
for o in all_imported:
|
try:
|
||||||
if o["id_comanda"] not in existing_ids:
|
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
|
||||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
except invoice_service.MassDeletionGuard as g:
|
||||||
orders_deleted += 1
|
logger.warning(f"Mass-deletion guard tripped during refresh: {g}")
|
||||||
|
to_delete = []
|
||||||
|
for o in to_delete:
|
||||||
|
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||||
|
orders_deleted += 1
|
||||||
|
|
||||||
# Cherry-pick A: Batch refresh Oracle addresses for all orders with stored address IDs
|
# Cherry-pick A: Batch refresh Oracle addresses for all orders with stored address IDs
|
||||||
addr_rows = await sqlite_service.get_orders_with_address_ids()
|
addr_rows = await sqlite_service.get_orders_with_address_ids()
|
||||||
|
|||||||
@@ -3,6 +3,39 @@ from .. import database
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── Mass-deletion safety guard ──────────────────────────────────────────────
|
||||||
|
# If ROA appears to have lost a large fraction of its orders, it is almost
|
||||||
|
# certainly a transient/recovery state (e.g. the DB just restarted after a power
|
||||||
|
# loss and COMENZI hasn't finished recovering), NOT real deletions. In that case
|
||||||
|
# we refuse to mass-mark orders as DELETED_IN_ROA — a sticky, hard-to-reverse
|
||||||
|
# operation that nulls id_comanda. See incident 2026-06-26 (3794 false deletes).
|
||||||
|
MASS_DELETION_ABORT_FRACTION = 0.30
|
||||||
|
MASS_DELETION_ABORT_MIN = 25
|
||||||
|
|
||||||
|
|
||||||
|
class MassDeletionGuard(Exception):
|
||||||
|
"""Raised when the number of orders that would be marked deleted is
|
||||||
|
suspiciously high, indicating ROA is unavailable rather than truly purged."""
|
||||||
|
|
||||||
|
|
||||||
|
def deletions_or_guard(all_imported: list, existing_ids: set) -> list:
|
||||||
|
"""Return the subset of all_imported whose id_comanda is missing from ROA,
|
||||||
|
or raise MassDeletionGuard if that subset is implausibly large.
|
||||||
|
|
||||||
|
`existing_ids` MUST come from a successful check_orders_exist call — that
|
||||||
|
function now raises on Oracle error rather than returning a partial set, so
|
||||||
|
an empty result here means ROA genuinely has none of these orders.
|
||||||
|
"""
|
||||||
|
missing = [o for o in all_imported if o["id_comanda"] not in existing_ids]
|
||||||
|
total = len(all_imported)
|
||||||
|
if total >= MASS_DELETION_ABORT_MIN and len(missing) > total * MASS_DELETION_ABORT_FRACTION:
|
||||||
|
raise MassDeletionGuard(
|
||||||
|
f"{len(missing)}/{total} comenzi par sterse din ROA "
|
||||||
|
f"(>{int(MASS_DELETION_ABORT_FRACTION * 100)}%) — posibil ROA "
|
||||||
|
f"indisponibil/in recuperare; marcarea DELETED_IN_ROA a fost ANULATA"
|
||||||
|
)
|
||||||
|
return missing
|
||||||
|
|
||||||
|
|
||||||
def check_invoices_for_orders(id_comanda_list: list) -> dict:
|
def check_invoices_for_orders(id_comanda_list: list) -> dict:
|
||||||
"""Check which orders have been invoiced in Oracle (vanzari table).
|
"""Check which orders have been invoiced in Oracle (vanzari table).
|
||||||
@@ -68,7 +101,11 @@ def check_orders_exist(id_comanda_list: list) -> set:
|
|||||||
for row in cur:
|
for row in cur:
|
||||||
existing.add(row[0])
|
existing.add(row[0])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
# Do NOT swallow: a partial/empty result on error would be misread by
|
||||||
|
# callers as "these orders were deleted in ROA" and trigger sticky
|
||||||
|
# DELETED_IN_ROA marking. Propagate so the caller skips deletion.
|
||||||
logger.warning(f"Order existence check failed: {e}")
|
logger.warning(f"Order existence check failed: {e}")
|
||||||
|
raise
|
||||||
finally:
|
finally:
|
||||||
database.pool.release(conn)
|
database.pool.release(conn)
|
||||||
|
|
||||||
|
|||||||
@@ -1081,10 +1081,15 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
|||||||
existing_ids = await asyncio.to_thread(
|
existing_ids = await asyncio.to_thread(
|
||||||
invoice_service.check_orders_exist, id_comanda_list
|
invoice_service.check_orders_exist, id_comanda_list
|
||||||
)
|
)
|
||||||
for o in all_imported:
|
try:
|
||||||
if o["id_comanda"] not in existing_ids:
|
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
|
||||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
except invoice_service.MassDeletionGuard as g:
|
||||||
orders_deleted += 1
|
_log_line(run_id, f"⚠ Protectie stergeri: {g}")
|
||||||
|
await _record_phase_err(run_id, "mass_deletion_guard", g)
|
||||||
|
to_delete = []
|
||||||
|
for o in to_delete:
|
||||||
|
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||||
|
orders_deleted += 1
|
||||||
|
|
||||||
if invoices_updated:
|
if invoices_updated:
|
||||||
_log_line(run_id, f"Facturi noi: {invoices_updated} comenzi facturate")
|
_log_line(run_id, f"Facturi noi: {invoices_updated} comenzi facturate")
|
||||||
|
|||||||
Reference in New Issue
Block a user