fix(sync): guard against mass DELETED_IN_ROA when ROA is recovering
After a power loss + reboot, COMENZI was queryable but not yet recovered; phase 4b-3 read it as empty and sticky-marked 3794 live orders DELETED_IN_ROA (nulling id_comanda). check_orders_exist also swallowed Oracle errors and returned a partial set, which callers misread as deletions. - check_orders_exist now re-raises on Oracle error instead of returning partial - new invoice_service.deletions_or_guard() raises MassDeletionGuard when the would-delete fraction is implausibly high (>30% of >=25 imported orders) - both deletion sites (auto sync + manual refresh) skip + log on guard trip Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -846,10 +846,14 @@ async def refresh_invoices():
|
||||
existing_ids = await asyncio.to_thread(
|
||||
invoice_service.check_orders_exist, id_comanda_list
|
||||
)
|
||||
for o in all_imported:
|
||||
if o["id_comanda"] not in existing_ids:
|
||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||
orders_deleted += 1
|
||||
try:
|
||||
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
|
||||
except invoice_service.MassDeletionGuard as g:
|
||||
logger.warning(f"Mass-deletion guard tripped during refresh: {g}")
|
||||
to_delete = []
|
||||
for o in to_delete:
|
||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||
orders_deleted += 1
|
||||
|
||||
# Cherry-pick A: Batch refresh Oracle addresses for all orders with stored address IDs
|
||||
addr_rows = await sqlite_service.get_orders_with_address_ids()
|
||||
|
||||
@@ -3,6 +3,39 @@ from .. import database
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Mass-deletion safety guard ──────────────────────────────────────────────
|
||||
# If ROA appears to have lost a large fraction of its orders, it is almost
|
||||
# certainly a transient/recovery state (e.g. the DB just restarted after a power
|
||||
# loss and COMENZI hasn't finished recovering), NOT real deletions. In that case
|
||||
# we refuse to mass-mark orders as DELETED_IN_ROA — a sticky, hard-to-reverse
|
||||
# operation that nulls id_comanda. See incident 2026-06-26 (3794 false deletes).
|
||||
MASS_DELETION_ABORT_FRACTION = 0.30
|
||||
MASS_DELETION_ABORT_MIN = 25
|
||||
|
||||
|
||||
class MassDeletionGuard(Exception):
|
||||
"""Raised when the number of orders that would be marked deleted is
|
||||
suspiciously high, indicating ROA is unavailable rather than truly purged."""
|
||||
|
||||
|
||||
def deletions_or_guard(all_imported: list, existing_ids: set) -> list:
|
||||
"""Return the subset of all_imported whose id_comanda is missing from ROA,
|
||||
or raise MassDeletionGuard if that subset is implausibly large.
|
||||
|
||||
`existing_ids` MUST come from a successful check_orders_exist call — that
|
||||
function now raises on Oracle error rather than returning a partial set, so
|
||||
an empty result here means ROA genuinely has none of these orders.
|
||||
"""
|
||||
missing = [o for o in all_imported if o["id_comanda"] not in existing_ids]
|
||||
total = len(all_imported)
|
||||
if total >= MASS_DELETION_ABORT_MIN and len(missing) > total * MASS_DELETION_ABORT_FRACTION:
|
||||
raise MassDeletionGuard(
|
||||
f"{len(missing)}/{total} comenzi par sterse din ROA "
|
||||
f"(>{int(MASS_DELETION_ABORT_FRACTION * 100)}%) — posibil ROA "
|
||||
f"indisponibil/in recuperare; marcarea DELETED_IN_ROA a fost ANULATA"
|
||||
)
|
||||
return missing
|
||||
|
||||
|
||||
def check_invoices_for_orders(id_comanda_list: list) -> dict:
|
||||
"""Check which orders have been invoiced in Oracle (vanzari table).
|
||||
@@ -68,7 +101,11 @@ def check_orders_exist(id_comanda_list: list) -> set:
|
||||
for row in cur:
|
||||
existing.add(row[0])
|
||||
except Exception as e:
|
||||
# Do NOT swallow: a partial/empty result on error would be misread by
|
||||
# callers as "these orders were deleted in ROA" and trigger sticky
|
||||
# DELETED_IN_ROA marking. Propagate so the caller skips deletion.
|
||||
logger.warning(f"Order existence check failed: {e}")
|
||||
raise
|
||||
finally:
|
||||
database.pool.release(conn)
|
||||
|
||||
|
||||
@@ -1081,10 +1081,15 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
|
||||
existing_ids = await asyncio.to_thread(
|
||||
invoice_service.check_orders_exist, id_comanda_list
|
||||
)
|
||||
for o in all_imported:
|
||||
if o["id_comanda"] not in existing_ids:
|
||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||
orders_deleted += 1
|
||||
try:
|
||||
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
|
||||
except invoice_service.MassDeletionGuard as g:
|
||||
_log_line(run_id, f"⚠ Protectie stergeri: {g}")
|
||||
await _record_phase_err(run_id, "mass_deletion_guard", g)
|
||||
to_delete = []
|
||||
for o in to_delete:
|
||||
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
|
||||
orders_deleted += 1
|
||||
|
||||
if invoices_updated:
|
||||
_log_line(run_id, f"Facturi noi: {invoices_updated} comenzi facturate")
|
||||
|
||||
Reference in New Issue
Block a user