fix(sync): guard against mass DELETED_IN_ROA when ROA is recovering

After a power loss + reboot, COMENZI was queryable but not yet recovered;
phase 4b-3 read it as empty and sticky-marked 3794 live orders DELETED_IN_ROA
(nulling id_comanda). check_orders_exist also swallowed Oracle errors and
returned a partial set, which callers misread as deletions.

- check_orders_exist now re-raises on Oracle error instead of returning partial
- new invoice_service.deletions_or_guard() raises MassDeletionGuard when the
  would-delete fraction is implausibly high (>30% of >=25 imported orders)
- both deletion sites (auto sync + manual refresh) skip + log on guard trip

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Agent
2026-06-26 07:18:08 +00:00
parent 4a03fe1016
commit 395e2b997a
3 changed files with 54 additions and 8 deletions

View File

@@ -846,8 +846,12 @@ async def refresh_invoices():
existing_ids = await asyncio.to_thread(
invoice_service.check_orders_exist, id_comanda_list
)
for o in all_imported:
if o["id_comanda"] not in existing_ids:
try:
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
except invoice_service.MassDeletionGuard as g:
logger.warning(f"Mass-deletion guard tripped during refresh: {g}")
to_delete = []
for o in to_delete:
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
orders_deleted += 1

View File

@@ -3,6 +3,39 @@ from .. import database
logger = logging.getLogger(__name__)
# ── Mass-deletion safety guard ──────────────────────────────────────────────
# If ROA appears to have lost a large fraction of its orders, it is almost
# certainly a transient/recovery state (e.g. the DB just restarted after a power
# loss and COMENZI hasn't finished recovering), NOT real deletions. In that case
# we refuse to mass-mark orders as DELETED_IN_ROA — a sticky, hard-to-reverse
# operation that nulls id_comanda. See incident 2026-06-26 (3794 false deletes).
MASS_DELETION_ABORT_FRACTION = 0.30
MASS_DELETION_ABORT_MIN = 25
class MassDeletionGuard(Exception):
"""Raised when the number of orders that would be marked deleted is
suspiciously high, indicating ROA is unavailable rather than truly purged."""
def deletions_or_guard(all_imported: list, existing_ids: set) -> list:
"""Return the subset of all_imported whose id_comanda is missing from ROA,
or raise MassDeletionGuard if that subset is implausibly large.
`existing_ids` MUST come from a successful check_orders_exist call — that
function now raises on Oracle error rather than returning a partial set, so
an empty result here means ROA genuinely has none of these orders.
"""
missing = [o for o in all_imported if o["id_comanda"] not in existing_ids]
total = len(all_imported)
if total >= MASS_DELETION_ABORT_MIN and len(missing) > total * MASS_DELETION_ABORT_FRACTION:
raise MassDeletionGuard(
f"{len(missing)}/{total} comenzi par sterse din ROA "
f"(>{int(MASS_DELETION_ABORT_FRACTION * 100)}%) — posibil ROA "
f"indisponibil/in recuperare; marcarea DELETED_IN_ROA a fost ANULATA"
)
return missing
def check_invoices_for_orders(id_comanda_list: list) -> dict:
"""Check which orders have been invoiced in Oracle (vanzari table).
@@ -68,7 +101,11 @@ def check_orders_exist(id_comanda_list: list) -> set:
for row in cur:
existing.add(row[0])
except Exception as e:
# Do NOT swallow: a partial/empty result on error would be misread by
# callers as "these orders were deleted in ROA" and trigger sticky
# DELETED_IN_ROA marking. Propagate so the caller skips deletion.
logger.warning(f"Order existence check failed: {e}")
raise
finally:
database.pool.release(conn)

View File

@@ -1081,8 +1081,13 @@ async def run_sync(id_pol: int = None, id_sectie: int = None, run_id: str = None
existing_ids = await asyncio.to_thread(
invoice_service.check_orders_exist, id_comanda_list
)
for o in all_imported:
if o["id_comanda"] not in existing_ids:
try:
to_delete = invoice_service.deletions_or_guard(all_imported, existing_ids)
except invoice_service.MassDeletionGuard as g:
_log_line(run_id, f"⚠ Protectie stergeri: {g}")
await _record_phase_err(run_id, "mass_deletion_guard", g)
to_delete = []
for o in to_delete:
await sqlite_service.mark_order_deleted_in_roa(o["order_number"])
orders_deleted += 1