roa2web-service-auto/backend/modules/data_entry/services/ocr/validation.py

"""
OCR Data Validation Module

Provides multi-layer validation for OCR extraction results to prevent
incorrect data from entering the system.

Validation Layers:
1. Absolute sanity checks (value ranges)
2. Cross-field validation (correlation between fields)
3. Inter-OCR consistency (compare multiple OCR results)
4. Auto-correction (fix obvious errors)

Usage:
    engine = OCRValidationEngine()
    validated_result = engine.validate_extraction(
        merged_result,
        light_ocr_result,
        medium_ocr_result
    )
"""

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional


@dataclass
class ValidationResult:
    """Result of a single validation rule execution.

    Attributes:
        is_valid: Whether the validation passed
        confidence_penalty: Penalty to apply to confidence score (0.0-1.0)
                          0.0 = no penalty, 1.0 = complete rejection
        message: Human-readable description of validation result
        severity: "info" | "warning" | "error"
    """
    is_valid: bool
    confidence_penalty: float = 0.0
    message: str = ""
    severity: str = "info"  # "info" | "warning" | "error"

    def __post_init__(self):
        """Validate penalty is in valid range."""
        if not 0.0 <= self.confidence_penalty <= 1.0:
            raise ValueError(f"Confidence penalty must be 0.0-1.0, got {self.confidence_penalty}")


class ValidationRule(ABC):
    """Abstract base class for OCR validation rules.

    Each rule implements a specific validation check and returns
    a ValidationResult indicating success/failure with optional
    confidence penalty.
    """

    @abstractmethod
    def validate(self, data: dict[str, Any]) -> ValidationResult:
        """Execute validation rule on extraction data.

        Args:
            data: Dictionary containing extraction fields to validate
                  Example: {"amount": 85.99, "tva": 14.92, ...}

        Returns:
            ValidationResult with is_valid flag and optional penalty
        """
        pass

    @property
    @abstractmethod
    def rule_name(self) -> str:
        """Human-readable name of this validation rule."""
        pass


# ============================================================================
# VALIDATION RULES
# ============================================================================


class AmountRangeRule(ValidationRule):
    """Validate amount is within reasonable bounds for Romanian receipts.

    Romanian receipts rarely exceed 100,000 RON. This catches obvious
    OCR errors like digit concatenation (85.99 → 859,762.16).

    Example:
        rule = AmountRangeRule(min_amount=0.01, max_amount=100_000.0)
        result = rule.validate({"amount": 859762.16})
        # result.is_valid = False, penalty = 0.5
    """

    def __init__(self, min_amount: float = 0.01, max_amount: float = 100_000.0):
        self.min_amount = min_amount
        self.max_amount = max_amount

    @property
    def rule_name(self) -> str:
        return "Amount Range Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        amount = data.get("amount")

        if amount is None:
            return ValidationResult(
                is_valid=True,
                message="No amount to validate"
            )

        if amount < self.min_amount:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.5,
                message=f"Amount {amount:.2f} RON below minimum {self.min_amount:.2f} RON",
                severity="error"
            )

        if amount > self.max_amount:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.5,
                message=f"Amount {amount:.2f} RON exceeds maximum {self.max_amount:.2f} RON (likely OCR error)",
                severity="error"
            )

        return ValidationResult(
            is_valid=True,
            message=f"Amount {amount:.2f} RON within valid range"
        )


class TVARatioRule(ValidationRule):
    """Validate TVA is reasonable percentage of TOTAL amount.

    Romanian TVA rates: 5%, 9%, 19%, 21% (most common: 19-21%)
    This catches errors where TVA > TOTAL (impossible).

    Example:
        rule = TVARatioRule(min_ratio=0.05, max_ratio=0.24)
        result = rule.validate({"amount": 85.99, "tva": 149.21})
        # result.is_valid = False (149.21 > 85.99!)
    """

    def __init__(self, min_ratio: float = 0.05, max_ratio: float = 0.24):
        self.min_ratio = min_ratio
        self.max_ratio = max_ratio

    @property
    def rule_name(self) -> str:
        return "TVA Ratio Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        amount = data.get("amount")
        tva = data.get("tva")

        if not amount or not tva:
            return ValidationResult(
                is_valid=True,
                message="Insufficient data for TVA correlation"
            )

        # Type safety: ensure numeric types before division
        if not isinstance(amount, (int, float)) or not isinstance(tva, (int, float)):
            return ValidationResult(
                is_valid=True,
                message="Non-numeric values, skipping TVA correlation"
            )

        # Avoid division by zero
        if amount <= 0:
            return ValidationResult(
                is_valid=True,
                message="Amount is zero or negative, skipping TVA ratio"
            )

        tva_ratio = tva / amount

        if tva_ratio < self.min_ratio or tva_ratio > self.max_ratio:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.3,
                message=f"TVA ratio {tva_ratio:.1%} outside valid range ({self.min_ratio:.0%}-{self.max_ratio:.0%})",
                severity="warning"
            )

        return ValidationResult(
            is_valid=True,
            message=f"TVA ratio {tva_ratio:.1%} valid"
        )


class PaymentSumRule(ValidationRule):
    """Validate CARD + NUMERAR = TOTAL BON (within tolerance).

    This is a CRITICAL validation that catches cases where OCR extracts
    wrong TOTAL but correct payment methods.

    Example:
        rule = PaymentSumRule(tolerance=0.02)
        result = rule.validate({
            "amount": 859762.16,  # Wrong from OCR
            "card_amount": 85.99,  # Correct
            "cash_amount": 0.0
        })
        # result.is_valid = False, suggests auto-correction
    """

    def __init__(self, tolerance: float = 0.02):
        self.tolerance = tolerance

    @property
    def rule_name(self) -> str:
        return "Payment Sum Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        total = data.get("amount")
        card = data.get("card_amount", 0.0) or 0.0
        cash = data.get("cash_amount", 0.0) or 0.0

        if not total:
            return ValidationResult(
                is_valid=True,
                message="No total amount to validate"
            )

        payment_sum = card + cash

        if payment_sum == 0:
            return ValidationResult(
                is_valid=True,
                message="No payment methods extracted"
            )

        diff = abs(total - payment_sum)

        if diff > self.tolerance:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.4,
                message=f"Payment sum {payment_sum:.2f} RON ≠ Total {total:.2f} RON (diff: {diff:.2f} RON). Consider auto-correction.",
                severity="error"
            )

        return ValidationResult(
            is_valid=True,
            message=f"Payment sum matches total (diff: {diff:.2f} RON)"
        )


class TVAEntriesSumRule(ValidationRule):
    """Validate Σ(TVA entries) = TVA TOTAL (within tolerance).

    TVA breakdown (A, B, C, D rates) should sum to total TVA.

    Example:
        rule = TVAEntriesSumRule(tolerance=0.02)
        result = rule.validate({
            "tva": 14.92,
            "tva_entries": {"A": 14.92, "B": 0.0}
        })
        # result.is_valid = True
    """

    def __init__(self, tolerance: float = 0.02):
        self.tolerance = tolerance

    @property
    def rule_name(self) -> str:
        return "TVA Entries Sum Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        tva_total = data.get("tva")
        tva_entries = data.get("tva_entries", {})

        if not tva_total:
            return ValidationResult(
                is_valid=True,
                message="No TVA total to validate"
            )

        if not tva_entries:
            return ValidationResult(
                is_valid=True,
                message="No TVA entries extracted"
            )

        entries_sum = sum(tva_entries.values())

        if entries_sum == 0:
            return ValidationResult(
                is_valid=True,
                message="TVA entries sum is zero"
            )

        diff = abs(tva_total - entries_sum)

        if diff > self.tolerance:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.2,
                message=f"TVA entries sum {entries_sum:.2f} RON ≠ TVA total {tva_total:.2f} RON (diff: {diff:.2f} RON)",
                severity="warning"
            )

        return ValidationResult(
            is_valid=True,
            message=f"TVA entries sum matches total (diff: {diff:.2f} RON)"
        )


class CUIFormatRule(ValidationRule):
    """Validate CUI format: RO + 6-10 digits.

    Romanian CUI (Cod Unic de Identificare) format:
    - Optional "RO" prefix (or "R0" from OCR errors)
    - 6-10 numeric digits

    Example:
        rule = CUIFormatRule()
        result = rule.validate({"cui": "RO10562600"})
        # result.is_valid = True
    """

    @property
    def rule_name(self) -> str:
        return "CUI Format Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        cui = data.get("cui")

        if not cui:
            return ValidationResult(
                is_valid=True,
                message="No CUI to validate"
            )

        # Normalize: remove RO/R0 prefix
        cui_clean = cui.strip().upper()
        if cui_clean.startswith("RO"):
            cui_clean = cui_clean[2:]
        elif cui_clean.startswith("R0"):
            cui_clean = cui_clean[2:]

        # Check if numeric
        if not cui_clean.isdigit():
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.3,
                message=f"CUI '{cui}' contains non-numeric characters",
                severity="warning"
            )

        # Check length
        if len(cui_clean) < 6 or len(cui_clean) > 10:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.3,
                message=f"CUI '{cui}' length {len(cui_clean)} outside valid range (6-10 digits)",
                severity="warning"
            )

        return ValidationResult(
            is_valid=True,
            message=f"CUI '{cui}' format valid"
        )


class CUIChecksumRule(ValidationRule):
    """Validate Romanian CIF/CUI using Mod 11 checksum algorithm.

    Algorithm:
    1. Remove RO prefix if present
    2. Extract last digit as declared checksum
    3. Apply multipliers [7,5,3,2,1,7,5,3,2] to first N-1 digits
    4. Calculate: (sum * 10) mod 11
    5. If result = 10, expected checksum = 0
    6. Else, expected checksum = result
    7. Compare with declared checksum

    Example:
        rule = CUIChecksumRule()
        result = rule.validate({"cui": "RO10562600"})
        # result.is_valid = True (checksum correct)

        result = rule.validate({"cui": "R01879855"})
        # result.is_valid = False (checksum mismatch)
    """

    @property
    def rule_name(self) -> str:
        return "CUI Checksum Check (Mod 11)"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        cui = data.get("cui")

        if not cui:
            return ValidationResult(
                is_valid=True,
                message="No CUI to validate"
            )

        # Normalize: remove RO/R0 prefix
        cui_clean = cui.strip().upper()
        if cui_clean.startswith("RO"):
            cui_clean = cui_clean[2:]
        elif cui_clean.startswith("R0"):
            cui_clean = cui_clean[2:]

        # Check format first
        if not cui_clean.isdigit():
            return ValidationResult(
                is_valid=True,  # Don't fail checksum if format invalid (handled by CUIFormatRule)
                message="CUI format invalid, skipping checksum"
            )

        if len(cui_clean) < 6 or len(cui_clean) > 10:
            return ValidationResult(
                is_valid=True,
                message="CUI length invalid, skipping checksum"
            )

        # Extract digits
        digits = [int(d) for d in cui_clean]
        checksum_declared = digits[-1]
        base_digits = digits[:-1]

        # Multipliers (trim to match base_digits length)
        multipliers = [7, 5, 3, 2, 1, 7, 5, 3, 2]
        multipliers = multipliers[:len(base_digits)]

        # Calculate weighted sum
        weighted_sum = sum(d * m for d, m in zip(base_digits, multipliers))

        # Calculate expected checksum
        checksum_calculated = (weighted_sum * 10) % 11
        if checksum_calculated == 10:
            checksum_calculated = 0

        if checksum_calculated != checksum_declared:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.3,
                message=f"CUI '{cui}' checksum mismatch: expected {checksum_calculated}, got {checksum_declared}",
                severity="warning"
            )

        return ValidationResult(
            is_valid=True,
            message=f"CUI '{cui}' checksum valid"
        )


class InterOCRConsistencyRule(ValidationRule):
    """Validate consistency between multiple OCR results.

    If Light OCR and Medium OCR produce values that differ by >10x,
    one is clearly wrong (likely digit concatenation error).

    Example:
        rule = InterOCRConsistencyRule(max_ratio=10.0)
        result = rule.validate({
            "light_amount": 85.99,
            "medium_amount": 859762.16
        })
        # result.is_valid = False (ratio = 10,000x!)
    """

    def __init__(self, max_ratio: float = 10.0):
        self.max_ratio = max_ratio

    @property
    def rule_name(self) -> str:
        return "Inter-OCR Consistency Check"

    def validate(self, data: dict[str, Any]) -> ValidationResult:
        light_value = data.get("light_value")
        medium_value = data.get("medium_value")
        field_name = data.get("field_name", "value")

        if not light_value or not medium_value:
            return ValidationResult(
                is_valid=True,
                message="Insufficient OCR results for consistency check"
            )

        # Avoid division by zero
        if light_value == 0 or medium_value == 0:
            return ValidationResult(
                is_valid=True,
                message="One value is zero, skipping consistency check"
            )

        ratio = max(light_value, medium_value) / min(light_value, medium_value)

        if ratio > self.max_ratio:
            return ValidationResult(
                is_valid=False,
                confidence_penalty=0.2,
                message=f"{field_name}: OCR results differ by {ratio:.1f}x (Light: {light_value}, Medium: {medium_value})",
                severity="warning"
            )

        return ValidationResult(
            is_valid=True,
            message=f"{field_name}: OCR results consistent (ratio: {ratio:.2f}x)"
        )


# ============================================================================
# VALIDATION ENGINE
# ============================================================================


@dataclass
class EnhancedExtractionResult:
    """Enhanced extraction result with validation metadata.

    This wraps the original extraction data and adds validation results.
    """
    # Original data
    data: dict[str, Any]

    # Validation results
    needs_manual_review: bool = False
    validation_warnings: list[str] = field(default_factory=list)
    validation_errors: list[str] = field(default_factory=list)
    confidence_adjustments: dict[str, float] = field(default_factory=dict)

    # Inter-OCR metadata
    inter_ocr_ratios: dict[str, float] = field(default_factory=dict)


class OCRValidationEngine:
    """Orchestrate all validation rules for OCR extraction results.

    This engine applies validation rules in order:
    1. Sanity checks (amount range, format checks)
    2. Cross-field correlation (TVA ratio, payment sum)
    3. Inter-OCR consistency checks

    Example:
        engine = OCRValidationEngine()
        result = engine.validate_extraction(
            extraction_result=merged_data,
            light_result=light_ocr_data,
            medium_result=medium_ocr_data
        )
    """

    def __init__(self):
        """Initialize validation engine with default rules."""
        # Sanity check rules (absolute value validation)
        self.sanity_rules = [
            AmountRangeRule(min_amount=0.01, max_amount=100_000.0),
            CUIFormatRule(),
            CUIChecksumRule(),
        ]

        # Cross-field validation rules (correlation between fields)
        self.cross_field_rules = [
            TVARatioRule(min_ratio=0.05, max_ratio=0.24),
            PaymentSumRule(tolerance=0.02),
            TVAEntriesSumRule(tolerance=0.02),
        ]

        # Inter-OCR consistency rules
        self.inter_ocr_rules = [
            InterOCRConsistencyRule(max_ratio=10.0),
        ]

    def validate_extraction(
        self,
        extraction_result: dict[str, Any],
        light_result: Optional[dict[str, Any]] = None,
        medium_result: Optional[dict[str, Any]] = None
    ) -> EnhancedExtractionResult:
        """Run all validation rules and return enhanced result.

        Args:
            extraction_result: Merged OCR extraction data (required)
            light_result: Light OCR preprocessing results (optional)
            medium_result: Medium OCR preprocessing results (optional)

        Returns:
            EnhancedExtractionResult with validation warnings and metadata
        """
        warnings = []
        errors = []
        confidence_adjustments = {}
        inter_ocr_ratios = {}

        # Step 1: Sanity checks
        print("\n[Validation] Step 1: Sanity checks...", flush=True)
        for rule in self.sanity_rules:
            result = rule.validate(extraction_result)

            if not result.is_valid:
                msg = f"[{rule.rule_name}] {result.message}"

                if result.severity == "error":
                    errors.append(msg)
                else:
                    warnings.append(msg)

                print(f"  ❌ {msg}", flush=True)

                # Track confidence penalty for the relevant field based on rule
                if result.confidence_penalty > 0:
                    rule_field_map = {
                        "Amount Range Check": ["amount"],
                        "CUI Format Check": ["cui"],
                        "CUI Checksum Check (Mod 11)": ["cui"],
                    }
                    fields = rule_field_map.get(rule.rule_name, ["amount", "tva", "cui"])
                    for f in fields:
                        if f in extraction_result:
                            confidence_adjustments[f] = result.confidence_penalty
            else:
                print(f"  ✅ {rule.rule_name}: {result.message}", flush=True)

        # Step 2: Cross-field validation
        print("\n[Validation] Step 2: Cross-field validation...", flush=True)
        for rule in self.cross_field_rules:
            result = rule.validate(extraction_result)

            if not result.is_valid:
                msg = f"[{rule.rule_name}] {result.message}"

                if result.severity == "error":
                    errors.append(msg)
                else:
                    warnings.append(msg)

                print(f"  ❌ {msg}", flush=True)

                # Track confidence penalty for the relevant field based on rule
                if result.confidence_penalty > 0:
                    rule_field_map = {
                        "TVA Ratio Check": ["tva"],
                        "Payment Sum Check": ["amount"],
                        "TVA Entries Sum Check": ["tva"],
                    }
                    fields = rule_field_map.get(rule.rule_name, ["amount", "tva"])
                    for f in fields:
                        if f in extraction_result:
                            confidence_adjustments[f] = result.confidence_penalty
            else:
                print(f"  ✅ {rule.rule_name}: {result.message}", flush=True)

        # Step 3: Inter-OCR consistency checks
        if light_result and medium_result:
            print("\n[Validation] Step 3: Inter-OCR consistency...", flush=True)

            # Check amount consistency
            if "amount" in light_result and "amount" in medium_result:
                consistency_data = {
                    "light_value": light_result["amount"],
                    "medium_value": medium_result["amount"],
                    "field_name": "amount"
                }

                result = self.inter_ocr_rules[0].validate(consistency_data)

                if not result.is_valid:
                    msg = f"[Inter-OCR] {result.message}"
                    warnings.append(msg)
                    print(f"  ❌ {msg}", flush=True)

                    # Store ratio for metadata
                    ratio = max(
                        light_result["amount"],
                        medium_result["amount"]
                    ) / min(light_result["amount"], medium_result["amount"])
                    inter_ocr_ratios["amount"] = ratio
                else:
                    print(f"  ✅ {result.message}", flush=True)

        # Determine if manual review is needed
        # Only flag for review if there are errors OR high-severity warnings
        high_severity_warnings = [w for w in warnings if "[Amount Range" in w or "[Payment Sum" in w or "[Inter-OCR]" in w]
        needs_manual_review = (
            len(errors) > 0 or
            len(high_severity_warnings) > 0 or
            any(ratio > 10.0 for ratio in inter_ocr_ratios.values())
        )

        print(f"\n[Validation] Summary:", flush=True)
        print(f"  Errors: {len(errors)}", flush=True)
        print(f"  Warnings: {len(warnings)}", flush=True)
        print(f"  Manual review needed: {needs_manual_review}", flush=True)

        return EnhancedExtractionResult(
            data=extraction_result,
            needs_manual_review=needs_manual_review,
            validation_warnings=warnings,
            validation_errors=errors,
            confidence_adjustments=confidence_adjustments,
            inter_ocr_ratios=inter_ocr_ratios
        )

    @staticmethod
    def normalize_cui(cui: Optional[str]) -> Optional[str]:
        """Normalize CUI to RO prefix + digits format.

        Examples:
            10562600 → RO10562600
            R010562600 → RO10562600 (fix R0 OCR error)
            RO10562600 → RO10562600 (unchanged)

        Args:
            cui: Raw CUI string from OCR

        Returns:
            Normalized CUI with RO prefix, or None if invalid
        """
        if not cui:
            return None

        cui = cui.strip().upper()

        # Remove existing prefix if present
        if cui.startswith("RO"):
            cui = cui[2:]
        elif cui.startswith("R0"):
            cui = cui[2:]

        # Remove any non-digit characters
        cui_digits = ''.join(c for c in cui if c.isdigit())

        # Validate length
        if len(cui_digits) < 6 or len(cui_digits) > 10:
            print(f"[CUI Normalize] Invalid length: {len(cui_digits)} digits (expected 6-10)", flush=True)
            return None

        # Add RO prefix
        return f"RO{cui_digits}"