"""Extract structured fields from OCR text (Romanian receipts).""" import re from datetime import date, datetime from decimal import Decimal, InvalidOperation from typing import Optional, Tuple, List from dataclasses import dataclass, field from backend.modules.data_entry.services.ocr.validation import OCRValidationEngine @dataclass class ExtractionResult: """Structured extraction result from receipt.""" receipt_type: str = 'bon_fiscal' receipt_number: Optional[str] = None receipt_series: Optional[str] = None receipt_date: Optional[date] = None amount: Optional[Decimal] = None partner_name: Optional[str] = None cui: Optional[str] = None description: Optional[str] = None # Additional extracted fields - Multiple TVA entries support tva_entries: List[dict] = field(default_factory=list) # [{code, percent, amount}] tva_total: Optional[Decimal] = None address: Optional[str] = None items_count: Optional[int] = None payment_methods: List[dict] = field(default_factory=list) # [{"method":"CARD","amount":Decimal}] suggested_payment_mode: Optional[str] = None # 'banca' if CARD detected, 'numerar' if cash only # Client data (for B2B receipts - buyer information) client_name: Optional[str] = None client_cui: Optional[str] = None client_address: Optional[str] = None confidence_amount: float = 0.0 confidence_date: float = 0.0 confidence_vendor: float = 0.0 confidence_client: float = 0.0 raw_text: str = "" ocr_engine: str = "" # OCR engine used: paddleocr or tesseract processing_time_ms: int = 0 # Processing time in milliseconds # Validation tracking (added by bon-ocr-validation feature) needs_manual_review: Optional[bool] = None # None=not validated, False=ok, True=needs review validation_warnings: List[str] = field(default_factory=list) validation_errors: List[str] = field(default_factory=list) confidence_adjustments: dict[str, float] = field(default_factory=dict) # Field -> penalty inter_ocr_ratios: dict[str, float] = field(default_factory=dict) # Field -> ratio @property def overall_confidence(self) -> float: """Calculate weighted overall confidence score.""" weights = {'amount': 0.4, 'date': 0.3, 'vendor': 0.3} return round( self.confidence_amount * weights['amount'] + self.confidence_date * weights['date'] + self.confidence_vendor * weights['vendor'], 2 ) class ReceiptExtractor: """Extract receipt fields using pattern matching for Romanian receipts.""" # Total amount patterns (most specific first) # Romanian receipts use various formats: TOTAL LEI, TOTAL:, TOTAL RON, etc. # OCR often produces errors, so patterns must be tolerant TOTAL_PATTERNS = [ # Most common: TOTAL LEI followed by amount (with OCR-tolerant variations) # Handles: TOTAL LEI, TOTAL. LE!, T0TAL LEI, TOTAL LE1, etc. (r'T[O0]TAL[.\s]+L[E3][I1!]\s*:?\s*([\d\s.,]+)', 0.98), # OCR-tolerant: TOTAL. LE!, T0TAL LEI (r'TOTAL\s+LEI\s*([\d\s.,]+)', 0.98), # Standard clean pattern (r'[OT]?OTAL\s+LEI\s*([\d\s.,]+)', 0.95), # OCR may miss first letter # Standard patterns (r'TOTAL\s*:?\s*([\d\s.,]+)\s*(?:RON|LEI)?', 0.95), (r'TOTAL\s+(?:RON|LEI)\s*([\d\s.,]+)', 0.95), # SUBTOTAL when TOTAL not found (r'SUBTOTAL\s*([\d\s.,]+)', 0.90), (r'[SB]?UBTOTAL\s*([\d\s.,]+)', 0.88), # OCR variations # Payment methods (r'DE\s+PLATA\s*:?\s*([\d\s.,]+)', 0.90), (r'SUMA\s*:?\s*([\d\s.,]+)', 0.85), (r'PLATA\s+CARD\s*:?\s*([\d\s.,]+)', 0.85), (r'NUMERAR\s*:?\s*([\d\s.,]+)', 0.80), (r'REST\s*:?\s*([\d\s.,]+)', 0.70), # Sometimes total is near REST ] # Fallback: Find the largest repeated amount (likely the total) # This handles cases where OCR doesn't capture "TOTAL" keyword # Date patterns - support dash, dot, and slash separators # OCR may produce DRTA instead of DATA, DAIA, etc. # OCR may also add spaces/commas in dates: "27. 10, 2025" instead of "27.10.2025" DATE_PATTERNS = [ # DATA/DRTA/DAIA: DD-MM-YYYY (OCR tolerant) (r'D[AR]TA\s*:?\s*(\d{2}[-./]\d{2}[-./]\d{4})', 0.98), (r'DATA\s*:?\s*(\d{2}[-./]\d{2}[-./]\d{4})', 0.98), # Date followed by ORA (time) - OCR may produce 0RA (r'(\d{2}[-./]\d{2}[-./]\d{4})\s+[O0]RA\s*:?\s*\d{2}:\d{2}', 0.95), # Date followed by time without ORA keyword (r'(\d{2}[-./]\d{2}[-./]\d{4})\s+\d{2}:\d{2}', 0.90), # Standalone date (r'(\d{2}[-./]\d{2}[-./]\d{4})', 0.80), # YYYY-MM-DD format (less common) (r'(\d{4}[-./]\d{2}[-./]\d{2})', 0.75), ] # OCR-corrupted date patterns with spaces/commas # Handles: "27. 10, 2025", "27, 10. 2025", "2025. 08. 14", etc. DATE_PATTERNS_OCR_SPACES = [ # YYYY. MM. DD format with spaces (OMV/Petrom receipts) - with time (r'(\d{4})[.,]\s*(\d{2})[.,]\s*(\d{2})\s+\d{2}:\d{2}', 0.92, 'ymd'), # YYYY. MM. DD format with spaces (standalone) (r'(\d{4})[.,]\s*(\d{2})[.,]\s*(\d{2})', 0.85, 'ymd'), # DD. MM, YYYY or DD, MM. YYYY (with time following) (r'(\d{2})[.,]\s*(\d{2})[.,]\s*(\d{4})\s+\d{2}:\d{2}', 0.92, 'dmy'), # DD. MM, YYYY or DD, MM. YYYY (standalone) (r'(\d{2})[.,]\s*(\d{2})[.,]\s*(\d{4})', 0.85, 'dmy'), ] # Receipt number patterns - Romanian fiscal receipt formats # OCR may produce N instead of : or other errors NUMBER_PATTERNS = [ # NDS format (common in Romanian POS) (r'NDS\s*:?\s*(\d+)', 0.98), # C3POS terminal format - OCR may have N instead of : (C3POS-CT2N1360760) (r'C3POS[-A-Z0-9]*[N:](\d{6,7})', 0.98), # CT2N1360760 format (r'C3POS.*?(\d{6,7})\b', 0.95), # Any C3POS followed by 6-7 digit number (r'CT2[N:]\s*(\d{6,})', 0.95), # CT2N prefix # BF (Bon Fiscal) number - high priority # Format: "Z:0864 BF:0018" - extract only the number after BF: (r'BF\s*:\s*(\d{4,})', 0.96), # BF: with colon (most specific) (r'BF\s+(\d{4,})', 0.93), # BF followed by space and number # NIVS format (r'NIVS\s*:?\s*(\d+)', 0.95), # Standard NR BON formats (r'NR\.?\s*BON\s*:?\s*(\d+)', 0.95), (r'BON\s+(?:FISCAL\s+)?NR\.?\s*:?\s*(\d+)', 0.95), (r'CHITANTA\s+NR\.?\s*:?\s*(\d+)', 0.95), # Document number (r'NR\.?\s+DOCUMENT\s*:?\s*(\d+)', 0.90), # ID BF format (r'ID\s*BF\s*:?\s*(\d+)', 0.90), # TD format (transaction ID) (r'TD\s*:?\s*(\d+)', 0.85), # 6-8 digit number (typical receipt number length) (r'\b(\d{6,8})\b', 0.70), # Generic long number at end (fallback) (r'NR\.?\s*:?\s*(\d{4,})', 0.65), ] # CUI (fiscal code) patterns - IMPORTANT: exclude CLIENT CUI # CIF = Cod de Identificare Fiscală (vendor's tax ID) # CLIENT C.U.I. = client's tax ID (should be ignored) # OCR errors: R0 instead of RO, C1F instead of CIF CUI_PATTERNS = [ # CIF at start of line (definitely vendor) - tolerant to OCR errors # NOTE: Capture full CUI including RO prefix: (R[O0]?\d{6,10}) or ((?:R[O0])?\d{6,10}) (r'^CIF\s*:?\s*(R[O0]?\d{6,10})', 0.98), (r'^CIF\s*:?\s*(\d{6,10})', 0.97), # Without RO prefix (r'^C[I1]F\s*:?\s*(R[O0]?\d{6,10})', 0.95), # C1F OCR error (r'^C[I1]F\s*:?\s*(\d{6,10})', 0.94), # C1F without RO # CIF not preceded by CLIENT (negative lookbehind) (r'(? ExtractionResult: """Extract all fields from OCR text.""" result = ExtractionResult() result.raw_text = text text_upper = text.upper() # Extract core fields result.amount, result.confidence_amount = self._extract_amount(text_upper) result.receipt_date, result.confidence_date = self._extract_date(text_upper) result.receipt_number, _ = self._extract_number(text_upper) result.receipt_series, _ = self._extract_series(text_upper) result.partner_name, result.confidence_vendor = self._extract_vendor(text) result.cui, _ = self._extract_cui(text_upper, text) # Normalize CUI: fix R0 → RO OCR error and validate format result.cui = OCRValidationEngine.normalize_cui(result.cui) # Extract additional fields - Multiple TVA entries result.tva_entries, result.tva_total = self._extract_tva_entries(text_upper) if not result.tva_entries: print(f"[TVA Debug] No TVA found. Checking patterns...", flush=True) # Debug: show what patterns see normalized = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text_upper) taxe_match = re.search(r'T?OTAL\s+TAXE', normalized, re.IGNORECASE) rev_match = re.search(r'([\d.,]+)\s*T?OTAL\s+TAXE', normalized, re.IGNORECASE) print(f"[TVA Debug] 'OTAL TAXE' found: {bool(taxe_match)}, reversed: {rev_match.group(1) if rev_match else None}", flush=True) # Log TVA vs TOTAL for debugging (validation happens in ocr_service._final_validation) # NOTE: We NO LONGER clear TVA here - the service will recalculate TOTAL from TVA if needed if result.tva_total and result.amount: if result.tva_total > result.amount: print(f"[TVA Extraction] TVA ({result.tva_total}) > TOTAL ({result.amount}) - will be corrected in final validation", flush=True) elif result.tva_total > result.amount * Decimal('0.5'): print(f"[TVA Extraction] Warning: TVA ({result.tva_total}) is > 50% of TOTAL ({result.amount}) - suspicious", flush=True) result.items_count = self._extract_items_count(text_upper) result.address = self._extract_address(text_upper) result.payment_methods = self._extract_payment_methods(text_upper) # Validate payment methods against extracted amount # If payment sum >> amount, clear invalid payments (likely OCR error) # Save original payment methods before validation (for payment mode detection) original_payment_methods = result.payment_methods.copy() if result.payment_methods else [] result.payment_methods = self._validate_payment_methods(result.payment_methods, result.amount) # Auto-suggest payment_mode based on detected payment methods # Use ORIGINAL payment_methods to detect CARD even if validation cleared them # (e.g., CARD 318.16 is valid even if total validation failed) payment_methods_for_mode = result.payment_methods if result.payment_methods else original_payment_methods if payment_methods_for_mode: card_amount = sum( pm.get('amount', Decimal('0')) for pm in payment_methods_for_mode if pm.get('method') == 'CARD' ) if card_amount > 0: result.suggested_payment_mode = 'banca' print(f"[Payment Mode] CARD detected ({card_amount}), suggesting 'banca'", flush=True) else: # Only cash payments detected result.suggested_payment_mode = 'numerar' print(f"[Payment Mode] Cash only detected, suggesting 'numerar'", flush=True) # Extract client data (B2B receipts) client_name, client_cui, client_address, confidence_client = self._extract_client_data(text_upper, text) result.client_name = client_name result.client_cui = OCRValidationEngine.normalize_cui(client_cui) # Fix R0 → RO OCR error result.client_address = client_address result.confidence_client = confidence_client # Detect receipt type result.receipt_type = self._detect_receipt_type(text_upper) # Reverse TVA validation if result.tva_entries and result.amount: is_valid, expected_total, msg = self._validate_tva_reverse(result.tva_entries, result.amount) if not is_valid: print(f"[TVA Reverse Validation] {msg}", flush=True) # Cross-validate amount using payment methods and TVA validated_amount, validated_confidence, source = self._cross_validate_and_calculate_amount( result.amount, result.confidence_amount, result.payment_methods, result.tva_entries, result.tva_total ) if validated_amount != result.amount: print(f"[Cross-Validation] Amount updated: {result.amount} -> {validated_amount} (source: {source})", flush=True) result.amount = validated_amount result.confidence_amount = validated_confidence return result def _extract_amount(self, text: str) -> Tuple[Optional[Decimal], float]: """Extract total amount from text.""" # PRE-FILTER: Remove lines containing REST (rest = change, not total) # When paid by card, there's no change - exact amount is paid lines = text.split('\n') filtered_lines = [] for line in lines: # Skip lines with REST pattern (change amount, not total) if re.search(r'\bREST\b', line, re.IGNORECASE): continue filtered_lines.append(line) text = '\n'.join(filtered_lines) # First try standard patterns (TOTAL, SUBTOTAL, etc.) for pattern, confidence in self.TOTAL_PATTERNS: match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE) if match: try: # IMPORTANT: Call _normalize_number FIRST to handle "190 60" → "190.60" # before stripping other characters amount_str = match.group(1).strip() amount_str = self._normalize_number(amount_str) # Now remove any remaining non-numeric chars (except decimal point) amount_str = re.sub(r'[^\d.]', '', amount_str) amount = Decimal(amount_str) if amount > 0: return amount, confidence except (InvalidOperation, ValueError): continue # Strategy 2: Find amounts AFTER product lines end # Products have pattern: "X BUC/ROLA X price = price" # Total appears after all products product_pattern = r'\d\s+(?:BUC|ROLA|ROLN|ROL)\s+X' product_matches = list(re.finditer(product_pattern, text, re.IGNORECASE)) if product_matches: # Get text after the last product line last_product_pos = product_matches[-1].end() after_products = text[last_product_pos:] # Find standalone amounts on their own line after products line_amount_pattern = r'^[\s]*(\d{2,4}[.,]\s*\d{2})[\s]*$' standalone_amounts = [] for match in re.finditer(line_amount_pattern, after_products, re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(amount_str) amount = Decimal(amount_str) if amount > 10: # Filter out small values standalone_amounts.append(amount) except (InvalidOperation, ValueError): continue if standalone_amounts: # The largest standalone amount after products is likely the total max_amount = max(standalone_amounts) # Higher confidence if it appears multiple times count = standalone_amounts.count(max_amount) confidence = 0.85 if count >= 2 else 0.75 return max_amount, confidence # Strategy 3: Find the most repeated large amount # Normalize spaces in numbers (OCR may produce "186. 16") normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) amount_pattern = r'(\d{2,4}[.,]\d{2})\b' amounts = re.findall(amount_pattern, normalized_text) if amounts: from collections import Counter amount_counts = Counter(amounts) # Filter amounts that appear 2+ times and are > 20 candidates = [] for amt_str, count in amount_counts.items(): try: amt = Decimal(self._normalize_number(amt_str)) if count >= 2 and amt > 20: candidates.append((amt, count)) except (InvalidOperation, ValueError): continue if candidates: # Return the LARGEST amount that appears multiple times candidates.sort(key=lambda x: x[0], reverse=True) return candidates[0][0], 0.65 # Last resort: Find any standalone large amount line_amount_pattern = r'^[\s]*(\d{2,4}[.,]\s*\d{2})[\s]*$' for match in re.finditer(line_amount_pattern, text, re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(amount_str) amount = Decimal(amount_str) if amount > 50: # Higher threshold for fallback return amount, 0.50 except (InvalidOperation, ValueError): continue return None, 0.0 def _normalize_number(self, num_str: str) -> str: """Normalize Romanian number format to standard decimal.""" # OCR often reads "." as " " (space) - handle "190 60" as "190.60" # Pattern: digits + space + exactly 2 digits at end space_decimal_match = re.match(r'^(\d+)\s+(\d{2})$', num_str.strip()) if space_decimal_match: num_str = f"{space_decimal_match.group(1)}.{space_decimal_match.group(2)}" else: # Handle "1 234 56" pattern (thousands + decimal with spaces) # Match: digits + space(s) + digits + space + 2 digits multi_space_match = re.match(r'^([\d\s]+?)\s+(\d{2})$', num_str.strip()) if multi_space_match: integer_part = multi_space_match.group(1).replace(' ', '') decimal_part = multi_space_match.group(2) num_str = f"{integer_part}.{decimal_part}" else: # Remove remaining spaces (thousands separators) num_str = num_str.replace(' ', '') # Handle comma as decimal separator if ',' in num_str and '.' in num_str: # Romanian format: 1.234,56 num_str = num_str.replace('.', '').replace(',', '.') elif ',' in num_str: # Could be 1,50 or 1,234 parts = num_str.split(',') if len(parts) == 2 and len(parts[1]) <= 2: # Decimal comma: 1,50 num_str = num_str.replace(',', '.') else: # Thousands comma: 1,234 num_str = num_str.replace(',', '') elif '.' in num_str: parts = num_str.split('.') if len(parts) > 2: # Multiple dots: 1.234.567 -> 1234567 num_str = ''.join(parts[:-1]) + '.' + parts[-1] return num_str def _cross_validate_and_calculate_amount( self, amount: Optional[Decimal], confidence_amount: float, payment_methods: List[dict], tva_entries: List[dict], tva_total: Optional[Decimal] ) -> Tuple[Optional[Decimal], float, str]: """ Cross-validate and potentially calculate total from payment methods and TVA. Returns: (amount, confidence, source_description) Logic: 1. If amount is valid (>0) with high confidence (>=0.8), use it directly 2. Calculate payment_sum = CARD + NUMERAR + other methods 3. Calculate tva_implied_total = tva_total * (100 + rate) / rate 4. Cross-validate: if payment_sum matches extracted amount, boost confidence 5. If amount is 0/None, use payment_sum as total 6. If payment_sum is 0, try to calculate from TVA """ # Calculate payment methods sum payment_sum = Decimal('0') if payment_methods: for pm in payment_methods: try: pm_amount = pm.get('amount') if pm_amount: payment_sum += Decimal(str(pm_amount)) except (InvalidOperation, ValueError, TypeError): continue # Calculate TVA-implied total: total = tva * (100 + rate) / rate tva_implied_total = None if tva_entries: # Use the main TVA entry (typically the largest or first one) main_entry = tva_entries[0] rate = main_entry.get('percent', 19) tva_amount = main_entry.get('amount') if tva_amount and rate > 0: try: tva_dec = Decimal(str(tva_amount)) # total = tva * (100 + rate) / rate tva_implied_total = (tva_dec * Decimal(100 + rate) / Decimal(rate)).quantize(Decimal('0.01')) except (InvalidOperation, ValueError, TypeError): pass # Case 1: Amount is valid with high confidence - validate against TVA and payments if amount and amount > 0 and confidence_amount >= 0.8: # First check TVA-implied total (most reliable when TVA is extracted correctly) if tva_implied_total and tva_implied_total > 0: tva_diff_percent = abs(float(amount) - float(tva_implied_total)) / float(tva_implied_total) * 100 if tva_diff_percent <= 1: # Near-perfect TVA match - highest confidence return amount, min(0.98, confidence_amount + 0.05), "extracted (validated by TVA)" elif tva_diff_percent > 10: # Significant mismatch - TVA-implied total is more reliable # This catches cases where wrong TOTAL line was extracted (e.g., REST, SUBTOTAL) print(f"[Cross-Validation] Amount mismatch with TVA: extracted={amount}, tva_implied={tva_implied_total} (diff={tva_diff_percent:.1f}%)", flush=True) return tva_implied_total, 0.90, "calculated from TVA (extracted amount mismatch)" # Cross-validate with payment methods if payment_sum > 0 and abs(amount - payment_sum) <= Decimal('0.02'): # Perfect match - boost confidence return amount, min(0.98, confidence_amount + 0.05), "extracted (validated by payment methods)" elif payment_sum > 0: payment_diff_percent = abs(float(amount) - float(payment_sum)) / float(payment_sum) * 100 if payment_diff_percent > 10: # Significant mismatch - payment sum is more reliable print(f"[Cross-Validation] Amount mismatch with payments: extracted={amount}, payments={payment_sum} (diff={payment_diff_percent:.1f}%)", flush=True) return payment_sum, 0.88, "calculated from payment methods (extracted amount mismatch)" return amount, confidence_amount, "extracted" # Case 2: Amount exists but low confidence - try to validate/correct if amount and amount > 0: # First check TVA-implied total (most reliable) if tva_implied_total and tva_implied_total > 0: tva_diff_percent = abs(float(amount) - float(tva_implied_total)) / float(tva_implied_total) * 100 if tva_diff_percent <= 2: # Close match - boost confidence return amount, 0.88, "extracted (validated by TVA)" elif tva_diff_percent > 10: # Significant mismatch - use TVA-implied total print(f"[Cross-Validation] Amount mismatch with TVA: extracted={amount}, tva_implied={tva_implied_total} (diff={tva_diff_percent:.1f}%)", flush=True) return tva_implied_total, 0.85, "calculated from TVA" # Check if payment methods sum matches if payment_sum > 0: payment_diff_percent = abs(float(amount) - float(payment_sum)) / float(payment_sum) * 100 if payment_diff_percent <= 0.5: # Close match - boost confidence return amount, 0.90, "extracted (validated by payment methods)" elif payment_diff_percent > 10: # Mismatch - prefer payment_sum as it's more reliable print(f"[Cross-Validation] Amount mismatch: extracted={amount}, payments={payment_sum}", flush=True) return payment_sum, 0.85, "calculated from payment methods" # No validation possible - return as-is return amount, confidence_amount, "extracted (unvalidated)" # Case 3: Amount is 0 or None - calculate from payment methods if payment_sum > 0: print(f"[Cross-Validation] Amount not found, using payment sum: {payment_sum}", flush=True) return payment_sum, 0.85, "calculated from payment methods" # Case 4: Try TVA-implied total as last resort if tva_implied_total and tva_implied_total > 0: print(f"[Cross-Validation] Amount not found, using TVA-implied total: {tva_implied_total}", flush=True) return tva_implied_total, 0.70, "calculated from TVA" # Nothing worked - return original return amount, confidence_amount, "not found" def _extract_date(self, text: str) -> Tuple[Optional[date], float]: """Extract receipt date from text.""" # First try standard patterns (clean dates) for pattern, confidence in self.DATE_PATTERNS: match = re.search(pattern, text) if match: try: # Normalize separators to dots date_str = match.group(1).replace('/', '.').replace('-', '.') # Try DD.MM.YYYY format first try: parsed = datetime.strptime(date_str, '%d.%m.%Y').date() except ValueError: # Try YYYY.MM.DD format parsed = datetime.strptime(date_str, '%Y.%m.%d').date() # Validate date range today = date.today() if parsed <= today and parsed.year >= 2020: return parsed, confidence except ValueError: continue # Then try OCR-corrupted patterns (dates with spaces/commas) # Handles: "27. 10, 2025", "27, 10. 2025", "2025. 08. 14", etc. for pattern, confidence, fmt in self.DATE_PATTERNS_OCR_SPACES: match = re.search(pattern, text) if match: try: if fmt == 'ymd': # YYYY. MM. DD format (OMV/Petrom) year = match.group(1) month = match.group(2) day = match.group(3) else: # DD. MM. YYYY format (default) day = match.group(1) month = match.group(2) year = match.group(3) date_str = f"{day}.{month}.{year}" parsed = datetime.strptime(date_str, '%d.%m.%Y').date() # Validate date range today = date.today() if parsed <= today and parsed.year >= 2020: return parsed, confidence except ValueError: continue return None, 0.0 def _extract_number(self, text: str) -> Tuple[Optional[str], float]: """Extract receipt number from text.""" for pattern, confidence in self.NUMBER_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: return match.group(1), confidence return None, 0.0 def _extract_series(self, text: str) -> Tuple[Optional[str], float]: """Extract receipt series from text.""" for pattern, confidence in self.SERIES_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: return match.group(1).upper(), confidence return None, 0.0 def _extract_vendor(self, text: str) -> Tuple[Optional[str], float]: """ Extract vendor/partner name from text. Uses multiple strategies: 1. Look for lines with company type indicators (S.R.L., S.A., etc.) 2. Look for company name + SRL on separate lines 3. Look for lines near CIF 4. Use first valid line as fallback """ lines = text.split('\n') skip_keywords = [ 'BON', 'FISCAL', 'TOTAL', 'DATA', 'NR', 'ORA', 'SUBTOTAL', 'TVA', 'PLATA', 'CARD', 'NUMERAR', 'RON', 'LEI', 'CHITANTA', 'REST', 'CLIENT', 'OPERATOR', 'CASIER', 'POS', 'AMEF', 'BINE ATI VENIT', 'VA RUGAM', 'PASTRATI', 'VOCEA', 'TIPARIT', 'DETERGENT', 'PROSOP', 'HARTIE', 'SACI', 'SPRAY', 'BUC', 'ROLA', 'CUMPARATOR', 'MAGAZIN', 'BRICK', 'NIVS', 'BENZINA', 'PETROM', 'OMV' ] # Strategy 0: Look for company name followed by SRL/SA on next line # Pattern: "COMPANY NAME\nSRL" or "COMPANY NAME\nS.R.L." for i, line in enumerate(lines[:15]): line = line.strip() if not line or len(line) < 3: continue line_upper = line.upper() # Skip lines with skip keywords if any(kw in line_upper for kw in skip_keywords): continue # Check if next line is standalone SRL, S.R.L., SA, S.A., etc. if i + 1 < len(lines): next_line = lines[i + 1].strip().upper() # Match standalone company type suffix if re.match(r'^S\.?\s*R\.?\s*L\.?$', next_line) or \ re.match(r'^S\.?\s*A\.?$', next_line) or \ re.match(r'^S\.?\s*N\.?\s*C\.?$', next_line) or \ re.match(r'^P\.?\s*F\.?\s*A\.?$', next_line) or \ re.match(r'^I\.?\s*I\.?$', next_line): # Combine: "COMPANY NAME" + " " + "SRL" vendor = self._clean_vendor_name(f"{line} {next_line}") if vendor and len(vendor) >= 5: return vendor, 0.95 # Strategy 1: Look for lines with vendor indicators (S.R.L., S.A., HOLDING, etc.) for i, line in enumerate(lines[:15]): # Check first 15 lines line = line.strip() if not line or len(line) < 3: continue line_upper = line.upper() # Skip lines with skip keywords (CUMPARATOR, CLIENT, etc.) if any(kw in line_upper for kw in skip_keywords): continue # Check for vendor indicators for indicator in self.VENDOR_INDICATORS: if re.search(indicator, line_upper): # Found a company name indicator vendor = self._clean_vendor_name(line) if vendor and len(vendor) >= 3: # High confidence for lines with company indicators return vendor, 0.95 # Strategy 2: Look for lines right before or after CIF for i, line in enumerate(lines[:15]): line_upper = line.upper() if 'CIF' in line_upper and 'CLIENT' not in line_upper: # Check line before if i > 0: prev_line = lines[i-1].strip() if prev_line and len(prev_line) >= 3: if not any(kw in prev_line.upper() for kw in skip_keywords): vendor = self._clean_vendor_name(prev_line) if vendor: return vendor, 0.85 # Strategy 3: First valid line as fallback for i, line in enumerate(lines[:10]): line = line.strip() # Skip empty lines if not line or len(line) < 3: continue # Skip lines that are just numbers or codes if re.match(r'^[\d.,\s:]+$', line): continue # Skip lines with barcodes/product codes if re.match(r'^[A-Z]*\d{6,}', line): continue # Skip lines with keywords if any(kw in line.upper() for kw in skip_keywords): continue # Clean the line vendor = self._clean_vendor_name(line) if vendor and len(vendor) >= 3: # Confidence decreases for lines further down confidence = max(0.3, 0.7 - (i * 0.05)) return vendor, confidence return None, 0.0 def _clean_vendor_name(self, name: str) -> Optional[str]: """Clean and normalize vendor name.""" if not name: return None # Remove common OCR artifacts name = re.sub(r'[^\w\s.,&\-()]', ' ', name) # Normalize whitespace name = re.sub(r'\s+', ' ', name).strip() # Skip if it looks like an address line only if re.match(r'^(STR|JUD|MUN|NR|BL|SC|ET|AP)\.?\s', name.upper()): return None # Skip if too short after cleaning if len(name) < 3: return None return name def _extract_cui(self, text_upper: str, original_text: str) -> Tuple[Optional[str], float]: """ Extract vendor CUI (fiscal identification code) from text. Excludes CLIENT CUI which appears as 'CLIENT C.U.I./C.I.F.:...' """ def get_cui_digit_count(cui: str) -> int: """Get the count of digits in CUI (excluding RO/R0 prefix).""" cui_upper = cui.upper().strip() if cui_upper.startswith('RO') or cui_upper.startswith('R0'): return len(cui_upper) - 2 return len(cui_upper) # Strategy 0: Check for reversed format (CIF NUMBER on line BEFORE "C.I.F." label) # This is common in some receipts: "RO11201891\nC. I. F." for pattern, confidence in self.CUI_REVERSED_PATTERNS: match = re.search(pattern, text_upper, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: # Verify this is not the CLIENT CUI by checking context start = match.start() # Check 50 chars before the match for CLIENT keyword context_start = max(0, start - 50) context = text_upper[context_start:start] if 'CLIENT' not in context and 'LIENT' not in context: return cui, confidence # Strategy 1: Try to find CIF on a line that doesn't contain CLIENT lines = text_upper.split('\n') for line in lines: # Skip lines that contain CLIENT (these are buyer's CUI, not vendor's) if 'CLIENT' in line or 'CUMPARATOR' in line or 'LIENT' in line: continue # Look for CIF in this line for pattern, confidence in self.CUI_PATTERNS: match = re.search(pattern, line, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: return cui, confidence # Strategy 2: Fallback - search entire text but exclude CLIENT patterns for pattern, confidence in self.CUI_PATTERNS: # Find all matches for match in re.finditer(pattern, text_upper, re.IGNORECASE | re.MULTILINE): cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: # Check if this match is preceded by CLIENT in the same line start = match.start() line_start = text_upper.rfind('\n', 0, start) + 1 line_text = text_upper[line_start:start] if 'CLIENT' not in line_text and 'LIENT' not in line_text: return cui, confidence return None, 0.0 def _detect_receipt_type(self, text: str) -> str: """Detect receipt type from text content. BON FISCAL variants: "BON FISCAL", "BON FISCAL.", "BON FISCAL" CHITANTA variants: "CHITANTA", "CHITANȚĂ" """ # Check for explicit BON FISCAL first (handles OCR spacing variations) if re.search(r'BON\s+FISCAL', text): return 'bon_fiscal' if 'CHITANTA' in text or 'CHITANȚĂ' in text: return 'chitanta' # Default to bon_fiscal if neither found return 'bon_fiscal' def _extract_tva_entries(self, text: str) -> Tuple[List[dict], Optional[Decimal]]: """ Extract multiple TVA (VAT) entries from text. Romanian receipts can have multiple TVA rates (A=19%, B=9%, C=5%, D=0%). Returns (tva_entries, tva_total) where tva_entries is a list of: {'code': 'A', 'percent': 19, 'amount': Decimal('15.20')} """ tva_entries = [] seen_entries = set() # To avoid duplicates # Check for non-VAT payer (NEPLATITOR DE TVA) - TVA = 0 # OCR variants: NEPLATTOR, NEPLATITOR, NEPLATOR, NEPLATTOR, ANEPLATHTOR, MEPLATITOR, etc. # Also handles: "TOTAL NEPLATITOR TVA", "(NEPLATITOR DE TVA)" non_vat_patterns = [ # Main pattern - flexible for OCR errors: NEPLAT + any chars + OR/R r'NEPLAT\w*OR', # NEPLATITOR, NEPLATTOR, NEPLATOR r'[ANM]EPLAT\w*O?R', # OCR errors: ANEPLATHTOR, MEPLATITOR r'TOTAL\s+NEPLAT', # TOTAL NEPLATITOR... r'TOTAL\s+[ANM]EPLAT', # TOTAL ANEPLAT... (OCR error) r'SCUTIT\s*(?:DE\s+)?T[VU]A', # SCUTIT DE TVA r'NEPLAT\w*\s+T[VU]A', # NEPLATITOR TVA r'NEPLAT\w*\s+DE\s+T', # NEPLATITOR DE T... (truncated) ] for pattern in non_vat_patterns: if re.search(pattern, text, re.IGNORECASE): # Non-VAT payer - return TVA = 0 return [{'code': 'D', 'percent': 0, 'amount': Decimal('0.00')}], Decimal('0.00') # Normalize spaces in numbers first (OCR may produce "32. 31" or "49, 58") normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) # Also normalize comma followed by space to comma (for "21, 00%" -> "21,00%") normalized_text = re.sub(r'(\d+),\s+(\d{2})\s*%', r'\1.\2%', normalized_text) # Pattern 0a: First try to get TVA from "TOTAL TAXE:" which is most reliable # Format: "TOTAL TAXE: 55,22" - this is always the TVA amount # OCR may cut "T" producing "OTAL TAXE:" instead of "TOTAL TAXE:" # OCR may also put amount BEFORE "OTAL TAXE": "55,22OTAL TAXE:" total_taxe_pattern = r'T?OTAL\s+TAXE\s*:?\s*([\d\s.,]+)' taxe_match = re.search(total_taxe_pattern, normalized_text, re.IGNORECASE) # Also try pattern where amount comes BEFORE "OTAL TAXE" (OCR line break issue) if not taxe_match: reversed_taxe_pattern = r'([\d.,]+)\s*T?OTAL\s+TAXE' taxe_match = re.search(reversed_taxe_pattern, normalized_text, re.IGNORECASE) if taxe_match: # Also need to find the TVA rate from the table # Pattern handles: "A-21%", "-21,00%", "21%" etc. rate_pattern = r'([A-D])?\s*[-:]?\s*(\d{1,2})[.,]?\s*\d{0,2}\s*%' rate_match = re.search(rate_pattern, normalized_text, re.IGNORECASE) if rate_match: try: code = rate_match.group(1).upper() if rate_match.group(1) else 'A' # Default to A if missing percent = int(rate_match.group(2)) amount_str = taxe_match.group(1).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 0b: Table format "A-21,00% 285,66 49,58" (code-percent base tva_amount) # This format appears after a TVA header line like "TVA TOTAL VALDARE" # The TVA amount position depends on header: VALDARE last = TVA last, VALOARE middle = TVA middle if not tva_entries: table_pattern = r'([A-D])\s*[-:]\s*(\d{1,2})[.,]\s*\d{2}\s*%\s*([\d\s.,]+)\s+([\d\s.,]+)' for match in re.finditer(table_pattern, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount1_str = match.group(3).replace(' ', '') amount2_str = match.group(4).replace(' ', '') amount1 = Decimal(self._normalize_number(re.sub(r'[^\d.,]', '', amount1_str))) amount2 = Decimal(self._normalize_number(re.sub(r'[^\d.,]', '', amount2_str))) # Determine which is TVA: the smaller amount is usually TVA # (TVA is a fraction of the total, so it's always smaller) tva_amount = min(amount1, amount2) if tva_amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': tva_amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 0c: REVERSED FORMAT "5.00% TUA*B" followed by amount on next line # This handles receipts where percentage comes BEFORE TVA code (e.g., books with 5% rate) # Matches: "5.00% TUA*B", "5% TVA B", "5.00% TVA", "9% TUA", "5% IVA" if not tva_entries: # Pattern: PERCENT% + TVA/IVA + optional code, then amount on next line reversed_tva_pattern = r'(\d{1,2})[.,]?\d{0,2}\s*%\s*(?:T[VU][AR]|IVA)\s*\*?([A-D])?' for match in re.finditer(reversed_tva_pattern, normalized_text, re.IGNORECASE): try: percent = int(match.group(1)) code = (match.group(2) or self._get_tva_code_from_percent(percent)).upper() # Look for amount on the next line(s) after the match after_match = normalized_text[match.end():] # Find standalone number (amount) - skip empty lines amount_match = re.search(r'^[\s\n]*([\d]+[.,]\d{2})\b', after_match) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 0d: "TOTAL TUA:", "TOTAL TVA:", "TOTAL IVA:" with amount (OCR variants) if not tva_entries: total_tva_simple = r'TOTAL\s+(?:T[VU][AR]|IVA)\s*:?\s*([\d.,]+)' match = re.search(total_tva_simple, normalized_text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: # Try to find the rate in nearby text percent = self._detect_tva_percent(text) if percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 0e: Multiline "TOTAL TUA\n198\n30.43" where: # - "TOTAL TUA" on one line # - "198" or similar (corrupted "19%") on next line (optional) # - "30.43" (TVA amount) on following line # OCR often splits this across multiple lines if not tva_entries: multiline_tva = r'TOTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s*\n\s*\d*\s*\n?\s*([\d]+[.,]\d{2})\b' match = re.search(multiline_tva, normalized_text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: percent = self._detect_tva_percent(text) if percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 1: "TVA A - 19%: 15.20" or "TVAA - 21% 32.31" or "IVA A - 19%" (with code) # OCR tolerant: TUA, TVR, IVA, etc. pattern_with_code = r'(?:T[VU][AR]|IVA)\s*([A-D])\s*[-:]\s*(\d{1,2})\s*%\s*:?\s*([\d\s.,]+)' for match in re.finditer(pattern_with_code, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount_str = match.group(3).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 2: "TVA - 21%: 32.31" or "IVA - 21%: 32.31" (without explicit code, assume 'A') if not tva_entries: pattern_no_code = r'(?:T[VU][AR]|IVA)\s*[-:]\s*(\d{1,2})\s*%\s*:?\s*([\d\s.,]+)' for match in re.finditer(pattern_no_code, normalized_text, re.IGNORECASE): try: percent = int(match.group(1)) amount_str = match.group(2).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: # Determine code based on percent code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3: "TOTAL TVA A - 21%" or "TOTAL IVA" with amount on same line or "TOTAL TVA BON" with amount if not tva_entries: # First try: "TOTAL TVA A - 21% 32.31" or "TOTAL IVA A - 21% 32.31" (amount on same line) tva_with_amount = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+([A-D])\s*[-:]\s*(\d{1,2})\s*%\s*([\d.,]+)' for match in re.finditer(tva_with_amount, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount_str = self._normalize_number(match.group(3)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3b: "TOTAL TVA A - 21%" or "TOTAL IVA A - 21%" on one line, look for "TOTAL TVA BON" amount if not tva_entries: tva_total_pattern = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+([A-D])\s*[-:]\s*(\d{1,2})\s*%' for match in re.finditer(tva_total_pattern, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) # Look for "TOTAL TVA BON" or "TOTAL IVA BON" followed by amount tva_bon_pattern = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+BON[:\s]*([\d.,]+)' tva_bon_match = re.search(tva_bon_pattern, normalized_text, re.IGNORECASE) if tva_bon_match: amount_str = self._normalize_number(tva_bon_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) continue # Fallback: Amount after TOTAL TVA BON or TOTAL IVA BON on next line tva_bon_pos = re.search(r'TOTAL\s+(?:T[VU][AR]|IVA)\s+BON', normalized_text, re.IGNORECASE) if tva_bon_pos: after_bon = normalized_text[tva_bon_pos.end():] # Find first standalone number (likely TVA amount) amount_match = re.search(r'[\s\n]*([\d]+[.,]\d{2})\s*\n', after_bon) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3c: "TVAA - 21%" or "IVA A - 21%" on one line, amount on next line (simpler format) if not tva_entries: tva_line_pattern = r'(?:T[VU][AR]|IVA)\s*([A-D])?\s*[-:]\s*(\d{1,2})\s*%' for match in re.finditer(tva_line_pattern, normalized_text, re.IGNORECASE): try: code = (match.group(1) or 'A').upper() percent = int(match.group(2)) # Look for amount on the next line or immediately after after_tva = normalized_text[match.end():] amount_match = re.search(r'^[\s\n]*([\d.,]+)', after_tva) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 4: Use TVA_PATTERNS for fallback if not tva_entries: for pattern, _ in self.TVA_PATTERNS: match = re.search(pattern, normalized_text, re.IGNORECASE) if match: try: # Some patterns have 2 groups (percent, amount), others just amount if match.lastindex >= 2: percent = int(match.group(1)) amount_str = match.group(2) else: amount_str = match.group(1) # Try to detect percent from text percent = self._detect_tva_percent(text) amount_str = amount_str.replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0 and percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) break # Only use first match from fallback except (ValueError, InvalidOperation): continue # Extract TOTAL TVA BON as reference (separate from individual entries) tva_bon_total = self._extract_total_tva_bon(normalized_text) # Calculate sum from entries entries_sum = None if tva_entries: entries_sum = sum(entry['amount'] for entry in tva_entries) # Validate and correct TVA values tva_entries, tva_total = self._validate_and_correct_tva( tva_entries, entries_sum, tva_bon_total ) # Sort by code (A, B, C, D) tva_entries.sort(key=lambda x: x.get('code', 'Z')) return tva_entries, tva_total def _get_tva_code_from_percent(self, percent: int) -> str: """Map TVA percentage to standard Romanian code. Romanian TVA rates changed in August 2025: - Standard rate: 19% → 21% - Reduced rate: 9% → 11% - Other rates (5%, 0%) remain unchanged Old rates (before Aug 2025): New rates (from Aug 2025): - A = 19% (standard) - A = 21% (standard) - B = 9% (reduced) - B = 11% (reduced) - C = 5% (reduced) - C = 5% (reduced) - D = 0% (exempt) - D = 0% (exempt) Both old and new rates are supported for historical receipts. """ if percent in (19, 21): return 'A' # Standard rate (19% old, 21% new from Aug 2025) elif percent in (9, 11): return 'B' # Reduced rate (9% old, 11% new from Aug 2025) elif percent == 5: return 'C' # Reduced rate (unchanged) elif percent == 0: return 'D' # Exempt (unchanged) else: return 'A' # Default to standard rate def _extract_total_tva_bon(self, text: str) -> Optional[Decimal]: """ Extract TOTAL TVA BON value separately as the reference. This is the authoritative total TVA on the receipt. Handles OCR variations: TOTAL TVA BON, OTAL TUA BON, TOTAL IVA BON, etc. """ # Pattern for TOTAL TVA BON or TOTAL IVA BON with amount after # OCR corruptions: TUAL (TVA+L merged), TVAL, TUAI, etc. patterns = [ # Standard: TOTAL TVA BON: 14.92 or TOTAL IVA BON: 14.92 # Handles: TUAL (TVA+L), TVAL, TUAI, etc. with optional trailing letters r'T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON\s*:?\s*([\d]+[.,]\d{2})\b', # Amount before: 14.92 OTAL TUA BON (OCR line break) r'([\d]+[.,]\d{2})\s*\n?\s*T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON', # Amount on next line after TOTAL TVA BON or TOTAL IVA BON r'T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON\s*\n\s*([\d]+[.,]\d{2})\b', ] for pattern in patterns: match = re.search(pattern, text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: return amount except (InvalidOperation, ValueError): continue return None def _validate_and_correct_tva( self, tva_entries: List[dict], entries_sum: Optional[Decimal], tva_bon_total: Optional[Decimal] ) -> Tuple[List[dict], Optional[Decimal]]: """ Validate and correct TVA values. Rules: 1. TVA cannot be greater than TOTAL amount (will be validated at higher level) 2. Sum of TVA A + TVA B + ... should equal TOTAL TVA BON 3. If single entry and sum != tva_bon_total, use tva_bon_total 4. Detect and fix OCR concatenation errors (e.g., 14.921492 from 14.92 + 14.92) """ if not tva_entries: return tva_entries, tva_bon_total # Check for OCR concatenation errors in individual entries # Pattern: X.XX followed by another decimal (e.g., 14.921492 from 14.92 + 14.92) corrected_entries = [] for entry in tva_entries: amount = entry['amount'] amount_str = str(amount) # Check if amount looks like concatenated decimals # e.g., 14.921492 could be 14.92 + 14.92 incorrectly joined # or 32.3132.31 from 32.31 + 32.31 if len(amount_str) > 6 and '.' in amount_str: int_part, dec_part = amount_str.split('.') # If decimal part > 2 digits, it's likely concatenation if len(dec_part) > 2: # Try to extract the first valid decimal amount # e.g., from 14.921492, extract 14.92 try: corrected_amount = Decimal(f"{int_part}.{dec_part[:2]}") print(f"[TVA Validation] Corrected concatenation error: {amount} -> {corrected_amount}", flush=True) entry['amount'] = corrected_amount except InvalidOperation: pass corrected_entries.append(entry) tva_entries = corrected_entries # Recalculate sum after corrections entries_sum = sum(entry['amount'] for entry in tva_entries) if tva_entries else None # Validate sum against TOTAL TVA BON if tva_bon_total and entries_sum: # Allow small tolerance for rounding (0.02) tolerance = Decimal('0.02') difference = abs(entries_sum - tva_bon_total) if difference > tolerance: print(f"[TVA Validation] Sum mismatch: entries_sum={entries_sum}, tva_bon_total={tva_bon_total}", flush=True) # If single entry and sum doesn't match, use TOTAL TVA BON as reference if len(tva_entries) == 1: print(f"[TVA Validation] Single entry - using TOTAL TVA BON as reference: {tva_bon_total}", flush=True) tva_entries[0]['amount'] = tva_bon_total entries_sum = tva_bon_total # If multiple entries and sum > tva_bon_total, likely double counting elif entries_sum > tva_bon_total: # Check if one entry is the duplicate of another amounts = [e['amount'] for e in tva_entries] unique_amounts = set(amounts) if len(unique_amounts) < len(amounts): # Duplicate detected - likely TOTAL TVA BON counted as separate entry print(f"[TVA Validation] Duplicate TVA detected, removing duplicates", flush=True) # Keep only unique entries seen = set() unique_entries = [] for entry in tva_entries: key = (entry.get('code'), entry['amount']) if key not in seen: seen.add(key) unique_entries.append(entry) tva_entries = unique_entries entries_sum = sum(e['amount'] for e in tva_entries) # Final total tva_total = entries_sum if entries_sum else tva_bon_total return tva_entries, tva_total def _detect_tva_percent(self, text: str) -> Optional[int]: """Detect TVA percentage from text content. IMPORTANT: Prioritize rates found near TVA markers over rates found elsewhere. E.g., "REDUCERE 5%" should not override "TVA A 19%". Also handle OCR corruptions like "194" for "19%" in "TOTAL TA F 194". """ import re as regex # First, look for percent NEAR TVA markers (most reliable) # This handles "TVA A 19%", "TVA 19,00%", "TOTAL TVA 19%" tva_context_patterns = [ r'T[VU][AR]\s*[A-D]?\s*[-:]?\s*(19|21|11|9|5)[.,]?\s*\d{0,2}\s*%', r'IVA\s*[A-D]?\s*[-:]?\s*(19|21|11|9|5)[.,]?\s*\d{0,2}\s*%', # OCR corruption: "TOTAL TA F 194" where 194 = 19% (4 is artifact) r'TOTAL\s+T[VA][AR]?\s*[F\s]?\s*(19|21)\d\b', ] for pattern in tva_context_patterns: match = regex.search(pattern, text, regex.IGNORECASE) if match: rate = int(match.group(1)) if rate in (19, 21, 11, 9, 5): return rate # Fallback: Look for common Romanian TVA percentages anywhere # But EXCLUDE patterns near "REDUCERE", "DISCOUNT", "RED." (these are discounts, not TVA) # Clean text by removing discount context # Handle OCR corruptions: RED.CERE (C instead of U), RED CERE, REDUC, etc. text_no_discount = regex.sub(r'(?:REDUC|DISCOUNT|RED)[.\sA-Z]*\d+[.,]?\d*\s*%', '', text, flags=regex.IGNORECASE) # Now search in cleaned text (priority order: 19% > 21% > 11% > 9% > 5%) if regex.search(r'\b19[.,]?\s*\d{0,2}\s*%', text_no_discount): return 19 elif regex.search(r'\b21[.,]?\s*\d{0,2}\s*%', text_no_discount): return 21 elif regex.search(r'\b11[.,]?\s*\d{0,2}\s*%', text_no_discount): return 11 elif regex.search(r'\b9[.,]?\s*\d{0,2}\s*%', text_no_discount): return 9 elif regex.search(r'\b5[.,]?\s*\d{0,2}\s*%', text_no_discount): return 5 # Default: If no percent found but we're in Romanian receipt context, # assume 19% (standard rate) if regex.search(r'T[VU][AR]|IVA', text, regex.IGNORECASE): return 19 return None def _validate_tva_reverse( self, tva_entries: List[dict], total_amount: Optional[Decimal] ) -> Tuple[bool, Optional[Decimal], str]: """ Reverse TVA validation: from TVA amount and rate, calculate expected total. Formula (CORRECT): For TVA that is INCLUDED in total (standard Romanian receipts): total = base + tva tva = base * rate/100 Therefore: base = tva * 100 / rate And: total = base + tva = tva * 100 / rate + tva = tva * (100 + rate) / rate Returns (is_valid, expected_total, message) """ if not tva_entries or not total_amount: return True, None, "Insufficient data for reverse validation" expected_total = Decimal('0') for entry in tva_entries: tva_amount = entry['amount'] rate = Decimal(str(entry['percent'])) print(f"[TVA Debug] Entry: amount={tva_amount}, rate={rate}%", flush=True) if rate > 0: # CORRECT formula: total = tva * (100 + rate) / rate # Example: tva=55.22, rate=21 → total = 55.22 * 121 / 21 = 318.16 gross_for_entry = tva_amount * (Decimal('100') + rate) / rate expected_total += gross_for_entry print(f"[TVA Debug] Calculated gross: {gross_for_entry}", flush=True) else: # 0% TVA - can't calculate base, skip pass if expected_total == 0: return True, None, "Cannot calculate expected total (0% TVA only)" # Tolerance: max(0.50 RON, 1% of total) tolerance = max(Decimal('0.50'), total_amount * Decimal('0.01')) difference = abs(expected_total - total_amount) if difference <= tolerance: return True, expected_total, f"TVA reverse validation passed (expected: {expected_total}, actual: {total_amount}, diff: {difference})" else: return False, expected_total, f"TVA reverse validation WARNING: expected {expected_total}, actual {total_amount}, diff {difference}" def _extract_items_count(self, text: str) -> Optional[int]: """Extract number of items/articles from receipt.""" for pattern, _ in self.ITEMS_COUNT_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: try: count = int(match.group(1)) if 0 < count < 1000: # Reasonable range return count except ValueError: continue return None def _extract_address(self, text: str) -> Optional[str]: """Extract vendor address from text.""" lines = text.split('\n') address_parts = [] for line in lines[:15]: # Check first 15 lines line = line.strip() if not line: continue # Check for address patterns line_upper = line.upper() # JUD. (county) pattern if re.search(r'\bJUD\.?\s+', line_upper): address_parts.append(line) continue # STR. (street) pattern if re.search(r'\bSTR\.?\s+', line_upper): address_parts.append(line) continue # MUN./OR./COM. (city/town) pattern if re.search(r'\b(MUN|OR|COM)\.?\s+', line_upper): address_parts.append(line) continue if address_parts: # Join and clean address parts address = ', '.join(address_parts) # Clean up address = re.sub(r'\s+', ' ', address).strip() address = re.sub(r',\s*,', ',', address) return address if len(address) >= 5 else None return None def _extract_payment_methods(self, text: str) -> List[dict]: """ Extract payment methods (CARD/NUMERAR) from receipt. These appear after TOTAL LEI and before TOTAL TVA section. Returns list of: {'method': 'CARD'/'NUMERAR', 'amount': Decimal} """ payment_methods = [] seen_methods = set() # Normalize spaces in numbers normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) # Find the region between TOTAL LEI and TOTAL TVA total_lei_match = re.search(r'TOTAL\s+LEI\s*([\d\s.,]+)', normalized_text, re.IGNORECASE) total_tva_match = re.search(r'TOTAL\s+(?:T[VU][AR]|IVA)', normalized_text, re.IGNORECASE) # Define search region (after TOTAL LEI, before TOTAL TVA if exists) if total_lei_match: start_pos = total_lei_match.end() end_pos = total_tva_match.start() if total_tva_match else len(normalized_text) search_region = normalized_text[start_pos:end_pos] else: search_region = normalized_text # Fallback to full text for pattern, method, confidence in self.PAYMENT_METHOD_PATTERNS: for match in re.finditer(pattern, search_region, re.IGNORECASE | re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) # Validate: amount must be positive and reasonable (< MAX_REASONABLE_PAYMENT) # This prevents OCR errors like CUI being parsed as payment if amount > 0 and amount < self.MAX_REASONABLE_PAYMENT and method not in seen_methods: payment_methods.append({ 'method': method, 'amount': amount }) seen_methods.add(method) print(f"[Payment] Found {method}: {amount} (pattern matched)", flush=True) elif amount >= self.MAX_REASONABLE_PAYMENT: print(f"[Payment] Rejected unreasonable amount {amount} for {method} (likely OCR error)", flush=True) except (InvalidOperation, ValueError): continue return payment_methods def _validate_payment_methods( self, payment_methods: List[dict], total: Optional[Decimal] ) -> List[dict]: """ Validate payment methods against extracted total. If payment sum is way larger than total (>10x), it's likely an OCR error (e.g., CUI number parsed as payment amount). Clear invalid payments. Args: payment_methods: List of {'method': str, 'amount': Decimal} total: Extracted total amount Returns: Validated payment methods (may be empty if all were invalid) """ if not total or not payment_methods: return payment_methods payment_sum = sum(pm.get('amount', Decimal('0')) for pm in payment_methods) # If payment sum > 10x total, it's definitely an error if payment_sum > total * 10: print(f"[Payment Validation] Payment sum {payment_sum} >> Total {total} (>10x), clearing invalid payments", flush=True) return [] # If payment sum > 2x total, it's suspicious but might be valid in some edge cases # Just log a warning if payment_sum > total * 2: print(f"[Payment Validation] Warning: Payment sum {payment_sum} > 2x Total {total}, possible OCR error", flush=True) return payment_methods def _extract_client_data( self, text_upper: str, original_text: str ) -> Tuple[Optional[str], Optional[str], Optional[str], float]: """ Extract client/buyer data from B2B receipts. Returns (client_name, client_cui, client_address, confidence) """ client_name = None client_cui = None client_address = None confidence = 0.0 # Step 1: Find CLIENT section marker client_section_start = None for marker in self.CLIENT_SECTION_MARKERS: match = re.search(marker, text_upper, re.IGNORECASE) if match: client_section_start = match.start() break if client_section_start is None: # No client section found return None, None, None, 0.0 # Step 2: Extract client CUI for pattern, conf in self.CLIENT_CUI_PATTERNS: match = re.search(pattern, text_upper, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) if 6 <= len(cui) <= 10: client_cui = cui confidence = max(confidence, conf) break # Step 3: Extract client name from CLIENT section # Look for company name after CLIENT: marker lines = original_text.split('\n') for i, line in enumerate(lines): line_upper = line.upper().strip() # Check if this line contains CLIENT marker if any(re.search(marker, line_upper) for marker in self.CLIENT_SECTION_MARKERS): # Check if name is on same line after ":" if ':' in line: name_part = line.split(':', 1)[1].strip() if name_part and len(name_part) >= 3: # Skip if it looks like a CUI (R/RO followed by digits) if re.match(r'^R[O0]?\d{6,10}$', name_part.upper()): # This is a CUI, not a name - extract it if not already found if not client_cui: cui_digits = re.sub(r'[^0-9]', '', name_part) if 6 <= len(cui_digits) <= 10: client_cui = cui_digits confidence = max(confidence, 0.90) continue # Check for company indicators if any(re.search(ind, name_part.upper()) for ind in self.VENDOR_INDICATORS): client_name = self._clean_vendor_name(name_part) confidence = max(confidence, 0.95) break elif len(name_part) >= 5 and not name_part.isdigit(): client_name = self._clean_vendor_name(name_part) confidence = max(confidence, 0.80) break # Check next line for company name if i + 1 < len(lines): next_line = lines[i + 1].strip() next_upper = next_line.upper() # Skip if it's a CUI/CIF line if not re.search(r'C\.?\s*[UI]\.?\s*[IF]\.?', next_upper): if any(re.search(ind, next_upper) for ind in self.VENDOR_INDICATORS): client_name = self._clean_vendor_name(next_line) confidence = max(confidence, 0.90) break elif len(next_line) >= 5 and not next_line.isdigit(): # Check if it looks like a company name if not any(kw in next_upper for kw in ['CUI', 'CIF', 'COD', 'FISCAL']): client_name = self._clean_vendor_name(next_line) confidence = max(confidence, 0.75) break # Step 4: Extract client address (if present after client section) if client_section_start: # Look for address patterns after client section client_region = text_upper[client_section_start:client_section_start + 500] for pattern, _ in self.ADDRESS_PATTERNS: match = re.search(pattern, client_region) if match: client_address = match.group(1).strip() break # Log extraction result if client_cui or client_name: print(f"[Client Extraction] Found: name={client_name}, cui={client_cui}, conf={confidence}", flush=True) return client_name, client_cui, client_address, confidence