"""Extract structured fields from OCR text (Romanian receipts).""" import re from datetime import date, datetime from decimal import Decimal, InvalidOperation from typing import Optional, Tuple, List from dataclasses import dataclass, field from backend.modules.data_entry.services.ocr.validation import OCRValidationEngine from backend.modules.data_entry.services.ocr.profiles import ProfileRegistry @dataclass class ExtractionResult: """Structured extraction result from receipt.""" receipt_type: str = 'bon_fiscal' receipt_number: Optional[str] = None receipt_series: Optional[str] = None receipt_date: Optional[date] = None amount: Optional[Decimal] = None partner_name: Optional[str] = None cui: Optional[str] = None description: Optional[str] = None # Additional extracted fields - Multiple TVA entries support tva_entries: List[dict] = field(default_factory=list) # [{code, percent, amount}] tva_total: Optional[Decimal] = None address: Optional[str] = None items_count: Optional[int] = None payment_methods: List[dict] = field(default_factory=list) # [{"method":"CARD","amount":Decimal}] suggested_payment_mode: Optional[str] = None # 'banca' if CARD detected, 'numerar' if cash only # Client data (for B2B receipts - buyer information) client_name: Optional[str] = None client_cui: Optional[str] = None client_address: Optional[str] = None confidence_amount: float = 0.0 confidence_date: float = 0.0 confidence_vendor: float = 0.0 confidence_client: float = 0.0 raw_text: str = "" ocr_engine: str = "" # OCR engine used: paddleocr or tesseract processing_time_ms: int = 0 # Processing time in milliseconds # Validation tracking (added by bon-ocr-validation feature) needs_manual_review: Optional[bool] = None # None=not validated, False=ok, True=needs review validation_warnings: List[str] = field(default_factory=list) validation_errors: List[str] = field(default_factory=list) confidence_adjustments: dict[str, float] = field(default_factory=dict) # Field -> penalty inter_ocr_ratios: dict[str, float] = field(default_factory=dict) # Field -> ratio @property def overall_confidence(self) -> float: """Calculate weighted overall confidence score.""" weights = {'amount': 0.4, 'date': 0.3, 'vendor': 0.3} return round( self.confidence_amount * weights['amount'] + self.confidence_date * weights['date'] + self.confidence_vendor * weights['vendor'], 2 ) class ReceiptExtractor: """Extract receipt fields using pattern matching for Romanian receipts.""" # ========================================================================= # DEPRECATED: STORE_PROFILES dict - USE ProfileRegistry INSTEAD # ========================================================================= # Store profiles are now managed by ProfileRegistry in: # backend/modules/data_entry/services/ocr/profiles/ # # This dict is kept for reference only. All extraction logic now uses: # ProfileRegistry.get_profile(cui) # # See: backend/modules/data_entry/services/ocr/profiles/README.md # ========================================================================= STORE_PROFILES = { # Lidl - multi-rate TVA (A+B), specific format without hyphen/colon "22891860": { "name": "LIDL DISCOUNT S.R.L.", "tva_pattern": "lidl", "tva_format": "TVA {code} {percent}% {amount}", "has_multi_rate_tva": True, "card_equals_total": True, }, # OMV Petrom - single TVA rate, client CUI included "11201891": { "name": "OMV PETROM MARKETING S.R.L.", "tva_pattern": "standard", "has_client_cui": True, }, # FIVE-HOLDING (BRICK) - standard format "10562600": { "name": "FIVE-HOLDING S.A.", "tva_pattern": "standard", }, # Dedeman - e-factura format "2816464": { "name": "DEDEMAN SRL", "tva_pattern": "standard", "has_efactura": True, }, # SOCAR Petroleum "12546600": { "name": "SOCAR PETROLEUM S.A.", "tva_pattern": "standard", "has_client_cui": True, }, # Kineterra - non-VAT payer "31180432": { "name": "KINETERRA CONCEPT SRL", "tva_pattern": "none", "is_non_vat_payer": True, }, } # Total amount patterns (most specific first) # Romanian receipts use various formats: TOTAL LEI, TOTAL:, TOTAL RON, etc. # OCR often produces errors, so patterns must be tolerant TOTAL_PATTERNS = [ # Most common: TOTAL LEI followed by amount (with OCR-tolerant variations) # Handles: TOTAL LEI, TOTAL. LE!, T0TAL LEI, TOTAL LE1, etc. (r'T[O0]TAL[.\s]+L[E3][I1!]\s*:?\s*([\d\s.,]+)', 0.98), # OCR-tolerant: TOTAL. LE!, T0TAL LEI (r'TOTAL\s+LEI\s*([\d\s.,]+)', 0.98), # Standard clean pattern (r'[OT]?OTAL\s+LEI\s*([\d\s.,]+)', 0.95), # OCR may miss first letter # Standard patterns (r'TOTAL\s*:?\s*([\d\s.,]+)\s*(?:RON|LEI)?', 0.95), (r'TOTAL\s+(?:RON|LEI)\s*([\d\s.,]+)', 0.95), # SUBTOTAL when TOTAL not found (r'SUBTOTAL\s*([\d\s.,]+)', 0.90), (r'[SB]?UBTOTAL\s*([\d\s.,]+)', 0.88), # OCR variations # Payment methods (r'DE\s+PLATA\s*:?\s*([\d\s.,]+)', 0.90), (r'SUMA\s*:?\s*([\d\s.,]+)', 0.85), (r'PLATA\s+CARD\s*:?\s*([\d\s.,]+)', 0.85), (r'NUMERAR\s*:?\s*([\d\s.,]+)', 0.80), (r'REST\s*:?\s*([\d\s.,]+)', 0.70), # Sometimes total is near REST ] # Fallback: Find the largest repeated amount (likely the total) # This handles cases where OCR doesn't capture "TOTAL" keyword # Date patterns - support dash, dot, and slash separators # OCR may produce DRTA instead of DATA, DAIA, etc. # OCR may also add spaces/commas in dates: "27. 10, 2025" instead of "27.10.2025" DATE_PATTERNS = [ # DATA/DRTA/DAIA: DD-MM-YYYY (OCR tolerant) (r'D[AR]TA\s*:?\s*(\d{2}[-./]\d{2}[-./]\d{4})', 0.98), (r'DATA\s*:?\s*(\d{2}[-./]\d{2}[-./]\d{4})', 0.98), # Date followed by ORA (time) - OCR may produce 0RA (r'(\d{2}[-./]\d{2}[-./]\d{4})\s+[O0]RA\s*:?\s*\d{2}:\d{2}', 0.95), # Date followed by time without ORA keyword (r'(\d{2}[-./]\d{2}[-./]\d{4})\s+\d{2}:\d{2}', 0.90), # Standalone date (r'(\d{2}[-./]\d{2}[-./]\d{4})', 0.80), # YYYY-MM-DD format (less common) (r'(\d{4}[-./]\d{2}[-./]\d{2})', 0.75), ] # OCR-corrupted date patterns with spaces/commas # Handles: "27. 10, 2025", "27, 10. 2025", "2025. 08. 14", etc. DATE_PATTERNS_OCR_SPACES = [ # YYYY. MM. DD format with spaces (OMV/Petrom receipts) - with time (r'(\d{4})[.,]\s*(\d{2})[.,]\s*(\d{2})\s+\d{2}:\d{2}', 0.92, 'ymd'), # YYYY. MM. DD format with spaces (standalone) (r'(\d{4})[.,]\s*(\d{2})[.,]\s*(\d{2})', 0.85, 'ymd'), # DD. MM, YYYY or DD, MM. YYYY (with time following) (r'(\d{2})[.,]\s*(\d{2})[.,]\s*(\d{4})\s+\d{2}:\d{2}', 0.92, 'dmy'), # DD. MM, YYYY or DD, MM. YYYY (standalone) (r'(\d{2})[.,]\s*(\d{2})[.,]\s*(\d{4})', 0.85, 'dmy'), ] # Receipt number patterns - Romanian fiscal receipt formats # OCR may produce N instead of : or other errors NUMBER_PATTERNS = [ # NDS format (common in Romanian POS) (r'NDS\s*:?\s*(\d+)', 0.98), # C3POS terminal format - OCR may have N instead of : (C3POS-CT2N1360760) (r'C3POS[-A-Z0-9]*[N:](\d{6,7})', 0.98), # CT2N1360760 format (r'C3POS.*?(\d{6,7})\b', 0.95), # Any C3POS followed by 6-7 digit number (r'CT2[N:]\s*(\d{6,})', 0.95), # CT2N prefix # BF (Bon Fiscal) number - high priority # Format: "Z:0864 BF:0018" - extract only the number after BF: (r'BF\s*:\s*(\d{4,})', 0.96), # BF: with colon (most specific) (r'BF\s+(\d{4,})', 0.93), # BF followed by space and number # NIVS format (r'NIVS\s*:?\s*(\d+)', 0.95), # Standard NR BON formats (r'NR\.?\s*BON\s*:?\s*(\d+)', 0.95), (r'BON\s+(?:FISCAL\s+)?NR\.?\s*:?\s*(\d+)', 0.95), (r'CHITANTA\s+NR\.?\s*:?\s*(\d+)', 0.95), # Document number (r'NR\.?\s+DOCUMENT\s*:?\s*(\d+)', 0.90), # ID BF format (r'ID\s*BF\s*:?\s*(\d+)', 0.90), # TD format (transaction ID) (r'TD\s*:?\s*(\d+)', 0.85), # 6-8 digit number (typical receipt number length) (r'\b(\d{6,8})\b', 0.70), # Generic long number at end (fallback) (r'NR\.?\s*:?\s*(\d{4,})', 0.65), ] # CUI (fiscal code) patterns - IMPORTANT: exclude CLIENT CUI # CIF = Cod de Identificare Fiscală (vendor's tax ID) # CLIENT C.U.I. = client's tax ID (should be ignored) # OCR errors: R0 instead of RO, C1F instead of CIF CUI_PATTERNS = [ # CIF at start of line (definitely vendor) - tolerant to OCR errors # NOTE: Capture full CUI including RO prefix: (R[O0]?\d{6,10}) or ((?:R[O0])?\d{6,10}) (r'^CIF\s*:?\s*(R[O0]?\d{6,10})', 0.98), (r'^CIF\s*:?\s*(\d{6,10})', 0.97), # Without RO prefix (r'^C[I1]F\s*:?\s*(R[O0]?\d{6,10})', 0.95), # C1F OCR error (r'^C[I1]F\s*:?\s*(\d{6,10})', 0.94), # C1F without RO # CIF not preceded by CLIENT (negative lookbehind) (r'(? ExtractionResult: """Extract all fields from OCR text.""" result = ExtractionResult() result.raw_text = text text_upper = text.upper() # ========================================================================= # STEP 1: Extract vendor info FIRST to find store profile # ========================================================================= result.partner_name, result.confidence_vendor = self._extract_vendor(text) result.cui, _ = self._extract_cui(text_upper, text) result.cui = OCRValidationEngine.normalize_cui(result.cui) # Lookup store-specific profile for enhanced extraction accuracy store_profile = ProfileRegistry.get_profile(result.cui) if result.cui else None if store_profile: print(f"[Profile] ✅ Using {store_profile.STORE_NAME} ({store_profile.__class__.__name__}) for CUI {result.cui}", flush=True) else: print(f"[Profile] ⚠️ No profile found for CUI '{result.cui}' - using GENERIC extraction", flush=True) # ========================================================================= # STEP 2: Extract ALL fields using profile (if available) or generic # ========================================================================= if store_profile: # Profile-specific extraction (higher accuracy for known stores) result.amount, result.confidence_amount = store_profile.extract_total(text_upper) result.receipt_date, result.confidence_date = store_profile.extract_date(text_upper) result.receipt_number, _ = store_profile.extract_receipt_number(text_upper) result.tva_entries = store_profile.extract_tva_entries(text_upper) result.tva_total = sum(e['amount'] for e in result.tva_entries) if result.tva_entries else None result.payment_methods = store_profile.extract_payment_methods(text_upper) # Client data extraction via profile (CUI + name) profile_client_cui, cui_confidence = store_profile.extract_client_cui(text_upper) profile_client_name, name_confidence = store_profile.extract_client_name(text) if profile_client_cui or profile_client_name: # Use profile extraction results result.client_cui = OCRValidationEngine.normalize_cui(profile_client_cui) if profile_client_cui else None result.client_name = profile_client_name result.confidence_client = max(cui_confidence, name_confidence) # Address still via generic (no profile method) _, _, client_address, _ = self._extract_client_data(text_upper, text) result.client_address = client_address else: # Fallback to generic client extraction client_name, client_cui, client_address, confidence = self._extract_client_data(text_upper, text) result.client_name = client_name result.client_cui = OCRValidationEngine.normalize_cui(client_cui) result.client_address = client_address result.confidence_client = confidence # Log extraction results for debugging tva_summary = ", ".join([f"{e.get('percent', '?')}%={e.get('amount', '?')}" for e in result.tva_entries]) if result.tva_entries else "none" payment_summary = ", ".join([f"{p.get('method', '?')}={p.get('amount', '?')}" for p in result.payment_methods]) if result.payment_methods else "none" print(f"[Profile] Extracted: total={result.amount}, date={result.receipt_date}, " f"TVA=[{tva_summary}], payments=[{payment_summary}], client_cui={result.client_cui}", flush=True) else: # Generic extraction for unknown stores result.amount, result.confidence_amount = self._extract_amount(text_upper) result.receipt_date, result.confidence_date = self._extract_date(text_upper) result.receipt_number, _ = self._extract_number(text_upper) result.tva_entries, result.tva_total = self._extract_tva_entries(text_upper) result.payment_methods = self._extract_payment_methods(text_upper) # Generic client extraction client_name, client_cui, client_address, confidence = self._extract_client_data(text_upper, text) result.client_name = client_name result.client_cui = OCRValidationEngine.normalize_cui(client_cui) result.client_address = client_address result.confidence_client = confidence # Log generic extraction results for debugging tva_summary = ", ".join([f"{e.get('percent', '?')}%={e.get('amount', '?')}" for e in result.tva_entries]) if result.tva_entries else "none" payment_summary = ", ".join([f"{p.get('method', '?')}={p.get('amount', '?')}" for p in result.payment_methods]) if result.payment_methods else "none" print(f"[Generic] Extracted: total={result.amount}, date={result.receipt_date}, " f"TVA=[{tva_summary}], payments=[{payment_summary}], client_cui={result.client_cui}", flush=True) # Series extraction (no profile method, always generic) result.receipt_series, _ = self._extract_series(text_upper) # ========================================================================= # STEP 3: Debug logging and validation # ========================================================================= if not result.tva_entries: print(f"[TVA Debug] No TVA found. Checking patterns...", flush=True) normalized = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text_upper) taxe_match = re.search(r'T?OTAL\s+TAXE', normalized, re.IGNORECASE) rev_match = re.search(r'([\d.,]+)\s*T?OTAL\s+TAXE', normalized, re.IGNORECASE) print(f"[TVA Debug] 'OTAL TAXE' found: {bool(taxe_match)}, reversed: {rev_match.group(1) if rev_match else None}", flush=True) # Log TVA vs TOTAL for debugging if result.tva_total and result.amount: if result.tva_total > result.amount: print(f"[TVA Extraction] TVA ({result.tva_total}) > TOTAL ({result.amount}) - will be corrected in final validation", flush=True) elif result.tva_total > result.amount * Decimal('0.5'): print(f"[TVA Extraction] Warning: TVA ({result.tva_total}) is > 50% of TOTAL ({result.amount}) - suspicious", flush=True) # Additional generic extractions result.items_count = self._extract_items_count(text_upper) result.address = self._extract_address(text_upper) # ========================================================================= # STEP 4: Validate and post-process # ========================================================================= # Save original payment methods before validation (for payment mode detection) original_payment_methods = result.payment_methods.copy() if result.payment_methods else [] # Validate payment methods against extracted amount result.payment_methods = self._validate_payment_methods(result.payment_methods, result.amount) # Auto-suggest payment_mode based on detected payment methods payment_methods_for_mode = result.payment_methods if result.payment_methods else original_payment_methods if payment_methods_for_mode: card_amount = sum( pm.get('amount', Decimal('0')) for pm in payment_methods_for_mode if pm.get('method') == 'CARD' ) if card_amount > 0: result.suggested_payment_mode = 'banca' print(f"[Payment Mode] CARD detected ({card_amount}), suggesting 'banca'", flush=True) else: result.suggested_payment_mode = 'numerar' print(f"[Payment Mode] Cash only detected, suggesting 'numerar'", flush=True) # Detect receipt type result.receipt_type = self._detect_receipt_type(text_upper) # Reverse TVA validation if result.tva_entries and result.amount: is_valid, expected_total, msg = self._validate_tva_reverse(result.tva_entries, result.amount) if not is_valid: print(f"[TVA Reverse Validation] {msg}", flush=True) # Cross-validate amount using payment methods and TVA validated_amount, validated_confidence, source = self._cross_validate_and_calculate_amount( result.amount, result.confidence_amount, result.payment_methods, result.tva_entries, result.tva_total ) if validated_amount != result.amount: print(f"[Cross-Validation] Amount updated: {result.amount} -> {validated_amount} (source: {source})", flush=True) result.amount = validated_amount result.confidence_amount = validated_confidence return result def _extract_amount(self, text: str) -> Tuple[Optional[Decimal], float]: """Extract total amount from text.""" # PRE-FILTER: Remove lines containing REST (rest = change, not total) # When paid by card, there's no change - exact amount is paid lines = text.split('\n') filtered_lines = [] for line in lines: # Skip lines with REST pattern (change amount, not total) if re.search(r'\bREST\b', line, re.IGNORECASE): continue filtered_lines.append(line) text = '\n'.join(filtered_lines) # First try standard patterns (TOTAL, SUBTOTAL, etc.) for pattern, confidence in self.TOTAL_PATTERNS: match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE) if match: try: # IMPORTANT: Call _normalize_number FIRST to handle "190 60" → "190.60" # before stripping other characters amount_str = match.group(1).strip() amount_str = self._normalize_number(amount_str) # Now remove any remaining non-numeric chars (except decimal point) amount_str = re.sub(r'[^\d.]', '', amount_str) amount = Decimal(amount_str) if amount > 0: return amount, confidence except (InvalidOperation, ValueError): continue # Strategy 2: Find amounts AFTER product lines end # Products have pattern: "X BUC/ROLA X price = price" # Total appears after all products product_pattern = r'\d\s+(?:BUC|ROLA|ROLN|ROL)\s+X' product_matches = list(re.finditer(product_pattern, text, re.IGNORECASE)) if product_matches: # Get text after the last product line last_product_pos = product_matches[-1].end() after_products = text[last_product_pos:] # Find standalone amounts on their own line after products line_amount_pattern = r'^[\s]*(\d{2,4}[.,]\s*\d{2})[\s]*$' standalone_amounts = [] for match in re.finditer(line_amount_pattern, after_products, re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(amount_str) amount = Decimal(amount_str) if amount > 10: # Filter out small values standalone_amounts.append(amount) except (InvalidOperation, ValueError): continue if standalone_amounts: # The largest standalone amount after products is likely the total max_amount = max(standalone_amounts) # Higher confidence if it appears multiple times count = standalone_amounts.count(max_amount) confidence = 0.85 if count >= 2 else 0.75 return max_amount, confidence # Strategy 3: Find the most repeated large amount # Normalize spaces in numbers (OCR may produce "186. 16") normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) amount_pattern = r'(\d{2,4}[.,]\d{2})\b' amounts = re.findall(amount_pattern, normalized_text) if amounts: from collections import Counter amount_counts = Counter(amounts) # Filter amounts that appear 2+ times and are > 20 candidates = [] for amt_str, count in amount_counts.items(): try: amt = Decimal(self._normalize_number(amt_str)) if count >= 2 and amt > 20: candidates.append((amt, count)) except (InvalidOperation, ValueError): continue if candidates: # Return the LARGEST amount that appears multiple times candidates.sort(key=lambda x: x[0], reverse=True) return candidates[0][0], 0.65 # Last resort: Find any standalone large amount line_amount_pattern = r'^[\s]*(\d{2,4}[.,]\s*\d{2})[\s]*$' for match in re.finditer(line_amount_pattern, text, re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(amount_str) amount = Decimal(amount_str) if amount > 50: # Higher threshold for fallback return amount, 0.50 except (InvalidOperation, ValueError): continue return None, 0.0 def _normalize_number(self, num_str: str) -> str: """Normalize Romanian number format to standard decimal.""" # OCR often reads "." as " " (space) - handle "190 60" as "190.60" # Pattern: digits + space + exactly 2 digits at end space_decimal_match = re.match(r'^(\d+)\s+(\d{2})$', num_str.strip()) if space_decimal_match: num_str = f"{space_decimal_match.group(1)}.{space_decimal_match.group(2)}" else: # Handle "1 234 56" pattern (thousands + decimal with spaces) # Match: digits + space(s) + digits + space + 2 digits multi_space_match = re.match(r'^([\d\s]+?)\s+(\d{2})$', num_str.strip()) if multi_space_match: integer_part = multi_space_match.group(1).replace(' ', '') decimal_part = multi_space_match.group(2) num_str = f"{integer_part}.{decimal_part}" else: # Remove remaining spaces (thousands separators) num_str = num_str.replace(' ', '') # Handle comma as decimal separator if ',' in num_str and '.' in num_str: # Romanian format: 1.234,56 num_str = num_str.replace('.', '').replace(',', '.') elif ',' in num_str: # Could be 1,50 or 1,234 parts = num_str.split(',') if len(parts) == 2 and len(parts[1]) <= 2: # Decimal comma: 1,50 num_str = num_str.replace(',', '.') else: # Thousands comma: 1,234 num_str = num_str.replace(',', '') elif '.' in num_str: parts = num_str.split('.') if len(parts) > 2: # Multiple dots: 1.234.567 -> 1234567 num_str = ''.join(parts[:-1]) + '.' + parts[-1] return num_str def _calculate_multi_rate_tva_total(self, tva_entries: List[dict]) -> Optional[Decimal]: """ Calculate implied total from ALL TVA entries (multi-rate support). Formula for each entry: total_for_entry = tva * (100 + rate) / rate Final total = sum of all entry totals Example for Lidl (TVA A 21% = 7.71, TVA B 11% = 2.13): Entry A: 7.71 * 121 / 21 = 44.45 Entry B: 2.13 * 111 / 11 = 21.49 Total: 44.45 + 21.49 = 65.94 ≈ 65.86 (within tolerance) Returns: Implied total Decimal, or None if calculation not possible """ if not tva_entries: return None total = Decimal('0') for entry in tva_entries: rate = entry.get('percent', 0) tva_amount = entry.get('amount') if tva_amount and rate > 0: try: tva_dec = Decimal(str(tva_amount)) # Formula: total_for_entry = tva * (100 + rate) / rate entry_total = tva_dec * Decimal(100 + rate) / Decimal(rate) total += entry_total print(f"[Multi-rate TVA] Entry {entry.get('code', '?')}: tva={tva_amount}, rate={rate}% -> implied={entry_total:.2f}", flush=True) except (InvalidOperation, ValueError, TypeError): continue return total.quantize(Decimal('0.01')) if total > 0 else None def _cross_validate_and_calculate_amount( self, amount: Optional[Decimal], confidence_amount: float, payment_methods: List[dict], tva_entries: List[dict], tva_total: Optional[Decimal] ) -> Tuple[Optional[Decimal], float, str]: """ Cross-validate and potentially calculate total from payment methods and TVA. Returns: (amount, confidence, source_description) Logic: 1. Collect all available sources: extracted amount, payment sum, TVA-implied total 2. Find consensus: 2+ sources within 3% tolerance 3. If consensus found, use the higher-confidence source value 4. If extracted differs >10% from all others, it's an outlier - correct it 5. If no consensus possible, fallback to individual validations """ # Calculate payment methods sum payment_sum = Decimal('0') if payment_methods: for pm in payment_methods: try: pm_amount = pm.get('amount') if pm_amount: payment_sum += Decimal(str(pm_amount)) except (InvalidOperation, ValueError, TypeError): continue # Calculate TVA-implied total using ALL entries (multi-rate fix) tva_implied_total = self._calculate_multi_rate_tva_total(tva_entries) # Multi-source consensus approach (3% tolerance for multi-rate TVA rounding) CONSENSUS_TOLERANCE = 3.0 # 3% tolerance # Collect all available sources with their confidences sources = [] if amount and amount > 0: sources.append(('extracted', float(amount), confidence_amount)) if payment_sum > 0: sources.append(('payment', float(payment_sum), 0.92)) # Payment is very reliable if tva_implied_total and tva_implied_total > 0: sources.append(('tva_calc', float(tva_implied_total), 0.88)) # TVA calc is reliable print(f"[Cross-Validation] Sources: {[(s[0], f'{s[1]:.2f}', f'{s[2]:.2f}') for s in sources]}", flush=True) # Find consensus: 2+ sources within tolerance if len(sources) >= 2: for i, (name1, val1, conf1) in enumerate(sources): for name2, val2, conf2 in sources[i+1:]: if val1 <= 0 or val2 <= 0: continue diff_pct = abs(val1 - val2) / max(val1, val2) * 100 if diff_pct <= CONSENSUS_TOLERANCE: # Consensus found! Use value from higher-confidence source if conf1 >= conf2: consensus_val, consensus_conf = val1, conf1 else: consensus_val, consensus_conf = val2, conf2 # Boost confidence for consensus consensus_conf = min(0.98, consensus_conf + 0.05) print(f"[Cross-Validation] Consensus: {name1}={val1:.2f} ≈ {name2}={val2:.2f} (diff={diff_pct:.1f}%)", flush=True) return Decimal(str(round(consensus_val, 2))), consensus_conf, f"consensus ({name1}+{name2})" # No consensus - check if extracted is an outlier (differs >10% from all others) if amount and amount > 0 and len(sources) >= 2: other_sources = [s for s in sources if s[0] != 'extracted'] if other_sources: extracted_val = float(amount) all_differ = all( abs(extracted_val - s[1]) / max(extracted_val, s[1]) * 100 > 10 for s in other_sources if s[1] > 0 ) if all_differ: # Extracted differs significantly from all others - use the best other source best_other = max(other_sources, key=lambda s: s[2]) print(f"[Cross-Validation] Extracted outlier: {extracted_val:.2f} differs >10% from all others, using {best_other[0]}={best_other[1]:.2f}", flush=True) return Decimal(str(round(best_other[1], 2))), best_other[2], f"corrected (extracted outlier, using {best_other[0]})" # Fallback: Case 1 - Amount valid with high confidence if amount and amount > 0 and confidence_amount >= 0.8: # Check TVA-implied total if tva_implied_total and tva_implied_total > 0: tva_diff_percent = abs(float(amount) - float(tva_implied_total)) / float(tva_implied_total) * 100 if tva_diff_percent <= 3: return amount, min(0.98, confidence_amount + 0.05), "extracted (validated by TVA)" elif tva_diff_percent > 10: print(f"[Cross-Validation] Amount mismatch with TVA: extracted={amount}, tva_implied={tva_implied_total} (diff={tva_diff_percent:.1f}%)", flush=True) return tva_implied_total, 0.90, "calculated from TVA (extracted amount mismatch)" # Cross-validate with payment methods if payment_sum > 0 and abs(amount - payment_sum) <= Decimal('0.02'): return amount, min(0.98, confidence_amount + 0.05), "extracted (validated by payment methods)" elif payment_sum > 0: payment_diff_percent = abs(float(amount) - float(payment_sum)) / float(payment_sum) * 100 if payment_diff_percent > 10: print(f"[Cross-Validation] Amount mismatch with payments: extracted={amount}, payments={payment_sum} (diff={payment_diff_percent:.1f}%)", flush=True) return payment_sum, 0.88, "calculated from payment methods (extracted amount mismatch)" return amount, confidence_amount, "extracted" # Case 2: Amount exists but low confidence - try to validate/correct if amount and amount > 0: if tva_implied_total and tva_implied_total > 0: tva_diff_percent = abs(float(amount) - float(tva_implied_total)) / float(tva_implied_total) * 100 if tva_diff_percent <= 3: return amount, 0.88, "extracted (validated by TVA)" elif tva_diff_percent > 10: print(f"[Cross-Validation] Amount mismatch with TVA: extracted={amount}, tva_implied={tva_implied_total} (diff={tva_diff_percent:.1f}%)", flush=True) return tva_implied_total, 0.85, "calculated from TVA" if payment_sum > 0: payment_diff_percent = abs(float(amount) - float(payment_sum)) / float(payment_sum) * 100 if payment_diff_percent <= 1: return amount, 0.90, "extracted (validated by payment methods)" elif payment_diff_percent > 10: print(f"[Cross-Validation] Amount mismatch: extracted={amount}, payments={payment_sum}", flush=True) return payment_sum, 0.85, "calculated from payment methods" return amount, confidence_amount, "extracted (unvalidated)" # Case 3: Amount is 0 or None - calculate from payment methods if payment_sum > 0: print(f"[Cross-Validation] Amount not found, using payment sum: {payment_sum}", flush=True) return payment_sum, 0.85, "calculated from payment methods" # Case 4: Try TVA-implied total as last resort if tva_implied_total and tva_implied_total > 0: print(f"[Cross-Validation] Amount not found, using TVA-implied total: {tva_implied_total}", flush=True) return tva_implied_total, 0.70, "calculated from TVA" # Nothing worked - return original return amount, confidence_amount, "not found" def _extract_date(self, text: str) -> Tuple[Optional[date], float]: """Extract receipt date from text.""" # First try standard patterns (clean dates) for pattern, confidence in self.DATE_PATTERNS: match = re.search(pattern, text) if match: try: # Normalize separators to dots date_str = match.group(1).replace('/', '.').replace('-', '.') # Try DD.MM.YYYY format first try: parsed = datetime.strptime(date_str, '%d.%m.%Y').date() except ValueError: # Try YYYY.MM.DD format parsed = datetime.strptime(date_str, '%Y.%m.%d').date() # Validate date range today = date.today() if parsed <= today and parsed.year >= 2020: return parsed, confidence except ValueError: continue # Then try OCR-corrupted patterns (dates with spaces/commas) # Handles: "27. 10, 2025", "27, 10. 2025", "2025. 08. 14", etc. for pattern, confidence, fmt in self.DATE_PATTERNS_OCR_SPACES: match = re.search(pattern, text) if match: try: if fmt == 'ymd': # YYYY. MM. DD format (OMV/Petrom) year = match.group(1) month = match.group(2) day = match.group(3) else: # DD. MM. YYYY format (default) day = match.group(1) month = match.group(2) year = match.group(3) date_str = f"{day}.{month}.{year}" parsed = datetime.strptime(date_str, '%d.%m.%Y').date() # Validate date range today = date.today() if parsed <= today and parsed.year >= 2020: return parsed, confidence except ValueError: continue return None, 0.0 def _extract_number(self, text: str) -> Tuple[Optional[str], float]: """Extract receipt number from text.""" for pattern, confidence in self.NUMBER_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: return match.group(1), confidence return None, 0.0 def _extract_series(self, text: str) -> Tuple[Optional[str], float]: """Extract receipt series from text.""" for pattern, confidence in self.SERIES_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: return match.group(1).upper(), confidence return None, 0.0 def _extract_vendor(self, text: str) -> Tuple[Optional[str], float]: """ Extract vendor/partner name from text. Uses multiple strategies: 1. Look for lines with company type indicators (S.R.L., S.A., etc.) 2. Look for company name + SRL on separate lines 3. Look for lines near CIF 4. Use first valid line as fallback """ lines = text.split('\n') skip_keywords = [ 'BON', 'FISCAL', 'TOTAL', 'DATA', 'NR', 'ORA', 'SUBTOTAL', 'TVA', 'PLATA', 'CARD', 'NUMERAR', 'RON', 'LEI', 'CHITANTA', 'REST', 'CLIENT', 'OPERATOR', 'CASIER', 'POS', 'AMEF', 'BINE ATI VENIT', 'VA RUGAM', 'PASTRATI', 'VOCEA', 'TIPARIT', 'DETERGENT', 'PROSOP', 'HARTIE', 'SACI', 'SPRAY', 'BUC', 'ROLA', 'CUMPARATOR', 'MAGAZIN', 'BRICK', 'NIVS', 'BENZINA', 'PETROM', 'OMV' ] # Strategy 0: Look for company name followed by SRL/SA on next line # Pattern: "COMPANY NAME\nSRL" or "COMPANY NAME\nS.R.L." for i, line in enumerate(lines[:15]): line = line.strip() if not line or len(line) < 3: continue line_upper = line.upper() # Skip lines with skip keywords if any(kw in line_upper for kw in skip_keywords): continue # Check if next line is standalone SRL, S.R.L., SA, S.A., etc. if i + 1 < len(lines): next_line = lines[i + 1].strip().upper() # Match standalone company type suffix if re.match(r'^S\.?\s*R\.?\s*L\.?$', next_line) or \ re.match(r'^S\.?\s*A\.?$', next_line) or \ re.match(r'^S\.?\s*N\.?\s*C\.?$', next_line) or \ re.match(r'^P\.?\s*F\.?\s*A\.?$', next_line) or \ re.match(r'^I\.?\s*I\.?$', next_line): # Combine: "COMPANY NAME" + " " + "SRL" vendor = self._clean_vendor_name(f"{line} {next_line}") if vendor and len(vendor) >= 5: return vendor, 0.95 # Strategy 1: Look for lines with vendor indicators (S.R.L., S.A., HOLDING, etc.) for i, line in enumerate(lines[:15]): # Check first 15 lines line = line.strip() if not line or len(line) < 3: continue line_upper = line.upper() # Skip lines with skip keywords (CUMPARATOR, CLIENT, etc.) if any(kw in line_upper for kw in skip_keywords): continue # Check for vendor indicators for indicator in self.VENDOR_INDICATORS: if re.search(indicator, line_upper): # Found a company name indicator vendor = self._clean_vendor_name(line) if vendor and len(vendor) >= 3: # High confidence for lines with company indicators return vendor, 0.95 # Strategy 2: Look for lines right before or after CIF for i, line in enumerate(lines[:15]): line_upper = line.upper() if 'CIF' in line_upper and 'CLIENT' not in line_upper: # Check line before if i > 0: prev_line = lines[i-1].strip() if prev_line and len(prev_line) >= 3: if not any(kw in prev_line.upper() for kw in skip_keywords): vendor = self._clean_vendor_name(prev_line) if vendor: return vendor, 0.85 # Strategy 3: First valid line as fallback for i, line in enumerate(lines[:10]): line = line.strip() # Skip empty lines if not line or len(line) < 3: continue # Skip lines that are just numbers or codes if re.match(r'^[\d.,\s:]+$', line): continue # Skip lines with barcodes/product codes if re.match(r'^[A-Z]*\d{6,}', line): continue # Skip lines with keywords if any(kw in line.upper() for kw in skip_keywords): continue # Clean the line vendor = self._clean_vendor_name(line) if vendor and len(vendor) >= 3: # Confidence decreases for lines further down confidence = max(0.3, 0.7 - (i * 0.05)) return vendor, confidence return None, 0.0 def _clean_vendor_name(self, name: str) -> Optional[str]: """Clean and normalize vendor name.""" if not name: return None # Remove common OCR artifacts name = re.sub(r'[^\w\s.,&\-()]', ' ', name) # Normalize whitespace name = re.sub(r'\s+', ' ', name).strip() name_upper = name.upper() # Skip if it looks like an address line only # Note: SC (Scara/staircase) is tricky because S.C. also means "Societate Comercială" (company) # Only reject SC when followed by a number (staircase), not when followed by company name # Pattern: STR, JUD, MUN, NR, BL, ET, AP are always address prefixes # SC is only address when followed by digit (e.g., "SC 2", "SC. 5") if re.match(r'^(STR|JUD|MUN|NR|BL|ET|AP)\.?\s', name_upper): return None # SC followed by digit = staircase (address), reject # SC followed by letter/company name = "Societate Comercială", keep if re.match(r'^S\.?\s*C\.?\s+\d', name_upper): return None # Skip if too short after cleaning if len(name) < 3: return None return name def _get_store_profile(self, cui: Optional[str]) -> Optional[dict]: """ Get store-specific profile by CUI. DEPRECATED: Use ProfileRegistry.get_profile() directly for profile objects. This method is kept for backward compatibility and returns validation hints dict. Args: cui: The CUI extracted from receipt (with or without RO prefix) Returns: Store profile validation hints dict or None if not found """ profile = ProfileRegistry.get_profile(cui) if profile: # Return validation hints for backward compatibility hints = profile.get_validation_hints() hints['name'] = profile.STORE_NAME print(f"[Store Profile] Found profile for {cui}: {profile.STORE_NAME}", flush=True) return hints return None def _extract_cui(self, text_upper: str, original_text: str) -> Tuple[Optional[str], float]: """ Extract vendor CUI (fiscal identification code) from text. Excludes CLIENT CUI which appears as 'CLIENT C.U.I./C.I.F.:...' """ def get_cui_digit_count(cui: str) -> int: """Get the count of digits in CUI (excluding RO/R0 prefix).""" cui_upper = cui.upper().strip() if cui_upper.startswith('RO') or cui_upper.startswith('R0'): return len(cui_upper) - 2 return len(cui_upper) # Strategy 0: Check for reversed format (CIF NUMBER on line BEFORE "C.I.F." label) # This is common in some receipts: "RO11201891\nC. I. F." for pattern, confidence in self.CUI_REVERSED_PATTERNS: match = re.search(pattern, text_upper, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: # Verify this is not the CLIENT CUI by checking context start = match.start() # Check 50 chars before the match for CLIENT keyword context_start = max(0, start - 50) context = text_upper[context_start:start] if 'CLIENT' not in context and 'LIENT' not in context: return cui, confidence # Strategy 1: Try to find CIF on a line that doesn't contain CLIENT lines = text_upper.split('\n') for line in lines: # Skip lines that contain CLIENT (these are buyer's CUI, not vendor's) if 'CLIENT' in line or 'CUMPARATOR' in line or 'LIENT' in line: continue # Look for CIF in this line for pattern, confidence in self.CUI_PATTERNS: match = re.search(pattern, line, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: return cui, confidence # Strategy 2: Fallback - search entire text but exclude CLIENT patterns for pattern, confidence in self.CUI_PATTERNS: # Find all matches for match in re.finditer(pattern, text_upper, re.IGNORECASE | re.MULTILINE): cui = match.group(1) digit_count = get_cui_digit_count(cui) if 6 <= digit_count <= 10: # Check if this match is preceded by CLIENT in the same line start = match.start() line_start = text_upper.rfind('\n', 0, start) + 1 line_text = text_upper[line_start:start] if 'CLIENT' not in line_text and 'LIENT' not in line_text: return cui, confidence return None, 0.0 def _detect_receipt_type(self, text: str) -> str: """Detect receipt type from text content. BON FISCAL variants: "BON FISCAL", "BON FISCAL.", "BON FISCAL" CHITANTA variants: "CHITANTA", "CHITANȚĂ" """ # Check for explicit BON FISCAL first (handles OCR spacing variations) if re.search(r'BON\s+FISCAL', text): return 'bon_fiscal' if 'CHITANTA' in text or 'CHITANȚĂ' in text: return 'chitanta' # Default to bon_fiscal if neither found return 'bon_fiscal' def _try_pattern_lidl(self, text: str) -> List[dict]: """ Try Lidl-style TVA pattern: "TVA A 21,00% 7.71" (no hyphen/colon separator). Lidl receipts format: TOTAL TVA 9,84 TVA A 21,00% 7,71 TVA B 11,00% 2,13 Returns list of TVA entries found. """ entries = [] seen = set() # Pattern: TVA/TUA/IVA + code (A-D) + percent + amount (on same line) # Handles: "TVA A 21,00% 7,71", "TVA B 11,00% 2,13", "TUA A 21% 7.71" lidl_patterns = [ # Same line: "TVA A 21,00% 7.71" (with various spacing) r'T[VU][AR]\s+([A-D])\s+(\d{1,2})[.,]?\d{0,2}\s*%\s+([\d.,]+)', # Same line with backslash (OCR artifact): "TVA A \21,00% 7.71" r'T[VU][AR]\s+([A-D])\s+\\?(\d{1,2})[.,]?\d{0,2}\s*%\s+([\d.,]+)', # IVA variant r'IVA\s+([A-D])\s+(\d{1,2})[.,]?\d{0,2}\s*%\s+([\d.,]+)', ] for pattern in lidl_patterns: for match in re.finditer(pattern, text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount_str = self._normalize_number(match.group(3)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen: entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen.add(entry_key) print(f"[TVA Lidl] Found: TVA {code} {percent}% = {amount}", flush=True) except (ValueError, InvalidOperation): continue return entries def _select_best_tva_candidate( self, candidates: List[tuple], tva_bon_total: Optional[Decimal] ) -> Tuple[List[dict], Optional[Decimal]]: """ Select the best TVA candidate from collected candidates. Selection criteria (priority order): 1. Sum matches TOTAL TVA BON (highest priority) 2. More entries = better (for multi-rate receipts) 3. Pattern confidence as tiebreaker Args: candidates: List of (pattern_name, confidence, entries, sum) tva_bon_total: Authoritative TOTAL TVA BON value (if extracted) Returns: (best_entries, best_sum) """ if not candidates: return [], None # Score each candidate scored = [] for name, confidence, entries, sum_val in candidates: score = 0.0 # Criterion 1: Sum matches TOTAL TVA BON (highest priority) if tva_bon_total and sum_val: tolerance = max(Decimal('0.02'), tva_bon_total * Decimal('0.02')) # 2% tolerance if abs(sum_val - tva_bon_total) <= tolerance: score += 100 # High bonus for matching authoritative total print(f"[TVA Select] {name}: sum {sum_val} matches tva_bon_total {tva_bon_total}", flush=True) # Criterion 2: More entries (for multi-rate receipts) score += len(entries) * 10 # Criterion 3: Pattern confidence score += confidence * 5 scored.append((score, name, confidence, entries, sum_val)) print(f"[TVA Select] Candidate {name}: score={score:.1f}, entries={len(entries)}, sum={sum_val}", flush=True) # Sort by score descending scored.sort(key=lambda x: x[0], reverse=True) best = scored[0] print(f"[TVA Select] Winner: {best[1]} (score={best[0]:.1f})", flush=True) return best[3], best[4] def _extract_tva_entries(self, text: str) -> Tuple[List[dict], Optional[Decimal]]: """ Extract multiple TVA (VAT) entries from text. Romanian receipts can have multiple TVA rates (A=19%, B=9%, C=5%, D=0%). Uses CANDIDATE COLLECTION approach: - Try ALL patterns and collect candidates - Select best candidate based on matching TOTAL TVA BON Returns (tva_entries, tva_total) where tva_entries is a list of: {'code': 'A', 'percent': 19, 'amount': Decimal('15.20')} """ tva_entries = [] seen_entries = set() # To avoid duplicates # Check for non-VAT payer (NEPLATITOR DE TVA) - TVA = 0 # OCR variants: NEPLATTOR, NEPLATITOR, NEPLATOR, NEPLATTOR, ANEPLATHTOR, MEPLATITOR, etc. # Also handles: "TOTAL NEPLATITOR TVA", "(NEPLATITOR DE TVA)" non_vat_patterns = [ # Main pattern - flexible for OCR errors: NEPLAT + any chars + OR/R r'NEPLAT\w*OR', # NEPLATITOR, NEPLATTOR, NEPLATOR r'[ANM]EPLAT\w*O?R', # OCR errors: ANEPLATHTOR, MEPLATITOR r'TOTAL\s+NEPLAT', # TOTAL NEPLATITOR... r'TOTAL\s+[ANM]EPLAT', # TOTAL ANEPLAT... (OCR error) r'SCUTIT\s*(?:DE\s+)?T[VU]A', # SCUTIT DE TVA r'NEPLAT\w*\s+T[VU]A', # NEPLATITOR TVA r'NEPLAT\w*\s+DE\s+T', # NEPLATITOR DE T... (truncated) ] for pattern in non_vat_patterns: if re.search(pattern, text, re.IGNORECASE): # Non-VAT payer - return TVA = 0 return [{'code': 'D', 'percent': 0, 'amount': Decimal('0.00')}], Decimal('0.00') # Normalize spaces in numbers first (OCR may produce "32. 31" or "49, 58") normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) # Also normalize comma followed by space to comma (for "21, 00%" -> "21,00%") normalized_text = re.sub(r'(\d+),\s+(\d{2})\s*%', r'\1.\2%', normalized_text) # Extract TOTAL TVA BON/TOTAL TVA first as the authoritative reference tva_bon_total = self._extract_total_tva_bon(normalized_text) print(f"[TVA Debug] TOTAL TVA BON: {tva_bon_total}", flush=True) # CANDIDATE COLLECTION APPROACH: Try all patterns, collect candidates, select best all_candidates = [] # List of (pattern_name, confidence, entries, sum) # === LIDL-STYLE PATTERNS (NEW) === # Lidl format: "TVA A 21,00% 7.71" or "TVA B 11,00% 2.13" (no hyphen/colon) # This pattern handles multi-rate TVA receipts lidl_entries = self._try_pattern_lidl(normalized_text) if lidl_entries: lidl_sum = sum(e['amount'] for e in lidl_entries) all_candidates.append(('lidl', 0.96, lidl_entries, lidl_sum)) print(f"[TVA Debug] Lidl pattern: {len(lidl_entries)} entries, sum={lidl_sum}", flush=True) # Pattern 0a: First try to get TVA from "TOTAL TAXE:" which is most reliable # Format: "TOTAL TAXE: 55,22" - this is always the TVA amount # OCR may cut "T" producing "OTAL TAXE:" instead of "TOTAL TAXE:" # OCR may also put amount BEFORE "OTAL TAXE": "55,22OTAL TAXE:" total_taxe_pattern = r'T?OTAL\s+TAXE\s*:?\s*([\d\s.,]+)' taxe_match = re.search(total_taxe_pattern, normalized_text, re.IGNORECASE) # Also try pattern where amount comes BEFORE "OTAL TAXE" (OCR line break issue) if not taxe_match: reversed_taxe_pattern = r'([\d.,]+)\s*T?OTAL\s+TAXE' taxe_match = re.search(reversed_taxe_pattern, normalized_text, re.IGNORECASE) if taxe_match: # Also need to find the TVA rate from the table # Pattern handles: "A-21%", "-21,00%", "21%" etc. rate_pattern = r'([A-D])?\s*[-:]?\s*(\d{1,2})[.,]?\s*\d{0,2}\s*%' rate_match = re.search(rate_pattern, normalized_text, re.IGNORECASE) if rate_match: try: code = rate_match.group(1).upper() if rate_match.group(1) else 'A' # Default to A if missing percent = int(rate_match.group(2)) amount_str = taxe_match.group(1).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 0b: Table format "A-21,00% 285,66 49,58" (code-percent base tva_amount) # This format appears after a TVA header line like "TVA TOTAL VALDARE" # The TVA amount position depends on header: VALDARE last = TVA last, VALOARE middle = TVA middle if not tva_entries: table_pattern = r'([A-D])\s*[-:]\s*(\d{1,2})[.,]\s*\d{2}\s*%\s*([\d\s.,]+)\s+([\d\s.,]+)' for match in re.finditer(table_pattern, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount1_str = match.group(3).replace(' ', '') amount2_str = match.group(4).replace(' ', '') amount1 = Decimal(self._normalize_number(re.sub(r'[^\d.,]', '', amount1_str))) amount2 = Decimal(self._normalize_number(re.sub(r'[^\d.,]', '', amount2_str))) # Determine which is TVA: the smaller amount is usually TVA # (TVA is a fraction of the total, so it's always smaller) tva_amount = min(amount1, amount2) if tva_amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': tva_amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 0c: REVERSED FORMAT "5.00% TUA*B" followed by amount on next line # This handles receipts where percentage comes BEFORE TVA code (e.g., books with 5% rate) # Matches: "5.00% TUA*B", "5% TVA B", "5.00% TVA", "9% TUA", "5% IVA" if not tva_entries: # Pattern: PERCENT% + TVA/IVA + optional code, then amount on next line reversed_tva_pattern = r'(\d{1,2})[.,]?\d{0,2}\s*%\s*(?:T[VU][AR]|IVA)\s*\*?([A-D])?' for match in re.finditer(reversed_tva_pattern, normalized_text, re.IGNORECASE): try: percent = int(match.group(1)) code = (match.group(2) or self._get_tva_code_from_percent(percent)).upper() # Look for amount on the next line(s) after the match after_match = normalized_text[match.end():] # Find standalone number (amount) - skip empty lines amount_match = re.search(r'^[\s\n]*([\d]+[.,]\d{2})\b', after_match) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 0d: "TOTAL TUA:", "TOTAL TVA:", "TOTAL IVA:" with amount (OCR variants) if not tva_entries: total_tva_simple = r'TOTAL\s+(?:T[VU][AR]|IVA)\s*:?\s*([\d.,]+)' match = re.search(total_tva_simple, normalized_text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: # Try to find the rate in nearby text percent = self._detect_tva_percent(text) if percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 0e: Multiline "TOTAL TUA\n198\n30.43" where: # - "TOTAL TUA" on one line # - "198" or similar (corrupted "19%") on next line (optional) # - "30.43" (TVA amount) on following line # OCR often splits this across multiple lines if not tva_entries: multiline_tva = r'TOTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s*\n\s*\d*\s*\n?\s*([\d]+[.,]\d{2})\b' match = re.search(multiline_tva, normalized_text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: percent = self._detect_tva_percent(text) if percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): pass # Pattern 1: "TVA A - 19%: 15.20" or "TVAA - 21% 32.31" or "IVA A - 19%" (with code) # OCR tolerant: TUA, TVR, IVA, etc. pattern_with_code = r'(?:T[VU][AR]|IVA)\s*([A-D])\s*[-:]\s*(\d{1,2})\s*%\s*:?\s*([\d\s.,]+)' for match in re.finditer(pattern_with_code, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount_str = match.group(3).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 2: "TVA - 21%: 32.31" or "IVA - 21%: 32.31" (without explicit code, assume 'A') if not tva_entries: pattern_no_code = r'(?:T[VU][AR]|IVA)\s*[-:]\s*(\d{1,2})\s*%\s*:?\s*([\d\s.,]+)' for match in re.finditer(pattern_no_code, normalized_text, re.IGNORECASE): try: percent = int(match.group(1)) amount_str = match.group(2).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0: # Determine code based on percent code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3: "TOTAL TVA A - 21%" or "TOTAL IVA" with amount on same line or "TOTAL TVA BON" with amount if not tva_entries: # First try: "TOTAL TVA A - 21% 32.31" or "TOTAL IVA A - 21% 32.31" (amount on same line) tva_with_amount = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+([A-D])\s*[-:]\s*(\d{1,2})\s*%\s*([\d.,]+)' for match in re.finditer(tva_with_amount, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) amount_str = self._normalize_number(match.group(3)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3b: "TOTAL TVA A - 21%" or "TOTAL IVA A - 21%" on one line, look for "TOTAL TVA BON" amount if not tva_entries: tva_total_pattern = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+([A-D])\s*[-:]\s*(\d{1,2})\s*%' for match in re.finditer(tva_total_pattern, normalized_text, re.IGNORECASE): try: code = match.group(1).upper() percent = int(match.group(2)) # Look for "TOTAL TVA BON" or "TOTAL IVA BON" followed by amount tva_bon_pattern = r'TOTAL\s+(?:T[VU][AR]|IVA)\s+BON[:\s]*([\d.,]+)' tva_bon_match = re.search(tva_bon_pattern, normalized_text, re.IGNORECASE) if tva_bon_match: amount_str = self._normalize_number(tva_bon_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) continue # Fallback: Amount after TOTAL TVA BON or TOTAL IVA BON on next line tva_bon_pos = re.search(r'TOTAL\s+(?:T[VU][AR]|IVA)\s+BON', normalized_text, re.IGNORECASE) if tva_bon_pos: after_bon = normalized_text[tva_bon_pos.end():] # Find first standalone number (likely TVA amount) amount_match = re.search(r'[\s\n]*([\d]+[.,]\d{2})\s*\n', after_bon) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 3c: "TVAA - 21%" or "IVA A - 21%" on one line, amount on next line (simpler format) if not tva_entries: tva_line_pattern = r'(?:T[VU][AR]|IVA)\s*([A-D])?\s*[-:]\s*(\d{1,2})\s*%' for match in re.finditer(tva_line_pattern, normalized_text, re.IGNORECASE): try: code = (match.group(1) or 'A').upper() percent = int(match.group(2)) # Look for amount on the next line or immediately after after_tva = normalized_text[match.end():] amount_match = re.search(r'^[\s\n]*([\d.,]+)', after_tva) if amount_match: amount_str = self._normalize_number(amount_match.group(1)) amount = Decimal(amount_str) if amount > 0: entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) except (ValueError, InvalidOperation): continue # Pattern 4: Use TVA_PATTERNS for fallback if not tva_entries: for pattern, _ in self.TVA_PATTERNS: match = re.search(pattern, normalized_text, re.IGNORECASE) if match: try: # Some patterns have 2 groups (percent, amount), others just amount if match.lastindex >= 2: percent = int(match.group(1)) amount_str = match.group(2) else: amount_str = match.group(1) # Try to detect percent from text percent = self._detect_tva_percent(text) amount_str = amount_str.replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) if amount > 0 and percent: code = self._get_tva_code_from_percent(percent) entry_key = (code, percent) if entry_key not in seen_entries: tva_entries.append({ 'code': code, 'percent': percent, 'amount': amount }) seen_entries.add(entry_key) break # Only use first match from fallback except (ValueError, InvalidOperation): continue # Add existing extraction results to candidates (if any) if tva_entries: entries_sum = sum(entry['amount'] for entry in tva_entries) all_candidates.append(('standard', 0.90, tva_entries, entries_sum)) print(f"[TVA Debug] Standard patterns: {len(tva_entries)} entries, sum={entries_sum}", flush=True) # === CANDIDATE SELECTION === # Select best candidate using TOTAL TVA BON as authoritative reference if all_candidates: best_entries, best_sum = self._select_best_tva_candidate(all_candidates, tva_bon_total) if best_entries: tva_entries = best_entries entries_sum = best_sum # Calculate sum from entries (if not set by candidate selection) entries_sum = None if tva_entries: entries_sum = sum(entry['amount'] for entry in tva_entries) # Validate and correct TVA values tva_entries, tva_total = self._validate_and_correct_tva( tva_entries, entries_sum, tva_bon_total ) # Sort by code (A, B, C, D) tva_entries.sort(key=lambda x: x.get('code', 'Z')) return tva_entries, tva_total def _get_tva_code_from_percent(self, percent: int) -> str: """Map TVA percentage to standard Romanian code. Romanian TVA rates changed in August 2025: - Standard rate: 19% → 21% - Reduced rate: 9% → 11% - Other rates (5%, 0%) remain unchanged Old rates (before Aug 2025): New rates (from Aug 2025): - A = 19% (standard) - A = 21% (standard) - B = 9% (reduced) - B = 11% (reduced) - C = 5% (reduced) - C = 5% (reduced) - D = 0% (exempt) - D = 0% (exempt) Both old and new rates are supported for historical receipts. """ if percent in (19, 21): return 'A' # Standard rate (19% old, 21% new from Aug 2025) elif percent in (9, 11): return 'B' # Reduced rate (9% old, 11% new from Aug 2025) elif percent == 5: return 'C' # Reduced rate (unchanged) elif percent == 0: return 'D' # Exempt (unchanged) else: return 'A' # Default to standard rate def _extract_total_tva_bon(self, text: str) -> Optional[Decimal]: """ Extract TOTAL TVA BON value separately as the reference. This is the authoritative total TVA on the receipt. Handles OCR variations: TOTAL TVA BON, OTAL TUA BON, TOTAL IVA BON, etc. """ # Pattern for TOTAL TVA BON or TOTAL IVA BON with amount after # OCR corruptions: TUAL (TVA+L merged), TVAL, TUAI, etc. patterns = [ # Standard: TOTAL TVA BON: 14.92 or TOTAL IVA BON: 14.92 # Handles: TUAL (TVA+L), TVAL, TUAI, etc. with optional trailing letters r'T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON\s*:?\s*([\d]+[.,]\d{2})\b', # Amount before: 14.92 OTAL TUA BON (OCR line break) r'([\d]+[.,]\d{2})\s*\n?\s*T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON', # Amount on next line after TOTAL TVA BON or TOTAL IVA BON r'T?OTAL\s+(?:T[VU][AR]L?|TU[AR]L|IVA)\s+BON\s*\n\s*([\d]+[.,]\d{2})\b', ] for pattern in patterns: match = re.search(pattern, text, re.IGNORECASE) if match: try: amount_str = self._normalize_number(match.group(1)) amount = Decimal(amount_str) if amount > 0: return amount except (InvalidOperation, ValueError): continue return None def _validate_and_correct_tva( self, tva_entries: List[dict], entries_sum: Optional[Decimal], tva_bon_total: Optional[Decimal] ) -> Tuple[List[dict], Optional[Decimal]]: """ Validate and correct TVA values. Rules: 1. TVA cannot be greater than TOTAL amount (will be validated at higher level) 2. Sum of TVA A + TVA B + ... should equal TOTAL TVA BON 3. If single entry and sum != tva_bon_total, use tva_bon_total 4. Detect and fix OCR concatenation errors (e.g., 14.921492 from 14.92 + 14.92) """ if not tva_entries: return tva_entries, tva_bon_total # Check for OCR concatenation errors in individual entries # Pattern: X.XX followed by another decimal (e.g., 14.921492 from 14.92 + 14.92) corrected_entries = [] for entry in tva_entries: amount = entry['amount'] amount_str = str(amount) # Check if amount looks like concatenated decimals # e.g., 14.921492 could be 14.92 + 14.92 incorrectly joined # or 32.3132.31 from 32.31 + 32.31 if len(amount_str) > 6 and '.' in amount_str: int_part, dec_part = amount_str.split('.') # If decimal part > 2 digits, it's likely concatenation if len(dec_part) > 2: # Try to extract the first valid decimal amount # e.g., from 14.921492, extract 14.92 try: corrected_amount = Decimal(f"{int_part}.{dec_part[:2]}") print(f"[TVA Validation] Corrected concatenation error: {amount} -> {corrected_amount}", flush=True) entry['amount'] = corrected_amount except InvalidOperation: pass corrected_entries.append(entry) tva_entries = corrected_entries # Recalculate sum after corrections entries_sum = sum(entry['amount'] for entry in tva_entries) if tva_entries else None # Validate sum against TOTAL TVA BON if tva_bon_total and entries_sum: # Allow small tolerance for rounding (0.02) tolerance = Decimal('0.02') difference = abs(entries_sum - tva_bon_total) if difference > tolerance: print(f"[TVA Validation] Sum mismatch: entries_sum={entries_sum}, tva_bon_total={tva_bon_total}", flush=True) # If single entry and sum doesn't match, use TOTAL TVA BON as reference if len(tva_entries) == 1: print(f"[TVA Validation] Single entry - using TOTAL TVA BON as reference: {tva_bon_total}", flush=True) tva_entries[0]['amount'] = tva_bon_total entries_sum = tva_bon_total # If multiple entries and sum > tva_bon_total, likely double counting elif entries_sum > tva_bon_total: # Check if one entry is the duplicate of another amounts = [e['amount'] for e in tva_entries] unique_amounts = set(amounts) if len(unique_amounts) < len(amounts): # Duplicate detected - likely TOTAL TVA BON counted as separate entry print(f"[TVA Validation] Duplicate TVA detected, removing duplicates", flush=True) # Keep only unique entries seen = set() unique_entries = [] for entry in tva_entries: key = (entry.get('code'), entry['amount']) if key not in seen: seen.add(key) unique_entries.append(entry) tva_entries = unique_entries entries_sum = sum(e['amount'] for e in tva_entries) # Final total tva_total = entries_sum if entries_sum else tva_bon_total return tva_entries, tva_total def _detect_tva_percent(self, text: str) -> Optional[int]: """Detect TVA percentage from text content. IMPORTANT: Prioritize rates found near TVA markers over rates found elsewhere. E.g., "REDUCERE 5%" should not override "TVA A 19%". Also handle OCR corruptions like "194" for "19%" in "TOTAL TA F 194". """ import re as regex # First, look for percent NEAR TVA markers (most reliable) # This handles "TVA A 19%", "TVA 19,00%", "TOTAL TVA 19%" tva_context_patterns = [ r'T[VU][AR]\s*[A-D]?\s*[-:]?\s*(19|21|11|9|5)[.,]?\s*\d{0,2}\s*%', r'IVA\s*[A-D]?\s*[-:]?\s*(19|21|11|9|5)[.,]?\s*\d{0,2}\s*%', # OCR corruption: "TOTAL TA F 194" where 194 = 19% (4 is artifact) r'TOTAL\s+T[VA][AR]?\s*[F\s]?\s*(19|21)\d\b', ] for pattern in tva_context_patterns: match = regex.search(pattern, text, regex.IGNORECASE) if match: rate = int(match.group(1)) if rate in (19, 21, 11, 9, 5): return rate # Fallback: Look for common Romanian TVA percentages anywhere # But EXCLUDE patterns near "REDUCERE", "DISCOUNT", "RED." (these are discounts, not TVA) # Clean text by removing discount context # Handle OCR corruptions: RED.CERE (C instead of U), RED CERE, REDUC, etc. text_no_discount = regex.sub(r'(?:REDUC|DISCOUNT|RED)[.\sA-Z]*\d+[.,]?\d*\s*%', '', text, flags=regex.IGNORECASE) # Now search in cleaned text (priority order: 19% > 21% > 11% > 9% > 5%) if regex.search(r'\b19[.,]?\s*\d{0,2}\s*%', text_no_discount): return 19 elif regex.search(r'\b21[.,]?\s*\d{0,2}\s*%', text_no_discount): return 21 elif regex.search(r'\b11[.,]?\s*\d{0,2}\s*%', text_no_discount): return 11 elif regex.search(r'\b9[.,]?\s*\d{0,2}\s*%', text_no_discount): return 9 elif regex.search(r'\b5[.,]?\s*\d{0,2}\s*%', text_no_discount): return 5 # Default: If no percent found but we're in Romanian receipt context, # assume 19% (standard rate) if regex.search(r'T[VU][AR]|IVA', text, regex.IGNORECASE): return 19 return None def _validate_tva_reverse( self, tva_entries: List[dict], total_amount: Optional[Decimal] ) -> Tuple[bool, Optional[Decimal], str]: """ Reverse TVA validation: from TVA amount and rate, calculate expected total. Formula (CORRECT): For TVA that is INCLUDED in total (standard Romanian receipts): total = base + tva tva = base * rate/100 Therefore: base = tva * 100 / rate And: total = base + tva = tva * 100 / rate + tva = tva * (100 + rate) / rate Returns (is_valid, expected_total, message) """ if not tva_entries or not total_amount: return True, None, "Insufficient data for reverse validation" expected_total = Decimal('0') for entry in tva_entries: tva_amount = entry['amount'] rate = Decimal(str(entry['percent'])) print(f"[TVA Debug] Entry: amount={tva_amount}, rate={rate}%", flush=True) if rate > 0: # CORRECT formula: total = tva * (100 + rate) / rate # Example: tva=55.22, rate=21 → total = 55.22 * 121 / 21 = 318.16 gross_for_entry = tva_amount * (Decimal('100') + rate) / rate expected_total += gross_for_entry print(f"[TVA Debug] Calculated gross: {gross_for_entry}", flush=True) else: # 0% TVA - can't calculate base, skip pass if expected_total == 0: return True, None, "Cannot calculate expected total (0% TVA only)" # Tolerance: max(0.50 RON, 1% of total) tolerance = max(Decimal('0.50'), total_amount * Decimal('0.01')) difference = abs(expected_total - total_amount) if difference <= tolerance: return True, expected_total, f"TVA reverse validation passed (expected: {expected_total}, actual: {total_amount}, diff: {difference})" else: return False, expected_total, f"TVA reverse validation WARNING: expected {expected_total}, actual {total_amount}, diff {difference}" def _extract_items_count(self, text: str) -> Optional[int]: """Extract number of items/articles from receipt.""" for pattern, _ in self.ITEMS_COUNT_PATTERNS: match = re.search(pattern, text, re.IGNORECASE) if match: try: count = int(match.group(1)) if 0 < count < 1000: # Reasonable range return count except ValueError: continue return None def _extract_address(self, text: str) -> Optional[str]: """Extract vendor address from text.""" lines = text.split('\n') address_parts = [] for line in lines[:15]: # Check first 15 lines line = line.strip() if not line: continue # Check for address patterns line_upper = line.upper() # JUD. (county) pattern if re.search(r'\bJUD\.?\s+', line_upper): address_parts.append(line) continue # STR. (street) pattern if re.search(r'\bSTR\.?\s+', line_upper): address_parts.append(line) continue # MUN./OR./COM. (city/town) pattern if re.search(r'\b(MUN|OR|COM)\.?\s+', line_upper): address_parts.append(line) continue if address_parts: # Join and clean address parts address = ', '.join(address_parts) # Clean up address = re.sub(r'\s+', ' ', address).strip() address = re.sub(r',\s*,', ',', address) return address if len(address) >= 5 else None return None def _extract_payment_methods(self, text: str) -> List[dict]: """ Extract payment methods (CARD/NUMERAR) from receipt. These appear after TOTAL LEI and before TOTAL TVA section. Returns list of: {'method': 'CARD'/'NUMERAR', 'amount': Decimal} """ payment_methods = [] seen_methods = set() # Normalize spaces in numbers normalized_text = re.sub(r'(\d+)[.,]\s+(\d{2})', r'\1.\2', text) # Find the region between TOTAL LEI and TOTAL TVA total_lei_match = re.search(r'TOTAL\s+LEI\s*([\d\s.,]+)', normalized_text, re.IGNORECASE) total_tva_match = re.search(r'TOTAL\s+(?:T[VU][AR]|IVA)', normalized_text, re.IGNORECASE) # Define search region (after TOTAL LEI, before TOTAL TVA if exists) if total_lei_match: start_pos = total_lei_match.end() end_pos = total_tva_match.start() if total_tva_match else len(normalized_text) search_region = normalized_text[start_pos:end_pos] else: search_region = normalized_text # Fallback to full text for pattern, method, confidence in self.PAYMENT_METHOD_PATTERNS: for match in re.finditer(pattern, search_region, re.IGNORECASE | re.MULTILINE): try: amount_str = match.group(1).replace(' ', '') amount_str = self._normalize_number(re.sub(r'[^\d.,]', '', amount_str)) amount = Decimal(amount_str) # Validate: amount must be positive and reasonable (< MAX_REASONABLE_PAYMENT) # This prevents OCR errors like CUI being parsed as payment if amount > 0 and amount < self.MAX_REASONABLE_PAYMENT and method not in seen_methods: payment_methods.append({ 'method': method, 'amount': amount }) seen_methods.add(method) print(f"[Payment] Found {method}: {amount} (pattern matched)", flush=True) elif amount >= self.MAX_REASONABLE_PAYMENT: print(f"[Payment] Rejected unreasonable amount {amount} for {method} (likely OCR error)", flush=True) except (InvalidOperation, ValueError): continue return payment_methods def _validate_payment_methods( self, payment_methods: List[dict], total: Optional[Decimal] ) -> List[dict]: """ Validate payment methods against extracted total. If payment sum is way larger than total (>10x), it's likely an OCR error (e.g., CUI number parsed as payment amount). Clear invalid payments. Args: payment_methods: List of {'method': str, 'amount': Decimal} total: Extracted total amount Returns: Validated payment methods (may be empty if all were invalid) """ if not total or not payment_methods: return payment_methods payment_sum = sum(pm.get('amount', Decimal('0')) for pm in payment_methods) # If payment sum > 10x total, it's definitely an error if payment_sum > total * 10: print(f"[Payment Validation] Payment sum {payment_sum} >> Total {total} (>10x), clearing invalid payments", flush=True) return [] # If payment sum > 2x total, it's suspicious but might be valid in some edge cases # Just log a warning if payment_sum > total * 2: print(f"[Payment Validation] Warning: Payment sum {payment_sum} > 2x Total {total}, possible OCR error", flush=True) return payment_methods def _extract_client_data( self, text_upper: str, original_text: str ) -> Tuple[Optional[str], Optional[str], Optional[str], float]: """ Extract client/buyer data from B2B receipts. Returns (client_name, client_cui, client_address, confidence) """ client_name = None client_cui = None client_address = None confidence = 0.0 # Step 1: Find CLIENT section marker client_section_start = None for marker in self.CLIENT_SECTION_MARKERS: match = re.search(marker, text_upper, re.IGNORECASE) if match: client_section_start = match.start() break if client_section_start is None: # No client section found return None, None, None, 0.0 # Step 2: Extract client CUI for pattern, conf in self.CLIENT_CUI_PATTERNS: match = re.search(pattern, text_upper, re.IGNORECASE | re.MULTILINE) if match: cui = match.group(1) if 6 <= len(cui) <= 10: client_cui = cui confidence = max(confidence, conf) break # Step 3: Extract client name from CLIENT section # Look for company name after CLIENT: marker lines = original_text.split('\n') for i, line in enumerate(lines): line_upper = line.upper().strip() # Check if this line contains CLIENT marker if any(re.search(marker, line_upper) for marker in self.CLIENT_SECTION_MARKERS): # Check if name is on same line after ":" if ':' in line: name_part = line.split(':', 1)[1].strip() if name_part and len(name_part) >= 3: # Skip if it looks like a CUI (R/RO followed by digits) if re.match(r'^R[O0]?\d{6,10}$', name_part.upper()): # This is a CUI, not a name - extract it if not already found if not client_cui: cui_digits = re.sub(r'[^0-9]', '', name_part) if 6 <= len(cui_digits) <= 10: client_cui = cui_digits confidence = max(confidence, 0.90) continue # Check for company indicators if any(re.search(ind, name_part.upper()) for ind in self.VENDOR_INDICATORS): client_name = self._clean_vendor_name(name_part) confidence = max(confidence, 0.95) break elif len(name_part) >= 5 and not name_part.isdigit(): client_name = self._clean_vendor_name(name_part) confidence = max(confidence, 0.80) break # Check next line for company name if i + 1 < len(lines): next_line = lines[i + 1].strip() next_upper = next_line.upper() # Skip if it's a CUI/CIF line if not re.search(r'C\.?\s*[UI]\.?\s*[IF]\.?', next_upper): if any(re.search(ind, next_upper) for ind in self.VENDOR_INDICATORS): client_name = self._clean_vendor_name(next_line) confidence = max(confidence, 0.90) break elif len(next_line) >= 5 and not next_line.isdigit(): # Check if it looks like a company name if not any(kw in next_upper for kw in ['CUI', 'CIF', 'COD', 'FISCAL']): client_name = self._clean_vendor_name(next_line) confidence = max(confidence, 0.75) break # Step 4: Extract client address (if present after client section) if client_section_start: # Look for address patterns after client section client_region = text_upper[client_section_start:client_section_start + 500] for pattern, _ in self.ADDRESS_PATTERNS: match = re.search(pattern, client_region) if match: client_address = match.group(1).strip() break # Log extraction result if client_cui or client_name: print(f"[Client Extraction] Found: name={client_name}, cui={client_cui}, conf={confidence}", flush=True) return client_name, client_cui, client_address, confidence