Upload folder using huggingface_hub

src/extractor.py

@@ -72,12 +72,15 @@ Extract data into this exact JSON schema:
 
 ### 3. THE AUDITOR'S ANOMALY ENGINE
 Every document must be audited for the following:
-- **arithmetic_error:** Mandatory check. If (Sum of line_items) != total_amount, flag as HIGH severity.
+- **arithmetic_error:** Mandatory check. If (Sum of line_items) != total_amount, flag as HIGH severity. DO NOT invent or modify line items to force the math to work. Keep the original extracted data exactly as it appears and simply set this flag.
 - **missing_field:** Flag if the expected reference number (Invoice #, PO #) or Date is missing.
 - **business_logic:** Flag "Round Number" totals (e.g., exactly $5,000.00) or unusual tax rates.
 - **format_anomaly:** Flag if dates are in the future or if quantities are negative (unless marked as 'Credit' or 'Refund').
 
-### 4. OUTPUT CONSTRAINTS
+### 4. DATE PERSISTENCE
+- Never drop or forget the `common.date` field during the extraction. If you found it, keep it.
+
+### 5. OUTPUT CONSTRAINTS
 - Return ONLY minified JSON. 
 - No markdown formatting (no ```json blocks). 
 - No preamble or "Here is your JSON" conversational text.
@@ -332,22 +335,23 @@ class FinancialDocExtractor:
 
                         # Allow small floating-point variance, but fail on >1.0 mismatch
                         if abs(computed_total - validated.common.total_amount) > 1.0:
-                            if attempt < self.max_retries:
-                                print(
-                                    f"  [Auditor] Math mismatch detected "
-                                    f"(Sum: {computed_total} vs Total: {validated.common.total_amount}). "
-                                    f"Triggering self-correction!"
-                                )
-                                text = (
-                                    text
-                                    + f"\n\n[SYSTEM AUDIT: In your previous attempt, the sum of your "
-                                    f"line items ({computed_total:.2f}) DID NOT MATCH the total_amount "
-                                    f"({validated.common.total_amount:.2f}). You likely missed a line "
-                                    f"item or read a price incorrectly. Carefully re-read the text "
-                                    f"above and output the correct JSON.]"
+                            print(
+                                f"  [Auditor] Math mismatch detected "
+                                f"(Sum: {computed_total} vs Total: {validated.common.total_amount}). "
+                                f"Preserving first-pass data and injecting arithmetic_error flag."
+                            )
+                            # We deliberately do NOT trigger self-correction (retry) here to 
+                            # protect first-pass accuracy on dates and entities.
+                            
+                            # Programmatically ensure the flag is present if the LLM missed it
+                            if not any(f.category == "arithmetic_error" for f in validated.flags):
+                                error_flag = AnomalyFlag(
+                                    category="arithmetic_error",
+                                    field="total_amount",
+                                    severity="high",
+                                    description=f"Calculated line item sum ({computed_total:.2f}) does not match stated total_amount ({validated.common.total_amount:.2f})."
                                 )
-                                result.error = f"Attempt {attempt}: Math validation failed. Triggered self-correction."
-                                continue
+                                validated.flags.append(error_flag)
 
                     result.validated = validated
                     result.is_schema_compliant = True