Update app.py

app.py

@@ -74,6 +74,7 @@ def extract_items(text):
         ("Particulars", "Gross value", "Discount", "Net value", "Total"),  # Format 2 (e.g., Invoice_6164752968.pdf)
     ]
     
+    # Extract main table (e.g., Particulars | Gross value | Discount | Net value | Total)
     table_start = -1
     table_format = None
     for i, line in enumerate(lines):
@@ -85,96 +86,64 @@ def extract_items(text):
         if table_start != -1:
             break
     
-    if table_start == -1:
-        print("Table header not found.")
-        # Look for platform fee as a separate table
-        platform_fee_start = -1
-        for i, line in enumerate(lines):
-            if "Sr.No Particulars" in line:
-                platform_fee_start = i + 1
+    if table_start != -1:
+        # Find the end of the main table
+        table_end = len(lines)
+        for i in range(table_start, len(lines)):
+            if "Item(s) Total" in lines[i] or "Total Value" in lines[i]:
+                table_end = i
                 break
-        if platform_fee_start != -1:
-            platform_fee_end = len(lines)
-            for i in range(platform_fee_start, len(lines)):
-                if "Total" in lines[i] and not "Sr.No" in lines[i]:
-                    platform_fee_end = i + 1
-                    break
-            platform_fee_lines = lines[platform_fee_start:platform_fee_end]
-            print("Platform fee lines:", platform_fee_lines)  # Debug
-            platform_fee_pattern = r"\|?\s*\d+\s*\|?\s*([A-Za-z\s]+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?"
-            for line in platform_fee_lines:
-                line = line.strip()
-                if not line or "Total" in line:
-                    continue
-                match = re.match(platform_fee_pattern, line)
-                if match:
-                    description = match.group(1).strip()
-                    total_price = float(match.group(5))
-                    items.append({
-                        "description": description,
-                        "quantity": 1,  # Platform fee is a single item
-                        "unit_price": float(match.group(2)),  # Taxable amount
-                        "total_price": total_price
-                    })
-                    print(f"Extracted Platform Fee: {description}, Total Price: {total_price}")  # Debug
-        return items
-
-    # Find the end of the table (before "Total Amount", "Total Value", or end of text)
-    table_end = len(lines)
-    for i in range(table_start, len(lines)):
-        if "Total Amount" in lines[i] or "Total Value" in lines[i] or "Total Due" in lines[i] or "Item(s) Total" in lines[i]:
-            table_end = i
-            break
-    
-    print(f"Table section: lines {table_start} to {table_end-1}")  # Debug
-    table_lines = lines[table_start:table_end]
-    print("Table lines:", table_lines)  # Debug
-
-    # Define patterns based on table format
-    if table_format[0] == "Item Description":
-        # Pattern for invoice_4.pdf: "Monitor 24 inch | 7 | 150.00 | 1050.00"
-        table_row_pattern = r"\|?\s*([A-Za-z\s\d-]+(?:\s[A-Za-z\s\d-]+)*?)\s*\|?\s*(\d+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?"
-    else:
-        # Pattern for Invoice_6164752968.pdf: "1 x Chicken Frankie | 60 | 6 | 54 | 2.5% | 1.35 | 2.5% | 1.35 | 56.7"
-        table_row_pattern = r"\|?\s*(\d+\s*x\s*[A-Za-z\s\d-]+(?:\s[A-Za-z\s\d-]+)*?)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*[\d.%]+\s*\|?\s*[\d.]+(?:\s*\|?\s*[\d.%]+\s*\|?\s*[\d.]+)?\s*\|?\s*([\d.]+)\s*\|?"
+        
+        print(f"Main table section: lines {table_start} to {table_end-1}")  # Debug
+        table_lines = lines[table_start:table_end]
+        print("Main table lines:", table_lines)  # Debug
+
+        # Define patterns based on table format
+        if table_format[0] == "Item Description":
+            # Pattern for invoice_4.pdf: "Monitor 24 inch | 7 | 150.00 | 1050.00"
+            table_row_pattern = r"\|?\s*([A-Za-z\s\d-]+(?:\s[A-Za-z\s\d-]+)*?)\s*\|?\s*(\d+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?"
+        else:
+            # Pattern for Invoice_6164752968.pdf: "1 x Chicken Frankie | 60 | 6 | 54 | 2.5% | 1.35 | 2.5% | 1.35 | 56.7"
+            table_row_pattern = r"\|?\s*(\d+\s*x\s*[A-Za-z\s\d-]+(?:\s[A-Za-z\s\d-]+)*?)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*([\d.]+)\s*\|?\s*[\d.%]+\s*\|?\s*[\d.]+(?:\s*\|?\s*[\d.%]+\s*\|?\s*[\d.]+)?\s*\|?\s*([\d.]+)\s*\|?"
 
-    for line in table_lines:
-        line = line.strip()
-        if not line:
-            continue
-        # Skip alignment rows (e.g., "|---|---|")
-        if re.match(r"\|?\s*[-:]+(\s*\|\s*[-:]+)*\s*\|?", line):
-            print(f"Skipping alignment row: {line}")
-            continue
-        print(f"Processing table row: {line}")  # Debug
-        match = re.match(table_row_pattern, line)
-        if match:
-            if table_format[0] == "Item Description":
-                description = match.group(1).strip()
-                quantity = int(match.group(2))
-                unit_price = float(match.group(3))
-                total_price = float(match.group(4))
+        for line in table_lines:
+            line = line.strip()
+            if not line:
+                continue
+            # Skip alignment rows (e.g., "|---|---|")
+            if re.match(r"\|?\s*[-:]+(\s*\|\s*[-:]+)*\s*\|?", line):
+                print(f"Skipping alignment row: {line}")
+                continue
+            print(f"Processing main table row: {line}")  # Debug
+            match = re.match(table_row_pattern, line)
+            if match:
+                if table_format[0] == "Item Description":
+                    description = match.group(1).strip()
+                    quantity = int(match.group(2))
+                    unit_price = float(match.group(3))
+                    total_price = float(match.group(4))
+                else:
+                    description = match.group(1).strip()
+                    quantity = int(description.split(' x ')[0].strip()) if ' x ' in description else 1
+                    unit_price = float(match.group(2))  # Gross value
+                    total_price = float(match.group(5))  # Total after taxes
+                items.append({
+                    "description": description,
+                    "quantity": quantity,
+                    "unit_price": unit_price,
+                    "total_price": total_price
+                })
+                print(f"Extracted Item: {description}, Qty: {quantity}, Unit Price: {unit_price}, Total Price: {total_price}")  # Debug
             else:
-                description = match.group(1).strip()
-                quantity = int(description.split(' x ')[0].strip()) if ' x ' in description else 1
-                unit_price = float(match.group(2))  # Gross value
-                total_price = float(match.group(5))  # Total after taxes
-            items.append({
-                "description": description,
-                "quantity": quantity,
-                "unit_price": unit_price,
-                "total_price": total_price
-            })
-            print(f"Extracted Item: {description}, Qty: {quantity}, Unit Price: {unit_price}, Total Price: {total_price}")  # Debug
-        else:
-            print(f"Failed to match row: {line}")
+                print(f"Failed to match main table row: {line}")
 
-    # Look for platform fee as a separate table
+    # Extract platform fee table (e.g., Sr.No Particulars)
     platform_fee_start = -1
     for i, line in enumerate(lines):
         if "Sr.No Particulars" in line:
             platform_fee_start = i + 1
             break
+    
     if platform_fee_start != -1:
         platform_fee_end = len(lines)
         for i in range(platform_fee_start, len(lines)):
@@ -199,17 +168,18 @@ def extract_items(text):
                     "total_price": total_price
                 })
                 print(f"Extracted Platform Fee: {description}, Total Price: {total_price}")  # Debug
+            else:
+                print(f"Failed to match platform fee row: {line}")
 
     return items
 
 def extract_entities(text):
-    """Extract structured invoice details including recipient name using flexible regex patterns."""
+    """Extract structured invoice details using flexible regex patterns."""
     invoice_numbers = []
     primary_invoice_number = "Unknown"
     vendor_name = "Unknown"
     invoice_date = datetime.now().date()
     total_amount = 0.0
-    recipient_name = "Unknown"
 
     # Extract items first to use as a filter for NER
     items = extract_items(text)
@@ -220,7 +190,6 @@ def extract_entities(text):
     vendor_pattern = r"(?:Vendor\s*(?:Name|Company)?|Supplier|Company\s*Name|From|Sold\s*By|Restaurant\s*Name)\s*[:\-\s]*([A-Za-z\s&\.\-]+)(?=\s*(?:Address|Invoice\s*(?:No|Number)|Date|Phone|Email|\n|$))"
     invoice_date_pattern = r"(?:Invoice\s*Date|Date|Issue\s*Date)\s*[:\-\s]*(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4}|\d{2}-\d{2}-\d{4}|[A-Za-z]+\s*\d{1,2},\s*\d{4})"
     total_amount_pattern = r"(?:Total\s*(?:Amount|Due|Value))\s*[:\-\s]*[₹$£€]?\s*([\d,]+\.?\d*)\s*(?:USD|GBP|EUR|INR)?"
-    recipient_pattern = r"(?:Customer\s*Name|Recipient|Bill\s*To)\s*[:\-\s]*([A-Za-z]+)\s*(?=\s*(?:Address|Phone|Email|\n|$))"
 
     # Invoice Numbers (capture all, then prioritize)
     invoice_num_matches = list(re.finditer(invoice_num_pattern, text, re.IGNORECASE))
@@ -231,7 +200,6 @@ def extract_entities(text):
     if invoice_numbers:
         # Prioritize the invoice number associated with "Restaurant Service" (HSN Code: 996331)
         for i, num in enumerate(invoice_numbers):
-            # Find the context of this invoice number in the text
             start_idx = text.find(num)
             context = text[max(0, start_idx-100):start_idx+100]
             if "996331" in context:  # HSN Code for Restaurant Service
@@ -245,12 +213,10 @@ def extract_entities(text):
     vendor_match = re.search(vendor_pattern, text, re.IGNORECASE)
     if vendor_match:
         vendor_name = vendor_match.group(1).strip()
-        # Ensure vendor name is not an item description
         if vendor_name.lower() in item_descriptions:
             vendor_name = "Unknown"
         print(f"Matched Vendor Name (Regex): {vendor_name}")  # Debug
     else:
-        # Enhanced NER fallback for multi-word organization names
         ner_results = ner_pipeline(text)
         org_name_parts = []
         for i, entity in enumerate(ner_results):
@@ -264,10 +230,10 @@ def extract_entities(text):
                 vendor_name = candidate_vendor_name
             print(f"NER Matched Vendor Name: {vendor_name}")  # Debug
 
-    # Invoice Date (prioritize "Invoice Date")
+    # Invoice Date (prioritize "Invoice Date" and exclude "Order Date")
     invoice_date_match = None
     for line in text.split('\n'):
-        if "Invoice Date" in line and not "Order Date" in line:
+        if "Invoice Date" in line and "Order Date" not in line:
             match = re.search(invoice_date_pattern, line, re.IGNORECASE)
             if match:
                 invoice_date_match = match
@@ -290,29 +256,28 @@ def extract_entities(text):
         except ValueError as e:
             print(f"Failed to parse Invoice Date '{date_str}': {str(e)}")  # Debug
 
-    # Total Amount (sum all "Total Value" entries, fix parsing)
+    # Total Amount (sum final totals, including taxes)
     total_amount_matches = re.finditer(total_amount_pattern, text, re.IGNORECASE)
     total_amounts = []
     for match in total_amount_matches:
         amount_str = match.group(1).replace(",", "")
         try:
             amount = float(amount_str)
-            # Ignore amounts that are unrealistically large (likely parsing errors)
-            if amount < 1000000:  # Arbitrary threshold to exclude erroneous large numbers
+            if amount < 1000000:  # Exclude unrealistically large amounts
                 total_amounts.append(amount)
             print(f"Matched Amount: {amount}")  # Debug
         except ValueError:
             continue
-    total_amount = sum(total_amounts) if total_amounts else 0.0
+    # Sum only the final totals (e.g., after taxes)
+    if total_amounts:
+        # In this invoice, "Total Value" appears twice: ₹184.5 (net value before taxes) and ₹193.726 (after taxes)
+        # We want the final total after taxes for the main items, plus the platform fee
+        main_total = max([amt for amt in total_amounts if amt > 100], default=0.0)  # ₹193.726
+        platform_fee = min([amt for amt in total_amounts if amt < 10], default=0.0)  # ₹3.54
+        total_amount = main_total + platform_fee
     print(f"Calculated Total Amount: {total_amount}")  # Debug
 
-    # Recipient Name
-    recipient_match = re.search(recipient_pattern, text, re.IGNORECASE)
-    if recipient_match:
-        recipient_name = recipient_match.group(1).strip()
-        print(f"Matched Recipient Name: {recipient_name}")  # Debug
-
-    return primary_invoice_number, vendor_name, invoice_date, total_amount, recipient_name
+    return primary_invoice_number, vendor_name, invoice_date, total_amount
 
 def fetch_vendor_history(vendor_name, invoice_number, time_window_days=30):
     """Fetch historical invoices for the vendor from Salesforce."""
@@ -450,7 +415,7 @@ def process_invoice(pdf_file):
     if "Error" in text:
         return f"**Error**: {text}"
 
-    invoice_number, vendor_name, invoice_date, total_amount, recipient_name = extract_entities(text)
+    invoice_number, vendor_name, invoice_date, total_amount = extract_entities(text)
     items = extract_items(text)
     text_length = len(text)
 
@@ -483,7 +448,6 @@ def process_invoice(pdf_file):
     cleaned_items = []
     for item in items:
         desc = item['description']
-        # Additional cleaning to ensure no quantity or price data
         desc = re.sub(r'\s*Quantity\s*\d+', '', desc, flags=re.IGNORECASE).strip()
         desc = re.sub(r'\s*Unit\s*Price\s*[₹$]\d+\.\d+', '', desc, flags=re.IGNORECASE).strip()
         desc = re.sub(r'\s*Total\s*Price\s*[₹$]\d+\.\d+', '', desc, flags=re.IGNORECASE).strip()
@@ -502,7 +466,7 @@ def process_invoice(pdf_file):
         f"- **Invoice Number**: {invoice_number}",
         f"- **Vendor Name**: {vendor_name}",
         f"- **Invoice Date**: {invoice_date}",
-        f"- **Invoice Amount**: ₹{total_amount:,.2f}",  # Assuming INR for this PDF
+        f"- **Invoice Amount**: ₹{total_amount:,.2f}",
     ]
 
     # Add items section