Update app.py

app.py

@@ -1,248 +1,15 @@
 import os
-from dotenv import load_dotenv
 import logging
-import pandas as pd
-import numpy as np
-from sklearn.ensemble import IsolationForest
-from sklearn.preprocessing import StandardScaler
-import uuid
-from datetime import datetime, timedelta
-import re
 import gradio as gr
-import time
-from simple_salesforce import Salesforce, SalesforceAuthenticationFailed
 from pdf_extraction import extract_text_from_pdf
 from image_extraction import extract_text_from_image
 
-# Load environment variables from .env file
-load_dotenv()
-
-# Configure environment for CPU usage
-os.environ["CUDA_VISIBLE_DEVICES"] = ""  # Disable GPU usage
-os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"  # Disable oneDNN optimizations
-
 # Set up logging
 logging.basicConfig(level=logging.DEBUG)  # Set to DEBUG for detailed logging
 logger = logging.getLogger(__name__)
 
-# Read Salesforce credentials from environment variables
-SF_USERNAME = os.getenv("SF_USERNAME")
-SF_PASSWORD = os.getenv("SF_PASSWORD")
-SF_SECURITY_TOKEN = os.getenv("SF_SECURITY_TOKEN")
-
-logger.info(f"Salesforce login info: username={SF_USERNAME}")
-
-# Salesforce connection with error handling
-try:
-    sf = Salesforce(
-        username=SF_USERNAME,
-        password=SF_PASSWORD,
-        security_token=SF_SECURITY_TOKEN
-    )
-    logger.info("Salesforce login successful.")
-except SalesforceAuthenticationFailed as e:
-    logger.error(f"Salesforce authentication failed: {e}")
-    sf = None
-
-def extract_basic_info(text):
-    """Extract minimal information for fraud detection without altering the raw text."""
-    invoice_number = "Unknown"
-    vendor_name = "Unknown"
-    invoice_date = datetime.now().date()
-    total_amount = 0.0
-
-    # Minimal regex patterns for fraud detection
-    invoice_num_pattern = r"(?:invoice\s*(?:number|no\.?|#)|order\s*(?:number|no\.?))\s*[:\-\s#]*([\w-]+)|(?:inv-|ord-)([\w-]+)"
-    vendor_pattern = r"(?:vendor\s*(?:name|company)?|supplier|company\s*name|from|sold\s*by|to)\s*[:\-\s]*([A-Za-z\s&\.\-]+)(?=\s*(?:address|invoice\s*(?:no|number)|date|phone|email|\n|$))"
-    invoice_date_pattern = r"(?:invoice\s*date|date|issue\s*date|order\s*date)\s*[:\-\s]*(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4}|\d{2}-\d{2}-\d{4}|\d{2}\s*[A-Za-z]+\s*\d{4}|[A-Za-z]+\s*\d{1,2}(?:st|nd|rd|th)?\s*,?\s*\d{4}|\d{1,2}\s*[A-Za-z]+\s*\d{4})"
-    total_amount_pattern = r"(?:total\s*(?:amount|due)?|amount\s*due|total|grand\s*total)\s*[:\-\s]*[$₹€]?\s*([\d,]+\.?\d*)\s*(?:usd|inr|gbp|eur)?"
-
-    # Invoice Number
-    invoice_num_match = re.search(invoice_num_pattern, text, re.IGNORECASE)
-    if invoice_num_match:
-        invoice_number = invoice_num_match.group(1) if invoice_num_match.group(1) else invoice_num_match.group(2)
-        logger.info(f"Matched Invoice Number: {invoice_number}")
-
-    # Vendor Name
-    vendor_match = re.search(vendor_pattern, text, re.IGNORECASE)
-    if vendor_match:
-        vendor_name = vendor_match.group(1).strip()
-        logger.info(f"Matched Vendor Name: {vendor_name}")
-
-    # Invoice Date
-    invoice_date_match = re.search(invoice_date_pattern, text, re.IGNORECASE)
-    if invoice_date_match:
-        date_str = invoice_date_match.group(1)
-        try:
-            if "/" in date_str:
-                invoice_date = datetime.strptime(date_str, "%m/%d/%Y").date()
-            elif "," in date_str:
-                date_str = re.sub(r'(st|nd|rd|th)', '', date_str)
-                invoice_date = datetime.strptime(date_str, "%B %d, %Y").date()
-            elif "-" in date_str:
-                try:
-                    invoice_date = datetime.strptime(date_str, "%Y-%m-%d").date()
-                except ValueError:
-                    invoice_date = datetime.strptime(date_str, "%d-%m-%Y").date()
-            else:
-                date_str = re.sub(r'(st|nd|rd|th)', '', date_str)
-                invoice_date = datetime.strptime(date_str, "%d %B %Y").date()
-            logger.info(f"Matched Invoice Date: {invoice_date}")
-        except ValueError as e:
-            logger.warning(f"Failed to parse Invoice Date '{date_str}': {str(e)}")
-
-    # Total Amount
-    total_amount_match = re.search(total_amount_pattern, text, re.IGNORECASE)
-    if total_amount_match:
-        total_amount = float(total_amount_match.group(1).replace(",", ""))
-        logger.info(f"Matched Total Amount: {total_amount}")
-
-    return invoice_number, vendor_name, invoice_date, total_amount
-
-def fetch_vendor_history(vendor_name, invoice_number, time_window_days=30):
-    """Fetch historical invoices for the vendor from Salesforce with retry logic."""
-    if sf is None:
-        logger.warning("Salesforce client not initialized. Skipping vendor history fetch.")
-        return pd.DataFrame()
-
-    max_retries = 3
-    retry_delay = 5  # seconds
-
-    for attempt in range(1, max_retries + 1):
-        try:
-            end_date = datetime.now().date()
-            start_date = end_date - timedelta(days=time_window_days)
-            
-            query = f"""
-                SELECT Invoice_Number__c, Invoice_Amount__c, Invoice_Date__c, Vendor_Name__c
-                FROM Invoice_Record__c
-                WHERE Invoice_Date__c >= {start_date} AND Invoice_Date__c <= {end_date}
-                AND Vendor_Name__c = '{vendor_name}'
-                LIMIT 100
-            """
-            logger.info(f"Fetching vendor history for {vendor_name} (Attempt {attempt}/{max_retries})...")
-            result = sf.query(query)
-            records = result['records']
-            
-            history_df = pd.DataFrame(records)
-            if not history_df.empty:
-                history_df['Invoice_Date__c'] = pd.to_datetime(history_df['Invoice_Date__c']).dt.date
-            logger.info(f"Successfully fetched vendor history for {vendor_name}.")
-            return history_df
-        except Exception as e:
-            logger.warning(f"Failed to fetch vendor history (Attempt {attempt}/{max_retries}): {str(e)}")
-            if attempt < max_retries:
-                logger.info(f"Retrying in {retry_delay} seconds...")
-                time.sleep(retry_delay)
-            else:
-                logger.error(f"Failed to fetch vendor history after all retries: {str(e)}")
-                return pd.DataFrame()
-
-def check_data_consistency(invoice_number, vendor_name, invoice_date, history_df):
-    """Check for data consistency issues like duplicates."""
-    consistency_issues = []
-
-    if not history_df.empty:
-        duplicate_invoices = history_df[history_df['Invoice_Number__c'] == invoice_number]
-        if not duplicate_invoices.empty:
-            consistency_issues.append(f"Duplicate invoice number '{invoice_number}' found for vendor '{vendor_name}'.")
-
-    return consistency_issues
-
-def detect_anomalies(df, history_df):
-    """Detect anomalies in amount, frequency, and vendor patterns."""
-    df["is_amount_anomaly"] = 0
-    df["is_frequency_anomaly"] = 0
-    df["is_vendor_pattern_anomaly"] = 0
-
-    if not df.empty:
-        scaler = StandardScaler()
-        X_scaled = scaler.fit_transform(df[["amount"]])
-        model = IsolationForest(contamination=0.05, random_state=42)
-        df["is_amount_anomaly"] = model.fit_predict(X_scaled)
-
-    if not history_df.empty:
-        history_df['Invoice_Date__c'] = pd.to_datetime(history_df['Invoice_Date__c'])
-        date_range = (history_df['Invoice_Date__c'].max() - history_df['Invoice_Date__c'].min()).days + 1
-        frequency = len(history_df) / max(date_range, 1)
-        
-        date_diffs = [(d - history_df['Invoice_Date__c'].min()).days for d in history_df['Invoice_Date__c']]
-        date_clustering = np.std(date_diffs) if len(date_diffs) > 1 else 0
-        
-        frequency_df = pd.DataFrame({
-            "frequency": [frequency],
-            "date_clustering": [date_clustering]
-        })
-        scaler = StandardScaler()
-        X_scaled = scaler.fit_transform(frequency_df[["frequency", "date_clustering"]])
-        model = IsolationForest(contamination=0.05, random_state=42)
-        df["is_frequency_anomaly"] = model.fit_predict(X_scaled)[0]
-    else:
-        df["is_frequency_anomaly"] = 1
-
-    if not history_df.empty and len(history_df) > 1:
-        historical_amounts = history_df["Invoice_Amount__c"].astype(float)
-        mean_amount = historical_amounts.mean()
-        std_amount = historical_amounts.std() if len(historical_amounts) > 1 else 1
-        amount_variance = historical_amounts.var() if len(historical_amounts) > 1 else 0
-        
-        current_amount = df["amount"].iloc[0]
-        deviation = abs(current_amount - mean_amount) / (std_amount if std_amount > 0 else 1)
-        invoice_count = len(history_df)
-        
-        vendor_pattern_df = pd.DataFrame({
-            "amount_deviation": [deviation],
-            "invoice_count": [invoice_count],
-            "amount_variance": [amount_variance]
-        })
-        scaler = StandardScaler()
-        X_scaled = scaler.fit_transform(vendor_pattern_df[["amount_deviation", "invoice_count", "amount_variance"]])
-        model = IsolationForest(contamination=0.05, random_state=42)
-        df["is_vendor_pattern_anomaly"] = model.fit_predict(X_scaled)[0]
-    else:
-        df["is_vendor_pattern_anomaly"] = 1
-
-    return df
-
-def calculate_fraud_score(amount, is_amount_anomaly, is_frequency_anomaly, is_vendor_pattern_anomaly, text_length, consistency_issues, invoice_date):
-    """Calculate fraud score based on amount, anomalies, text length, consistency issues, and invoice date."""
-    score = 0.0
-    reasoning = []
-    today = datetime.now().date()
-
-    if amount > 5000:
-        score += 40
-        reasoning.append("High invoice amount detected.")
-    elif amount < 10:
-        score += 20
-        reasoning.append("Unusually low invoice amount.")
-
-    if invoice_date > today:
-        score += 10
-        reasoning.append("Invoice date is in the future.")
-
-    if is_amount_anomaly == -1:
-        score += 30
-        reasoning.append("Amount flagged as an anomaly.")
-    if is_frequency_anomaly == -1:
-        score += 25
-        reasoning.append("Unusual invoice submission frequency or clustering detected.")
-    if is_vendor_pattern_anomaly == -1:
-        score += 25
-        reasoning.append("Unusual vendor pattern detected (amount deviation, frequency, or variance).")
-
-    if text_length > 500:
-        score += 10
-        reasoning.append("Excessive text length in invoice.")
-
-    if consistency_issues:
-        score += 15 * len(consistency_issues)
-        reasoning.extend(consistency_issues)
-
-    return min(score, 100), reasoning
-
 def process_invoice(file):
-    """Process a single invoice (PDF or image) and return the raw extracted text with fraud detection."""
+    """Extract text from a single invoice (PDF or image) and return it as is."""
     # Determine file type and extract text accordingly
     file_extension = os.path.splitext(file.name)[1].lower()
     if file_extension == '.pdf':
@@ -250,98 +17,26 @@ def process_invoice(file):
     elif file_extension in ['.png', '.jpg', '.jpeg']:
         text = extract_text_from_image(file.name)
     else:
-        return f"**Error**: Unsupported file type '{file_extension}'. Please upload a PDF, PNG, or JPG file."
+        return f"Error: Unsupported file type '{file_extension}'. Please upload a PDF, PNG, or JPG file."
 
     if "Error" in text:
-        return f"**Error**: {text}"
-
-    # Extract basic info for fraud detection
-    invoice_number, vendor_name, invoice_date, total_amount = extract_basic_info(text)
-    text_length = len(text)
-
-    history_df = fetch_vendor_history(vendor_name, invoice_number)
-    consistency_issues = check_data_consistency(invoice_number, vendor_name, invoice_date, history_df)
-
-    data = {
-        "invoice_id": str(uuid.uuid4()),
-        "invoice_number": invoice_number,
-        "vendor_name": vendor_name,
-        "amount": total_amount,
-        "invoice_date": invoice_date,
-        "text_length": text_length
-    }
-    df = pd.DataFrame([data])
-
-    df = detect_anomalies(df, history_df)
-
-    fraud_score, fraud_reasoning = calculate_fraud_score(
-        df["amount"].iloc[0],
-        df["is_amount_anomaly"].iloc[0],
-        df["is_frequency_anomaly"].iloc[0],
-        df["is_vendor_pattern_anomaly"].iloc[0],
-        text_length,
-        consistency_issues,
-        invoice_date
-    )
-
-    # Prepare the output with raw text only
-    output = [
-        "## Raw Extracted Text",
-        "```",
-        text,
-        "```",
-        "",
-        "## Fraud Detection Summary",
-        f"- **Invoice Number**: {invoice_number}",
-        f"- **Vendor Name**: {vendor_name}",
-        f"- **Invoice Date**: {invoice_date}",
-        f"- **Invoice Amount**: ${total_amount:,.2f}",
-        f"- **Fraud Score**: {fraud_score}",
-        f"- **Status**: {'Flagged' if fraud_score > 50 else 'Cleared'}",
-        f"- **Flagged**: {fraud_score > 50}",
-        "",
-        "## Fraud Reasoning"
-    ]
-
-    if fraud_reasoning:
-        output.extend([f"- {reason}" for reason in fraud_reasoning])
-    else:
-        output.append("- No specific fraud indicators detected")
-
-    if sf is not None:
-        try:
-            record_data = {
-                "Invoice_Number__c": invoice_number,
-                "Vendor_Name__c": vendor_name,
-                "Invoice_Amount__c": total_amount,
-                "Invoice_Date__c": str(invoice_date),
-                "Fraud_Score__c": fraud_score,
-                "Fraud_Reason__c": "; ".join(fraud_reasoning),
-                "Flagged__c": fraud_score > 50,
-                "Status__c": "Flagged" if fraud_score > 50 else "Cleared",
-                "Items_Selected__c": "Not extracted"  # Since we're not parsing items
-            }
-            logger.debug(f"Record data being sent to Salesforce: {record_data}")
-            sf.Invoice_Record__c.create(record_data)
-            logger.info("Successfully created Salesforce record.")
-        except Exception as e:
-            logger.error(f"Failed to create Salesforce record: {str(e)}")
-            pass
+        return text
 
-    return "\n".join(output)
+    # Return the raw extracted text without any additional formatting
+    return text
 
 def gradio_interface(file):
-    """Gradio interface to process uploaded file (PDF or image) and display raw text with fraud detection."""
+    """Gradio interface to process uploaded file (PDF or image) and display raw text."""
     if file is None:
         return "Please upload a PDF or image file."
     result = process_invoice(file)
     return result
 
 with gr.Blocks(css=".prose a[href*='share']:has(svg) {display:none !important;}") as iface:
-    gr.Markdown("# Invoice Fraud Detection")
+    gr.Markdown("# Invoice Text Extraction")
     with gr.Row():
         file_input = gr.File(label="Upload Invoice (PDF, PNG, JPG)")
-    result_output = gr.Markdown(label="Fraud Detection Results")
+    result_output = gr.Textbox(label="Extracted Text")
     file_input.change(fn=gradio_interface, inputs=file_input, outputs=result_output)
 
 if __name__ == "__main__":