Update backend/app/api/ingest.py

backend/app/api/ingest.py

@@ -1,11 +1,4 @@
-"""
-Complete ingest pipeline with AUTONOMOUS AGENT INTEGRATION
-✅ Step 1: HF agents extract raw text
-✅ Step 2: HF NER finds entities
-✅ Step 3: Gemini maps to structured invoice fields
-✅ NEW: Autonomous agent orchestrates, validates, and self-corrects
-✅ UPDATED: Retry logic with exponential backoff + Local OCR fallback
-"""
+
 
 import os
 import uuid
@@ -27,36 +20,46 @@ logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s',
     handlers=[logging.StreamHandler(sys.stdout)],
-    force=True  # Override any existing config
+    force=True
 )
 
 logger = logging.getLogger(__name__)
 
-# ============================================
-# FORCE DATABASE INITIALIZATION ON MODULE LOAD
-# ============================================
 
-def init_tables_now():
-    """Force create tables on module import"""
+BASE_DIR = Path(__file__).parent.parent.parent.parent
+STORAGE_PATH = Path(os.getenv('STORAGE_PATH', str(BASE_DIR / "data" / "docs")))
+DB_PATH = Path(os.getenv('DB_PATH', str(BASE_DIR / "data" / "invoices.db")))
+LOCK_PATH = BASE_DIR / "data" / "invoices.db.lock"
+PREDICT_ENDPOINT = 'http://localhost:7860/predict'
+
+STORAGE_PATH.mkdir(parents=True, exist_ok=True)
+
+router = APIRouter(prefix="/api", tags=["ingest"])
+
+
+
+def _init_db_tables():
+    """Create tables on module import - ensures HF Space has tables"""
     try:
+        logger.info("🔍 Checking if database tables exist...")
         DB_PATH.parent.mkdir(parents=True, exist_ok=True)
         
-        with FileLock(str(LOCK_PATH), timeout=30):
+        with FileLock(str(LOCK_PATH), timeout=10):
             conn = sqlite3.connect(str(DB_PATH))
             cursor = conn.cursor()
             
-            # Check if table exists
+            # Quick check
             cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='ingest_jobs'")
             if cursor.fetchone():
                 conn.close()
                 logger.info("✅ Database tables already exist")
                 return
             
-            logger.warning("⚠️  Creating database tables...")
+            logger.warning("⚠️  Database tables not found, creating...")
             
-            # Create all tables (abbreviated version shown)
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS ingest_jobs (
+            # Create all tables
+            tables_sql = [
+                """CREATE TABLE IF NOT EXISTS ingest_jobs (
                     job_id TEXT PRIMARY KEY,
                     doc_id TEXT,
                     filename TEXT NOT NULL,
@@ -64,34 +67,30 @@ def init_tables_now():
                     error_text TEXT,
                     created_at TEXT DEFAULT CURRENT_TIMESTAMP,
                     updated_at TEXT DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS documents (
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS documents (
                     doc_id TEXT PRIMARY KEY,
                     job_id TEXT NOT NULL,
                     path TEXT NOT NULL,
                     filename TEXT NOT NULL,
                     content_type TEXT NOT NULL,
-                    uploaded_at TEXT DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS extractions (
+                    uploaded_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (job_id) REFERENCES ingest_jobs(job_id)
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS extractions (
                     doc_id TEXT PRIMARY KEY,
                     raw_text TEXT,
                     tables_json TEXT,
                     entities_json TEXT,
                     classification_json TEXT,
                     summary_text TEXT,
-                    extracted_at TEXT DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS invoice_fields (
+                    extracted_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (doc_id) REFERENCES documents(doc_id)
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS invoice_fields (
                     invoice_id INTEGER PRIMARY KEY AUTOINCREMENT,
                     doc_id TEXT NOT NULL,
                     cust_number TEXT,
@@ -100,30 +99,81 @@ def init_tables_now():
                     business_code TEXT,
                     cust_payment_terms TEXT,
                     confidence_map TEXT,
-                    created_at TEXT DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS batch_jobs (
+                    created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (doc_id) REFERENCES documents(doc_id)
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS batch_jobs (
                     batch_id TEXT PRIMARY KEY,
                     total_files INTEGER,
                     message TEXT,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            
-            cursor.execute("""
-                CREATE TABLE IF NOT EXISTS batch_job_mapping (
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS batch_job_mapping (
                     batch_id TEXT,
-                    job_id TEXT
-                )
-            """)
+                    job_id TEXT,
+                    FOREIGN KEY (batch_id) REFERENCES batch_jobs(batch_id),
+                    FOREIGN KEY (job_id) REFERENCES ingest_jobs(job_id)
+                )""",
+                
+                # ML feature tables
+                """CREATE TABLE IF NOT EXISTS customer_aggregates (
+                    cust_number TEXT PRIMARY KEY,
+                    cust_avg_days REAL,
+                    cust_median_days REAL,
+                    cust_invoice_count INTEGER,
+                    last_updated TEXT DEFAULT CURRENT_TIMESTAMP
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS payment_terms_aggregates (
+                    cust_payment_terms TEXT PRIMARY KEY,
+                    terms_avg_days REAL,
+                    terms_median_days REAL,
+                    terms_invoice_count INTEGER,
+                    last_updated TEXT DEFAULT CURRENT_TIMESTAMP
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS business_code_aggregates (
+                    business_code TEXT PRIMARY KEY,
+                    bc_avg_days REAL,
+                    bc_median_days REAL,
+                    bc_invoice_count INTEGER,
+                    last_updated TEXT DEFAULT CURRENT_TIMESTAMP
+                )""",
+                
+                """CREATE TABLE IF NOT EXISTS predictions_log (
+                    prediction_id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    invoice_id INTEGER,
+                    cust_number TEXT,
+                    posting_date TEXT,
+                    total_open_amount REAL,
+                    business_code TEXT,
+                    cust_payment_terms TEXT,
+                    predicted_days_to_clear REAL,
+                    predicted_clear_date TEXT,
+                    model_version TEXT,
+                    features_json TEXT,
+                    predicted_at TEXT DEFAULT CURRENT_TIMESTAMP
+                )"""
+            ]
+            
+            # Execute all CREATE TABLE statements
+            for sql in tables_sql:
+                cursor.execute(sql)
             
             # Create indexes
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_ingest_jobs_status ON ingest_jobs(status)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_documents_job_id ON documents(job_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_invoice_fields_doc_id ON invoice_fields(doc_id)")
+            indexes_sql = [
+                "CREATE INDEX IF NOT EXISTS idx_ingest_jobs_status ON ingest_jobs(status)",
+                "CREATE INDEX IF NOT EXISTS idx_ingest_jobs_created ON ingest_jobs(created_at DESC)",
+                "CREATE INDEX IF NOT EXISTS idx_documents_job_id ON documents(job_id)",
+                "CREATE INDEX IF NOT EXISTS idx_invoice_fields_doc_id ON invoice_fields(doc_id)",
+                "CREATE INDEX IF NOT EXISTS idx_batch_mapping_batch ON batch_job_mapping(batch_id)",
+                "CREATE INDEX IF NOT EXISTS idx_predictions_log_cust ON predictions_log(cust_number)"
+            ]
+            
+            for sql in indexes_sql:
+                cursor.execute(sql)
             
             conn.commit()
             conn.close()
@@ -135,21 +185,16 @@ def init_tables_now():
         import traceback
         logger.error(traceback.format_exc())
 
-# Run immediately on import
-logger.info("🔍 Initializing database on module load...")
-init_tables_now()
-
-# Setup
-BASE_DIR = Path(__file__).parent.parent.parent.parent
-STORAGE_PATH = Path(os.getenv('STORAGE_PATH', str(BASE_DIR / "data" / "docs")))
-DB_PATH = Path(os.getenv('DB_PATH', str(BASE_DIR / "data" / "invoices.db")))
-LOCK_PATH = BASE_DIR / "data" / "invoices.db.lock"
-PREDICT_ENDPOINT = 'http://localhost:7860/predict'
 
-STORAGE_PATH.mkdir(parents=True, exist_ok=True)
+# Run on module import
+logger.info("🔍 Initializing database on module load...")
+try:
+    _init_db_tables()
+    logger.info("✅ Database initialization complete")
+except Exception as e:
+    logger.error(f"❌ Database initialization failed: {e}")
+    logger.warning("⚠️  Application may not work correctly!")
 
-logger = logging.getLogger(__name__)
-router = APIRouter(prefix="/api", tags=["ingest"])
 
 
 # ============================================