Deploy FastAPI Hugging Face

Dockerfile

@@ -1,9 +1,8 @@
 # Use a newer Python version for better wheel support
 FROM python:3.10-slim
 
-# Install a LIGHTER Java Runtime (Headless is much faster to install)
+# Install system dependencies (No Java needed for pdfplumber)
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends openjdk-17-jre-headless && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -24,8 +23,8 @@ RUN pip install --no-cache-dir --user -r requirements.txt
 # Copy the rest of the application
 COPY --chown=user . .
 
-# Create necessary directories
-RUN mkdir -p data/raw_csv_temp data/output_process_data data/strictly_cleaned_data data/output data/save_pdf logs
+# Pre-create directories and set permissions (important for the 'user' account)
+RUN mkdir -p data/input data/output data/save_pdf logs data/raw_csv_temp data/output_process_data data/strictly_cleaned_data
 
 # Expose port 7860 (Hugging Face default)
 EXPOSE 7860

requirements.txt

@@ -2,7 +2,7 @@
 pandas>=1.3.0
 numpy>=1.21.0
 scikit-learn>=1.0.0
-tabula-py>=2.4.0
+pdfplumber>=0.7.0
 
 # API
 fastapi>=0.95.0

telecom_anomaly/utils/pdf_processor.py

@@ -1,12 +1,13 @@
-import tabula
+import pdfplumber
 import pandas as pd
 import io
 import re
 import os
 import tempfile
+import csv
 from pathlib import Path
 
-# --- CONFIGURATION (Restored) ---
+# --- CONFIGURATION ---
 DATA_BASE = Path("data")
 RAW_CSV_DIR = DATA_BASE / "raw_csv_temp"
 PROCESS_OUTPUT_DIR = DATA_BASE / "output_process_data"
@@ -17,23 +18,63 @@ def setup_folders():
     for folder in [RAW_CSV_DIR, PROCESS_OUTPUT_DIR, STRICT_CLEAN_DIR]:
         try:
             if not os.path.exists(folder):
-                os.makedirs(folder)
+                os.makedirs(folder, exist_ok=True)
         except Exception as e:
             print(f"Warning: Could not create folder {folder}: {e}")
 
-# --- ORIGINAL FUNCTIONS (Restored) ---
+# --- UPDATED EXTRACTION LOGIC (using pdfplumber) ---
+
+def extract_pdf_to_lines(pdf_file_path):
+    """Stage 1: Returns raw CSV lines using pdfplumber (no Java required)."""
+    setup_folders()
+    output_lines = []
+    
+    try:
+        with pdfplumber.open(pdf_file_path) as pdf:
+            for page in pdf.pages:
+                tables = page.extract_tables()
+                if not tables:
+                    # If no tables found, try extracting text directly as a fallback
+                    text = page.extract_text()
+                    if text:
+                        output_lines.extend([line + "\n" for line in text.splitlines()])
+                    continue
+                
+                for table in tables:
+                    # Write each table to a CSV-formatted string
+                    si = io.StringIO()
+                    cw = csv.writer(si)
+                    # Clean the data: remove None and replace newlines with spaces
+                    cleaned_table = []
+                    for row in table:
+                        if row:
+                            cleaned_row = [str(cell).replace('\n', ' ').strip() if cell is not None else "" for cell in row]
+                            cleaned_table.append(cleaned_row)
+                    
+                    cw.writerows(cleaned_table)
+                    output_lines.extend(si.getvalue().splitlines(True))
+                    si.close()
+                    
+        return output_lines
+    except Exception as e:
+        print(f"Error extracting {pdf_file_path} with pdfplumber: {e}")
+        return []
 
 def extract_pdf_to_csv(pdf_path, base_name):
-    """Stage 1: Convert PDF tables to a raw CSV file."""
+    """Stage 1: Convert PDF to a raw CSV file using pdfplumber."""
     setup_folders()
     csv_path = os.path.join(RAW_CSV_DIR, f"{base_name}_raw.csv")
     try:
-        tabula.convert_into(pdf_path, csv_path, output_format="csv", pages="all", lattice=True)
+        lines = extract_pdf_to_lines(pdf_path)
+        with open(csv_path, 'w', encoding='utf-8') as f:
+            f.writelines(lines)
         return csv_path
     except Exception as e:
-        print(f"Error extracting {pdf_path}: {e}")
+        print(f"Error saving CSV for {pdf_path}: {e}")
         return None
 
+# --- REMAINING LOGIC (Exactly same as before) ---
+
 def process_telecom_data_complete(input_path, base_name):
     """Stage 2: Parse raw CSV and repair fragmented rows."""
     try:
@@ -54,21 +95,6 @@ def strict_clean_data(file_path, base_name):
     df_cleaned.to_csv(final_path, index=False)
     return final_path
 
-# --- IN-MEMORY VERSIONS (Exact same logic) ---
-
-def extract_pdf_to_lines(pdf_file_path):
-    """Stage 1 (In-Memory): Returns raw CSV lines."""
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp:
-        tmp_path = tmp.name
-    try:
-        tabula.convert_into(pdf_file_path, tmp_path, output_format="csv", pages="all", lattice=True)
-        with open(tmp_path, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-        return lines
-    finally:
-        if os.path.exists(tmp_path):
-            os.remove(tmp_path)
-
 def process_telecom_data_df(lines, base_name):
     """Stage 2 (In-Memory): Returns DataFrame."""
     return _process_lines_logic(lines, base_name, save_files=False)
@@ -77,8 +103,6 @@ def strict_clean_df(df):
     """Stage 3 (In-Memory): Returns Cleaned DataFrame."""
     return _strict_clean_logic(df)
 
-# --- SHARED LOGIC (To ensure 100% parity) ---
-
 def _process_lines_logic(lines, base_name, save_files=False):
     header_idx = -1
     for i, line in enumerate(lines):
@@ -92,13 +116,21 @@ def _process_lines_logic(lines, base_name, save_files=False):
     if save_files:
         setup_folders()
         customer_text = "".join(lines[:header_idx])
-        df_customer = pd.read_csv(io.StringIO(customer_text))
-        df_customer.columns = [re.sub(r'[\r\n\s]+', ' ', c).strip() for c in df_customer.columns]
-        cust_path = os.path.join(PROCESS_OUTPUT_DIR, f'{base_name}_customer_info.csv')
-        df_customer.to_csv(cust_path, index=False)
+        if customer_text.strip():
+            try:
+                df_customer = pd.read_csv(io.StringIO(customer_text))
+                df_customer.columns = [re.sub(r'[\r\n\s]+', ' ', c).strip() for c in df_customer.columns]
+                cust_path = os.path.join(PROCESS_OUTPUT_DIR, f'{base_name}_customer_info.csv')
+                df_customer.to_csv(cust_path, index=False)
+            except:
+                pass
 
     call_text = "".join(lines[header_idx:])
-    df_raw = pd.read_csv(io.StringIO(call_text))
+    try:
+        df_raw = pd.read_csv(io.StringIO(call_text))
+    except:
+        return None
+        
     df_raw.columns = [c.strip() for c in df_raw.columns]
     main_col = df_raw.columns[0] 
     df_raw = df_raw[df_raw[main_col].astype(str).str.contains("event_date_time") == False]
@@ -122,6 +154,9 @@ def _process_lines_logic(lines, base_name, save_files=False):
     if current_row is not None:
         fixed_rows.append(current_row)
         
+    if not fixed_rows:
+        return None
+        
     df_calls = pd.DataFrame(fixed_rows)
     df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
     dt_series = pd.to_datetime(df_calls[main_col], errors='coerce')