Optimize for Hugging Face Spaces with Java support

.dockerignore

@@ -17,14 +17,3 @@ Task2.txt
 Task3.txt
 error.txt
 tests/
-.ipynb_checkpoints/
-.cache/
-node_modules/
-*.exe
-*.dll
-*.so
-*.bak
-*.tmp
-*.log
-.DS_Store
-Thumbs.db

Dockerfile

@@ -1,37 +1,50 @@
-FROM python:3.10-slim
-
-# Set environment variables
-ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    HOME=/home/user
-
-WORKDIR /app
-
-# Create user
+# Use official Python slim image
+FROM python:3.9-slim
+
+# Install Java Runtime Environment (REQUIRED for tabula-py to process PDFs)
+# Also install other potentially useful system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    default-jre \
+    gcc \
+    python3-dev \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set up a new user named "user" with user ID 1000
 RUN useradd -m -u 1000 user
 
-# Copy and install requirements first
+# Set environment variables
+# HF Spaces needs the port to be 7860
+# _JAVA_OPTIONS limits Java's heap size to avoid crashing the free tier (16GB RAM)
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=/home/user/app \
+    JAVA_HOME=/usr/lib/jvm/default-java \
+    _JAVA_OPTIONS="-Xmx4g"
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the requirements file and install dependencies as the "user"
 COPY --chown=user requirements.txt .
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir --user -r requirements.txt
-
-# Add user local bin to path
-ENV PATH="/home/user/.local/bin:${PATH}"
+RUN pip install --no-cache-dir --user -r requirements.txt
 
-# surgically copy only what is needed
-COPY --chown=user telecom_anomaly/ telecom_anomaly/
-COPY --chown=user config/ config/
-COPY --chown=user main.py .
+# Copy the rest of the application and set ownership
+COPY --chown=user . .
 
-# Create empty directories with right permissions
-RUN mkdir -p data/input data/output data/save_pdf logs data/raw_csv_temp data/output_process_data data/strictly_cleaned_data && \
-    chown -R user:user /app
+# Create necessary directories and ensure they are writable
+# We create them under the app directory to ensure they are within the user's scope
+RUN mkdir -p data/input data/output data/save_pdf data/raw_csv_temp data/output_process_data data/strictly_cleaned_data logs && \
+    chmod -R 777 data logs
 
+# Switch to the "user"
 USER user
 
-# Expose HF default port
+# Expose port 7860 (Hugging Face default)
 EXPOSE 7860
 
-# Run API
-CMD ["python3", "-m", "uvicorn", "telecom_anomaly.api:app", "--host", "0.0.0.0", "--port", "7860"]
+# Command to run the API
+# We use --workers 1 to save memory on free tier
+# We use --timeout-keep-alive 60 to handle potentially slow PDF processing
+CMD ["uvicorn", "telecom_anomaly.api:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--timeout-keep-alive", "60"]

error.txt

@@ -0,0 +1,24 @@
+INFO:telecom_anomaly.core.detector:Initialized TelecomAnomalyDetector with 5 detectors
+INFO:telecom_anomaly.io.reader:Reading data from C:\Users\winsweb\AppData\Local\Temp\tmpdwrf9kgk.csv
+INFO:telecom_anomaly.io.reader:Read 551 valid records from 551 total rows
+INFO:telecom_anomaly.core.detector:Loaded 551 valid records
+INFO:telecom_anomaly.core.detector:Starting anomaly detection
+INFO:telecom_anomaly.core.detector:DurationDetector: 21 anomalies
+INFO:telecom_anomaly.core.detector:FrequencyDetector: 0 anomalies
+INFO:telecom_anomaly.core.detector:SMSDetector: 0 anomalies
+INFO:telecom_anomaly.core.detector:LocationDetector: 0 anomalies
+INFO:telecom_anomaly.core.detector:TimePatternDetector: 9 anomalies
+INFO:telecom_anomaly.core.detector:Detection completed in 0.10s
+INFO:telecom_anomaly.core.detector:Rows with anomalies: 30/551
+INFO:telecom_anomaly.io.exporter:Generating trackable data: C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\data\output\telecom_data_with_anomalies_20260324_161213.csv
+INFO:telecom_anomaly.io.exporter:Exported 551 rows, 30 with anomalies
+INFO:telecom_anomaly.io.exporter:Results exported to C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\data\output\telecom_anomaly_results_20260324_161213.json
+INFO:telecom_anomaly.io.exporter:Results exported to C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\data\output\telecom_anomaly_results_20260324_161213.csv
+INFO:telecom_anomaly.io.exporter:Report generated: C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\data\output\telecom_anomaly_report_20260324_161213.txt
+INFO:telecom_anomaly.io.exporter:Suspicious subscribers exported to C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\data\output\suspicious_subscribers_20260324_161213.csv
+ERROR:telecom_anomaly.api:Analysis failed: name 'Path' is not defined
+Traceback (most recent call last):
+  File "C:\my_project\work\GH_Police\Backend\telecom_anomaly_detection\telecom_anomaly\api.py", line 72, in analyze_cdr
+    "saved_files": [Path(p).name for p in outputs.values()] if save_results else []
+                    ^^^^
+NameError: name 'Path' is not defined

requirements.txt

@@ -1,14 +1,23 @@
 # Core Analysis
-pandas==2.0.3
-numpy==1.24.3
-scikit-learn==1.3.0
-pdfplumber==0.10.2
+pandas>=1.3.0
+numpy>=1.21.0
+scikit-learn>=1.0.0
+tabula-py>=2.4.0
 
 # API
-fastapi==0.103.1
-uvicorn==0.23.2
-python-multipart==0.0.6
-pydantic==1.10.12
+fastapi>=0.95.0
+uvicorn>=0.21.0
+python-multipart>=0.0.6
+pydantic>=1.10.0
 
 # Export & PDF
-fpdf2==2.7.4
+fpdf2>=2.7.0
+
+# Visualization (if needed for exports)
+matplotlib>=3.4.0
+seaborn>=0.11.0
+
+# Development
+pytest>=6.2.0
+black>=21.0.0
+flake8>=3.9.0

telecom_anomaly/utils/pdf_processor.py

@@ -1,13 +1,12 @@
-import pdfplumber
+import tabula
 import pandas as pd
 import io
 import re
 import os
 import tempfile
-import csv
 from pathlib import Path
 
-# --- CONFIGURATION ---
+# --- CONFIGURATION (Restored) ---
 DATA_BASE = Path("data")
 RAW_CSV_DIR = DATA_BASE / "raw_csv_temp"
 PROCESS_OUTPUT_DIR = DATA_BASE / "output_process_data"
@@ -18,63 +17,23 @@ def setup_folders():
     for folder in [RAW_CSV_DIR, PROCESS_OUTPUT_DIR, STRICT_CLEAN_DIR]:
         try:
             if not os.path.exists(folder):
-                os.makedirs(folder, exist_ok=True)
+                os.makedirs(folder)
         except Exception as e:
             print(f"Warning: Could not create folder {folder}: {e}")
 
-# --- UPDATED EXTRACTION LOGIC (using pdfplumber) ---
-
-def extract_pdf_to_lines(pdf_file_path):
-    """Stage 1: Returns raw CSV lines using pdfplumber (no Java required)."""
-    setup_folders()
-    output_lines = []
-    
-    try:
-        with pdfplumber.open(pdf_file_path) as pdf:
-            for page in pdf.pages:
-                tables = page.extract_tables()
-                if not tables:
-                    # If no tables found, try extracting text directly as a fallback
-                    text = page.extract_text()
-                    if text:
-                        output_lines.extend([line + "\n" for line in text.splitlines()])
-                    continue
-                
-                for table in tables:
-                    # Write each table to a CSV-formatted string
-                    si = io.StringIO()
-                    cw = csv.writer(si)
-                    # Clean the data: remove None and replace newlines with spaces
-                    cleaned_table = []
-                    for row in table:
-                        if row:
-                            cleaned_row = [str(cell).replace('\n', ' ').strip() if cell is not None else "" for cell in row]
-                            cleaned_table.append(cleaned_row)
-                    
-                    cw.writerows(cleaned_table)
-                    output_lines.extend(si.getvalue().splitlines(True))
-                    si.close()
-                    
-        return output_lines
-    except Exception as e:
-        print(f"Error extracting {pdf_file_path} with pdfplumber: {e}")
-        return []
+# --- ORIGINAL FUNCTIONS (Restored) ---
 
 def extract_pdf_to_csv(pdf_path, base_name):
-    """Stage 1: Convert PDF to a raw CSV file using pdfplumber."""
+    """Stage 1: Convert PDF tables to a raw CSV file."""
     setup_folders()
     csv_path = os.path.join(RAW_CSV_DIR, f"{base_name}_raw.csv")
     try:
-        lines = extract_pdf_to_lines(pdf_path)
-        with open(csv_path, 'w', encoding='utf-8') as f:
-            f.writelines(lines)
+        tabula.convert_into(pdf_path, csv_path, output_format="csv", pages="all", lattice=True)
         return csv_path
     except Exception as e:
-        print(f"Error saving CSV for {pdf_path}: {e}")
+        print(f"Error extracting {pdf_path}: {e}")
         return None
 
-# --- REMAINING LOGIC (Exactly same as before) ---
-
 def process_telecom_data_complete(input_path, base_name):
     """Stage 2: Parse raw CSV and repair fragmented rows."""
     try:
@@ -95,6 +54,21 @@ def strict_clean_data(file_path, base_name):
     df_cleaned.to_csv(final_path, index=False)
     return final_path
 
+# --- IN-MEMORY VERSIONS (Exact same logic) ---
+
+def extract_pdf_to_lines(pdf_file_path):
+    """Stage 1 (In-Memory): Returns raw CSV lines."""
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp:
+        tmp_path = tmp.name
+    try:
+        tabula.convert_into(pdf_file_path, tmp_path, output_format="csv", pages="all", lattice=True)
+        with open(tmp_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+        return lines
+    finally:
+        if os.path.exists(tmp_path):
+            os.remove(tmp_path)
+
 def process_telecom_data_df(lines, base_name):
     """Stage 2 (In-Memory): Returns DataFrame."""
     return _process_lines_logic(lines, base_name, save_files=False)
@@ -103,6 +77,8 @@ def strict_clean_df(df):
     """Stage 3 (In-Memory): Returns Cleaned DataFrame."""
     return _strict_clean_logic(df)
 
+# --- SHARED LOGIC (To ensure 100% parity) ---
+
 def _process_lines_logic(lines, base_name, save_files=False):
     header_idx = -1
     for i, line in enumerate(lines):
@@ -111,99 +87,48 @@ def _process_lines_logic(lines, base_name, save_files=False):
             break
     
     if header_idx == -1:
-        print(f"DEBUG: Header 'event_date_time' not found in lines")
         return None
 
+    if save_files:
+        setup_folders()
+        customer_text = "".join(lines[:header_idx])
+        df_customer = pd.read_csv(io.StringIO(customer_text))
+        df_customer.columns = [re.sub(r'[\r\n\s]+', ' ', c).strip() for c in df_customer.columns]
+        cust_path = os.path.join(PROCESS_OUTPUT_DIR, f'{base_name}_customer_info.csv')
+        df_customer.to_csv(cust_path, index=False)
+
     call_text = "".join(lines[header_idx:])
-    try:
-        df_raw = pd.read_csv(io.StringIO(call_text))
-    except Exception as e:
-        print(f"DEBUG: Failed to read CSV from lines: {e}")
-        return None
-        
+    df_raw = pd.read_csv(io.StringIO(call_text))
     df_raw.columns = [c.strip() for c in df_raw.columns]
-    
-    # Intelligently find the date-time column
-    main_col = None
-    for col in df_raw.columns:
-        if "event_date_time" in col.lower():
-            main_col = col
-            break
-    
-    if main_col is None:
-        main_col = df_raw.columns[0]
-        print(f"DEBUG: 'event_date_time' column not found, falling back to first column: {main_col}")
-
-    # Remove header repetitions that can happen with pdfplumber page-by-page extraction
-    df_raw = df_raw[df_raw[main_col].astype(str).str.contains("event_date_time|date_time") == False]
+    main_col = df_raw.columns[0] 
+    df_raw = df_raw[df_raw[main_col].astype(str).str.contains("event_date_time") == False]
 
     fixed_rows = []
     current_row = None
-    # Support multiple date formats (YYYY-MM-DD or DD-MM-YYYY or DD/MM/YYYY)
-    date_regex = r'^(\d{4}-\d{2}-\d{2}|\d{2}-\d{2}-\d{4}|\d{2}/\d{2}/\d{4})'
-    
     for _, row in df_raw.iterrows():
         val = str(row[main_col]).strip()
-        if re.match(date_regex, val):
+        if re.match(r'^\d{4}-\d{2}-\d{2}', val):
             if current_row is not None:
                 fixed_rows.append(current_row)
             current_row = row.copy()
         elif current_row is not None:
-            # Append fragments to current row
-            # CRITICAL FIX: If the current value ends in a colon, don't add a space when joining
-            if str(current_row[main_col]).strip().endswith(':'):
-                current_row[main_col] = (str(current_row[main_col]).strip() + val).replace('\r', '').replace('\n', '')
-            else:
-                current_row[main_col] = (str(current_row[main_col]) + " " + val).replace('\r', '').replace('\n', '')
-            
+            current_row[main_col] = (str(current_row[main_col]) + " " + val).replace('\r', '').replace('\n', '')
             for col in df_raw.columns:
                 if col != main_col and pd.api.types.is_string_dtype(df_raw[col]):
                     fragment = str(row[col]).strip()
-                    if fragment.lower() != 'nan' and fragment != "":
+                    if fragment.lower() != 'nan':
                         current_row[col] = (str(current_row[col]) + " " + fragment).replace('\r', '').replace('\n', ' ')
 
     if current_row is not None:
         fixed_rows.append(current_row)
         
-    if not fixed_rows:
-        print(f"DEBUG: No rows matched date regex {date_regex}")
-        return None
-        
     df_calls = pd.DataFrame(fixed_rows)
-    # Clean up the joined column: remove any "01: 52:44" -> "01:52:44"
-    df_calls[main_col] = df_calls[main_col].str.replace(r'(\d{2}:)\s+(\d{2})', r'\1\2', regex=True)
     df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
-    
-    # Robust date parsing
-    try:
-        dt_series = pd.to_datetime(df_calls[main_col], errors='coerce')
-        # If too many NaT, try with dayfirst
-        if dt_series.isna().sum() > len(dt_series) * 0.5:
-            dt_series = pd.to_datetime(df_calls[main_col], errors='coerce', dayfirst=True)
-    except:
-        dt_series = pd.Series([None] * len(df_calls))
-
+    dt_series = pd.to_datetime(df_calls[main_col], errors='coerce')
     df_calls['event_date'] = dt_series.dt.date
     df_calls['event_time'] = dt_series.dt.time
-    
-    # Manual fallback for the specific "2024-01-01 01:52:44" format if pandas fails
-    mask = df_calls['event_date'].isna()
-    if mask.any():
-        for i in df_calls[mask].index:
-            try:
-                # Try simple split
-                parts = str(df_calls.loc[i, main_col]).split(' ')
-                if len(parts) >= 2:
-                    df_calls.loc[i, 'event_date'] = pd.to_datetime(parts[0]).date()
-                    df_calls.loc[i, 'event_time'] = pd.to_datetime(parts[1]).time()
-            except:
-                pass
-    
-    # If the main column was NOT the first one, we should keep it for debugging or drop it if it's already split
-    if main_col in df_calls.columns:
-        df_calls.drop(columns=[main_col], inplace=True)
+    df_calls.drop(columns=[main_col], inplace=True)
 
-    # Standardize column types and casing
     uppercase_cols = ['district', 'city', 'usage_type', 'usage_sub_type', 'calling_no', 'called_no', 'imei', 'location_id', 'msisdn']
     for col in df_calls.columns:
         c_low = col.lower()
@@ -219,22 +144,16 @@ def _process_lines_logic(lines, base_name, save_files=False):
     ordered_cols = ['event_date', 'event_time'] + [c for c in df_calls.columns if c not in ['event_date', 'event_time']]
     df_calls = df_calls[ordered_cols]
     
+    if save_files:
+        interim_path = os.path.join(PROCESS_OUTPUT_DIR, f'{base_name}_call_records_final.csv')
+        df_calls.to_csv(interim_path, index=False)
+        return interim_path
     return df_calls
 
 def _strict_clean_logic(df):
-    """Refined strict cleaning to be more resilient to parsing warnings."""
-    df_temp = df.copy()
-    
-    # Ensure duration is numeric
-    if 'duration' in df_temp.columns:
-        if df_temp['duration'].dtype == 'object':
-            df_temp['duration'] = df_temp['duration'].astype(str).str.replace(',', '.').str.extract('(\d+\.?\d*)')[0].astype(float)
-    
-    # Drop rows where we absolutely don't have a date or time
-    # But be careful not to drop everything if just the time format check is strict
-    df_temp = df_temp.dropna(subset=['event_date', 'event_time'])
-    
-    if df_temp.empty:
-        return df_temp
-        
-    return df_temp
+    if 'duration' in df.columns and df['duration'].dtype == 'object':
+        df['duration'] = df['duration'].str.replace(',', '.').astype(float)
+    df['event_date_copy'] = pd.to_datetime(df['event_date'], errors='coerce')
+    df['event_time_check'] = pd.to_datetime(df['event_time'].astype(str), format='%H:%M:%S', errors='coerce')
+    df_cleaned = df.dropna().copy()
+    return df_cleaned.drop(columns=['event_date_copy', 'event_time_check'])

telecom_anomaly/validation/validator.py

@@ -53,39 +53,23 @@ class DataValidator:
     
     @staticmethod
     def parse_datetime(date_str: str, time_str: str) -> Optional[datetime]:
-        """Parse datetime from date and time strings with robust fallback"""
-        if not date_str or date_str == 'NULL' or not time_str or time_str == 'NULL':
-            return None
-            
+        """Parse datetime from date and time strings"""
         try:
-            # Try standard format
-            return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
-        except:
-            try:
-                # Fallback to pandas for flexible parsing
-                import pandas as pd
-                combined = f"{date_str} {time_str}"
-                dt = pd.to_datetime(combined, errors='coerce')
-                if pd.notna(dt):
-                    return dt.to_pydatetime()
-            except:
-                pass
+            if date_str and time_str and date_str != 'NULL' and time_str != 'NULL':
+                return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError) as e:
+            logger.debug(f"Failed to parse datetime: {date_str} {time_str} - {e}")
         return None
     
     @staticmethod
     def is_night_call(time_str: str, night_start: int = 23, night_end: int = 5) -> bool:
         """Check if call occurred at night"""
         try:
-            if not time_str or time_str == 'NULL' or time_str == 'None':
+            if not time_str or time_str == 'NULL':
                 return False
-            # Handle both "HH:MM:SS" and full "YYYY-MM-DD HH:MM:SS"
-            ts = str(time_str).strip()
-            if ' ' in ts:
-                ts = ts.split(' ')[1]
-            
-            hour = int(ts.split(':')[0])
+            hour = int(time_str.split(':')[0])
             return hour >= night_start or hour < night_end
-        except:
+        except (ValueError, IndexError):
             return False
     
     @staticmethod