Upload app.py

app.py

@@ -4,6 +4,7 @@ import numpy as np
 import joblib
 from model_wrapper import FraudDetectionModel
 from preprocessor import FraudDataPreprocessor
+from feature_utils import fill_missing_features
 import os
 
 # Initialize the fraud detection model
@@ -11,6 +12,10 @@ fraud_model = FraudDetectionModel()
 
 # Load model if files exist
 try:
+    # First, ensure the FraudDataPreprocessor class is available
+    import sys
+    sys.modules['__main__'].FraudDataPreprocessor = FraudDataPreprocessor
+    
     # Load the specific XGBoost model files from your training
     model_path = "fraud_detection_model_xgboost_20250727_145448.joblib"
     preprocessor_path = "preprocessor_20250727_145448.joblib"
@@ -22,83 +27,61 @@ try:
         else:
             fraud_model.load_model(model_path, preprocessor_path)
         model_loaded = True
+        print(f"✅ Model loaded successfully!")
     else:
         model_loaded = False
-        print("Model files not found. Please upload the following files:")
+        print("❌ Model files not found. Please upload the following files:")
         print("- fraud_detection_model_xgboost_20250727_145448.joblib")
         print("- preprocessor_20250727_145448.joblib") 
         print("- model_metadata_20250727_145448.joblib")
 except Exception as e:
     model_loaded = False
-    print(f"Error loading model: {e}")
+    print(f"❌ Error loading model: {e}")
 
-def predict_fraud_risk(
-    transaction_amount,
-    card_type,
-    email_domain,
-    transaction_hour,
-    addr1,
-    addr2,
+def predict_single_transaction(
+    transaction_id,
+    transaction_dt,
+    transaction_amt,
+    product_cd,
     card1,
     card2,
-    dist1,
-    c1, c2, c3, c4, c5, c6,
-    d1, d2, d3, d4, d5,
-    m1, m2, m3, m4, m5, m6
+    card3,
+    card4,
+    card5,
+    card6,
+    addr1,
+    addr2,
+    p_emaildomain
 ):
-    """Predict fraud risk for a transaction"""
+    """Predict fraud risk for a single transaction with exact API fields"""
     
     if not model_loaded:
         return "❌ Model not loaded. Please contact administrator.", "", "", ""
     
     try:
-        # Prepare transaction data
+        # Prepare transaction data exactly as API expects
         transaction_data = {
-            'TransactionAmt': float(transaction_amount),
-            'card4': card_type,
-            'P_emaildomain': email_domain,
-            'R_emaildomain': email_domain,
+            'TransactionID': int(transaction_id) if transaction_id else 123456,
+            'TransactionDT': int(transaction_dt) if transaction_dt else 18403200,
+            'TransactionAmt': float(transaction_amt),
+            'ProductCD': product_cd,
+            'card1': int(card1) if card1 else None,
+            'card2': float(card2) if card2 else None,
+            'card3': float(card3) if card3 else None,
+            'card4': card4,
+            'card5': float(card5) if card5 else None,
+            'card6': card6,
             'addr1': float(addr1) if addr1 else None,
             'addr2': float(addr2) if addr2 else None,
-            'card1': float(card1) if card1 else None,
-            'card2': float(card2) if card2 else None,
-            'card3': float(transaction_amount),  # Often similar to transaction amount
-            'card5': 142.0,  # Default value
-            'card6': 'credit',  # Default value
-            'dist1': float(dist1) if dist1 else None,
-            'dist2': float(dist1) if dist1 else None,  # Often similar to dist1
-            'C1': float(c1),
-            'C2': float(c2),
-            'C3': float(c3),
-            'C4': float(c4),
-            'C5': float(c5),
-            'C6': float(c6),
-            'C7': 0.0,
-            'C8': 0.0,
-            'C9': 1.0,
-            'C10': 0.0,
-            'C11': 1.0,
-            'C12': 1.0,
-            'C13': 1.0,
-            'C14': 1.0,
-            'D1': float(d1),
-            'D2': float(d2),
-            'D3': float(d3),
-            'D4': float(d4),
-            'D5': float(d5),
-            'D10': 0.0,
-            'D15': 0.0,
-            'M1': m1,
-            'M2': m2,
-            'M3': m3,
-            'M4': m4,
-            'M5': m5,
-            'M6': m6,
-            'TransactionDT': transaction_hour * 3600  # Convert hour to seconds
+            'P_emaildomain': p_emaildomain,
+            'R_emaildomain': p_emaildomain  # Often same as P_emaildomain
         }
         
+        # Fill missing features with defaults
+        complete_data = fill_missing_features(transaction_data)
+        
         # Make prediction
-        result = fraud_model.predict_single_transaction(transaction_data)
+        result = fraud_model.predict_single_transaction(complete_data)
         
         if 'error' in result:
             return f"❌ {result['error']}", "", "", ""
@@ -123,44 +106,111 @@ def predict_fraud_risk(
     except Exception as e:
         return f"❌ Error: {str(e)}", "", "", ""
 
-def predict_from_csv(file):
+def predict_batch_from_csv(file):
     """Predict fraud risk for multiple transactions from CSV"""
     
     if not model_loaded:
-        return "❌ Model not loaded. Please contact administrator."
+        return "❌ Model not loaded. Please contact administrator.", None
     
     if file is None:
-        return "❌ Please upload a CSV file."
+        return "❌ Please upload a CSV file.", None
     
     try:
         # Read CSV file
         df = pd.read_csv(file.name)
         
-        # Make batch predictions
-        results_df = fraud_model.predict_batch(df)
+        # Validate required columns
+        required_cols = ['TransactionAmt']
+        missing_cols = [col for col in required_cols if col not in df.columns]
+        
+        if missing_cols:
+            return f"❌ Missing required columns: {missing_cols}. Please ensure your CSV has at least 'TransactionAmt' column.", None
+        
+        # Add default TransactionID if not present
+        if 'TransactionID' not in df.columns:
+            df['TransactionID'] = range(1, len(df) + 1)
+        
+        # Process each row and make predictions
+        results = []
+        
+        for idx, row in df.iterrows():
+            try:
+                # Fill missing features for this row
+                transaction_data = row.to_dict()
+                complete_data = fill_missing_features(transaction_data)
+                
+                # Make prediction
+                result = fraud_model.predict_single_transaction(complete_data)
+                
+                if 'error' not in result:
+                    # Add results to original row data
+                    row_result = row.copy()
+                    row_result['fraud_probability'] = result['fraud_probability']
+                    row_result['risk_level'] = result['risk_level']
+                    row_result['recommendation'] = result['recommendation']
+                    row_result['is_suspicious'] = result['is_suspicious']
+                else:
+                    # Handle prediction error
+                    row_result = row.copy()
+                    row_result['fraud_probability'] = None
+                    row_result['risk_level'] = 'Error'
+                    row_result['recommendation'] = result.get('error', 'Prediction failed')
+                    row_result['is_suspicious'] = False
+                
+                results.append(row_result)
+                
+            except Exception as e:
+                # Handle row processing error
+                row_result = row.copy()
+                row_result['fraud_probability'] = None
+                row_result['risk_level'] = 'Error'
+                row_result['recommendation'] = f'Processing error: {str(e)}'
+                row_result['is_suspicious'] = False
+                results.append(row_result)
+        
+        # Create results DataFrame
+        results_df = pd.DataFrame(results)
         
         # Save results
-        output_path = "fraud_predictions.csv"
+        output_path = "fraud_predictions_batch.csv"
         results_df.to_csv(output_path, index=False)
         
         # Create summary
+        valid_predictions = results_df[results_df['fraud_probability'].notna()]
         total_transactions = len(results_df)
-        high_risk = len(results_df[results_df['fraud_probability'] >= 0.8])
-        medium_risk = len(results_df[(results_df['fraud_probability'] >= 0.5) & (results_df['fraud_probability'] < 0.8)])
-        low_risk = len(results_df[(results_df['fraud_probability'] >= 0.2) & (results_df['fraud_probability'] < 0.5)])
-        very_low_risk = len(results_df[results_df['fraud_probability'] < 0.2])
-        
-        summary = f"""
-        📊 **Batch Prediction Summary**
+        valid_count = len(valid_predictions)
         
-        Total Transactions: {total_transactions}
-        🔴 High Risk: {high_risk} ({high_risk/total_transactions:.1%})
-        🟡 Medium Risk: {medium_risk} ({medium_risk/total_transactions:.1%})
-        🟠 Low Risk: {low_risk} ({low_risk/total_transactions:.1%})
-        🟢 Very Low Risk: {very_low_risk} ({very_low_risk/total_transactions:.1%})
-        
-        Results saved to: {output_path}
-        """
+        if valid_count > 0:
+            high_risk = len(valid_predictions[valid_predictions['fraud_probability'] >= 0.8])
+            medium_risk = len(valid_predictions[(valid_predictions['fraud_probability'] >= 0.5) & (valid_predictions['fraud_probability'] < 0.8)])
+            low_risk = len(valid_predictions[(valid_predictions['fraud_probability'] >= 0.2) & (valid_predictions['fraud_probability'] < 0.5)])
+            very_low_risk = len(valid_predictions[valid_predictions['fraud_probability'] < 0.2])
+            
+            summary = f"""
+📊 **Batch Prediction Summary**
+
+Total Transactions: {total_transactions}
+Successfully Processed: {valid_count}
+Errors: {total_transactions - valid_count}
+
+**Risk Distribution:**
+🔴 High Risk: {high_risk} ({high_risk/valid_count:.1%})
+🟡 Medium Risk: {medium_risk} ({medium_risk/valid_count:.1%})
+🟠 Low Risk: {low_risk} ({low_risk/valid_count:.1%})
+🟢 Very Low Risk: {very_low_risk} ({very_low_risk/valid_count:.1%})
+
+Results saved to: {output_path}
+            """
+        else:
+            summary = f"""
+❌ **Batch Processing Failed**
+
+Total Transactions: {total_transactions}
+Successfully Processed: 0
+All transactions encountered errors.
+
+Please check your CSV format and try again.
+            """
         
         return summary, output_path
         
@@ -173,8 +223,8 @@ with gr.Blocks(title="Fraud Detection System", theme=gr.themes.Soft()) as app:
     gr.Markdown("""
     # 🔒 Credit Card Fraud Detection System
     
-    This system uses machine learning to assess the risk of credit card transactions being fraudulent.
-    Enter transaction details below to get a risk assessment.
+    This system uses **XGBoost machine learning** to assess the risk of credit card transactions being fraudulent.
+    Enter transaction details for single prediction or upload CSV for batch processing.
     
     **Risk Levels:**
     - 🔴 High Risk (≥80%): Block transaction immediately
@@ -186,52 +236,53 @@ with gr.Blocks(title="Fraud Detection System", theme=gr.themes.Soft()) as app:
     with gr.Tabs():
         
         # Single Transaction Tab
-        with gr.TabItem("Single Transaction"):
+        with gr.TabItem("🔍 Single Transaction"):
+            gr.Markdown("""
+            ### Single Transaction Fraud Detection
+            Enter the transaction details below for instant fraud risk assessment.
+            """)
+            
             with gr.Row():
                 with gr.Column():
-                    gr.Markdown("### Transaction Details")
-                    transaction_amount = gr.Number(label="Transaction Amount ($)", value=100.0)
-                    card_type = gr.Dropdown(
-                        choices=["visa", "mastercard", "american express", "discover"], 
-                        label="Card Type", 
-                        value="visa"
+                    gr.Markdown("### 📝 Transaction Information")
+                    transaction_id = gr.Number(label="Transaction ID", value=123456, precision=0)
+                    transaction_dt = gr.Number(label="Transaction DateTime (seconds)", value=18403200, precision=0)
+                    transaction_amt = gr.Number(label="Transaction Amount ($)", value=150.00)
+                    product_cd = gr.Dropdown(
+                        choices=["W", "C", "S", "R", "H"],
+                        label="Product Code",
+                        value="W"
                     )
-                    email_domain = gr.Textbox(label="Email Domain", value="gmail.com")
-                    transaction_hour = gr.Slider(0, 23, label="Transaction Hour", value=12)
                     
-                    gr.Markdown("### Address & Card Info")
-                    addr1 = gr.Number(label="Address 1", value=325.0)
-                    addr2 = gr.Number(label="Address 2", value=87.0)
-                    card1 = gr.Number(label="Card 1", value=13553)
-                    card2 = gr.Number(label="Card 2", value=150.0)
-                    dist1 = gr.Number(label="Distance 1", value=19.0)
+                    gr.Markdown("### 💳 Card Information")
+                    card1 = gr.Number(label="Card 1", value=4532015112830366, precision=0)
+                    card2 = gr.Number(label="Card 2", value=404.0)
+                    card3 = gr.Number(label="Card 3", value=150.0)
                 
                 with gr.Column():
-                    gr.Markdown("### Transaction Counts")
-                    c1 = gr.Number(label="C1", value=1.0)
-                    c2 = gr.Number(label="C2", value=1.0)
-                    c3 = gr.Number(label="C3", value=0.0)
-                    c4 = gr.Number(label="C4", value=0.0)
-                    c5 = gr.Number(label="C5", value=0.0)
-                    c6 = gr.Number(label="C6", value=1.0)
+                    gr.Markdown("### 💳 Card Details")
+                    card4 = gr.Dropdown(
+                        choices=["visa", "mastercard", "american express", "discover"],
+                        label="Card Type",
+                        value="visa"
+                    )
+                    card5 = gr.Number(label="Card 5", value=142.0)
+                    card6 = gr.Dropdown(
+                        choices=["credit", "debit"],
+                        label="Card Category",
+                        value="credit"
+                    )
                     
-                    gr.Markdown("### Time Deltas")
-                    d1 = gr.Number(label="D1", value=0.0)
-                    d2 = gr.Number(label="D2", value=0.0)
-                    d3 = gr.Number(label="D3", value=0.0)
-                    d4 = gr.Number(label="D4", value=0.0)
-                    d5 = gr.Number(label="D5", value=20.0)
+                    gr.Markdown("### 📍 Address Information") 
+                    addr1 = gr.Number(label="Address 1", value=315.0)
+                    addr2 = gr.Number(label="Address 2", value=87.0)
                     
-                    gr.Markdown("### Match Features")
-                    m1 = gr.Dropdown(choices=["T", "F"], label="M1", value="T")
-                    m2 = gr.Dropdown(choices=["T", "F"], label="M2", value="T")
-                    m3 = gr.Dropdown(choices=["T", "F"], label="M3", value="T")
-                    m4 = gr.Dropdown(choices=["M0", "M1", "M2"], label="M4", value="M0")
-                    m5 = gr.Dropdown(choices=["T", "F"], label="M5", value="F")
-                    m6 = gr.Dropdown(choices=["T", "F"], label="M6", value="F")
+                    gr.Markdown("### 📧 Email Information")
+                    p_emaildomain = gr.Textbox(label="Email Domain", value="gmail.com")
             
             predict_btn = gr.Button("🔍 Analyze Transaction", variant="primary", size="lg")
             
+            gr.Markdown("### 📊 Prediction Results")
             with gr.Row():
                 risk_output = gr.Textbox(label="Risk Assessment", lines=1)
                 probability_output = gr.Textbox(label="Fraud Probability", lines=1)
@@ -241,74 +292,146 @@ with gr.Blocks(title="Fraud Detection System", theme=gr.themes.Soft()) as app:
                 recommendation_output = gr.Textbox(label="Recommendation", lines=2)
             
             predict_btn.click(
-                predict_fraud_risk,
+                predict_single_transaction,
                 inputs=[
-                    transaction_amount, card_type, email_domain, transaction_hour,
-                    addr1, addr2, card1, card2, dist1,
-                    c1, c2, c3, c4, c5, c6,
-                    d1, d2, d3, d4, d5,
-                    m1, m2, m3, m4, m5, m6
+                    transaction_id, transaction_dt, transaction_amt, product_cd,
+                    card1, card2, card3, card4, card5, card6,
+                    addr1, addr2, p_emaildomain
                 ],
                 outputs=[risk_output, probability_output, risk_level_output, recommendation_output]
             )
         
         # Batch Processing Tab
-        with gr.TabItem("Batch Processing"):
+        with gr.TabItem("📁 Batch Processing"):
             gr.Markdown("""
-            ### Upload CSV File for Batch Processing
+            ### CSV Batch Processing
+            
+            Upload a CSV file containing multiple transactions for batch fraud detection.
             
-            Upload a CSV file containing multiple transactions. The file should include the same columns
-            as used in single transaction prediction.
+            **Required CSV Columns:**
+            - `TransactionAmt` (required)
+            - `TransactionID` (optional - will be auto-generated)
+            - `TransactionDT`, `ProductCD`, `card1-6`, `addr1-2`, `P_emaildomain` (optional - smart defaults used)
+            
+            **Example CSV Format:**
+            ```
+            TransactionID,TransactionDT,TransactionAmt,ProductCD,card1,card2,card3,card4,card5,card6,addr1,addr2,P_emaildomain
+            123456,18403200,150.00,W,4532015112830366,404.0,150.0,visa,142.0,credit,315.0,87.0,gmail.com
+            123457,18403300,2500.00,C,5555555555554444,555.0,200.0,mastercard,224.0,credit,420.0,95.0,yahoo.com
+            ```
             """)
             
-            file_upload = gr.File(label="Upload CSV File", file_types=[".csv"])
-            batch_btn = gr.Button("🔍 Process Batch", variant="primary")
+            file_upload = gr.File(
+                label="Upload CSV File", 
+                file_types=[".csv"],
+                elem_id="csv-upload"
+            )
+            batch_btn = gr.Button("🔍 Process Batch", variant="primary", size="lg")
             
-            batch_output = gr.Textbox(label="Batch Results", lines=10)
-            download_file = gr.File(label="Download Results")
+            gr.Markdown("### 📊 Batch Results")
+            batch_output = gr.Textbox(label="Processing Summary", lines=12)
+            download_file = gr.File(label="Download Results CSV")
             
             batch_btn.click(
-                predict_from_csv,
+                predict_batch_from_csv,
                 inputs=[file_upload],
                 outputs=[batch_output, download_file]
             )
         
+        # Sample Data Tab
+        with gr.TabItem("📋 Sample Data"):
+            gr.Markdown("""
+            ### Sample Transaction Data
+            
+            Use these examples to test the system or as a template for your CSV files.
+            """)
+            
+            gr.Markdown("""
+            #### Example 1: Low Risk Transaction
+            ```json
+            {
+                "TransactionID": 123456,
+                "TransactionDT": 18403200,
+                "TransactionAmt": 150.00,
+                "ProductCD": "W",
+                "card1": 4532015112830366,
+                "card2": 404.0,
+                "card3": 150.0,
+                "card4": "visa",
+                "card5": 142.0,
+                "card6": "credit",
+                "addr1": 315.0,
+                "addr2": 87.0,
+                "P_emaildomain": "gmail.com"
+            }
+            ```
+            
+            #### Example 2: Higher Risk Transaction
+            ```json
+            {
+                "TransactionID": 123457,
+                "TransactionDT": 18403300,
+                "TransactionAmt": 2500.00,
+                "ProductCD": "C",
+                "card1": 5555555555554444,
+                "card2": 555.0,
+                "card3": 200.0,
+                "card4": "mastercard",
+                "card5": 224.0,
+                "card6": "credit",
+                "addr1": 420.0,
+                "addr2": 95.0,
+                "P_emaildomain": "yahoo.com"
+            }
+            ```
+            
+            #### CSV Sample File
+            You can copy this into a CSV file for batch testing:
+            ```
+            TransactionID,TransactionDT,TransactionAmt,ProductCD,card1,card2,card3,card4,card5,card6,addr1,addr2,P_emaildomain
+            123456,18403200,150.00,W,4532015112830366,404.0,150.0,visa,142.0,credit,315.0,87.0,gmail.com
+            123457,18403300,2500.00,C,5555555555554444,555.0,200.0,mastercard,224.0,credit,420.0,95.0,yahoo.com
+            123458,18403400,75.50,W,4111111111111111,300.0,75.0,visa,100.0,debit,200.0,50.0,hotmail.com
+            ```
+            """)
+        
         # Model Info Tab
-        with gr.TabItem("Model Information"):
+        with gr.TabItem("ℹ️ Model Information"):
             if model_loaded and fraud_model.metadata:
                 model_info = fraud_model.get_model_info()
                 gr.Markdown(f"""
                 ### Model Status
-                **Status:** ✅ {model_info.get('model_name', 'XGBoost')} Model Loaded
-                **AUC Score:** {model_info.get('auc_score', 'N/A')}
-                **Training Date:** {model_info.get('training_timestamp', 'N/A')}
-                **Features:** {model_info.get('feature_count', 'N/A')}
+                **Status:** ✅ {model_info.get('model_name', 'XGBoost')} Model Loaded  
+                **AUC Score:** {model_info.get('auc_score', 'N/A')}  
+                **Training Date:** {model_info.get('training_timestamp', 'N/A')}  
+                **Features:** {model_info.get('feature_count', 'N/A')}  
                 
                 ### About This Model
                 This fraud detection system uses an **XGBoost classifier** trained on a comprehensive dataset 
                 of credit card transactions. The model achieved high performance with advanced feature engineering
                 and ensemble learning techniques.
                 
+                ### API Compatible Interface
+                This interface matches the exact field structure expected by the fraud detection API:
+                
+                **Single Prediction Endpoint:** `/v1/predict`  
+                **Batch Prediction Endpoint:** `/v1/predict/batch`
+                
+                ### Supported Fields
+                - **TransactionID**: Unique transaction identifier
+                - **TransactionDT**: Transaction datetime (seconds)
+                - **TransactionAmt**: Transaction amount in USD
+                - **ProductCD**: Product code (W, C, S, R, H)
+                - **card1-6**: Card-related features
+                - **addr1-2**: Address information
+                - **P_emaildomain**: Primary email domain
+                
                 ### Model Performance
                 - **Algorithm**: XGBoost (Extreme Gradient Boosting)
                 - **AUC Score**: {model_info.get('auc_score', 'N/A')}
                 - **Features Used**: {model_info.get('feature_count', 'N/A')} engineered features
                 - **Training Method**: Cross-validation with stratified sampling
                 - **Speed**: Real-time predictions (<100ms)
-                
-                ### Features Used
-                The model processes over 40 features including:
-                - **Transaction Details**: Amount, timing, frequency patterns
-                - **Card Information**: Type, issuer details, security features  
-                - **User Behaviour**: Email domains, address patterns, historical counts
-                - **Device & Session**: Geographic data, device fingerprinting
-                - **Engineered Features**: Ratios, transformations, temporal patterns
-                
-                ### XGBoost Advantages
-                - **High Accuracy**: Excellent performance on tabular data
-                - **Feature Importance**: Clear understanding of decision factors
-                - **Robustness**: Handles missing values and outliers well
-                - **Scalability**: Efficient training and inference
                 """)
             else:
                 gr.Markdown(f"""
@@ -317,21 +440,13 @@ with gr.Blocks(title="Fraud Detection System", theme=gr.themes.Soft()) as app:
                 
                 ### About This Model
                 This fraud detection system uses advanced machine learning algorithms to assess transaction risk.
-                The model was trained on a large dataset of credit card transactions and uses multiple features
-                including transaction amount, card details, user behaviour patterns, and timing information.
-                
-                ### Features Used
-                - Transaction amount and timing
-                - Card information (type, numbers)
-                - Email domain patterns  
-                - Address information
-                - User behaviour counts
-                - Device and session data
+                The model processes transactions with the same field structure as the API endpoints.
                 
-                ### Model Performance
-                - **Algorithm**: Ensemble methods (Random Forest, XGBoost, LightGBM)
-                - **Accuracy**: High precision in detecting fraudulent transactions
-                - **Speed**: Real-time predictions
+                ### Features
+                - Single transaction analysis
+                - Batch CSV processing
+                - Real-time risk assessment
+                - API-compatible field structure
                 """)
 
 # Launch the app