{ "cells": [ { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2025-12-27T14:35:45.060410Z", "start_time": "2025-12-27T14:35:44.968192Z" } }, "source": "import numpy as np", "outputs": [], "execution_count": 1 }, { "metadata": { "ExecuteTime": { "end_time": "2025-12-27T14:41:14.878582Z", "start_time": "2025-12-27T14:37:02.228757Z" } }, "cell_type": "code", "source": [ "import pandas as pd\n", "import numpy as np\n", "import xgboost as xgb\n", "from sklearn.ensemble import IsolationForest\n", "from sklearn.metrics import classification_report, average_precision_score, roc_auc_score\n", "from sklearn.preprocessing import LabelEncoder\n", "import joblib # For saving models\n", "import warnings\n", "\n", "warnings.filterwarnings('ignore')\n", "\n", "# ==========================================\n", "# 1. CONFIGURATION\n", "# ==========================================\n", "DATA_PATH = r'E:\\TechFiesta\\Datasets\\Paysim\\PS_20174392719_1491204439457_log.csv' # Path to your PaySim file\n", "TRAIN_SPLIT_STEP = 600 # PaySim has 744 steps. We train on first 600 (approx 25 days)\n", "MODEL_SAVE_DIR = 'models/'\n", "\n", "# ==========================================\n", "# 2. DATA LOADING & CLEANING\n", "# ==========================================\n", "print(\"Loading data...\")\n", "df = pd.read_csv(DATA_PATH)\n", "\n", "# Fraud in PaySim predominantly happens in TRANSFER and CASH_OUT types.\n", "# We filter to focus the model on these high-risk areas.\n", "df = df[df['type'].isin(['TRANSFER', 'CASH_OUT'])]\n", "\n", "print(f\"Data Loaded. Shape: {df.shape}\")\n", "\n", "# ==========================================\n", "# 3. FEATURE ENGINEERING\n", "# ==========================================\n", "print(\"Generating Features...\")\n", "\n", "# A. ENCODING CATEGORICAL DATA\n", "# We need to convert 'type' (TRANSFER/CASH_OUT) to numbers.\n", "le = LabelEncoder()\n", "df['type'] = le.fit_transform(df['type'])\n", "\n", "# B. ERROR FEATURES (Specific to PaySim)\n", "# Fraudsters often try to empty accounts. We check for discrepancy in balances.\n", "df['errorBalanceOrig'] = df['newbalanceOrig'] + df['amount'] - df['oldbalanceOrg']\n", "df['errorBalanceDest'] = df['oldbalanceDest'] + df['amount'] - df['newbalanceDest']\n", "\n", "# C. VELOCITY FEATURES (Time since last transaction)\n", "# We sort by customer and time to calculate how fast they are transacting.\n", "df = df.sort_values(['nameOrig', 'step'])\n", "df['step_diff'] = df.groupby('nameOrig')['step'].diff().fillna(0) # 0 means first transaction\n", "\n", "# D. DROPPING UNUSABLE COLUMNS\n", "# We drop ID columns (nameOrig, nameDest) because they don't generalize to future users.\n", "# We keep 'isFraud' as our target.\n", "features_to_drop = ['nameOrig', 'nameDest', 'isFlaggedFraud']\n", "df = df.drop(columns=features_to_drop)\n", "\n", "# ==========================================\n", "# 4. TIME-BASED TRAIN/TEST SPLIT\n", "# ==========================================\n", "# We do NOT shuffle. We split by time (step).\n", "print(\"Splitting data by time...\")\n", "\n", "train_df = df[df['step'] <= TRAIN_SPLIT_STEP]\n", "test_df = df[df['step'] > TRAIN_SPLIT_STEP]\n", "\n", "# Define Feature Columns (Exclude Target and Step)\n", "X_train = train_df.drop(['isFraud', 'step'], axis=1)\n", "y_train = train_df['isFraud']\n", "\n", "X_test = test_df.drop(['isFraud', 'step'], axis=1)\n", "y_test = test_df['isFraud']\n", "\n", "print(f\"Training Sets: {X_train.shape}\")\n", "print(f\"Testing Sets: {X_test.shape}\")\n", "\n", "# ==========================================\n", "# 5. ISOLATION FOREST (The Watchdog)\n", "# ==========================================\n", "print(\"\\n--- Phase 1: Training Isolation Forest ---\")\n", "\n", "# We use a subset of features for anomaly detection (usually continuous ones)\n", "iso_features = ['amount', 'errorBalanceOrig', 'errorBalanceDest', 'step_diff']\n", "\n", "# n_jobs=-1 uses all CPU cores\n", "iso_model = IsolationForest(n_estimators=100, contamination=0.01, random_state=42, n_jobs=-1)\n", "iso_model.fit(X_train[iso_features])\n", "\n", "# Generate Anomaly Scores\n", "# IMPORTANT: We use 'decision_function'.\n", "# Lower scores = More Anomalous. We multiply by -1 so Higher = More Anomalous.\n", "print(\"Augmenting data with Anomaly Scores...\")\n", "X_train['anomaly_score'] = -1 * iso_model.decision_function(X_train[iso_features])\n", "X_test['anomaly_score'] = -1 * iso_model.decision_function(X_test[iso_features])\n", "\n", "# ==========================================\n", "# 6. XGBOOST (The Specialist)\n", "# ==========================================\n", "print(\"\\n--- Phase 2: Training XGBoost ---\")\n", "\n", "# Calculate Scale Pos Weight (Imbalance handling)\n", "# Count(Negatives) / Count(Positives)\n", "negatives = len(y_train) - sum(y_train)\n", "positives = sum(y_train)\n", "scale_pos_weight = negatives / positives\n", "\n", "print(f\"Class Imbalance Ratio: {scale_pos_weight:.2f}\")\n", "\n", "xgb_model = xgb.XGBClassifier(\n", " objective='binary:logistic',\n", " n_estimators=200,\n", " learning_rate=0.1,\n", " max_depth=5, # Prevent Overfitting\n", " scale_pos_weight=scale_pos_weight, # Critical for fraud\n", " n_jobs=-1,\n", " random_state=42\n", ")\n", "\n", "xgb_model.fit(X_train, y_train)\n", "\n", "# ==========================================\n", "# 7. EVALUATION\n", "# ==========================================\n", "print(\"\\n--- Evaluation on Future Data (Test Set) ---\")\n", "\n", "preds = xgb_model.predict(X_test)\n", "probs = xgb_model.predict_proba(X_test)[:, 1]\n", "\n", "print(classification_report(y_test, preds))\n", "print(f\"ROC AUC Score: {roc_auc_score(y_test, probs):.4f}\")\n", "print(f\"PR AUC Score: {average_precision_score(y_test, probs):.4f}\")\n", "\n", "# ==========================================\n", "# 8. SAVE MODELS\n", "# ==========================================\n", "import os\n", "if not os.path.exists(MODEL_SAVE_DIR):\n", " os.makedirs(MODEL_SAVE_DIR)\n", "\n", "print(f\"\\nSaving models to {MODEL_SAVE_DIR}...\")\n", "\n", "# Save Models\n", "joblib.dump(iso_model, f'{MODEL_SAVE_DIR}/iso_forest.pkl')\n", "joblib.dump(xgb_model, f'{MODEL_SAVE_DIR}/xgboost_fraud.pkl')\n", "# Save the Label Encoder (You need this for the pipeline!)\n", "joblib.dump(le, f'{MODEL_SAVE_DIR}/label_encoder.pkl')\n", "\n", "print(\"Done! Ready for Ingestion Pipeline.\")" ], "id": "bd7d71f965ea8b75", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading data...\n", "Data Loaded. Shape: (2770409, 11)\n", "Generating Features...\n", "Splitting data by time...\n", "Training Sets: (2726859, 9)\n", "Testing Sets: (43550, 9)\n", "\n", "--- Phase 1: Training Isolation Forest ---\n", "Augmenting data with Anomaly Scores...\n", "\n", "--- Phase 2: Training XGBoost ---\n", "Class Imbalance Ratio: 411.35\n", "\n", "--- Evaluation on Future Data (Test Set) ---\n", " precision recall f1-score support\n", "\n", " 0 1.00 1.00 1.00 41950\n", " 1 1.00 1.00 1.00 1600\n", "\n", " accuracy 1.00 43550\n", " macro avg 1.00 1.00 1.00 43550\n", "weighted avg 1.00 1.00 1.00 43550\n", "\n", "ROC AUC Score: 1.0000\n", "PR AUC Score: 1.0000\n", "\n", "Saving models to models/...\n", "Done! Ready for Ingestion Pipeline.\n" ] } ], "execution_count": 2 }, { "metadata": { "ExecuteTime": { "end_time": "2025-12-27T14:45:48.688441Z", "start_time": "2025-12-27T14:45:48.637360Z" } }, "cell_type": "code", "source": [ "import pandas as pd\n", "import numpy as np\n", "import joblib\n", "import warnings\n", "\n", "warnings.filterwarnings('ignore')\n", "\n", "# ==========================================\n", "# 1. LOAD SAVED MODELS\n", "# ==========================================\n", "print(\"Loading models...\")\n", "try:\n", " iso_model = joblib.load('models/iso_forest.pkl')\n", " xgb_model = joblib.load('models/xgboost_fraud.pkl')\n", " le = joblib.load('models/label_encoder.pkl')\n", " print(\"✅ Models loaded successfully.\")\n", "except FileNotFoundError:\n", " print(\"❌ Error: Models not found. Run the training script first!\")\n", " exit()\n", "\n", "# ==========================================\n", "# 2. DEFINE TEST DATA (MOCK TRANSACTIONS)\n", "# ==========================================\n", "# We simulate 2 scenarios.\n", "# Case A: Normal Transfer (Small amount, balances match)\n", "# Case B: Fraud Transfer (High amount, empties account, balances don't match)\n", "\n", "mock_transactions = [\n", " {\n", " \"id\": \"TXN_NORMAL_001\",\n", " \"step\": 601,\n", " \"type\": \"PAYMENT\", # Note: Will be filtered out or encoded\n", " \"amount\": 200.00,\n", " \"oldbalanceOrg\": 5000.00,\n", " \"newbalanceOrig\": 4800.00, # Correct math (5000 - 200 = 4800)\n", " \"oldbalanceDest\": 0.00,\n", " \"newbalanceDest\": 200.00,\n", " \"prev_step\": 600 # Needed to calc 'step_diff'\n", " },\n", " {\n", " \"id\": \"TXN_FRAUD_999\",\n", " \"step\": 602,\n", " \"type\": \"TRANSFER\", # High risk type\n", " \"amount\": 1000000.00,\n", " \"oldbalanceOrg\": 1000000.00,\n", " \"newbalanceOrig\": 0.00, # EMPTIED ACCOUNT!\n", " \"oldbalanceDest\": 0.00,\n", " \"newbalanceDest\": 0.00, # Suspicious: Destination didn't get the money?\n", " \"prev_step\": 550 # Long silence before big transaction\n", " }\n", "]\n", "\n", "# ==========================================\n", "# 3. THE INFERENCE PIPELINE FUNCTION\n", "# ==========================================\n", "def predict_transaction(txn_dict):\n", " \"\"\"\n", " Takes a raw transaction dictionary and returns Fraud Probability.\n", " This mimics what your production API will do.\n", " \"\"\"\n", "\n", " # A. Convert to DataFrame (Model expects a batch, even if size 1)\n", " df = pd.DataFrame([txn_dict])\n", "\n", " # B. Filter Type (Remember we only trained on TRANSFER and CASH_OUT)\n", " # If it's PAYMENT/DEBIT, our model doesn't know it.\n", " if df['type'].iloc[0] not in ['TRANSFER', 'CASH_OUT']:\n", " return \"SKIPPED (Low Risk Type)\"\n", "\n", " # C. Feature Engineering (MUST MATCH TRAINING EXACTLY)\n", "\n", " # 1. Encode Type\n", " # Handle unseen labels safely\n", " try:\n", " df['type'] = le.transform(df['type'])\n", " except ValueError:\n", " # If label wasn't in training (e.g., 'DEBIT'), handle gracefully\n", " return \"ERROR: Unknown Transaction Type\"\n", "\n", " # 2. Error Features (Math Checks)\n", " df['errorBalanceOrig'] = df['newbalanceOrig'] + df['amount'] - df['oldbalanceOrg']\n", " df['errorBalanceDest'] = df['oldbalanceDest'] + df['amount'] - df['newbalanceDest']\n", "\n", " # 3. Velocity Feature (Step Diff)\n", " # In production, you'd fetch 'prev_step' from a Database/Redis.\n", " # Here we used the mocked 'prev_step' from the input.\n", " df['step_diff'] = df['step'] - df['prev_step']\n", "\n", " # D. Prepare Input for Isolation Forest\n", " iso_features = ['amount', 'errorBalanceOrig', 'errorBalanceDest', 'step_diff']\n", "\n", " # E. Get Anomaly Score\n", " # Remember: We multiplied by -1 in training!\n", " iso_score = iso_model.decision_function(df[iso_features])\n", " df['anomaly_score'] = -1 * iso_score\n", "\n", " # F. Prepare Final Input for XGBoost\n", " # Ensure columns are in the EXACT same order as training\n", " # Check your training script for the final feature list order\n", " xgb_features = ['type', 'amount', 'oldbalanceOrg', 'newbalanceOrig',\n", " 'oldbalanceDest', 'newbalanceDest', 'errorBalanceOrig',\n", " 'errorBalanceDest', 'step_diff', 'anomaly_score']\n", "\n", " # G. Final Prediction\n", " prob = xgb_model.predict_proba(df[xgb_features])[:, 1][0]\n", " is_fraud = prob > 0.5 # Threshold\n", "\n", " return {\n", " \"Anomaly_Score\": round(df['anomaly_score'].iloc[0], 4),\n", " \"Fraud_Probability\": round(prob, 4),\n", " \"Is_Fraud\": \"🚨 YES\" if is_fraud else \"✅ NO\"\n", " }\n", "\n", "# ==========================================\n", "# 4. RUN TESTS\n", "# ==========================================\n", "print(\"\\n--- Starting Inference Tests ---\\n\")\n", "\n", "for txn in mock_transactions:\n", " print(f\"Testing Transaction: {txn['id']} ({txn['type']})\")\n", " result = predict_transaction(txn)\n", " print(f\"Result: {result}\")\n", " print(\"-\" * 30)" ], "id": "74cfecbc30658aaa", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loading models...\n", "✅ Models loaded successfully.\n", "\n", "--- Starting Inference Tests ---\n", "\n", "Testing Transaction: TXN_NORMAL_001 (PAYMENT)\n", "Result: SKIPPED (Low Risk Type)\n", "------------------------------\n", "Testing Transaction: TXN_FRAUD_999 (TRANSFER)\n", "Result: {'Anomaly_Score': -0.0407, 'Fraud_Probability': 1.0, 'Is_Fraud': '🚨 YES'}\n", "------------------------------\n" ] } ], "execution_count": 3 } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }