scripts/train.py

import time
from pathlib import Path
import joblib
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score, classification_report, f1_score
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.utils import resample
from xgboost import XGBClassifier

# ==============================================================================
# 0. DIRECTORY & DATASET SETUP
# ==============================================================================
MODEL_DIR = Path("models")
MODEL_DIR.mkdir(parents=True, exist_ok=True)

DATASET_CSV = "pose_benchmark_priority1_dataset.csv"

# ==============================================================================
# 1. LOAD AND BALANCE DATASET
# ==============================================================================
print(f"[*] Loading Priority 1 dataset: {DATASET_CSV}...")
df = pd.read_csv(DATASET_CSV)

# Separate by split
raw_train_df = df[df["split"] == "train"]
test_df = df[df["split"] == "test"]  # Keep original test set intact

# --- AUTOMATIC 1:1 CLASS BALANCING FOR TRAINING ---
train_fall = raw_train_df[raw_train_df["label"] == 1]
train_normal = raw_train_df[raw_train_df["label"] == 0]

min_samples = min(len(train_fall), len(train_normal))

print(
    f"[*] Raw Training Counts -> Normal (0): {len(train_normal)} | Fall (1):"
    f" {len(train_fall)}"
)
print(
    "[*] Balancing training set to exactly"
    f" {min_samples} samples per class..."
)

# Undersample both classes to min_samples to guarantee exact 1:1 match
train_fall_balanced = resample(
    train_fall, replace=False, n_samples=min_samples, random_state=42
)
train_normal_balanced = resample(
    train_normal, replace=False, n_samples=min_samples, random_state=42
)

# Combine and shuffle balanced dataset
train_df = (
    pd.concat([train_fall_balanced, train_normal_balanced])
    .sample(frac=1, random_state=42)
    .reset_index(drop=True)
)

print(
    f"[*] Balanced Training Set Ready: {len(train_df)} total samples (50% Fall"
    " / 50% Normal)\n"
)

# Prepare Feature & Label Arrays (Dropping metadata columns)
X_train = train_df.drop(columns=["split", "label"])
y_train = train_df["label"]
X_test = test_df.drop(columns=["split", "label"])
y_test = test_df["label"]

print(f"[*] Total Input Features per Sample: {X_train.shape[1]}")

# ==============================================================================
# 2. PREPARE DATA PIPELINES (FOR NON-XGBOOST MODELS)
# ==============================================================================
imputer_scaler = Pipeline(
    [("imputer", SimpleImputer(strategy="mean")), ("scaler", StandardScaler())]
)

print("[*] Preprocessing data for standard models (imputation & scaling)...")
X_train_processed = imputer_scaler.fit_transform(X_train)
X_test_processed = imputer_scaler.transform(X_test)

# ==============================================================================
# 3. DEFINE MODELS
# ==============================================================================
models = {
    "XGBoost": XGBClassifier(
        n_estimators=150,
        max_depth=5,
        learning_rate=0.03,
        scale_pos_weight=1.0,  # 1:1 balanced
        missing=np.nan,        # Native NaN handling for missing keypoints
        random_state=42,
        eval_metric="logloss"
    ),
    "Random_Forest": RandomForestClassifier(n_estimators=100, max_depth=10, random_state=42, n_jobs=-1),
    "SVM_(RBF)": SVC(kernel="rbf", probability=True, random_state=42),
    "MLP_(Neural_Net)": MLPClassifier(
        hidden_layer_sizes=(64, 32), max_iter=500, random_state=42
    ),
}

# ==============================================================================
# 4. TRAIN, BENCHMARK, AND EXPORT MODELS
# ==============================================================================
results = []

print("\n[*] Starting Benchmark Training on Priority 1 Dataset...\n")

# Save preprocessing pipeline inside models/
pipeline_path = MODEL_DIR / "imputer_scaler_pipeline.pkl"
joblib.dump(imputer_scaler, pipeline_path)
print(f"[*] Exported preprocessing pipeline to: {pipeline_path}")

for name, model in models.items():
    print(f"\nTraining {name}...")

    # XGBoost uses raw data with NaNs; others use imputed/scaled data
    X_train_curr = X_train if name == "XGBoost" else X_train_processed
    X_test_curr = X_test if name == "XGBoost" else X_test_processed

    # Train
    start_train = time.time()
    model.fit(X_train_curr, y_train)
    train_time = time.time() - start_train

    # Export Model File into models/
    model_filename = f"{name.lower().replace('_(rbf)', '').replace('_(neural_net)', '')}_priority1_fall_model.pkl"
    model_path = MODEL_DIR / model_filename
    joblib.dump(model, model_path)
    print(f"[*] Saved trained model to: {model_path}")

    # Predict & Measure Inference Latency
    start_infer = time.time()
    y_pred = model.predict(X_test_curr)
    infer_time = time.time() - start_infer

    # Calculate Latency per sample in milliseconds
    latency_ms = (infer_time / len(X_test_curr)) * 1000

    # Evaluate Metrics
    acc = accuracy_score(y_test, y_pred)
    f1 = f1_score(y_test, y_pred, average="macro")

    results.append({
        "Model": name,
        "Accuracy": f"{acc:.4f}",
        "Macro F1": f"{f1:.4f}",
        "Train Time (s)": f"{train_time:.2f}",
        "Inference Latency (ms)": f"{latency_ms:.4f}",
        "Saved Path": str(model_path),
    })

    print(f"--- {name} Classification Report ---")
    print(classification_report(y_test, y_pred, target_names=["Normal (0)", "Fall (1)"]))

# ==============================================================================
# 5. DISPLAY BENCHMARK SUMMARY
# ==============================================================================
results_df = pd.DataFrame(results)
print(
    "\n=========================================================================================="
)
print(
    "                                   BENCHMARK RESULTS                                      "
)
print(
    "=========================================================================================="
)
print(results_df.to_string(index=False))