feat(ml): evaluation pipeline and improved XGBoost training from diagnostic report

scripts/evaluate_model.py

@@ -0,0 +1,228 @@
+"""
+Model Evaluation and Benchmarking Script.
+
+Implements diagnostics from the ML accuracy report:
+- Confusion matrix, precision, recall, F1, AUROC
+- Class balance analysis
+- Data leakage check (train/val/test path overlap)
+- RandomizedSearchCV hyperparameter search
+
+Usage:
+    python scripts/evaluate_model.py
+    python scripts/evaluate_model.py --hparam-search
+"""
+import csv
+import json
+import logging
+import argparse
+import numpy as np
+from pathlib import Path
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)s  %(message)s",
+    datefmt="%H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+ROOT        = Path(__file__).parents[1]
+MANIFEST    = ROOT / "data" / "manifest.csv"
+FEATURES    = ROOT / "data" / "features.csv"
+RESULTS_OUT = ROOT / "data" / "reference" / "eval_results.json"
+
+
+def check_class_balance(manifest_path: Path) -> dict:
+    counts = {}
+    with open(manifest_path, newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            label = row.get("label", "")
+            counts[label] = counts.get(label, 0) + 1
+
+    total         = sum(counts.values())
+    balance_ratio = min(counts.values()) / max(counts.values(), 1)
+    imbalanced    = balance_ratio < 0.30
+
+    logger.info(f"Class balance: {counts} (ratio={balance_ratio:.3f})")
+    if imbalanced:
+        logger.warning(
+            f"Class imbalance detected (ratio={balance_ratio:.3f}). "
+            "Consider resampling or scale_pos_weight."
+        )
+    return {"counts": counts, "total": total,
+            "balance_ratio": round(balance_ratio, 4), "imbalanced": imbalanced}
+
+
+def check_data_leakage(manifest_path: Path) -> dict:
+    split_paths: dict = {}
+    with open(manifest_path, newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            split = row.get("split", "train")
+            path  = row.get("path", "")
+            if split not in split_paths:
+                split_paths[split] = set()
+            split_paths[split].add(path)
+
+    overlaps = {}
+    splits   = list(split_paths.keys())
+    for i in range(len(splits)):
+        for j in range(i + 1, len(splits)):
+            a, b    = splits[i], splits[j]
+            overlap = split_paths[a] & split_paths[b]
+            if overlap:
+                key           = f"{a}_vs_{b}"
+                overlaps[key] = len(overlap)
+                logger.warning(f"Data leakage: {len(overlap)} duplicate paths between {a} and {b}")
+
+    leakage = len(overlaps) > 0
+    if not leakage:
+        logger.info("No path-based data leakage detected.")
+    return {"split_sizes": {k: len(v) for k, v in split_paths.items()},
+            "leakage_detected": leakage, "overlapping_paths": overlaps}
+
+
+def evaluate_xgboost(features_path: Path, threshold: float = 0.5) -> dict:
+    import pickle
+    from sklearn.model_selection import StratifiedShuffleSplit
+    from sklearn.metrics import (
+        accuracy_score, precision_score, recall_score,
+        f1_score, roc_auc_score, confusion_matrix, classification_report,
+    )
+
+    model_path = ROOT / "data" / "reference" / "ensemble_xgb.pkl"
+    if not model_path.exists():
+        logger.warning("ensemble_xgb.pkl not found. Run scripts/train_ensemble.py first.")
+        return {"error": "Model not found"}
+
+    with open(model_path, "rb") as f:
+        pkg = pickle.load(f)
+    model         = pkg["model"]
+    feature_names = pkg["feature_names"]
+
+    rows, labels = [], []
+    with open(features_path, newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            labels.append(int(row["label"]))
+            rows.append([float(row.get(k, 0.5)) for k in feature_names])
+
+    X = np.array(rows)
+    y = np.array(labels)
+
+    sss = StratifiedShuffleSplit(n_splits=1, test_size=0.20, random_state=42)
+    _, test_idx    = next(sss.split(X, y))
+    X_test, y_test = X[test_idx], y[test_idx]
+
+    y_pred  = (model.predict_proba(X_test)[:, 1] >= threshold).astype(int)
+    y_score = model.predict_proba(X_test)[:, 1]
+
+    acc   = float(accuracy_score(y_test, y_pred))
+    prec  = float(precision_score(y_test, y_pred, zero_division=0))
+    rec   = float(recall_score(y_test, y_pred, zero_division=0))
+    f1    = float(f1_score(y_test, y_pred, zero_division=0))
+    auroc = float(roc_auc_score(y_test, y_score))
+    cm    = confusion_matrix(y_test, y_pred).tolist()
+
+    alarms = []
+    if acc   < 0.90: alarms.append(f"Accuracy {acc:.3f} below threshold 0.90")
+    if prec  < 0.85: alarms.append(f"Precision {prec:.3f} below threshold 0.85")
+    if rec   < 0.80: alarms.append(f"Recall {rec:.3f} below threshold 0.80")
+    if f1    < 0.83: alarms.append(f"F1 {f1:.3f} below threshold 0.83")
+    if auroc < 0.92: alarms.append(f"AUROC {auroc:.3f} below threshold 0.92")
+
+    logger.info(f"Evaluation on {len(y_test)} held-out samples:")
+    logger.info(f"  Accuracy:  {acc:.4f}")
+    logger.info(f"  Precision: {prec:.4f}")
+    logger.info(f"  Recall:    {rec:.4f}")
+    logger.info(f"  F1:        {f1:.4f}")
+    logger.info(f"  AUROC:     {auroc:.4f}")
+    for alarm in alarms:
+        logger.warning(f"ALARM: {alarm}")
+    if not alarms:
+        logger.info("All metrics above alarm thresholds.")
+
+    return {
+        "n_test_samples": len(y_test), "threshold": threshold,
+        "accuracy": round(acc, 4), "precision_ai": round(prec, 4),
+        "recall_ai": round(rec, 4), "f1_ai": round(f1, 4),
+        "auroc": round(auroc, 4), "confusion_matrix": cm,
+        "classification_report": classification_report(
+            y_test, y_pred, target_names=["real", "ai"], output_dict=True),
+        "alarms": alarms,
+        "targets":    {"accuracy": 0.95, "precision_ai": 0.90,
+                       "recall_ai": 0.88, "f1_ai": 0.89, "auroc": 0.97},
+        "thresholds": {"accuracy": 0.90, "precision_ai": 0.85,
+                       "recall_ai": 0.80, "f1_ai": 0.83, "auroc": 0.92},
+    }
+
+
+def hyperparameter_search(features_path: Path, n_iter: int = 20) -> dict:
+    import xgboost as xgb
+    from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold
+
+    rows, labels, feature_names = [], [], None
+    with open(features_path, newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            if feature_names is None:
+                feature_names = [k for k in row if k not in ("label", "path")]
+            labels.append(int(row["label"]))
+            rows.append([float(row[k]) for k in feature_names])
+
+    X = np.array(rows)
+    y = np.array(labels)
+
+    param_grid = {
+        "learning_rate":    [0.01, 0.05, 0.10, 0.15, 0.20],
+        "max_depth":        [3, 4, 5, 6],
+        "n_estimators":     [100, 200, 300, 400],
+        "subsample":        [0.6, 0.7, 0.8, 0.9],
+        "colsample_bytree": [0.6, 0.7, 0.8, 0.9],
+        "min_child_weight": [1, 3, 5],
+        "gamma":            [0, 0.1, 0.2, 0.5],
+    }
+
+    model  = xgb.XGBClassifier(eval_metric="logloss", random_state=42)
+    cv     = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+    logger.info(f"Running RandomizedSearchCV with {n_iter} iterations")
+    search = RandomizedSearchCV(model, param_grid, n_iter=n_iter,
+                                cv=cv, scoring="roc_auc",
+                                random_state=42, n_jobs=-1, verbose=1)
+    search.fit(X, y)
+    logger.info(f"Best params: {search.best_params_}")
+    logger.info(f"Best CV AUC: {search.best_score_:.4f}")
+    return {"best_params": search.best_params_,
+            "best_cv_auc": round(search.best_score_, 4), "n_iterations": n_iter}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--threshold",    type=float, default=0.5)
+    parser.add_argument("--hparam-search", action="store_true")
+    parser.add_argument("--n-iter",       type=int, default=20)
+    args = parser.parse_args()
+
+    results = {}
+
+    if MANIFEST.exists():
+        logger.info("=== CLASS BALANCE CHECK ===")
+        results["class_balance"] = check_class_balance(MANIFEST)
+        logger.info("=== DATA LEAKAGE CHECK ===")
+        results["leakage_check"] = check_data_leakage(MANIFEST)
+    else:
+        logger.warning("manifest.csv not found")
+
+    if FEATURES.exists():
+        logger.info("=== MODEL EVALUATION ===")
+        results["evaluation"] = evaluate_xgboost(FEATURES, threshold=args.threshold)
+        if args.hparam_search:
+            logger.info("=== HYPERPARAMETER SEARCH ===")
+            results["hparam_search"] = hyperparameter_search(FEATURES, n_iter=args.n_iter)
+    else:
+        logger.warning("features.csv not found — run scripts/extract_features.py first")
+
+    RESULTS_OUT.parent.mkdir(parents=True, exist_ok=True)
+    with open(RESULTS_OUT, "w") as f:
+        json.dump(results, f, indent=2, default=str)
+    logger.info(f"Saved to {RESULTS_OUT}")
+
+
+if __name__ == "__main__":
+    main()

scripts/train_ensemble.py

@@ -1,11 +1,23 @@
-import sys
+"""
+Train XGBoost ensemble classifier with ML accuracy improvements:
+- scale_pos_weight for class imbalance
+- Early stopping on validation AUC
+- Regularization: gamma, min_child_weight
+- Separate held-out test evaluation
+"""
 import csv
 import pickle
 import logging
+import argparse
 import numpy as np
 from pathlib import Path
+import json
 
-logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)s  %(message)s", datefmt="%H:%M:%S")
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)s  %(message)s",
+    datefmt="%H:%M:%S",
+)
 logger = logging.getLogger(__name__)
 
 ROOT        = Path(__file__).parents[1]
@@ -17,16 +29,19 @@ RESULTS_OUT = ROOT / "data" / "reference" / "ensemble_results.json"
 def main():
     import xgboost as xgb
     import shap
-    from sklearn.model_selection import StratifiedKFold, cross_validate
-    from sklearn.metrics import roc_auc_score, f1_score
-    import json
+    from sklearn.model_selection import StratifiedKFold, cross_validate, train_test_split
+    from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--test-size",  type=float, default=0.15)
+    parser.add_argument("--early-stop", type=int,   default=20)
+    args = parser.parse_args()
 
     logger.info("Loading feature matrix")
     rows, labels, feature_names = [], [], None
 
     with open(FEATURES, newline="", encoding="utf-8") as f:
-        reader = csv.DictReader(f)
-        for row in reader:
+        for row in csv.DictReader(f):
             if feature_names is None:
                 feature_names = [k for k in row if k not in ("label", "path")]
             labels.append(int(row["label"]))
@@ -36,41 +51,62 @@ def main():
     y = np.array(labels)
     logger.info(f"Feature matrix: {X.shape} | Positives: {y.sum()}/{len(y)}")
 
-    model = xgb.XGBClassifier(
-        n_estimators=300,
-        max_depth=4,
-        learning_rate=0.05,
-        subsample=0.8,
-        colsample_bytree=0.8,
-        eval_metric="logloss",
-        random_state=42,
+    neg_count        = (y == 0).sum()
+    pos_count        = y.sum()
+    scale_pos_weight = neg_count / max(pos_count, 1)
+    logger.info(f"scale_pos_weight: {scale_pos_weight:.3f}")
+
+    X_dev, X_test, y_dev, y_test = train_test_split(
+        X, y, test_size=args.test_size, stratify=y, random_state=42
+    )
+
+    cv_model = xgb.XGBClassifier(
+        n_estimators=300, max_depth=4, learning_rate=0.05,
+        subsample=0.8, colsample_bytree=0.8,
+        min_child_weight=3, gamma=0.1,
+        scale_pos_weight=scale_pos_weight,
+        eval_metric="logloss", random_state=42,
     )
 
     logger.info("Cross-validating (5-fold stratified)")
-    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-    scores = cross_validate(
-        model, X, y, cv=cv,
-        scoring=["roc_auc", "f1"],
-        return_train_score=True,
+    cv     = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+    scores = cross_validate(cv_model, X_dev, y_dev, cv=cv,
+                            scoring=["roc_auc", "f1"], return_train_score=True)
+    auc_cv = scores["test_roc_auc"].mean()
+    f1_cv  = scores["test_f1"].mean()
+    logger.info(f"CV AUC: {auc_cv:.4f} +/- {scores['test_roc_auc'].std():.4f}")
+    logger.info(f"CV F1:  {f1_cv:.4f}  +/- {scores['test_f1'].std():.4f}")
+
+    X_tr, X_val, y_tr, y_val = train_test_split(
+        X_dev, y_dev, test_size=0.15, stratify=y_dev, random_state=0
     )
 
-    auc  = scores["test_roc_auc"].mean()
-    f1   = scores["test_f1"].mean()
-    logger.info(f"CV AUC:  {auc:.4f} +/- {scores['test_roc_auc'].std():.4f}")
-    logger.info(f"CV F1:   {f1:.4f} +/- {scores['test_f1'].std():.4f}")
+    model = xgb.XGBClassifier(
+        n_estimators=500, max_depth=4, learning_rate=0.05,
+        subsample=0.8, colsample_bytree=0.8,
+        min_child_weight=3, gamma=0.1,
+        scale_pos_weight=scale_pos_weight,
+        eval_metric="auc",
+        early_stopping_rounds=args.early_stop,
+        random_state=42,
+    )
+    logger.info("Fitting final model with early stopping")
+    model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], verbose=50)
+    logger.info(f"Best iteration: {model.best_iteration}")
 
-    logger.info("Fitting final model on all data")
-    model.fit(X, y)
+    y_score   = model.predict_proba(X_test)[:, 1]
+    y_pred    = (y_score >= 0.5).astype(int)
+    test_auc  = float(roc_auc_score(y_test, y_score))
+    test_f1   = float(f1_score(y_test, y_pred, zero_division=0))
+    test_acc  = float(accuracy_score(y_test, y_pred))
+    logger.info(f"Test AUC: {test_auc:.4f}  F1: {test_f1:.4f}  Acc: {test_acc:.4f}")
 
     logger.info("Computing SHAP values")
-    explainer   = shap.TreeExplainer(model)
-    shap_values = explainer.shap_values(X)
-    mean_shap   = np.abs(shap_values).mean(axis=0)
-
-    signal_importance = sorted(
-        zip(feature_names, mean_shap.tolist()),
-        key=lambda x: x[1], reverse=True
-    )
+    explainer        = shap.TreeExplainer(model)
+    shap_values      = explainer.shap_values(X_dev)
+    mean_shap        = np.abs(shap_values).mean(axis=0)
+    signal_importance = sorted(zip(feature_names, mean_shap.tolist()),
+                               key=lambda x: x[1], reverse=True)
 
     logger.info("Top 10 signals by SHAP importance:")
     for name, imp in signal_importance[:10]:
@@ -78,22 +114,27 @@ def main():
 
     MODEL_OUT.parent.mkdir(parents=True, exist_ok=True)
     with open(MODEL_OUT, "wb") as f:
-        pickle.dump({"model": model, "feature_names": feature_names, "explainer": explainer}, f)
+        pickle.dump({"model": model, "feature_names": feature_names,
+                     "explainer": explainer}, f)
     logger.info(f"Model saved to {MODEL_OUT}")
 
     results = {
-        "cv_auc_mean":  round(auc, 4),
-        "cv_auc_std":   round(scores["test_roc_auc"].std(), 4),
-        "cv_f1_mean":   round(f1, 4),
-        "cv_f1_std":    round(scores["test_f1"].std(), 4),
-        "n_features":   len(feature_names),
-        "n_samples":    len(y),
+        "cv_auc_mean":       round(auc_cv, 4),
+        "cv_auc_std":        round(scores["test_roc_auc"].std(), 4),
+        "cv_f1_mean":        round(f1_cv, 4),
+        "cv_f1_std":         round(scores["test_f1"].std(), 4),
+        "test_auc":          round(test_auc, 4),
+        "test_f1":           round(test_f1, 4),
+        "test_accuracy":     round(test_acc, 4),
+        "best_iteration":    int(model.best_iteration),
+        "scale_pos_weight":  round(float(scale_pos_weight), 4),
+        "n_features":        len(feature_names),
+        "n_samples":         len(y),
         "feature_importance": {k: round(v, 6) for k, v in signal_importance},
     }
     with open(RESULTS_OUT, "w") as f:
         json.dump(results, f, indent=2)
     logger.info(f"Results saved to {RESULTS_OUT}")
-    logger.info("Phase 4 complete.")
 
 
 if __name__ == "__main__":