Create generate_supplementary_calibration_figure.py

src/generate_supplementary_calibration_figure.py

@@ -0,0 +1,166 @@
+# generate_supplementary_calibration_figure.py
+"""
+Standalone script — produces the multi-cohort calibration figure for
+the manuscript supplementary section.
+ 
+Inputs:
+  - The frozen acute / chronic GVHD models (loaded via existing
+    src.model_utils.load_latest_*_single helpers)
+  - External cohort CSVs (paths configured below)
+ 
+Outputs:
+  - PNG figure: supplementary_calibration_<target>_300dpi.png
+  - CSV table: supplementary_calibration_stats_<target>.csv
+ 
+Run from the project root:
+  python generate_supplementary_calibration_figure.py
+"""
+ 
+import os
+import sys
+from pathlib import Path
+ 
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+ 
+# Make src importable
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT / "src"))
+ 
+from model_utils import load_latest_acute_single, load_latest_chronic_single
+from inference_utils import align_to_saved_features, sanitize_inference_matrix
+from preprocess_utils import preprocess_pipeline as preprocess
+from calibration_utils import (
+    compute_calibration_stats,
+    plot_multi_cohort_calibration,
+)
+ 
+ 
+# ----------------------------------------------------------------------
+# CONFIG — edit the cohort paths to point to your external CSVs.
+# ----------------------------------------------------------------------
+EXTERNAL_COHORTS = {
+    "P5356":      "external_cohorts/P5356.csv",
+    "P5441":      "external_cohorts/P5441.csv",
+    "P5373":      "external_cohorts/P5373.csv",
+    "P5178":      "external_cohorts/P5178.csv",
+    "HS1502":     "external_cohorts/HS1502.csv",
+    "UAE+Jordan": "external_cohorts/UAE_Jordan.csv",
+}
+ 
+OUTPUT_DIR = ROOT / "manuscript_outputs"
+OUTPUT_DIR.mkdir(exist_ok=True)
+ 
+ 
+# ----------------------------------------------------------------------
+# Load OOF predictions from saved model metadata
+# ----------------------------------------------------------------------
+ 
+def load_oof_from_model(model_dict):
+    extra = model_dict.get("extra_metadata", {}) or {}
+    y_true = extra.get("oof_true")
+    y_prob = extra.get("oof_probs")
+    if y_true is None or y_prob is None:
+        return None, None
+    return np.asarray(y_true), np.asarray(y_prob)
+ 
+ 
+def score_cohort(model_dict, cohort_csv_path, target_col):
+    """Load a cohort CSV, preprocess, score with frozen model."""
+    df = pd.read_csv(cohort_csv_path, header=1)
+    df_full, df_model = preprocess(df, target_col=target_col)
+ 
+    extra = model_dict.get("extra_metadata", {}) or {}
+    train_features = extra.get("train_features", [])
+    cat_features = extra.get("cat_features", [])
+ 
+    X = align_to_saved_features(df_model, train_features, cat_features)
+ 
+    y = pd.to_numeric(df_full[target_col], errors="coerce")
+    valid = y.notna()
+    X = X.loc[valid]
+    y = y.loc[valid].astype(int).to_numpy()
+ 
+    model = model_dict["model"]
+    preds = model.predict_proba(X)[:, 1]
+    return y, preds
+ 
+ 
+# ----------------------------------------------------------------------
+# Build figure for one target (acute or chronic)
+# ----------------------------------------------------------------------
+ 
+def build_figure_for_target(target_label, target_col):
+    print(f"\n=== {target_label} ===")
+    if "acute" in target_col.lower():
+        model_dict = load_latest_acute_single()
+    elif "chronic" in target_col.lower():
+        model_dict = load_latest_chronic_single()
+    else:
+        raise ValueError(f"Unknown target_col: {target_col}")
+ 
+    cohort_results = {}
+ 
+    # Training OOF
+    oof_true, oof_probs = load_oof_from_model(model_dict)
+    if oof_true is not None and oof_probs is not None:
+        cohort_results["Training (OOF)"] = (oof_true, oof_probs)
+        print(f"  Training OOF: N={len(oof_true)}")
+    else:
+        print("  WARNING: no OOF arrays in model metadata. Retrain to capture them.")
+ 
+    # External cohorts
+    for cohort_name, cohort_path in EXTERNAL_COHORTS.items():
+        if not Path(cohort_path).exists():
+            print(f"  SKIP {cohort_name}: file not found at {cohort_path}")
+            continue
+        try:
+            y, preds = score_cohort(model_dict, cohort_path, target_col)
+            if len(y) < 20:
+                print(f"  SKIP {cohort_name}: too few rows ({len(y)})")
+                continue
+            cohort_results[cohort_name] = (y, preds)
+            print(f"  {cohort_name}: N={len(y)}, events={int(y.sum())}")
+        except Exception as e:
+            print(f"  ERROR scoring {cohort_name}: {e}")
+ 
+    # Composite figure
+    fig = plot_multi_cohort_calibration(cohort_results, ncols=3)
+    fig_path = OUTPUT_DIR / f"supplementary_calibration_{target_label}_300dpi.png"
+    fig.savefig(fig_path, dpi=300, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Figure saved: {fig_path}")
+ 
+    # Stats table
+    rows = []
+    for cohort_name, (y, preds) in cohort_results.items():
+        stats = compute_calibration_stats(y, preds, n_bins=10)
+        rows.append({
+            "Cohort": cohort_name,
+            "N": stats["N"],
+            "Events": stats["Events"],
+            "Prevalence": stats["Prevalence"],
+            "Brier": stats["Brier"],
+            "CalibrationInTheLarge": stats["CalibrationInTheLarge"],
+            "Intercept": stats["CalibrationIntercept"],
+            "Intercept_CI_low": stats["CalibrationIntercept_CI_low"],
+            "Intercept_CI_high": stats["CalibrationIntercept_CI_high"],
+            "Slope": stats["CalibrationSlope"],
+            "Slope_CI_low": stats["CalibrationSlope_CI_low"],
+            "Slope_CI_high": stats["CalibrationSlope_CI_high"],
+        })
+ 
+    table_path = OUTPUT_DIR / f"supplementary_calibration_stats_{target_label}.csv"
+    pd.DataFrame(rows).to_csv(table_path, index=False)
+    print(f"  Stats table saved: {table_path}")
+ 
+ 
+# ----------------------------------------------------------------------
+# Run for both targets
+# ----------------------------------------------------------------------
+ 
+if __name__ == "__main__":
+    build_figure_for_target("acute", "Acute GVHD(<100 days)")
+    build_figure_for_target("chronic", "Chronic GVHD>100 days")
+    print("\nDone.")