| """Figure 6 — Rank-cutoff vs probability-threshold decision robustness. |
| |
| Left: F1 vs positive ratio (stack_ratio_analysis) — flat-topped near 0.5, so the |
| rank cutoff sits on the robust plateau while the probability-threshold-transferred |
| test ratio (0.524) lands off-peak. Right: predicted-positive-ratio drift across the |
| two strategies. Argues why rank cutoff beats a transferred probability threshold. |
| """ |
| from pathlib import Path |
| import pandas as pd |
| import numpy as np |
| import matplotlib.pyplot as plt |
| from style import apply, save, PALETTE as C, COL2 |
|
|
| KEY = "fig6_rank_vs_threshold" |
| TITLE = "Figure 6. Rank-cutoff vs probability-threshold robustness" |
|
|
| |
| TEST_POS_RATIO_PROBTHRESH = 0.524195 |
|
|
|
|
| def make(root, out): |
| apply() |
| ratio_csv = root / "validation_runs" / "stack_ratio_analysis.csv" |
| thr_csv = root / "validation_runs" / "stack_threshold_summary.csv" |
| VR = root / "validation_runs" / "dynamic_seed202" / "high_order_graph_stack" / "submission_summary.csv" |
| ratio = pd.read_csv(ratio_csv).sort_values("ratio") if ratio_csv.exists() else None |
| thr = pd.read_csv(thr_csv) if thr_csv.exists() else None |
| status = "ok" if ratio is not None else "fallback" |
| sources = [str(p) for p in (ratio_csv, thr_csv, VR) if p.exists()] |
| if not sources: |
| sources = ["reported numbers (CSVs missing)"] |
|
|
| fig, (axL, axR) = plt.subplots(1, 2, figsize=(COL2, 3.3), gridspec_kw={"width_ratios": [1.35, 1]}) |
|
|
| |
| if ratio is not None: |
| axL.plot(ratio.ratio, ratio.f1_mean, "-o", color=C[0], ms=3.5, lw=1.6, label="F1 (mean over seeds)") |
| axL.fill_between(ratio.ratio, ratio.f1_min, ratio.f1_max, color=C[0], alpha=0.15, label="min–max band") |
| axL.axvline(0.50, color=C[2], lw=1.6) |
| axL.plot([0.50], [ratio.f1_mean.iloc[(ratio.ratio - 0.5).abs().argmin()]] if ratio is not None else [0.9556], |
| marker="*", ms=15, color=C[2], mec="black", mew=0.5, zorder=6, label="rank cutoff (0.50)") |
| axL.axvline(TEST_POS_RATIO_PROBTHRESH, color=C[3], lw=1.4, ls="--") |
| axL.plot([TEST_POS_RATIO_PROBTHRESH], |
| [np.interp(TEST_POS_RATIO_PROBTHRESH, ratio.ratio, ratio.f1_mean)] if ratio is not None else [0.9520], |
| marker="D", ms=7, color=C[3], label="prob. threshold → test ratio") |
| axL.set_xlabel("predicted-positive ratio"); axL.set_ylabel("F1") |
| axL.set_title("(a) Rank cutoff sits on the robust plateau", fontsize=8.5) |
| axL.legend(fontsize=6.6, loc="lower left") |
|
|
| |
| strategies = ["rank cutoff", "probability\nthreshold"] |
| val_ratio = [0.500, 0.500] |
| test_ratio = [0.500, TEST_POS_RATIO_PROBTHRESH] |
| x = np.arange(2); w = 0.34 |
| axR.bar(x - w / 2, val_ratio, w, color=C[2], alpha=0.85, label="validation") |
| axR.bar(x + w / 2, test_ratio, w, color=C[3], alpha=0.85, label="test (transferred)") |
| for xi, v, t in zip(x, val_ratio, test_ratio): |
| axR.text(xi - w / 2, v + 0.004, f"{v:.3f}", ha="center", fontsize=6.8) |
| axR.text(xi + w / 2, t + 0.004, f"{t:.3f}", ha="center", fontsize=6.8, |
| color=C[3] if abs(t - v) > 0.01 else "black") |
| axR.set_xticks(x); axR.set_xticklabels(strategies, fontsize=7.5) |
| axR.set_ylabel("predicted-positive ratio"); axR.set_ylim(0.45, 0.55) |
| axR.set_title("(b) Positive-ratio drift on test", fontsize=8.5) |
| axR.legend(fontsize=6.8, loc="upper left") |
| if thr is not None: |
| axR.text(0.5, -0.30, |
| f"val-optimal prob. threshold varies by seed: " |
| f"{thr.threshold.min():.3f}–{thr.threshold.max():.3f}", |
| transform=axR.transAxes, ha="center", fontsize=6.4, color="dimgray") |
|
|
| fig.suptitle("Decision rule robustness: rank cutoff vs transferred probability threshold", |
| fontsize=9.5, y=1.02) |
| save(fig, KEY, out) |
| return dict(key=KEY, title=TITLE, status=status, files=[f"{KEY}.pdf", f"{KEY}.png", f"{KEY}.svg"], |
| sources=sources + ["final_report: test positive ratio 0.524195 at val-opt threshold"], |
| caption=( |
| "Rank-cutoff vs probability-threshold robustness. The validation split is artificially " |
| "1:1 positive/negative, so the LightGBM probability does not transfer to the imbalanced " |
| "test set: the validation-optimal probability threshold (0.462, and varying 0.44–0.49 " |
| "across seeds) drifts to a 0.524 predicted-positive ratio on test, off the F1 peak (a). " |
| "The rank-cutoff rule (predict the top-50% plus known positives) is class-balance invariant " |
| "and stays exactly at 0.50 on both splits (b), making it the more robust decision rule.")) |
|
|
|
|
| if __name__ == "__main__": |
| from style import ensure_dirs |
| r = make(Path("."), ensure_dirs(Path("."))) |
| print(r["key"], r["status"]) |
|
|
