Add pooled-protocol provenance scripts (regenerate summary, 9-model McNemar, per-class, ensemble from fold predictions)

code/make_ensemble_pooled.py

@@ -0,0 +1,107 @@
+"""Compute the nine-model soft-vote ensemble on the pooled five-fold predictions.
+
+Every other result in the thesis reduces each model to a single pooled prediction
+per image (the probabilities of its five fold checkpoints averaged on the fixed
+3,208-image test set). The original ensemble report shipped from the training VM
+was instead computed on the single-checkpoint predictions, a different and
+non-comparable aggregation. This script recomputes the ensemble in the same
+pooled paradigm so Section 4.8 is consistent with Table 4.1 and the McNemar
+analysis: each model is first pooled across its folds, the nine pooled
+probability vectors are then combined by a soft vote weighted by each model's
+macro-F1, and the result is compared against the best single pooled model with
+the same exact-binomial McNemar test used in Section 4.5.
+
+Writes research_v2_latest/analysis/ensemble_pooled.json.
+
+Run with the project .venv interpreter:
+    .venv/bin/python thesis_build/make_ensemble_pooled.py
+"""
+import json
+import numpy as np
+from pathlib import Path
+from scipy.stats import binom
+from sklearn.metrics import (
+    accuracy_score, precision_recall_fscore_support, roc_auc_score, cohen_kappa_score,
+)
+
+ROOT = Path(__file__).resolve().parent.parent
+KF = ROOT / "kfold"
+OUT = ROOT / "analysis" / "ensemble_pooled.json"
+
+CNN_CLIP = {"inception_v3", "clip_openai", "vgg19", "resnet101", "densenet121", "resnet50"}
+ALL = ["inception_v3", "clip_openai", "vgg19", "resnet101", "dinov2_l",
+       "densenet121", "resnet50", "swin_b", "retfound"]
+NUM_CLASSES = 10
+LABELS = list(range(NUM_CLASSES))
+
+
+def pooled(model):
+    if model in CNN_CLIP:
+        d = json.load(open(KF / "cnn_clip" / f"{model}_test_preds.json"))
+        return np.array(d["labels"]), np.array(d["probs"])
+    probs, labels = [], None
+    for k in range(5):
+        d = json.load(open(KF / f"foundation_fold{k}_{model}_preds.json"))
+        probs.append(np.array(d["probs"])); labels = np.array(d["labels"])
+    return labels, np.mean(probs, axis=0)
+
+
+def ece(probs, labels, n_bins=15):
+    conf = probs.max(1); pred = probs.argmax(1); correct = (pred == labels).astype(float)
+    bins = np.linspace(0, 1, n_bins + 1); e = 0.0
+    for i in range(n_bins):
+        m = (conf > bins[i]) & (conf <= bins[i + 1])
+        if m.sum():
+            e += m.mean() * abs(correct[m].mean() - conf[m].mean())
+    return float(e)
+
+
+def mcnemar(pred_a, pred_b, labels):
+    ca = pred_a == labels; cb = pred_b == labels
+    b = int(np.sum(ca & ~cb)); c = int(np.sum(~ca & cb)); n = b + c
+    p = 1.0 if n == 0 else float(min(1.0, 2 * binom.cdf(min(b, c), n, 0.5)))
+    return b, c, p
+
+
+def main():
+    P = {}; L = None
+    for m in ALL:
+        L, P[m] = pooled(m)
+    f1 = {m: precision_recall_fscore_support(L, P[m].argmax(1), average="macro", zero_division=0)[2]
+          for m in ALL}
+
+    # soft vote weighted by each model's pooled macro-F1
+    wsum = sum(f1.values())
+    ens_probs = sum(f1[m] * P[m] for m in ALL) / wsum
+    ens_pred = ens_probs.argmax(1)
+
+    acc = accuracy_score(L, ens_pred) * 100
+    prec, rec, fm, _ = precision_recall_fscore_support(L, ens_pred, average="macro", zero_division=0)
+    roc = roc_auc_score(L, ens_probs, multi_class="ovr", average="macro", labels=LABELS)
+
+    # best single pooled model
+    best = max(ALL, key=lambda m: accuracy_score(L, P[m].argmax(1)))
+    best_acc = accuracy_score(L, P[best].argmax(1)) * 100
+    b, c, p = mcnemar(ens_pred, P[best].argmax(1), L)
+
+    report = {
+        "protocol": "pooled five-fold soft vote, weight = pooled macro-F1, all nine models",
+        "members": ALL,
+        "ensemble": {
+            "acc": round(acc, 2), "f1": round(fm * 100, 2), "roc_auc": round(roc, 4),
+            "ece": round(ece(ens_probs, L), 4), "kappa": round(cohen_kappa_score(L, ens_pred), 3),
+        },
+        "best_single": {"model": best, "acc": round(best_acc, 2), "f1": round(f1[best] * 100, 2)},
+        "ensemble_vs_best_mcnemar": {"b": b, "c": c, "p": round(p, 3),
+                                     "significant_005": bool(p < 0.05)},
+        "n_test": int(len(L)),
+    }
+    with open(OUT, "w") as fh:
+        json.dump(report, fh, indent=2)
+    with open("/tmp/ensemble_pooled_check.txt", "w") as fh:
+        fh.write(json.dumps(report, indent=2))
+    print("wrote", OUT.relative_to(ROOT))
+
+
+if __name__ == "__main__":
+    main()

code/make_kfold_summary.py

@@ -0,0 +1,105 @@
+"""Regenerate the three foundation rows of research_v2_latest/kfold/kfold_v2_summary.csv
+under the uniform pooled-ensemble protocol (Table 4.1 source).
+
+The six convolutional/CLIP rows were produced on the training VM as a pooled
+five-fold ensemble (probabilities averaged across the five fold checkpoints,
+scored once on the fixed 3,208-image test set). The three foundation models were
+originally summarised by the mean of their per-fold scores instead, a different,
+non-comparable aggregation that also denied them the same ensemble treatment.
+This script brings the foundation models into the identical pooled-ensemble
+protocol so all nine rows are directly comparable and every model carries a
+single aligned per-image prediction (which also lets the McNemar test in
+make_mcnemar9.py cover all nine).
+
+The six CNN/CLIP rows are kept verbatim from the authoritative VM summary; only
+the three foundation rows are recomputed from
+kfold/foundation_fold<k>_<model>_preds.json. Confidence intervals use the
+bootstrap documented in research_v2_latest/code/ensemble_and_stats.py: 2000
+resamples, seed 42, 2.5/97.5 percentile. Brier uses the element-wise mean
+convention of the original summary.
+
+Run with the project .venv interpreter:
+    .venv/bin/python thesis_build/make_kfold_summary.py
+"""
+import csv
+import json
+import numpy as np
+from pathlib import Path
+from sklearn.metrics import (
+    accuracy_score, precision_recall_fscore_support, roc_auc_score,
+    average_precision_score, cohen_kappa_score,
+)
+
+ROOT = Path(__file__).resolve().parent.parent
+KF = ROOT / "kfold"
+CSV = KF / "kfold_v2_summary.csv"
+FOUNDATION = ["dinov2_l", "swin_b", "retfound"]
+NUM_CLASSES = 10
+LABELS = list(range(NUM_CLASSES))
+
+
+def pooled_foundation(model):
+    probs, labels = [], None
+    for k in range(5):
+        d = json.load(open(KF / f"foundation_fold{k}_{model}_preds.json"))
+        probs.append(np.array(d["probs"])); labels = np.array(d["labels"])
+    P = np.mean(probs, axis=0)
+    return labels, P.argmax(1), P
+
+
+def ece(probs, labels, n_bins=15):
+    conf = probs.max(1); pred = probs.argmax(1); correct = (pred == labels).astype(float)
+    bins = np.linspace(0, 1, n_bins + 1); e = 0.0
+    for i in range(n_bins):
+        m = (conf > bins[i]) & (conf <= bins[i + 1])
+        if m.sum():
+            e += m.mean() * abs(correct[m].mean() - conf[m].mean())
+    return float(e)
+
+
+def boot_ci(fn, labels, preds, n=2000, seed=42):
+    rng = np.random.default_rng(seed); N = len(labels); vals = []
+    for _ in range(n):
+        idx = rng.integers(0, N, N)
+        vals.append(fn(labels[idx], preds[idx]))
+    return float(np.percentile(vals, 2.5)), float(np.percentile(vals, 97.5))
+
+
+def foundation_row(model):
+    y, p, pr = pooled_foundation(model)
+    acc = accuracy_score(y, p) * 100
+    prec, rec, f1, _ = precision_recall_fscore_support(y, p, average="macro", zero_division=0)
+    acc_lo, acc_hi = boot_ci(lambda a, b: accuracy_score(a, b) * 100, y, p)
+    f1_lo, f1_hi = boot_ci(
+        lambda a, b: precision_recall_fscore_support(a, b, average="macro", zero_division=0)[2] * 100, y, p)
+    roc = roc_auc_score(y, pr, multi_class="ovr", average="macro", labels=LABELS)
+    oh = np.eye(NUM_CLASSES)[y]
+    prauc = average_precision_score(oh, pr, average="macro")
+    brier = float(((pr - oh) ** 2).mean())
+    return [model, "Foundation", round(acc, 2), round(acc_lo, 2), round(acc_hi, 2),
+            round(f1 * 100, 2), round(f1_lo, 2), round(f1_hi, 2),
+            round(prec * 100, 2), round(rec * 100, 2), round(roc, 4), round(prauc, 4),
+            round(ece(pr, y), 4), round(cohen_kappa_score(y, p), 3), round(brier, 3),
+            5, "five-fold pooled preds"]
+
+
+def main():
+    with open(CSV) as fh:
+        rdr = csv.reader(fh); header = next(rdr); rows = list(rdr)
+    kept = [r for r in rows if r[1] != "Foundation"]
+    for r in kept:
+        r[16] = "five-fold pooled preds"
+    new_found = [foundation_row(m) for m in FOUNDATION]
+    allrows = kept + new_found
+    allrows.sort(key=lambda r: -float(r[2]))
+    with open(CSV, "w", newline="") as fh:
+        w = csv.writer(fh); w.writerow(header); w.writerows(allrows)
+    with open("/tmp/summary_check.txt", "w") as fh:
+        for r in allrows:
+            fh.write(f"{r[0]:14s} acc {r[2]:6} ({r[3]}-{r[4]})  f1 {r[5]:6}  "
+                     f"roc {r[10]}  ece {r[12]}  kappa {r[13]}  brier {r[14]}  {r[16]}\n")
+    print("wrote", CSV.relative_to(ROOT))
+
+
+if __name__ == "__main__":
+    main()

code/make_mcnemar9.py

@@ -0,0 +1,83 @@
+"""Regenerate research_v2_latest/kfold/cnn_clip/mcnemar.json for ALL NINE models.
+
+Under the uniform pooled-ensemble protocol every model carries one aligned
+per-image prediction on the fixed 3,208-image test set: the five convolutional
+networks and CLIP from kfold/cnn_clip/<model>_test_preds.json, the three
+foundation models from the probability average of their five
+kfold/foundation_fold<k>_<model>_preds.json checkpoints. Because all nine share
+the same test images in the same order, the paired McNemar test now covers every
+pair (36 in total).
+
+Test: exact binomial two-sided on the discordant pairs (b, c), identical to
+research_v2_latest/code/ensemble_and_stats.py. The file keeps the original
+"<a>_vs_<b>": {"b","c","p"} layout used by thesis_build/figures/make_fig4_5.py;
+a sibling key "_meta" records the model order, pair count and Bonferroni factor.
+
+Run with the project .venv interpreter:
+    .venv/bin/python thesis_build/make_mcnemar9.py
+"""
+import json
+import numpy as np
+from pathlib import Path
+from scipy.stats import binom
+
+ROOT = Path(__file__).resolve().parent.parent
+KF = ROOT / "kfold"
+OUT = KF / "cnn_clip" / "mcnemar.json"
+
+# accuracy-descending order (matches Table 4.1)
+ORDER = ["inception_v3", "clip_openai", "vgg19", "resnet101", "dinov2_l",
+         "densenet121", "resnet50", "swin_b", "retfound"]
+CNN_CLIP = {"inception_v3", "clip_openai", "vgg19", "resnet101", "densenet121", "resnet50"}
+
+
+def preds(model):
+    if model in CNN_CLIP:
+        d = json.load(open(KF / "cnn_clip" / f"{model}_test_preds.json"))
+        return np.array(d["labels"]), np.array(d["preds"])
+    probs, labels = [], None
+    for k in range(5):
+        d = json.load(open(KF / f"foundation_fold{k}_{model}_preds.json"))
+        probs.append(np.array(d["probs"])); labels = np.array(d["labels"])
+    return labels, np.mean(probs, axis=0).argmax(1)
+
+
+def main():
+    P = {m: preds(m) for m in ORDER}
+    L = P[ORDER[0]][0]
+    for m in ORDER:
+        assert np.array_equal(P[m][0], L), f"{m} label order differs"
+
+    out = {}
+    pairs = []
+    for i in range(len(ORDER)):
+        for j in range(i + 1, len(ORDER)):
+            a, b = ORDER[i], ORDER[j]
+            ca = P[a][1] == L; cb = P[b][1] == L
+            bb = int(np.sum(ca & ~cb)); cc = int(np.sum(~ca & cb)); n = bb + cc
+            p = 1.0 if n == 0 else float(min(1.0, 2 * binom.cdf(min(bb, cc), n, 0.5)))
+            out[f"{a}_vs_{b}"] = {"b": bb, "c": cc, "p": p}
+            pairs.append((a, b, bb, cc, p))
+
+    npairs = len(pairs)
+    out["_meta"] = {"models": ORDER, "n_pairs": npairs, "bonferroni_factor": npairs,
+                    "test": "exact binomial two-sided McNemar"}
+    with open(OUT, "w") as fh:
+        json.dump(out, fh, indent=1)
+
+    lines = []
+    nsig = 0
+    for a, b, bb, cc, p in sorted(pairs, key=lambda x: x[4]):
+        adj = min(1.0, p * npairs); sig = adj < 0.05; nsig += sig
+        lines.append(f"{'*' if sig else ' '} {a:13s} vs {b:13s}  b={bb:4d} c={cc:4d}  p={p:.3g}  adj={adj:.3g}")
+    top = [x for x in pairs if x[0] not in {"swin_b", "retfound"} and x[1] not in {"swin_b", "retfound"}]
+    with open("/tmp/mcnemar9_check.txt", "w") as fh:
+        fh.write(f"wrote {OUT.relative_to(ROOT)}  n_pairs={npairs}  significant(Bonferroni)={nsig}\n")
+        fh.write(f"top-7 pairs={len(top)}  min raw p among top-7={min(x[4] for x in top):.4f}  "
+                 f"significant among top-7={sum(1 for x in top if x[4]*npairs<0.05)}\n\n")
+        fh.write("\n".join(lines) + "\n")
+    print("wrote", OUT.relative_to(ROOT))
+
+
+if __name__ == "__main__":
+    main()

code/make_per_class_pooled_csv.py

@@ -0,0 +1,89 @@
+"""Generate per_class_metrics_pooled.csv from the latest five-fold predictions.
+
+This is the provenance script for research_v2_latest/analysis/per_class_metrics_pooled.csv,
+the per-class precision/recall/F1/support table that drives Appendix B and Figure 4.2.
+
+Aggregation matches Table 4.1 exactly under the uniform pooled-ensemble protocol:
+  * the five convolutional networks and CLIP use their pooled five-fold test
+    predictions (kfold/cnn_clip/<model>_test_preds.json), i.e. the probabilities
+    averaged across the five fold checkpoints, scored once on the fixed test set;
+  * the three foundation models are pooled the same way, averaging the probability
+    vectors of their five fold checkpoints (kfold/foundation_fold<k>_<model>_preds.json)
+    into one aligned per-image prediction before the per-class metrics are computed.
+
+Run with the project .venv interpreter (needs numpy + scikit-learn):
+    .venv/bin/python thesis_build/make_per_class_pooled_csv.py
+"""
+import csv
+import json
+import numpy as np
+from pathlib import Path
+from sklearn.metrics import precision_recall_fscore_support
+
+ROOT = Path(__file__).resolve().parent.parent
+KF = ROOT / "kfold"
+OUT = ROOT / "analysis" / "per_class_metrics_pooled.csv"
+
+# Class index -> display name (the dataset's alphabetical label order).
+CLASSES = [
+    "Central Serous Chorioretinopathy",  # 0
+    "Diabetic Retinopathy",              # 1
+    "Disc Edema",                        # 2
+    "Glaucoma",                          # 3
+    "Healthy",                           # 4
+    "Macular Scar",                      # 5
+    "Myopia",                            # 6
+    "Pterygium",                         # 7
+    "Retinal Detachment",                # 8
+    "Retinitis Pigmentosa",              # 9
+]
+LABELS = list(range(len(CLASSES)))
+
+# Model write order = accuracy-descending order of Table 4.1 (DINOv2-L is 5th).
+CNN_CLIP = ["inception_v3", "clip_openai", "vgg19", "resnet101", "densenet121", "resnet50"]
+FOUNDATION = ["dinov2_l", "swin_b", "retfound"]
+MODEL_ORDER = ["inception_v3", "clip_openai", "vgg19", "resnet101", "dinov2_l",
+               "densenet121", "resnet50", "swin_b", "retfound"]
+
+
+def pooled_cnn_clip(model):
+    """Per-class metrics from the single pooled five-fold prediction file."""
+    d = json.load(open(KF / "cnn_clip" / f"{model}_test_preds.json"))
+    y, p = np.array(d["labels"]), np.array(d["preds"])
+    P, R, F, S = precision_recall_fscore_support(y, p, labels=LABELS, zero_division=0)
+    return P, R, F, S
+
+
+def pooled_foundation(model):
+    """Per-class metrics from the pooled five-fold ensemble (probabilities averaged
+    across the five fold checkpoints, then argmax), matching Table 4.1."""
+    files = sorted(KF.glob(f"foundation_fold*_{model}_preds.json"))
+    assert len(files) == 5, f"expected 5 folds for {model}, found {len(files)}"
+    probs, y = [], None
+    for f in files:
+        d = json.load(open(f))
+        probs.append(np.array(d["probs"])); y = np.array(d["labels"])
+    p = np.mean(probs, axis=0).argmax(1)
+    P, R, F, S = precision_recall_fscore_support(y, p, labels=LABELS, zero_division=0)
+    return P, R, F, S
+
+
+def main():
+    rows = []
+    for model in MODEL_ORDER:
+        if model in CNN_CLIP:
+            P, R, F, S = pooled_cnn_clip(model)
+        else:
+            P, R, F, S = pooled_foundation(model)
+        for i in LABELS:
+            rows.append([model, CLASSES[i], P[i], R[i], F[i], int(S[i])])
+
+    with open(OUT, "w", newline="") as fh:
+        w = csv.writer(fh)
+        w.writerow(["model", "class", "precision", "recall", "f1", "support"])
+        w.writerows(rows)
+    print(f"wrote {OUT.relative_to(ROOT)} ({len(rows)} rows)")
+
+
+if __name__ == "__main__":
+    main()