code/make_mcnemar9.py

"""Regenerate research_v2_latest/kfold/cnn_clip/mcnemar.json for ALL NINE models.

Under the uniform pooled-ensemble protocol every model carries one aligned
per-image prediction on the fixed 3,208-image test set: the five convolutional
networks and CLIP from kfold/cnn_clip/<model>_test_preds.json, the three
foundation models from the probability average of their five
kfold/foundation_fold<k>_<model>_preds.json checkpoints. Because all nine share
the same test images in the same order, the paired McNemar test now covers every
pair (36 in total).

Test: exact binomial two-sided on the discordant pairs (b, c), identical to
research_v2_latest/code/ensemble_and_stats.py. The file keeps the original
"<a>_vs_<b>": {"b","c","p"} layout used by thesis_build/figures/make_fig4_5.py;
a sibling key "_meta" records the model order, pair count and Bonferroni factor.

Run with the project .venv interpreter:
    .venv/bin/python thesis_build/make_mcnemar9.py
"""
import json
import numpy as np
from pathlib import Path
from scipy.stats import binom

ROOT = Path(__file__).resolve().parent.parent
KF = ROOT / "kfold"
OUT = KF / "cnn_clip" / "mcnemar.json"

# accuracy-descending order (matches Table 4.1)
ORDER = ["inception_v3", "clip_openai", "vgg19", "resnet101", "dinov2_l",
         "densenet121", "resnet50", "swin_b", "retfound"]
CNN_CLIP = {"inception_v3", "clip_openai", "vgg19", "resnet101", "densenet121", "resnet50"}


def preds(model):
    if model in CNN_CLIP:
        d = json.load(open(KF / "cnn_clip" / f"{model}_test_preds.json"))
        return np.array(d["labels"]), np.array(d["preds"])
    probs, labels = [], None
    for k in range(5):
        d = json.load(open(KF / f"foundation_fold{k}_{model}_preds.json"))
        probs.append(np.array(d["probs"])); labels = np.array(d["labels"])
    return labels, np.mean(probs, axis=0).argmax(1)


def main():
    P = {m: preds(m) for m in ORDER}
    L = P[ORDER[0]][0]
    for m in ORDER:
        assert np.array_equal(P[m][0], L), f"{m} label order differs"

    out = {}
    pairs = []
    for i in range(len(ORDER)):
        for j in range(i + 1, len(ORDER)):
            a, b = ORDER[i], ORDER[j]
            ca = P[a][1] == L; cb = P[b][1] == L
            bb = int(np.sum(ca & ~cb)); cc = int(np.sum(~ca & cb)); n = bb + cc
            p = 1.0 if n == 0 else float(min(1.0, 2 * binom.cdf(min(bb, cc), n, 0.5)))
            out[f"{a}_vs_{b}"] = {"b": bb, "c": cc, "p": p}
            pairs.append((a, b, bb, cc, p))

    npairs = len(pairs)
    out["_meta"] = {"models": ORDER, "n_pairs": npairs, "bonferroni_factor": npairs,
                    "test": "exact binomial two-sided McNemar"}
    with open(OUT, "w") as fh:
        json.dump(out, fh, indent=1)

    lines = []
    nsig = 0
    for a, b, bb, cc, p in sorted(pairs, key=lambda x: x[4]):
        adj = min(1.0, p * npairs); sig = adj < 0.05; nsig += sig
        lines.append(f"{'*' if sig else ' '} {a:13s} vs {b:13s}  b={bb:4d} c={cc:4d}  p={p:.3g}  adj={adj:.3g}")
    top = [x for x in pairs if x[0] not in {"swin_b", "retfound"} and x[1] not in {"swin_b", "retfound"}]
    with open("/tmp/mcnemar9_check.txt", "w") as fh:
        fh.write(f"wrote {OUT.relative_to(ROOT)}  n_pairs={npairs}  significant(Bonferroni)={nsig}\n")
        fh.write(f"top-7 pairs={len(top)}  min raw p among top-7={min(x[4] for x in top):.4f}  "
                 f"significant among top-7={sum(1 for x in top if x[4]*npairs<0.05)}\n\n")
        fh.write("\n".join(lines) + "\n")
    print("wrote", OUT.relative_to(ROOT))


if __name__ == "__main__":
    main()