File size: 3,569 Bytes
1ea7ba6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | #!/usr/bin/env python
"""Two statistics the reviewer panel asked for, computed from stored artifacts.
1. An image-level cluster bootstrap for the studio-to-wild collapse. The three seeds
share one 1218-image wild test set, so resampling all 3x1218 rows independently
(as the previous pooled bootstrap did) triple-counts each image and shrinks the
interval by about sqrt(3). Here we resample the 1218 image indices once per
bootstrap draw and average the per-seed collapse over that shared resample.
2. Seed-level paired t-tests for the single-source table. With three seeds the paired
difference per seed is the right unit; this is both valid and, for the ARC-V and SD
gains over ERM, stronger than the per-seed McNemar we already report.
Inputs are the per-sample correctness arrays in outputs/shortcut_evidence_s*.json and
the per-seed accuracies. No model is run.
"""
import json
from pathlib import Path
import numpy as np
from scipy import stats
BASE = Path("/mnt/d/SpiceNet") if Path("/mnt/d/SpiceNet").exists() else Path("D:/SpiceNet")
SEEDS = [42, 1337, 2024]
RNG = np.random.default_rng(20260721)
B = 10000
# Per-seed held-out wild accuracy, single-source (studio-trained), all four methods.
# From eval_baseline_mcnemar.py (CPU); reproduces the paper's Table means to 0.03.
ACC = {
"erm": [61.99, 59.85, 63.38],
"rsc": [65.35, 61.00, 64.70],
"arcv": [67.73, 65.44, 68.80],
"sd": [68.31, 66.67, 67.82],
}
def cluster_bootstrap_collapse():
# correct[seed] = studio-trained correctness on the 1218 wild-test images.
# ss_test is the wild test set; in_trained_correct is the studio-trained model on it.
correct = []
for s in SEEDS:
j = json.loads((BASE / "outputs" / f"shortcut_evidence_s{s}.json").read_text())
correct.append(np.asarray(j["per_sample"]["ss_test"]["in_trained_correct"], float))
correct = np.vstack(correct) # (3, 1218); studio within-source is 100%
n = correct.shape[1]
point = float(np.mean([100 * (1 - c.mean()) for c in correct]))
draws = np.empty(B)
for b in range(B):
idx = RNG.integers(0, n, n) # one shared resample across seeds
draws[b] = np.mean([100 * (1 - c[idx].mean()) for c in correct])
lo, hi = np.percentile(draws, [2.5, 97.5])
# naive pooled bootstrap for contrast (resample 3*n rows independently)
flat = correct.reshape(-1)
m = flat.size
pooled = np.empty(B)
for b in range(B):
pooled[b] = 100 * (1 - flat[RNG.integers(0, m, m)].mean())
plo, phi = np.percentile(pooled, [2.5, 97.5])
print("=== studio->wild collapse, bootstrap ===")
print(f"point estimate : {point:.2f} pp")
print(f"cluster (image-level) 95%: [{lo:.2f}, {hi:.2f}] width {hi-lo:.2f}")
print(f"pooled (rows) 95% : [{plo:.2f}, {phi:.2f}] width {phi-plo:.2f} (invalid, shown for contrast)")
def paired_t(a, b):
a, b = np.asarray(a), np.asarray(b)
d = a - b
t, p = stats.ttest_rel(a, b)
return d.mean(), d.std(ddof=1), float(t), float(p)
def seed_level_tests():
print("\n=== seed-level paired t-tests (single-source held-out wild) ===")
for m in ("rsc", "arcv", "sd"):
md, sd_, t, p = paired_t(ACC[m], ACC["erm"])
print(f"{m:>4} - erm : {md:+.2f} +- {sd_:.2f} t(2)={t:.1f} p={p:.4f}")
md, sd_, t, p = paired_t(ACC["sd"], ACC["arcv"])
print(f" sd - arcv : {md:+.2f} +- {sd_:.2f} t(2)={t:.2f} p={p:.4f} (the near-tie)")
if __name__ == "__main__":
cluster_bootstrap_collapse()
seed_level_tests()
|