"""Summarise paper_protocol.py results as paired statistics against official e-means. Usage: python benchmarks/summarize.py results/deep1m_cpu.jsonl [--md OUT.md] """ import argparse, collections, json import numpy as np from scipy.stats import wilcoxon # Published reference points for DEEP* 1M, k = 16,384 (arXiv:2611.09501 and faiss benchs/e_means): PAPER = { "deep1b-1M": { "Lloyd 25 it., faiss CPU (paper_targets.json, median of 10 seeds)": 0.38921320633218964, "Flash-KMeans 25 it., H100 (paper_targets.json, median)": 0.38925347141324373, "e-means 3 ep., H100 bf16 (paper Table 1, median of 30 seeds)": 0.3880}, "fb-ssnpp-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 113555.28402890626}, "spacev-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 7228.99001443477}, } def boot_ci(d, reps=10000, seed=0): r = np.random.default_rng(seed) m = r.choice(d, (reps, len(d))).mean(1) return np.percentile(m, [2.5, 97.5]) def main(): ap = argparse.ArgumentParser(); ap.add_argument("jsonl"); ap.add_argument("--md") a = ap.parse_args() rows = [json.loads(l) for l in open(a.jsonl)] by = collections.defaultdict(dict) for r in rows: by[(r["epochs"], r["method"])][r["seed"]] = r out = ["| epochs | method | seeds | test MSE (mean ± sd) | paired change vs official | 95% CI | wins | Wilcoxon p | seconds |", "|---|---|---|---|---|---|---|---|---|"] for T in sorted({k[0] for k in by}): base = by.get((T, "official"), {}) for m in sorted({k[1] for k in by if k[0] == T}, key=lambda s: (s != "official", s)): cur = by[(T, m)] seeds = sorted(set(cur) & set(base)) v = np.array([cur[s]["test_mse"] for s in sorted(cur)]) sec = np.mean([cur[s]["seconds"] for s in cur]) if m == "official" or len(seeds) < 2: out.append(f"| {T:g} | {m} | {len(cur)} | {v.mean():.5f} ± {v.std():.5f} | — | — | — | — | {sec:.0f} |") continue d = np.array([100 * (cur[s]["test_mse"] - base[s]["test_mse"]) / base[s]["test_mse"] for s in seeds]) lo, hi = boot_ci(d) p = wilcoxon(d).pvalue if len(d) >= 5 and np.any(d) else float("nan") out.append(f"| {T:g} | {m} | {len(seeds)} | {v.mean():.5f} ± {v.std():.5f} | {d.mean():+.3f}% | " f"[{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {p:.3g} | {sec:.0f} |") # cross-budget: PE-means at a smaller budget vs official at a larger one (equal or less time) cross = [] for (Tp, m) in sorted(by): if m == "official": continue for (To, mo) in sorted(by): if mo != "official" or To <= Tp: continue seeds = sorted(set(by[(Tp, m)]) & set(by[(To, mo)])) if len(seeds) < 2: continue d = np.array([100 * (by[(Tp, m)][s]["test_mse"] - by[(To, mo)][s]["test_mse"]) / by[(To, mo)][s]["test_mse"] for s in seeds]) t = np.mean([by[(Tp, m)][s]["seconds"] / by[(To, mo)][s]["seconds"] for s in seeds]) lo, hi = boot_ci(d) cross.append(f"| {m} @ {Tp:g} ep. | official @ {To:g} ep. | {len(seeds)} | {d.mean():+.3f}% | [{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {t:.2f}x |") if cross: out += ["", "Cross-budget comparison (fewer epochs for PE-means):", "", "| method | vs | seeds | paired change | 95% CI | wins | time ratio |", "|---|---|---|---|---|---|---|"] + cross ds = rows[0].get("dataset", "deep1b-1M") if ds in PAPER: out += ["", "Published reference points (different hardware and starting rows; for orientation only):", ""] for k, v in PAPER[ds].items(): out.append(f"- {k}: {v:.5f}") text = "\n".join(out) print(text) if a.md: open(a.md, "w").write(text + "\n") if __name__ == "__main__": main()