PISCES / benchmarks /summarize.py
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
4 kB
"""Summarise paper_protocol.py results as paired statistics against official e-means.
Usage: python benchmarks/summarize.py results/deep1m_cpu.jsonl [--md OUT.md]
"""
import argparse, collections, json
import numpy as np
from scipy.stats import wilcoxon
# Published reference points for DEEP* 1M, k = 16,384 (arXiv:2611.09501 and faiss benchs/e_means):
PAPER = {
"deep1b-1M": {
"Lloyd 25 it., faiss CPU (paper_targets.json, median of 10 seeds)": 0.38921320633218964,
"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 0.38925347141324373,
"e-means 3 ep., H100 bf16 (paper Table 1, median of 30 seeds)": 0.3880},
"fb-ssnpp-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 113555.28402890626},
"spacev-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 7228.99001443477},
}
def boot_ci(d, reps=10000, seed=0):
r = np.random.default_rng(seed)
m = r.choice(d, (reps, len(d))).mean(1)
return np.percentile(m, [2.5, 97.5])
def main():
ap = argparse.ArgumentParser(); ap.add_argument("jsonl"); ap.add_argument("--md")
a = ap.parse_args()
rows = [json.loads(l) for l in open(a.jsonl)]
by = collections.defaultdict(dict)
for r in rows:
by[(r["epochs"], r["method"])][r["seed"]] = r
out = ["| epochs | method | seeds | test MSE (mean ± sd) | paired change vs official | 95% CI | wins | Wilcoxon p | seconds |",
"|---|---|---|---|---|---|---|---|---|"]
for T in sorted({k[0] for k in by}):
base = by.get((T, "official"), {})
for m in sorted({k[1] for k in by if k[0] == T}, key=lambda s: (s != "official", s)):
cur = by[(T, m)]
seeds = sorted(set(cur) & set(base))
v = np.array([cur[s]["test_mse"] for s in sorted(cur)])
sec = np.mean([cur[s]["seconds"] for s in cur])
if m == "official" or len(seeds) < 2:
out.append(f"| {T:g} | {m} | {len(cur)} | {v.mean():.5f} ± {v.std():.5f} | — | — | — | — | {sec:.0f} |")
continue
d = np.array([100 * (cur[s]["test_mse"] - base[s]["test_mse"]) / base[s]["test_mse"] for s in seeds])
lo, hi = boot_ci(d)
p = wilcoxon(d).pvalue if len(d) >= 5 and np.any(d) else float("nan")
out.append(f"| {T:g} | {m} | {len(seeds)} | {v.mean():.5f} ± {v.std():.5f} | {d.mean():+.3f}% | "
f"[{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {p:.3g} | {sec:.0f} |")
# cross-budget: PE-means at a smaller budget vs official at a larger one (equal or less time)
cross = []
for (Tp, m) in sorted(by):
if m == "official":
continue
for (To, mo) in sorted(by):
if mo != "official" or To <= Tp:
continue
seeds = sorted(set(by[(Tp, m)]) & set(by[(To, mo)]))
if len(seeds) < 2:
continue
d = np.array([100 * (by[(Tp, m)][s]["test_mse"] - by[(To, mo)][s]["test_mse"]) / by[(To, mo)][s]["test_mse"] for s in seeds])
t = np.mean([by[(Tp, m)][s]["seconds"] / by[(To, mo)][s]["seconds"] for s in seeds])
lo, hi = boot_ci(d)
cross.append(f"| {m} @ {Tp:g} ep. | official @ {To:g} ep. | {len(seeds)} | {d.mean():+.3f}% | [{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {t:.2f}x |")
if cross:
out += ["", "Cross-budget comparison (fewer epochs for PE-means):", "",
"| method | vs | seeds | paired change | 95% CI | wins | time ratio |", "|---|---|---|---|---|---|---|"] + cross
ds = rows[0].get("dataset", "deep1b-1M")
if ds in PAPER:
out += ["", "Published reference points (different hardware and starting rows; for orientation only):", ""]
for k, v in PAPER[ds].items():
out.append(f"- {k}: {v:.5f}")
text = "\n".join(out)
print(text)
if a.md:
open(a.md, "w").write(text + "\n")
if __name__ == "__main__":
main()