Download benchmarks/summarize.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 4 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/summarize.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/summarize.py
-
curl -L -o summarize.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/summarize.py
4 kB
| """Summarise paper_protocol.py results as paired statistics against official e-means. | |
| Usage: python benchmarks/summarize.py results/deep1m_cpu.jsonl [--md OUT.md] | |
| """ | |
| import argparse, collections, json | |
| import numpy as np | |
| from scipy.stats import wilcoxon | |
| # Published reference points for DEEP* 1M, k = 16,384 (arXiv:2611.09501 and faiss benchs/e_means): | |
| PAPER = { | |
| "deep1b-1M": { | |
| "Lloyd 25 it., faiss CPU (paper_targets.json, median of 10 seeds)": 0.38921320633218964, | |
| "Flash-KMeans 25 it., H100 (paper_targets.json, median)": 0.38925347141324373, | |
| "e-means 3 ep., H100 bf16 (paper Table 1, median of 30 seeds)": 0.3880}, | |
| "fb-ssnpp-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 113555.28402890626}, | |
| "spacev-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 7228.99001443477}, | |
| } | |
| def boot_ci(d, reps=10000, seed=0): | |
| r = np.random.default_rng(seed) | |
| m = r.choice(d, (reps, len(d))).mean(1) | |
| return np.percentile(m, [2.5, 97.5]) | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("jsonl"); ap.add_argument("--md") | |
| a = ap.parse_args() | |
| rows = [json.loads(l) for l in open(a.jsonl)] | |
| by = collections.defaultdict(dict) | |
| for r in rows: | |
| by[(r["epochs"], r["method"])][r["seed"]] = r | |
| out = ["| epochs | method | seeds | test MSE (mean ± sd) | paired change vs official | 95% CI | wins | Wilcoxon p | seconds |", | |
| "|---|---|---|---|---|---|---|---|---|"] | |
| for T in sorted({k[0] for k in by}): | |
| base = by.get((T, "official"), {}) | |
| for m in sorted({k[1] for k in by if k[0] == T}, key=lambda s: (s != "official", s)): | |
| cur = by[(T, m)] | |
| seeds = sorted(set(cur) & set(base)) | |
| v = np.array([cur[s]["test_mse"] for s in sorted(cur)]) | |
| sec = np.mean([cur[s]["seconds"] for s in cur]) | |
| if m == "official" or len(seeds) < 2: | |
| out.append(f"| {T:g} | {m} | {len(cur)} | {v.mean():.5f} ± {v.std():.5f} | — | — | — | — | {sec:.0f} |") | |
| continue | |
| d = np.array([100 * (cur[s]["test_mse"] - base[s]["test_mse"]) / base[s]["test_mse"] for s in seeds]) | |
| lo, hi = boot_ci(d) | |
| p = wilcoxon(d).pvalue if len(d) >= 5 and np.any(d) else float("nan") | |
| out.append(f"| {T:g} | {m} | {len(seeds)} | {v.mean():.5f} ± {v.std():.5f} | {d.mean():+.3f}% | " | |
| f"[{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {p:.3g} | {sec:.0f} |") | |
| # cross-budget: PE-means at a smaller budget vs official at a larger one (equal or less time) | |
| cross = [] | |
| for (Tp, m) in sorted(by): | |
| if m == "official": | |
| continue | |
| for (To, mo) in sorted(by): | |
| if mo != "official" or To <= Tp: | |
| continue | |
| seeds = sorted(set(by[(Tp, m)]) & set(by[(To, mo)])) | |
| if len(seeds) < 2: | |
| continue | |
| d = np.array([100 * (by[(Tp, m)][s]["test_mse"] - by[(To, mo)][s]["test_mse"]) / by[(To, mo)][s]["test_mse"] for s in seeds]) | |
| t = np.mean([by[(Tp, m)][s]["seconds"] / by[(To, mo)][s]["seconds"] for s in seeds]) | |
| lo, hi = boot_ci(d) | |
| cross.append(f"| {m} @ {Tp:g} ep. | official @ {To:g} ep. | {len(seeds)} | {d.mean():+.3f}% | [{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {t:.2f}x |") | |
| if cross: | |
| out += ["", "Cross-budget comparison (fewer epochs for PE-means):", "", | |
| "| method | vs | seeds | paired change | 95% CI | wins | time ratio |", "|---|---|---|---|---|---|---|"] + cross | |
| ds = rows[0].get("dataset", "deep1b-1M") | |
| if ds in PAPER: | |
| out += ["", "Published reference points (different hardware and starting rows; for orientation only):", ""] | |
| for k, v in PAPER[ds].items(): | |
| out.append(f"- {k}: {v:.5f}") | |
| text = "\n".join(out) | |
| print(text) | |
| if a.md: | |
| open(a.md, "w").write(text + "\n") | |
| if __name__ == "__main__": | |
| main() | |