File size: 3,995 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
"""Summarise paper_protocol.py results as paired statistics against official e-means.

Usage: python benchmarks/summarize.py results/deep1m_cpu.jsonl [--md OUT.md]
"""
import argparse, collections, json
import numpy as np
from scipy.stats import wilcoxon

# Published reference points for DEEP* 1M, k = 16,384 (arXiv:2611.09501 and faiss benchs/e_means):
PAPER = {
    "deep1b-1M": {
        "Lloyd 25 it., faiss CPU (paper_targets.json, median of 10 seeds)": 0.38921320633218964,
        "Flash-KMeans 25 it., H100 (paper_targets.json, median)": 0.38925347141324373,
        "e-means 3 ep., H100 bf16 (paper Table 1, median of 30 seeds)": 0.3880},
    "fb-ssnpp-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 113555.28402890626},
    "spacev-1M": {"Flash-KMeans 25 it., H100 (paper_targets.json, median)": 7228.99001443477},
}


def boot_ci(d, reps=10000, seed=0):
    r = np.random.default_rng(seed)
    m = r.choice(d, (reps, len(d))).mean(1)
    return np.percentile(m, [2.5, 97.5])


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("jsonl"); ap.add_argument("--md")
    a = ap.parse_args()
    rows = [json.loads(l) for l in open(a.jsonl)]
    by = collections.defaultdict(dict)
    for r in rows:
        by[(r["epochs"], r["method"])][r["seed"]] = r
    out = ["| epochs | method | seeds | test MSE (mean ± sd) | paired change vs official | 95% CI | wins | Wilcoxon p | seconds |",
           "|---|---|---|---|---|---|---|---|---|"]
    for T in sorted({k[0] for k in by}):
        base = by.get((T, "official"), {})
        for m in sorted({k[1] for k in by if k[0] == T}, key=lambda s: (s != "official", s)):
            cur = by[(T, m)]
            seeds = sorted(set(cur) & set(base))
            v = np.array([cur[s]["test_mse"] for s in sorted(cur)])
            sec = np.mean([cur[s]["seconds"] for s in cur])
            if m == "official" or len(seeds) < 2:
                out.append(f"| {T:g} | {m} | {len(cur)} | {v.mean():.5f} ± {v.std():.5f} | — | — | — | — | {sec:.0f} |")
                continue
            d = np.array([100 * (cur[s]["test_mse"] - base[s]["test_mse"]) / base[s]["test_mse"] for s in seeds])
            lo, hi = boot_ci(d)
            p = wilcoxon(d).pvalue if len(d) >= 5 and np.any(d) else float("nan")
            out.append(f"| {T:g} | {m} | {len(seeds)} | {v.mean():.5f} ± {v.std():.5f} | {d.mean():+.3f}% | "
                       f"[{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {p:.3g} | {sec:.0f} |")
    # cross-budget: PE-means at a smaller budget vs official at a larger one (equal or less time)
    cross = []
    for (Tp, m) in sorted(by):
        if m == "official":
            continue
        for (To, mo) in sorted(by):
            if mo != "official" or To <= Tp:
                continue
            seeds = sorted(set(by[(Tp, m)]) & set(by[(To, mo)]))
            if len(seeds) < 2:
                continue
            d = np.array([100 * (by[(Tp, m)][s]["test_mse"] - by[(To, mo)][s]["test_mse"]) / by[(To, mo)][s]["test_mse"] for s in seeds])
            t = np.mean([by[(Tp, m)][s]["seconds"] / by[(To, mo)][s]["seconds"] for s in seeds])
            lo, hi = boot_ci(d)
            cross.append(f"| {m} @ {Tp:g} ep. | official @ {To:g} ep. | {len(seeds)} | {d.mean():+.3f}% | [{lo:+.3f}, {hi:+.3f}] | {int((d < 0).sum())}/{len(d)} | {t:.2f}x |")
    if cross:
        out += ["", "Cross-budget comparison (fewer epochs for PE-means):", "",
                "| method | vs | seeds | paired change | 95% CI | wins | time ratio |", "|---|---|---|---|---|---|---|"] + cross
    ds = rows[0].get("dataset", "deep1b-1M")
    if ds in PAPER:
        out += ["", "Published reference points (different hardware and starting rows; for orientation only):", ""]
        for k, v in PAPER[ds].items():
            out.append(f"- {k}: {v:.5f}")
    text = "\n".join(out)
    print(text)
    if a.md:
        open(a.md, "w").write(text + "\n")


if __name__ == "__main__":
    main()