File size: 7,153 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
"""e-means paper protocol on CPU: DEEP* 1M (k = 16,384), test MSE on the paper's test rows.

Data: run ``python benchmarks/get_deep1m.py DATA_DIR`` first. It downloads the first 1,010,000
rows of Deep1B and checks them against the fingerprints shipped with faiss/benchs/e_means
(paper_splits/fingerprints.json), so train and test rows are byte-identical to the paper's.

Protocol (Bessa et al. 2026, arXiv:2611.09501, Sec. 4 and benchs/e_means/paper_xb.py):
  * k = 16,384 at n = 1M, batch law "throughput", no per-dataset tuning
  * starting centroids: the first k rows of a seeded permutation, identical for every method
  * test error: nearest centroid ranked in float32, distance recomputed in float64
Differences from the paper, stated up front: CPU float32 assignment (paper: H100 bfloat16), and
NumPy's seeded permutation for the starting rows (paper: a CUDA generator), shared by all methods.

Usage: python benchmarks/paper_protocol.py DATA_DIR OUT.jsonl --epochs 3 --seeds 10 11 12 13 14
          [--methods official pemeans ...] [--threads 4]
Each finished run is appended to OUT.jsonl; runs already there are skipped (resumable).
"""
import argparse, json, os, sys, time
import numpy as np
import faiss

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from faiss.contrib.e_means import NumpyEmeans  # noqa: E402
from aries.pemeans import PEmeans  # noqa: E402

K = 16384

METHODS = {
    "official": lambda d, T, s: NumpyEmeans(d, K, nepochs=T, seed=s, batch_law="throughput"),
    "pemeans": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
                                       split="principal", gain="greedy"),
    "pe_split_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
                                             split="principal", gain="usage"),
    "pe_gain_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
                                            split="faiss", gain="greedy"),
    "pe_sample": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
                                         split="principal", gain="sample"),
}


def test_mse(c, test):
    _, idx = faiss.knn(np.ascontiguousarray(test), np.ascontiguousarray(c, dtype="float32"), 1)
    diff = test.astype(np.float64) - c[idx[:, 0]].astype(np.float64)
    return float((diff * diff).sum() / len(test))


def train_mse(c, x, chunk=1 << 17):
    tot = 0.0
    for s in range(0, len(x), chunk):
        D, _ = faiss.knn(x[s:s + chunk], c, 1)
        tot += float(D.sum(dtype=np.float64))
    return tot / len(x)


def make_variant(kw, n):
    """A PEmeans configuration; alpha_scale / batch_scale rescale the e-means fitted laws."""
    kw = dict(kw)
    bs, asc = kw.pop("batch_scale", None), kw.pop("alpha_scale", None)

    def build(d, T, s):
        extra = {}
        if bs is not None or asc is not None:
            from faiss.contrib.e_means.base import _resolve_recipe, _step_law
            B, _ = _resolve_recipe(n, K, T, "throughput", None, None)
            B = int(B * (bs or 1.0))
            extra = dict(batch_size=B, alpha0=min(1.0, _step_law(n, K, T, B) * (asc or 1.0)))
        return PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", **kw, **extra)
    build.kwargs = dict(kw, batch_scale=bs, alpha_scale=asc)
    return build


def code_version():
    import subprocess
    try:
        root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        rev = subprocess.run(["git", "-C", root, "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
        dirty = subprocess.run(["git", "-C", root, "status", "--porcelain", "aries"], capture_output=True, text=True).stdout.strip()
        return rev + ("+dirty" if dirty else "")
    except Exception:
        return "unknown"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("data"); ap.add_argument("out")
    ap.add_argument("--epochs", type=float, nargs="+", default=[3])
    ap.add_argument("--seeds", type=int, nargs="+", default=[10, 11, 12, 13, 14])
    ap.add_argument("--methods", nargs="+", default=["official", "pemeans"])
    ap.add_argument("--threads", type=int, default=4)
    ap.add_argument("--train-mse", action="store_true")
    ap.add_argument("--dataset", default="deep1b-1M")
    ap.add_argument("--k", type=int, default=16384, help="number of centroids (paper: 16,384)")
    ap.add_argument("--variant", nargs=2, action="append", default=[],
                    help="name and PEmeans kwargs as JSON; may include alpha_scale / batch_scale")
    ap.add_argument("--eval", choices=["test", "val"], default="test",
                    help="val for design decisions, test for reported results")
    a = ap.parse_args()
    global K
    K = a.k
    faiss.omp_set_num_threads(a.threads)
    x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy")))
    test = np.load(os.path.join(a.data, a.eval + ".npy"))
    from aries.pemeans import PE_V2, PE_V3, PISCES, PISCES_V10, PISCES_POLISH, PISCES_FUSED, PISCES_FAST
    METHODS["pisces_v10"] = make_variant(PISCES_V10, len(x))
    METHODS["pisces_fused"] = make_variant(PISCES_FUSED, len(x))
    METHODS["pisces_fast"] = make_variant(PISCES_FAST, len(x))
    METHODS["pisces_polish"] = make_variant(PISCES_POLISH, len(x))
    METHODS["pemeans_v2"] = make_variant(PE_V2, len(x))
    METHODS["pemeans_v3"] = make_variant(PE_V3, len(x))
    METHODS["pisces"] = make_variant(PISCES, len(x))
    METHODS["emeans_approx"] = make_variant(dict(split="faiss", gain="usage", approx="auto"), len(x))
    for name, js in a.variant:
        METHODS[name] = make_variant(json.loads(js), len(x))
        a.methods.append(name) if name not in a.methods else None
    done = set()
    if os.path.exists(a.out):
        for line in open(a.out):
            r = json.loads(line)
            done.add((r.get("dataset"), r["method"], r["epochs"], r["seed"], r.get("eval", "test")))
    for T in a.epochs:
        for seed in a.seeds:
            init = x[np.random.RandomState(seed).permutation(len(x))[:K]].copy()
            for m in a.methods:
                if (a.dataset, m, T, seed, a.eval) in done:
                    continue
                t0 = time.time()
                km = METHODS[m](x.shape[1], T, seed)
                c, _ = km.train(x, init_centroids=init.copy())
                sec = time.time() - t0
                r = dict(dataset=a.dataset, k=K, method=m, epochs=T, seed=seed, seconds=sec,
                         eval=a.eval, test_mse=test_mse(c, test), splits=int(km.n_splits),
                         batch=int(km.recipe["batch_size"]), alpha0=float(km.recipe["alpha0"]),
                         threads=a.threads, faiss=faiss.__version__, code=code_version(),
                         kwargs=getattr(METHODS[m], "kwargs", None))
                if a.train_mse:
                    r["train_mse"] = train_mse(c, x)
                with open(a.out, "a") as f:
                    f.write(json.dumps(r) + "\n")
                print(json.dumps(r), flush=True)


if __name__ == "__main__":
    main()