File size: 7,153 Bytes
c09a3f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | """e-means paper protocol on CPU: DEEP* 1M (k = 16,384), test MSE on the paper's test rows.
Data: run ``python benchmarks/get_deep1m.py DATA_DIR`` first. It downloads the first 1,010,000
rows of Deep1B and checks them against the fingerprints shipped with faiss/benchs/e_means
(paper_splits/fingerprints.json), so train and test rows are byte-identical to the paper's.
Protocol (Bessa et al. 2026, arXiv:2611.09501, Sec. 4 and benchs/e_means/paper_xb.py):
* k = 16,384 at n = 1M, batch law "throughput", no per-dataset tuning
* starting centroids: the first k rows of a seeded permutation, identical for every method
* test error: nearest centroid ranked in float32, distance recomputed in float64
Differences from the paper, stated up front: CPU float32 assignment (paper: H100 bfloat16), and
NumPy's seeded permutation for the starting rows (paper: a CUDA generator), shared by all methods.
Usage: python benchmarks/paper_protocol.py DATA_DIR OUT.jsonl --epochs 3 --seeds 10 11 12 13 14
[--methods official pemeans ...] [--threads 4]
Each finished run is appended to OUT.jsonl; runs already there are skipped (resumable).
"""
import argparse, json, os, sys, time
import numpy as np
import faiss
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from faiss.contrib.e_means import NumpyEmeans # noqa: E402
from aries.pemeans import PEmeans # noqa: E402
K = 16384
METHODS = {
"official": lambda d, T, s: NumpyEmeans(d, K, nepochs=T, seed=s, batch_law="throughput"),
"pemeans": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="greedy"),
"pe_split_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="usage"),
"pe_gain_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="faiss", gain="greedy"),
"pe_sample": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="sample"),
}
def test_mse(c, test):
_, idx = faiss.knn(np.ascontiguousarray(test), np.ascontiguousarray(c, dtype="float32"), 1)
diff = test.astype(np.float64) - c[idx[:, 0]].astype(np.float64)
return float((diff * diff).sum() / len(test))
def train_mse(c, x, chunk=1 << 17):
tot = 0.0
for s in range(0, len(x), chunk):
D, _ = faiss.knn(x[s:s + chunk], c, 1)
tot += float(D.sum(dtype=np.float64))
return tot / len(x)
def make_variant(kw, n):
"""A PEmeans configuration; alpha_scale / batch_scale rescale the e-means fitted laws."""
kw = dict(kw)
bs, asc = kw.pop("batch_scale", None), kw.pop("alpha_scale", None)
def build(d, T, s):
extra = {}
if bs is not None or asc is not None:
from faiss.contrib.e_means.base import _resolve_recipe, _step_law
B, _ = _resolve_recipe(n, K, T, "throughput", None, None)
B = int(B * (bs or 1.0))
extra = dict(batch_size=B, alpha0=min(1.0, _step_law(n, K, T, B) * (asc or 1.0)))
return PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", **kw, **extra)
build.kwargs = dict(kw, batch_scale=bs, alpha_scale=asc)
return build
def code_version():
import subprocess
try:
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
rev = subprocess.run(["git", "-C", root, "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
dirty = subprocess.run(["git", "-C", root, "status", "--porcelain", "aries"], capture_output=True, text=True).stdout.strip()
return rev + ("+dirty" if dirty else "")
except Exception:
return "unknown"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("data"); ap.add_argument("out")
ap.add_argument("--epochs", type=float, nargs="+", default=[3])
ap.add_argument("--seeds", type=int, nargs="+", default=[10, 11, 12, 13, 14])
ap.add_argument("--methods", nargs="+", default=["official", "pemeans"])
ap.add_argument("--threads", type=int, default=4)
ap.add_argument("--train-mse", action="store_true")
ap.add_argument("--dataset", default="deep1b-1M")
ap.add_argument("--k", type=int, default=16384, help="number of centroids (paper: 16,384)")
ap.add_argument("--variant", nargs=2, action="append", default=[],
help="name and PEmeans kwargs as JSON; may include alpha_scale / batch_scale")
ap.add_argument("--eval", choices=["test", "val"], default="test",
help="val for design decisions, test for reported results")
a = ap.parse_args()
global K
K = a.k
faiss.omp_set_num_threads(a.threads)
x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy")))
test = np.load(os.path.join(a.data, a.eval + ".npy"))
from aries.pemeans import PE_V2, PE_V3, PISCES, PISCES_V10, PISCES_POLISH, PISCES_FUSED, PISCES_FAST
METHODS["pisces_v10"] = make_variant(PISCES_V10, len(x))
METHODS["pisces_fused"] = make_variant(PISCES_FUSED, len(x))
METHODS["pisces_fast"] = make_variant(PISCES_FAST, len(x))
METHODS["pisces_polish"] = make_variant(PISCES_POLISH, len(x))
METHODS["pemeans_v2"] = make_variant(PE_V2, len(x))
METHODS["pemeans_v3"] = make_variant(PE_V3, len(x))
METHODS["pisces"] = make_variant(PISCES, len(x))
METHODS["emeans_approx"] = make_variant(dict(split="faiss", gain="usage", approx="auto"), len(x))
for name, js in a.variant:
METHODS[name] = make_variant(json.loads(js), len(x))
a.methods.append(name) if name not in a.methods else None
done = set()
if os.path.exists(a.out):
for line in open(a.out):
r = json.loads(line)
done.add((r.get("dataset"), r["method"], r["epochs"], r["seed"], r.get("eval", "test")))
for T in a.epochs:
for seed in a.seeds:
init = x[np.random.RandomState(seed).permutation(len(x))[:K]].copy()
for m in a.methods:
if (a.dataset, m, T, seed, a.eval) in done:
continue
t0 = time.time()
km = METHODS[m](x.shape[1], T, seed)
c, _ = km.train(x, init_centroids=init.copy())
sec = time.time() - t0
r = dict(dataset=a.dataset, k=K, method=m, epochs=T, seed=seed, seconds=sec,
eval=a.eval, test_mse=test_mse(c, test), splits=int(km.n_splits),
batch=int(km.recipe["batch_size"]), alpha0=float(km.recipe["alpha0"]),
threads=a.threads, faiss=faiss.__version__, code=code_version(),
kwargs=getattr(METHODS[m], "kwargs", None))
if a.train_mse:
r["train_mse"] = train_mse(c, x)
with open(a.out, "a") as f:
f.write(json.dumps(r) + "\n")
print(json.dumps(r), flush=True)
if __name__ == "__main__":
main()
|