PISCES / benchmarks /paper_protocol.py
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
7.15 kB
"""e-means paper protocol on CPU: DEEP* 1M (k = 16,384), test MSE on the paper's test rows.
Data: run ``python benchmarks/get_deep1m.py DATA_DIR`` first. It downloads the first 1,010,000
rows of Deep1B and checks them against the fingerprints shipped with faiss/benchs/e_means
(paper_splits/fingerprints.json), so train and test rows are byte-identical to the paper's.
Protocol (Bessa et al. 2026, arXiv:2611.09501, Sec. 4 and benchs/e_means/paper_xb.py):
* k = 16,384 at n = 1M, batch law "throughput", no per-dataset tuning
* starting centroids: the first k rows of a seeded permutation, identical for every method
* test error: nearest centroid ranked in float32, distance recomputed in float64
Differences from the paper, stated up front: CPU float32 assignment (paper: H100 bfloat16), and
NumPy's seeded permutation for the starting rows (paper: a CUDA generator), shared by all methods.
Usage: python benchmarks/paper_protocol.py DATA_DIR OUT.jsonl --epochs 3 --seeds 10 11 12 13 14
[--methods official pemeans ...] [--threads 4]
Each finished run is appended to OUT.jsonl; runs already there are skipped (resumable).
"""
import argparse, json, os, sys, time
import numpy as np
import faiss
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from faiss.contrib.e_means import NumpyEmeans # noqa: E402
from aries.pemeans import PEmeans # noqa: E402
K = 16384
METHODS = {
"official": lambda d, T, s: NumpyEmeans(d, K, nepochs=T, seed=s, batch_law="throughput"),
"pemeans": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="greedy"),
"pe_split_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="usage"),
"pe_gain_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="faiss", gain="greedy"),
"pe_sample": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput",
split="principal", gain="sample"),
}
def test_mse(c, test):
_, idx = faiss.knn(np.ascontiguousarray(test), np.ascontiguousarray(c, dtype="float32"), 1)
diff = test.astype(np.float64) - c[idx[:, 0]].astype(np.float64)
return float((diff * diff).sum() / len(test))
def train_mse(c, x, chunk=1 << 17):
tot = 0.0
for s in range(0, len(x), chunk):
D, _ = faiss.knn(x[s:s + chunk], c, 1)
tot += float(D.sum(dtype=np.float64))
return tot / len(x)
def make_variant(kw, n):
"""A PEmeans configuration; alpha_scale / batch_scale rescale the e-means fitted laws."""
kw = dict(kw)
bs, asc = kw.pop("batch_scale", None), kw.pop("alpha_scale", None)
def build(d, T, s):
extra = {}
if bs is not None or asc is not None:
from faiss.contrib.e_means.base import _resolve_recipe, _step_law
B, _ = _resolve_recipe(n, K, T, "throughput", None, None)
B = int(B * (bs or 1.0))
extra = dict(batch_size=B, alpha0=min(1.0, _step_law(n, K, T, B) * (asc or 1.0)))
return PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", **kw, **extra)
build.kwargs = dict(kw, batch_scale=bs, alpha_scale=asc)
return build
def code_version():
import subprocess
try:
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
rev = subprocess.run(["git", "-C", root, "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
dirty = subprocess.run(["git", "-C", root, "status", "--porcelain", "aries"], capture_output=True, text=True).stdout.strip()
return rev + ("+dirty" if dirty else "")
except Exception:
return "unknown"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("data"); ap.add_argument("out")
ap.add_argument("--epochs", type=float, nargs="+", default=[3])
ap.add_argument("--seeds", type=int, nargs="+", default=[10, 11, 12, 13, 14])
ap.add_argument("--methods", nargs="+", default=["official", "pemeans"])
ap.add_argument("--threads", type=int, default=4)
ap.add_argument("--train-mse", action="store_true")
ap.add_argument("--dataset", default="deep1b-1M")
ap.add_argument("--k", type=int, default=16384, help="number of centroids (paper: 16,384)")
ap.add_argument("--variant", nargs=2, action="append", default=[],
help="name and PEmeans kwargs as JSON; may include alpha_scale / batch_scale")
ap.add_argument("--eval", choices=["test", "val"], default="test",
help="val for design decisions, test for reported results")
a = ap.parse_args()
global K
K = a.k
faiss.omp_set_num_threads(a.threads)
x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy")))
test = np.load(os.path.join(a.data, a.eval + ".npy"))
from aries.pemeans import PE_V2, PE_V3, PISCES, PISCES_V10, PISCES_POLISH, PISCES_FUSED, PISCES_FAST
METHODS["pisces_v10"] = make_variant(PISCES_V10, len(x))
METHODS["pisces_fused"] = make_variant(PISCES_FUSED, len(x))
METHODS["pisces_fast"] = make_variant(PISCES_FAST, len(x))
METHODS["pisces_polish"] = make_variant(PISCES_POLISH, len(x))
METHODS["pemeans_v2"] = make_variant(PE_V2, len(x))
METHODS["pemeans_v3"] = make_variant(PE_V3, len(x))
METHODS["pisces"] = make_variant(PISCES, len(x))
METHODS["emeans_approx"] = make_variant(dict(split="faiss", gain="usage", approx="auto"), len(x))
for name, js in a.variant:
METHODS[name] = make_variant(json.loads(js), len(x))
a.methods.append(name) if name not in a.methods else None
done = set()
if os.path.exists(a.out):
for line in open(a.out):
r = json.loads(line)
done.add((r.get("dataset"), r["method"], r["epochs"], r["seed"], r.get("eval", "test")))
for T in a.epochs:
for seed in a.seeds:
init = x[np.random.RandomState(seed).permutation(len(x))[:K]].copy()
for m in a.methods:
if (a.dataset, m, T, seed, a.eval) in done:
continue
t0 = time.time()
km = METHODS[m](x.shape[1], T, seed)
c, _ = km.train(x, init_centroids=init.copy())
sec = time.time() - t0
r = dict(dataset=a.dataset, k=K, method=m, epochs=T, seed=seed, seconds=sec,
eval=a.eval, test_mse=test_mse(c, test), splits=int(km.n_splits),
batch=int(km.recipe["batch_size"]), alpha0=float(km.recipe["alpha0"]),
threads=a.threads, faiss=faiss.__version__, code=code_version(),
kwargs=getattr(METHODS[m], "kwargs", None))
if a.train_mse:
r["train_mse"] = train_mse(c, x)
with open(a.out, "a") as f:
f.write(json.dumps(r) + "\n")
print(json.dumps(r), flush=True)
if __name__ == "__main__":
main()