Download benchmarks/paper_protocol.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 7.15 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/paper_protocol.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/paper_protocol.py
-
curl -L -o paper_protocol.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/paper_protocol.py
7.15 kB
| """e-means paper protocol on CPU: DEEP* 1M (k = 16,384), test MSE on the paper's test rows. | |
| Data: run ``python benchmarks/get_deep1m.py DATA_DIR`` first. It downloads the first 1,010,000 | |
| rows of Deep1B and checks them against the fingerprints shipped with faiss/benchs/e_means | |
| (paper_splits/fingerprints.json), so train and test rows are byte-identical to the paper's. | |
| Protocol (Bessa et al. 2026, arXiv:2611.09501, Sec. 4 and benchs/e_means/paper_xb.py): | |
| * k = 16,384 at n = 1M, batch law "throughput", no per-dataset tuning | |
| * starting centroids: the first k rows of a seeded permutation, identical for every method | |
| * test error: nearest centroid ranked in float32, distance recomputed in float64 | |
| Differences from the paper, stated up front: CPU float32 assignment (paper: H100 bfloat16), and | |
| NumPy's seeded permutation for the starting rows (paper: a CUDA generator), shared by all methods. | |
| Usage: python benchmarks/paper_protocol.py DATA_DIR OUT.jsonl --epochs 3 --seeds 10 11 12 13 14 | |
| [--methods official pemeans ...] [--threads 4] | |
| Each finished run is appended to OUT.jsonl; runs already there are skipped (resumable). | |
| """ | |
| import argparse, json, os, sys, time | |
| import numpy as np | |
| import faiss | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from faiss.contrib.e_means import NumpyEmeans # noqa: E402 | |
| from aries.pemeans import PEmeans # noqa: E402 | |
| K = 16384 | |
| METHODS = { | |
| "official": lambda d, T, s: NumpyEmeans(d, K, nepochs=T, seed=s, batch_law="throughput"), | |
| "pemeans": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", | |
| split="principal", gain="greedy"), | |
| "pe_split_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", | |
| split="principal", gain="usage"), | |
| "pe_gain_only": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", | |
| split="faiss", gain="greedy"), | |
| "pe_sample": lambda d, T, s: PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", | |
| split="principal", gain="sample"), | |
| } | |
| def test_mse(c, test): | |
| _, idx = faiss.knn(np.ascontiguousarray(test), np.ascontiguousarray(c, dtype="float32"), 1) | |
| diff = test.astype(np.float64) - c[idx[:, 0]].astype(np.float64) | |
| return float((diff * diff).sum() / len(test)) | |
| def train_mse(c, x, chunk=1 << 17): | |
| tot = 0.0 | |
| for s in range(0, len(x), chunk): | |
| D, _ = faiss.knn(x[s:s + chunk], c, 1) | |
| tot += float(D.sum(dtype=np.float64)) | |
| return tot / len(x) | |
| def make_variant(kw, n): | |
| """A PEmeans configuration; alpha_scale / batch_scale rescale the e-means fitted laws.""" | |
| kw = dict(kw) | |
| bs, asc = kw.pop("batch_scale", None), kw.pop("alpha_scale", None) | |
| def build(d, T, s): | |
| extra = {} | |
| if bs is not None or asc is not None: | |
| from faiss.contrib.e_means.base import _resolve_recipe, _step_law | |
| B, _ = _resolve_recipe(n, K, T, "throughput", None, None) | |
| B = int(B * (bs or 1.0)) | |
| extra = dict(batch_size=B, alpha0=min(1.0, _step_law(n, K, T, B) * (asc or 1.0))) | |
| return PEmeans(d, K, nepochs=T, seed=s, batch_law="throughput", **kw, **extra) | |
| build.kwargs = dict(kw, batch_scale=bs, alpha_scale=asc) | |
| return build | |
| def code_version(): | |
| import subprocess | |
| try: | |
| root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| rev = subprocess.run(["git", "-C", root, "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip() | |
| dirty = subprocess.run(["git", "-C", root, "status", "--porcelain", "aries"], capture_output=True, text=True).stdout.strip() | |
| return rev + ("+dirty" if dirty else "") | |
| except Exception: | |
| return "unknown" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("data"); ap.add_argument("out") | |
| ap.add_argument("--epochs", type=float, nargs="+", default=[3]) | |
| ap.add_argument("--seeds", type=int, nargs="+", default=[10, 11, 12, 13, 14]) | |
| ap.add_argument("--methods", nargs="+", default=["official", "pemeans"]) | |
| ap.add_argument("--threads", type=int, default=4) | |
| ap.add_argument("--train-mse", action="store_true") | |
| ap.add_argument("--dataset", default="deep1b-1M") | |
| ap.add_argument("--k", type=int, default=16384, help="number of centroids (paper: 16,384)") | |
| ap.add_argument("--variant", nargs=2, action="append", default=[], | |
| help="name and PEmeans kwargs as JSON; may include alpha_scale / batch_scale") | |
| ap.add_argument("--eval", choices=["test", "val"], default="test", | |
| help="val for design decisions, test for reported results") | |
| a = ap.parse_args() | |
| global K | |
| K = a.k | |
| faiss.omp_set_num_threads(a.threads) | |
| x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy"))) | |
| test = np.load(os.path.join(a.data, a.eval + ".npy")) | |
| from aries.pemeans import PE_V2, PE_V3, PISCES, PISCES_V10, PISCES_POLISH, PISCES_FUSED, PISCES_FAST | |
| METHODS["pisces_v10"] = make_variant(PISCES_V10, len(x)) | |
| METHODS["pisces_fused"] = make_variant(PISCES_FUSED, len(x)) | |
| METHODS["pisces_fast"] = make_variant(PISCES_FAST, len(x)) | |
| METHODS["pisces_polish"] = make_variant(PISCES_POLISH, len(x)) | |
| METHODS["pemeans_v2"] = make_variant(PE_V2, len(x)) | |
| METHODS["pemeans_v3"] = make_variant(PE_V3, len(x)) | |
| METHODS["pisces"] = make_variant(PISCES, len(x)) | |
| METHODS["emeans_approx"] = make_variant(dict(split="faiss", gain="usage", approx="auto"), len(x)) | |
| for name, js in a.variant: | |
| METHODS[name] = make_variant(json.loads(js), len(x)) | |
| a.methods.append(name) if name not in a.methods else None | |
| done = set() | |
| if os.path.exists(a.out): | |
| for line in open(a.out): | |
| r = json.loads(line) | |
| done.add((r.get("dataset"), r["method"], r["epochs"], r["seed"], r.get("eval", "test"))) | |
| for T in a.epochs: | |
| for seed in a.seeds: | |
| init = x[np.random.RandomState(seed).permutation(len(x))[:K]].copy() | |
| for m in a.methods: | |
| if (a.dataset, m, T, seed, a.eval) in done: | |
| continue | |
| t0 = time.time() | |
| km = METHODS[m](x.shape[1], T, seed) | |
| c, _ = km.train(x, init_centroids=init.copy()) | |
| sec = time.time() - t0 | |
| r = dict(dataset=a.dataset, k=K, method=m, epochs=T, seed=seed, seconds=sec, | |
| eval=a.eval, test_mse=test_mse(c, test), splits=int(km.n_splits), | |
| batch=int(km.recipe["batch_size"]), alpha0=float(km.recipe["alpha0"]), | |
| threads=a.threads, faiss=faiss.__version__, code=code_version(), | |
| kwargs=getattr(METHODS[m], "kwargs", None)) | |
| if a.train_mse: | |
| r["train_mse"] = train_mse(c, x) | |
| with open(a.out, "a") as f: | |
| f.write(json.dumps(r) + "\n") | |
| print(json.dumps(r), flush=True) | |
| if __name__ == "__main__": | |
| main() | |