PISCES / benchmarks /dev_loop.py
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
4.08 kB
"""Development loop on DEEP* (validation rows only; test rows are reserved for final reports).
Usage: python benchmarks/dev_loop.py DATA_DIR TAG --n 250000 --k 4096 --epochs 1 3 --seeds 1 2 3 4 5
--variant name '{"split": "principal", ...}' [--variant ...]
Each variant is PEmeans keyword arguments as JSON; "official" (faiss NumpyEmeans) is always run.
Results go to results/dev/TAG.jsonl and a paired summary is printed.
"""
import argparse, json, os, sys, time
import numpy as np
import faiss
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from faiss.contrib.e_means import NumpyEmeans # noqa: E402
from aries.pemeans import PEmeans # noqa: E402
def mse(c, rows):
_, idx = faiss.knn(rows, np.ascontiguousarray(c, dtype="float32"), 1)
diff = rows.astype(np.float64) - c[idx[:, 0]].astype(np.float64)
return float((diff * diff).sum() / len(rows))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("data"); ap.add_argument("tag")
ap.add_argument("--n", type=int, default=250000); ap.add_argument("--k", type=int, default=4096)
ap.add_argument("--epochs", type=float, nargs="+", default=[1, 3])
ap.add_argument("--seeds", type=int, nargs="+", default=[1, 2, 3, 4, 5])
ap.add_argument("--variant", nargs=2, action="append", default=[])
ap.add_argument("--threads", type=int, default=4)
a = ap.parse_args()
faiss.omp_set_num_threads(a.threads)
x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy"), mmap_mode="r")[:a.n])
val = np.load(os.path.join(a.data, "val.npy"))
variants = [("official", None)] + [(n, json.loads(j)) for n, j in a.variant]
os.makedirs(os.path.join(ROOT, "results", "dev"), exist_ok=True)
final = os.path.join(ROOT, "results", "dev", a.tag + ".jsonl")
out = os.path.join(os.environ.get("DEV_TMP", "/tmp"), a.tag + ".jsonl.partial")
res = {}
for T in a.epochs:
for s in a.seeds:
init = x[np.random.RandomState(s).permutation(len(x))[:a.k]].copy()
for name, kw in variants:
t0 = time.time()
extra = {}
kw2 = dict(kw or {})
bs, asc = kw2.pop("batch_scale", None), kw2.pop("alpha_scale", None)
if bs is not None or asc is not None: # LawFit: rescale the fitted batch / step
from faiss.contrib.e_means.base import _resolve_recipe, _step_law
B, a0 = _resolve_recipe(len(x), a.k, T, "throughput", None, None)
B = int(B * (bs or 1.0))
a0 = min(1.0, _step_law(len(x), a.k, T, B) * (asc or 1.0))
extra = dict(batch_size=B, alpha0=a0)
base = kw2.pop("base", None)
km = (NumpyEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **extra)
if kw is None or base == "official"
else PEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **kw2, **extra))
c, _ = km.train(x, init_centroids=init.copy())
r = dict(tag=a.tag, n=a.n, k=a.k, epochs=T, seed=s, variant=name, kwargs=kw,
seconds=time.time() - t0, val_mse=mse(c, val), splits=int(km.n_splits))
res[(T, name, s)] = r
with open(out, "a") as f:
f.write(json.dumps(r) + "\n")
import shutil
shutil.copyfile(out, final)
for T in a.epochs:
print(f"--- n={a.n} k={a.k} epochs={T} seeds={a.seeds}")
for name, _ in variants:
v = np.array([res[(T, name, s)]["val_mse"] for s in a.seeds])
b = np.array([res[(T, "official", s)]["val_mse"] for s in a.seeds])
d = 100 * (v - b) / b
sec = np.mean([res[(T, name, s)]["seconds"] for s in a.seeds])
print(f"{name:28s} val {v.mean():.5f} vs official {d.mean():+.3f}% "
f"(sd {d.std():.3f}, wins {int((d < 0).sum())}/{len(d)}) {sec:.1f}s", flush=True)
if __name__ == "__main__":
main()