Download benchmarks/dev_loop.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 4.08 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/dev_loop.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/dev_loop.py
-
curl -L -o dev_loop.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/dev_loop.py
4.08 kB
| """Development loop on DEEP* (validation rows only; test rows are reserved for final reports). | |
| Usage: python benchmarks/dev_loop.py DATA_DIR TAG --n 250000 --k 4096 --epochs 1 3 --seeds 1 2 3 4 5 | |
| --variant name '{"split": "principal", ...}' [--variant ...] | |
| Each variant is PEmeans keyword arguments as JSON; "official" (faiss NumpyEmeans) is always run. | |
| Results go to results/dev/TAG.jsonl and a paired summary is printed. | |
| """ | |
| import argparse, json, os, sys, time | |
| import numpy as np | |
| import faiss | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, ROOT) | |
| from faiss.contrib.e_means import NumpyEmeans # noqa: E402 | |
| from aries.pemeans import PEmeans # noqa: E402 | |
| def mse(c, rows): | |
| _, idx = faiss.knn(rows, np.ascontiguousarray(c, dtype="float32"), 1) | |
| diff = rows.astype(np.float64) - c[idx[:, 0]].astype(np.float64) | |
| return float((diff * diff).sum() / len(rows)) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("data"); ap.add_argument("tag") | |
| ap.add_argument("--n", type=int, default=250000); ap.add_argument("--k", type=int, default=4096) | |
| ap.add_argument("--epochs", type=float, nargs="+", default=[1, 3]) | |
| ap.add_argument("--seeds", type=int, nargs="+", default=[1, 2, 3, 4, 5]) | |
| ap.add_argument("--variant", nargs=2, action="append", default=[]) | |
| ap.add_argument("--threads", type=int, default=4) | |
| a = ap.parse_args() | |
| faiss.omp_set_num_threads(a.threads) | |
| x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy"), mmap_mode="r")[:a.n]) | |
| val = np.load(os.path.join(a.data, "val.npy")) | |
| variants = [("official", None)] + [(n, json.loads(j)) for n, j in a.variant] | |
| os.makedirs(os.path.join(ROOT, "results", "dev"), exist_ok=True) | |
| final = os.path.join(ROOT, "results", "dev", a.tag + ".jsonl") | |
| out = os.path.join(os.environ.get("DEV_TMP", "/tmp"), a.tag + ".jsonl.partial") | |
| res = {} | |
| for T in a.epochs: | |
| for s in a.seeds: | |
| init = x[np.random.RandomState(s).permutation(len(x))[:a.k]].copy() | |
| for name, kw in variants: | |
| t0 = time.time() | |
| extra = {} | |
| kw2 = dict(kw or {}) | |
| bs, asc = kw2.pop("batch_scale", None), kw2.pop("alpha_scale", None) | |
| if bs is not None or asc is not None: # LawFit: rescale the fitted batch / step | |
| from faiss.contrib.e_means.base import _resolve_recipe, _step_law | |
| B, a0 = _resolve_recipe(len(x), a.k, T, "throughput", None, None) | |
| B = int(B * (bs or 1.0)) | |
| a0 = min(1.0, _step_law(len(x), a.k, T, B) * (asc or 1.0)) | |
| extra = dict(batch_size=B, alpha0=a0) | |
| base = kw2.pop("base", None) | |
| km = (NumpyEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **extra) | |
| if kw is None or base == "official" | |
| else PEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **kw2, **extra)) | |
| c, _ = km.train(x, init_centroids=init.copy()) | |
| r = dict(tag=a.tag, n=a.n, k=a.k, epochs=T, seed=s, variant=name, kwargs=kw, | |
| seconds=time.time() - t0, val_mse=mse(c, val), splits=int(km.n_splits)) | |
| res[(T, name, s)] = r | |
| with open(out, "a") as f: | |
| f.write(json.dumps(r) + "\n") | |
| import shutil | |
| shutil.copyfile(out, final) | |
| for T in a.epochs: | |
| print(f"--- n={a.n} k={a.k} epochs={T} seeds={a.seeds}") | |
| for name, _ in variants: | |
| v = np.array([res[(T, name, s)]["val_mse"] for s in a.seeds]) | |
| b = np.array([res[(T, "official", s)]["val_mse"] for s in a.seeds]) | |
| d = 100 * (v - b) / b | |
| sec = np.mean([res[(T, name, s)]["seconds"] for s in a.seeds]) | |
| print(f"{name:28s} val {v.mean():.5f} vs official {d.mean():+.3f}% " | |
| f"(sd {d.std():.3f}, wins {int((d < 0).sum())}/{len(d)}) {sec:.1f}s", flush=True) | |
| if __name__ == "__main__": | |
| main() | |