File size: 4,083 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
"""Development loop on DEEP* (validation rows only; test rows are reserved for final reports).

Usage: python benchmarks/dev_loop.py DATA_DIR TAG --n 250000 --k 4096 --epochs 1 3 --seeds 1 2 3 4 5
           --variant name '{"split": "principal", ...}' [--variant ...]
Each variant is PEmeans keyword arguments as JSON; "official" (faiss NumpyEmeans) is always run.
Results go to results/dev/TAG.jsonl and a paired summary is printed.
"""
import argparse, json, os, sys, time
import numpy as np
import faiss

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)
from faiss.contrib.e_means import NumpyEmeans  # noqa: E402
from aries.pemeans import PEmeans  # noqa: E402


def mse(c, rows):
    _, idx = faiss.knn(rows, np.ascontiguousarray(c, dtype="float32"), 1)
    diff = rows.astype(np.float64) - c[idx[:, 0]].astype(np.float64)
    return float((diff * diff).sum() / len(rows))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("data"); ap.add_argument("tag")
    ap.add_argument("--n", type=int, default=250000); ap.add_argument("--k", type=int, default=4096)
    ap.add_argument("--epochs", type=float, nargs="+", default=[1, 3])
    ap.add_argument("--seeds", type=int, nargs="+", default=[1, 2, 3, 4, 5])
    ap.add_argument("--variant", nargs=2, action="append", default=[])
    ap.add_argument("--threads", type=int, default=4)
    a = ap.parse_args()
    faiss.omp_set_num_threads(a.threads)
    x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy"), mmap_mode="r")[:a.n])
    val = np.load(os.path.join(a.data, "val.npy"))
    variants = [("official", None)] + [(n, json.loads(j)) for n, j in a.variant]
    os.makedirs(os.path.join(ROOT, "results", "dev"), exist_ok=True)
    final = os.path.join(ROOT, "results", "dev", a.tag + ".jsonl")
    out = os.path.join(os.environ.get("DEV_TMP", "/tmp"), a.tag + ".jsonl.partial")
    res = {}
    for T in a.epochs:
        for s in a.seeds:
            init = x[np.random.RandomState(s).permutation(len(x))[:a.k]].copy()
            for name, kw in variants:
                t0 = time.time()
                extra = {}
                kw2 = dict(kw or {})
                bs, asc = kw2.pop("batch_scale", None), kw2.pop("alpha_scale", None)
                if bs is not None or asc is not None:   # LawFit: rescale the fitted batch / step
                    from faiss.contrib.e_means.base import _resolve_recipe, _step_law
                    B, a0 = _resolve_recipe(len(x), a.k, T, "throughput", None, None)
                    B = int(B * (bs or 1.0))
                    a0 = min(1.0, _step_law(len(x), a.k, T, B) * (asc or 1.0))
                    extra = dict(batch_size=B, alpha0=a0)
                base = kw2.pop("base", None)
                km = (NumpyEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **extra)
                      if kw is None or base == "official"
                      else PEmeans(x.shape[1], a.k, nepochs=T, seed=s, batch_law="throughput", **kw2, **extra))
                c, _ = km.train(x, init_centroids=init.copy())
                r = dict(tag=a.tag, n=a.n, k=a.k, epochs=T, seed=s, variant=name, kwargs=kw,
                         seconds=time.time() - t0, val_mse=mse(c, val), splits=int(km.n_splits))
                res[(T, name, s)] = r
                with open(out, "a") as f:
                    f.write(json.dumps(r) + "\n")
    import shutil
    shutil.copyfile(out, final)
    for T in a.epochs:
        print(f"--- n={a.n} k={a.k} epochs={T} seeds={a.seeds}")
        for name, _ in variants:
            v = np.array([res[(T, name, s)]["val_mse"] for s in a.seeds])
            b = np.array([res[(T, "official", s)]["val_mse"] for s in a.seeds])
            d = 100 * (v - b) / b
            sec = np.mean([res[(T, name, s)]["seconds"] for s in a.seeds])
            print(f"{name:28s} val {v.mean():.5f}  vs official {d.mean():+.3f}% "
                  f"(sd {d.std():.3f}, wins {int((d < 0).sum())}/{len(d)})  {sec:.1f}s", flush=True)


if __name__ == "__main__":
    main()