File size: 3,583 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""Where the time goes: cProfile of one 3-epoch run of official e-means, PISCES v1.0 and PISCES on
DEEP* 1M (k = 16,384), grouped into assignment search, statistics, index building and the rest.

Usage: python benchmarks/time_breakdown.py DATA/deep1b OUT.json [--seed 1]
"""
import argparse, cProfile, json, os, pstats, re, sys, time
import numpy as np
import faiss

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT); sys.path.insert(0, os.path.join(ROOT, "benchmarks"))
from faiss.contrib.e_means import NumpyEmeans  # noqa: E402
from paper_protocol import make_variant  # noqa: E402
from aries.pemeans import PISCES, PISCES_V10  # noqa: E402

GROUPS = [  # (group, substrings of "file:function" in the profile; first match wins)
    ("assignment search", ["IndexIVF_search", "IndexRefine_search", "IndexRefineFlat_search", "knn", "IndexFlat_search", "IndexFlatCodes_search",
                           "bruteForceKnn", "Index_search", "search_preassigned"]),
    ("index build", ["IndexIVF_add", "Clustering_train", "IndexIVFScalarQuantizer_train", "Index_add", "IndexFlatCodes_add",
                     "IndexScalarQuantizer", "Index_train"]),
    ("statistics", ["ufunc' objects}", "_fused_stats_nb", "fused_stats", "argsort", "bincount", "einsum", "unique",
                    "_apply_statistics", "reduceat", "fill"]),
]


def short(key):
    """Drop machine-specific path prefixes from a profile key."""
    key = re.sub(r"^.*/site-packages/", "", key)
    key = re.sub(r"^.*/(aries|benchmarks)/", r"\1/", key)
    return re.sub(r"^/usr/.*?/lib/python3\.\d+/", "stdlib/", key)


def profile(build, x, init):
    pr = cProfile.Profile(); t0 = time.time(); pr.enable()
    build().train(x, init_centroids=init.copy())
    pr.disable(); wall = time.time() - t0
    st = pstats.Stats(pr).stats
    out = {g: 0.0 for g, _ in GROUPS}
    for (fn, line, name), (cc, nc, tt, ct, callers) in st.items():
        key = f"{fn}:{name}"
        for g, pats in GROUPS:
            if any(p in key for p in pats):
                out[g] += tt
                break
    out["other"] = max(0.0, wall - sum(out.values()))
    out["wall"] = wall
    top = sorted(st.items(), key=lambda t: -t[1][2])[:25]      # for auditing the grouping
    out["top"] = [[short(f"{fn}:{name}"), round(tt, 3)] for (fn, line, name), (cc, nc, tt, ct, callers) in top]
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("data"); ap.add_argument("out"); ap.add_argument("--seed", type=int, default=1)
    ap.add_argument("--epochs", type=float, default=3); ap.add_argument("--threads", type=int, default=4)
    a = ap.parse_args()
    faiss.omp_set_num_threads(a.threads)
    x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy")))
    k = 16384
    init = x[np.random.RandomState(a.seed).permutation(len(x))[:k]].copy()
    make_variant(PISCES, len(x))(x.shape[1], 0.02, 0).train(x[:40000], init_centroids=init[:k].copy())  # JIT warm-up
    res = {}
    for name, build in [("official", lambda: NumpyEmeans(x.shape[1], k, nepochs=a.epochs, seed=a.seed, batch_law="throughput")),
                        ("pisces_v10", lambda: make_variant(PISCES_V10, len(x))(x.shape[1], a.epochs, a.seed)),
                        ("pisces", lambda: make_variant(PISCES, len(x))(x.shape[1], a.epochs, a.seed))]:
        res[name] = profile(build, x, init)
        print(name, {g: round(v, 1) for g, v in res[name].items() if g != "top"}, flush=True)
    json.dump(res, open(a.out, "w"), indent=1)


if __name__ == "__main__":
    main()