File size: 3,583 Bytes
c09a3f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """Where the time goes: cProfile of one 3-epoch run of official e-means, PISCES v1.0 and PISCES on
DEEP* 1M (k = 16,384), grouped into assignment search, statistics, index building and the rest.
Usage: python benchmarks/time_breakdown.py DATA/deep1b OUT.json [--seed 1]
"""
import argparse, cProfile, json, os, pstats, re, sys, time
import numpy as np
import faiss
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT); sys.path.insert(0, os.path.join(ROOT, "benchmarks"))
from faiss.contrib.e_means import NumpyEmeans # noqa: E402
from paper_protocol import make_variant # noqa: E402
from aries.pemeans import PISCES, PISCES_V10 # noqa: E402
GROUPS = [ # (group, substrings of "file:function" in the profile; first match wins)
("assignment search", ["IndexIVF_search", "IndexRefine_search", "IndexRefineFlat_search", "knn", "IndexFlat_search", "IndexFlatCodes_search",
"bruteForceKnn", "Index_search", "search_preassigned"]),
("index build", ["IndexIVF_add", "Clustering_train", "IndexIVFScalarQuantizer_train", "Index_add", "IndexFlatCodes_add",
"IndexScalarQuantizer", "Index_train"]),
("statistics", ["ufunc' objects}", "_fused_stats_nb", "fused_stats", "argsort", "bincount", "einsum", "unique",
"_apply_statistics", "reduceat", "fill"]),
]
def short(key):
"""Drop machine-specific path prefixes from a profile key."""
key = re.sub(r"^.*/site-packages/", "", key)
key = re.sub(r"^.*/(aries|benchmarks)/", r"\1/", key)
return re.sub(r"^/usr/.*?/lib/python3\.\d+/", "stdlib/", key)
def profile(build, x, init):
pr = cProfile.Profile(); t0 = time.time(); pr.enable()
build().train(x, init_centroids=init.copy())
pr.disable(); wall = time.time() - t0
st = pstats.Stats(pr).stats
out = {g: 0.0 for g, _ in GROUPS}
for (fn, line, name), (cc, nc, tt, ct, callers) in st.items():
key = f"{fn}:{name}"
for g, pats in GROUPS:
if any(p in key for p in pats):
out[g] += tt
break
out["other"] = max(0.0, wall - sum(out.values()))
out["wall"] = wall
top = sorted(st.items(), key=lambda t: -t[1][2])[:25] # for auditing the grouping
out["top"] = [[short(f"{fn}:{name}"), round(tt, 3)] for (fn, line, name), (cc, nc, tt, ct, callers) in top]
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("data"); ap.add_argument("out"); ap.add_argument("--seed", type=int, default=1)
ap.add_argument("--epochs", type=float, default=3); ap.add_argument("--threads", type=int, default=4)
a = ap.parse_args()
faiss.omp_set_num_threads(a.threads)
x = np.ascontiguousarray(np.load(os.path.join(a.data, "train_1M.npy")))
k = 16384
init = x[np.random.RandomState(a.seed).permutation(len(x))[:k]].copy()
make_variant(PISCES, len(x))(x.shape[1], 0.02, 0).train(x[:40000], init_centroids=init[:k].copy()) # JIT warm-up
res = {}
for name, build in [("official", lambda: NumpyEmeans(x.shape[1], k, nepochs=a.epochs, seed=a.seed, batch_law="throughput")),
("pisces_v10", lambda: make_variant(PISCES_V10, len(x))(x.shape[1], a.epochs, a.seed)),
("pisces", lambda: make_variant(PISCES, len(x))(x.shape[1], a.epochs, a.seed))]:
res[name] = profile(build, x, init)
print(name, {g: round(v, 1) for g, v in res[name].items() if g != "top"}, flush=True)
json.dump(res, open(a.out, "w"), indent=1)
if __name__ == "__main__":
main()
|