Download benchmarks/plot.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 1.82 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/plot.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/plot.py
-
curl -L -o plot.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/plot.py
1.82 kB
| """Test MSE vs wall-clock for each method (mean over seeds, bars = sd). | |
| Usage: python benchmarks/plot.py results/deep1m_cpu.jsonl results/deep1m_cpu.png | |
| """ | |
| import collections, json, sys | |
| import numpy as np | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| rows = [json.loads(l) for l in open(sys.argv[1])] | |
| by = collections.defaultdict(list) | |
| for r in rows: | |
| by[(r["method"], r["epochs"])].append(r) | |
| style = {"official": ("#16233F", "o", "e-means (official faiss 1.16)"), | |
| "pemeans": ("#9FD3D1", "D", "PE-means v1"), | |
| "pemeans_v2": ("#3FA7A5", "s", "PE-means v2"), | |
| "pemeans_v3": ("#0E8C8C", "^", "PE-means v3 (ours, final)")} | |
| fig, ax = plt.subplots(figsize=(6.4, 4.0), dpi=160) | |
| for m, (c, mk, lab) in style.items(): | |
| pts = sorted((T, np.mean([r["seconds"] for r in v]), np.mean([r["test_mse"] for r in v]), np.std([r["test_mse"] for r in v])) | |
| for (mm, T), v in by.items() if mm == m) | |
| if not pts: | |
| continue | |
| T, s, mu, sd = map(np.array, zip(*pts)) | |
| ax.errorbar(s, mu, yerr=sd, color=c, marker=mk, lw=2, ms=6, capsize=3, label=lab) | |
| for t, x, y in zip(T, s, mu): | |
| ax.annotate(f"{t:g} ep.", (x, y), xytext=(6, 4), textcoords="offset points", fontsize=8, color=c) | |
| ref = {"deep1b-1M": 0.38921320633218964}.get(rows[0].get("dataset")) | |
| if ref: | |
| ax.axhline(ref, color="#C4413A", ls="--", lw=1) | |
| ax.text(ax.get_xlim()[1], ref, "Lloyd 25 it. (paper, CPU median) ", color="#C4413A", fontsize=8, ha="right", va="bottom") | |
| ax.set_xlabel("wall-clock seconds (4 CPU threads)"); ax.set_ylabel("test MSE (lower is better)") | |
| ax.set_title(f"{rows[0].get('dataset', '')}, k = 16,384, mean ± sd over seeds", fontsize=10) | |
| ax.grid(alpha=0.3); ax.legend(frameon=False) | |
| fig.tight_layout(); fig.savefig(sys.argv[2]) | |
| print("wrote", sys.argv[2]) | |