PISCES / benchmarks /plot.py
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
1.82 kB
"""Test MSE vs wall-clock for each method (mean over seeds, bars = sd).
Usage: python benchmarks/plot.py results/deep1m_cpu.jsonl results/deep1m_cpu.png
"""
import collections, json, sys
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
rows = [json.loads(l) for l in open(sys.argv[1])]
by = collections.defaultdict(list)
for r in rows:
by[(r["method"], r["epochs"])].append(r)
style = {"official": ("#16233F", "o", "e-means (official faiss 1.16)"),
"pemeans": ("#9FD3D1", "D", "PE-means v1"),
"pemeans_v2": ("#3FA7A5", "s", "PE-means v2"),
"pemeans_v3": ("#0E8C8C", "^", "PE-means v3 (ours, final)")}
fig, ax = plt.subplots(figsize=(6.4, 4.0), dpi=160)
for m, (c, mk, lab) in style.items():
pts = sorted((T, np.mean([r["seconds"] for r in v]), np.mean([r["test_mse"] for r in v]), np.std([r["test_mse"] for r in v]))
for (mm, T), v in by.items() if mm == m)
if not pts:
continue
T, s, mu, sd = map(np.array, zip(*pts))
ax.errorbar(s, mu, yerr=sd, color=c, marker=mk, lw=2, ms=6, capsize=3, label=lab)
for t, x, y in zip(T, s, mu):
ax.annotate(f"{t:g} ep.", (x, y), xytext=(6, 4), textcoords="offset points", fontsize=8, color=c)
ref = {"deep1b-1M": 0.38921320633218964}.get(rows[0].get("dataset"))
if ref:
ax.axhline(ref, color="#C4413A", ls="--", lw=1)
ax.text(ax.get_xlim()[1], ref, "Lloyd 25 it. (paper, CPU median) ", color="#C4413A", fontsize=8, ha="right", va="bottom")
ax.set_xlabel("wall-clock seconds (4 CPU threads)"); ax.set_ylabel("test MSE (lower is better)")
ax.set_title(f"{rows[0].get('dataset', '')}, k = 16,384, mean ± sd over seeds", fontsize=10)
ax.grid(alpha=0.3); ax.legend(frameon=False)
fig.tight_layout(); fig.savefig(sys.argv[2])
print("wrote", sys.argv[2])