"""Test MSE vs wall-clock for each method (mean over seeds, bars = sd). Usage: python benchmarks/plot.py results/deep1m_cpu.jsonl results/deep1m_cpu.png """ import collections, json, sys import numpy as np import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt rows = [json.loads(l) for l in open(sys.argv[1])] by = collections.defaultdict(list) for r in rows: by[(r["method"], r["epochs"])].append(r) style = {"official": ("#16233F", "o", "e-means (official faiss 1.16)"), "pemeans": ("#9FD3D1", "D", "PE-means v1"), "pemeans_v2": ("#3FA7A5", "s", "PE-means v2"), "pemeans_v3": ("#0E8C8C", "^", "PE-means v3 (ours, final)")} fig, ax = plt.subplots(figsize=(6.4, 4.0), dpi=160) for m, (c, mk, lab) in style.items(): pts = sorted((T, np.mean([r["seconds"] for r in v]), np.mean([r["test_mse"] for r in v]), np.std([r["test_mse"] for r in v])) for (mm, T), v in by.items() if mm == m) if not pts: continue T, s, mu, sd = map(np.array, zip(*pts)) ax.errorbar(s, mu, yerr=sd, color=c, marker=mk, lw=2, ms=6, capsize=3, label=lab) for t, x, y in zip(T, s, mu): ax.annotate(f"{t:g} ep.", (x, y), xytext=(6, 4), textcoords="offset points", fontsize=8, color=c) ref = {"deep1b-1M": 0.38921320633218964}.get(rows[0].get("dataset")) if ref: ax.axhline(ref, color="#C4413A", ls="--", lw=1) ax.text(ax.get_xlim()[1], ref, "Lloyd 25 it. (paper, CPU median) ", color="#C4413A", fontsize=8, ha="right", va="bottom") ax.set_xlabel("wall-clock seconds (4 CPU threads)"); ax.set_ylabel("test MSE (lower is better)") ax.set_title(f"{rows[0].get('dataset', '')}, k = 16,384, mean ± sd over seeds", fontsize=10) ax.grid(alpha=0.3); ax.legend(frameon=False) fig.tight_layout(); fig.savefig(sys.argv[2]) print("wrote", sys.argv[2])