File size: 1,822 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
"""Test MSE vs wall-clock for each method (mean over seeds, bars = sd).

Usage: python benchmarks/plot.py results/deep1m_cpu.jsonl results/deep1m_cpu.png
"""
import collections, json, sys
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

rows = [json.loads(l) for l in open(sys.argv[1])]
by = collections.defaultdict(list)
for r in rows:
    by[(r["method"], r["epochs"])].append(r)
style = {"official": ("#16233F", "o", "e-means (official faiss 1.16)"),
         "pemeans": ("#9FD3D1", "D", "PE-means v1"),
         "pemeans_v2": ("#3FA7A5", "s", "PE-means v2"),
         "pemeans_v3": ("#0E8C8C", "^", "PE-means v3 (ours, final)")}
fig, ax = plt.subplots(figsize=(6.4, 4.0), dpi=160)
for m, (c, mk, lab) in style.items():
    pts = sorted((T, np.mean([r["seconds"] for r in v]), np.mean([r["test_mse"] for r in v]), np.std([r["test_mse"] for r in v]))
                 for (mm, T), v in by.items() if mm == m)
    if not pts:
        continue
    T, s, mu, sd = map(np.array, zip(*pts))
    ax.errorbar(s, mu, yerr=sd, color=c, marker=mk, lw=2, ms=6, capsize=3, label=lab)
    for t, x, y in zip(T, s, mu):
        ax.annotate(f"{t:g} ep.", (x, y), xytext=(6, 4), textcoords="offset points", fontsize=8, color=c)
ref = {"deep1b-1M": 0.38921320633218964}.get(rows[0].get("dataset"))
if ref:
    ax.axhline(ref, color="#C4413A", ls="--", lw=1)
    ax.text(ax.get_xlim()[1], ref, "Lloyd 25 it. (paper, CPU median)  ", color="#C4413A", fontsize=8, ha="right", va="bottom")
ax.set_xlabel("wall-clock seconds (4 CPU threads)"); ax.set_ylabel("test MSE (lower is better)")
ax.set_title(f"{rows[0].get('dataset', '')}, k = 16,384, mean ± sd over seeds", fontsize=10)
ax.grid(alpha=0.3); ax.legend(frameon=False)
fig.tight_layout(); fig.savefig(sys.argv[2])
print("wrote", sys.argv[2])