eval/plot_perf.py

"""Plot the system measurements from perf_results.json.

Two panels: (1) quality (nDCG@10) × median latency Pareto front, each
point annotated with its build cost; (2) throughput (req/s) by concurrency.
Writes docs/perf-pareto.svg. Requires the [notebooks] extra (matplotlib).

    python -m eval.plot_perf
"""

import json
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402

ROOT = Path(__file__).resolve().parent.parent
COLORS = {"vector": "#3b82f6", "hybrid": "#22c55e", "graph": "#a855f7"}
SHORT = {"vector": "Vector", "hybrid": "Hybrid", "graph": "Graph"}
BUILD_KEY = {"vector": "vector_total", "hybrid": "hybrid_total", "graph": "graph_total"}


def main() -> None:
    data = json.loads((ROOT / "eval" / "perf_results.json").read_text("utf-8"))
    stacks, build = data["stacks"], data["build_seconds"]
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4.5))

    # Panel 1: quality × latency Pareto (ideal = top-left).
    for kind, m in stacks.items():
        x, y = m["latency"]["median_ms"], m["ndcg@10"]
        ax1.scatter(x, y, s=160, color=COLORS[kind], zorder=3, edgecolor="white", linewidth=1.5)
        ax1.annotate(f"{SHORT[kind]}\n{build[BUILD_KEY[kind]]:.0f}s build",
                     (x, y), textcoords="offset points", xytext=(10, 6), fontsize=9)
    ax1.set_xlabel("median retrieval latency (ms) — lower is better →", fontsize=9)
    ax1.set_ylabel("nDCG@10 — higher is better ↑", fontsize=9)
    ax1.set_title("Quality × latency Pareto\n(↖ ideal; label = index build time)")
    ax1.invert_xaxis()
    ax1.grid(alpha=0.3)

    # Panel 2: throughput by concurrency.
    conc = sorted((int(w) for w in next(iter(stacks.values()))["throughput_qps"]))
    for kind, m in stacks.items():
        ys = [m["throughput_qps"][str(w)] for w in conc]
        ax2.plot(conc, ys, "-o", color=COLORS[kind], label=SHORT[kind])
    ax2.set_xlabel("concurrent threads", fontsize=9)
    ax2.set_ylabel("throughput (queries / second)", fontsize=9)
    ax2.set_xticks(conc)
    ax2.set_title("Throughput vs concurrency\n(only Vector scales — FAISS releases the GIL)")
    ax2.legend(fontsize=9)
    ax2.grid(alpha=0.3)

    cfg = data["config"]
    fig.suptitle(f"Retrieval systems profile — {cfg['dataset']}, {cfg['n_docs']} docs "
                 f"(no LLM)", fontsize=12)
    fig.tight_layout()
    out = ROOT / "docs" / "perf-pareto.svg"
    fig.savefig(out, bbox_inches="tight")
    print(f"✅ {out}")


if __name__ == "__main__":
    main()