feat(bench): auto-detect latest run and plot per-stage latency + metrics summary

benchmarks/plot_results.py

@@ -1,145 +1,101 @@
 """
-Plot and summarize results from benchmarks/results_pro/<run>/summary.json
+Plot evaluation summaries for NL2SQL Copilot benchmark runs.
 
-- Auto-detects the latest run directory (unless --run-dir is provided).
-- Prints a compact textual report (EM/SM/ExecAcc, latency, success rate).
-- Saves two charts next to summary.json:
-    - latency_per_stage.png
-    - metrics_overview.png (EM/SM/ExecAcc as a bar chart)
+Automatically detects the latest results folder under benchmarks/results_pro/,
+reads summary.json + eval.jsonl, and plots:
+  1. Average latency per pipeline stage (ms)
+  2. EM / SM / ExecAcc overview
 
-Usage:
-    PYTHONPATH=$PWD python benchmarks/plot_results.py
-    PYTHONPATH=$PWD python benchmarks/plot_results.py --run-dir benchmarks/results_pro/20251108-105442
+If summary.json lacks per-stage averages, they are derived from eval.jsonl traces.
 """
 
-from __future__ import annotations
-
-import argparse
 import json
+import time
 from pathlib import Path
-from typing import Dict, Any, List
-
 import matplotlib.pyplot as plt
 
-
-STAGES: List[str] = [
-    "detector",
-    "planner",
-    "generator",
-    "safety",
-    "executor",
-    "verifier",
-]
-
-
-def _find_latest_run(results_root: Path) -> Path:
-    runs = sorted([p for p in results_root.iterdir() if p.is_dir()])
-    if not runs:
-        raise FileNotFoundError(f"No runs found under {results_root}")
-    return runs[-1]
-
-
-def _load_summary(run_dir: Path) -> Dict[str, Any]:
-    summary_path = run_dir / "summary.json"
-    if not summary_path.exists():
-        # Back-compat (legacy name used by tests)
-        summary_path = run_dir / "metrics_summary.json"
-    if not summary_path.exists():
-        raise FileNotFoundError(f"Missing summary JSON in {run_dir}")
-    return json.loads(summary_path.read_text(encoding="utf-8"))
-
-
-def _print_report(summary: Dict[str, Any]) -> None:
-    # Gracefully read metrics (demo runs may not have EM/SM/ExecAcc)
-    em = summary.get("EM", 0.0)
-    sm = summary.get("SM", 0.0)
-    exec_acc = summary.get("ExecAcc", 0.0)
-    success_rate = summary.get("success_rate", 0.0)
-    avg_ms = summary.get("avg_latency_ms", 0.0)
-    p95_ms = summary.get("p95_latency_ms", None)
-
-    total = summary.get("queries_total", summary.get("total", 0))
-    src = summary.get("pipeline_source", "adapter")
-    ts = summary.get("timestamp", "-")
-
-    print("\n================ Benchmark Summary ================")
-    print(f"Timestamp        : {ts}")
-    print(f"Pipeline source  : {src}")
-    print(f"Queries total    : {total}")
-    print(f"Success rate     : {success_rate:.0%}")
-    print(f"EM / SM / ExecAcc: {em:.2f} / {sm:.2f} / {exec_acc:.2f}")
-    print(f"Avg latency (ms) : {avg_ms:.1f}")
-    if p95_ms is not None:
-        print(f"p95 latency (ms) : {p95_ms:.1f}")
-    print("===================================================\n")
-
-
-def _plot_latency_per_stage(run_dir: Path, summary: Dict[str, Any]) -> Path:
-    latencies = [summary.get(f"{s}_avg_ms", 0.0) for s in STAGES]
-    out_path = run_dir / "latency_per_stage.png"
-
-    # Single-plot bar chart (no explicit colors)
-    plt.figure()
-    plt.bar(STAGES, latencies)
-    plt.title("Average Latency per Stage (ms)")
-    plt.xlabel("Stage")
-    plt.ylabel("Latency (ms)")
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=160)
-    plt.close()
-
-    return out_path
-
-
-def _plot_metrics_overview(run_dir: Path, summary: Dict[str, Any]) -> Path:
-    # Even if zeros (demo mode), chart is still useful in README.
-    em = summary.get("EM", 0.0)
-    sm = summary.get("SM", 0.0)
-    exec_acc = summary.get("ExecAcc", 0.0)
-    out_path = run_dir / "metrics_overview.png"
-
-    labels = ["EM", "SM", "ExecAcc"]
-    values = [em, sm, exec_acc]
-
-    plt.figure()
-    plt.bar(labels, values)
-    plt.title("EM / SM / ExecAcc")
-    plt.xlabel("Metric")
-    plt.ylabel("Score")
-    plt.ylim(0, 1)  # normalized range
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=160)
-    plt.close()
-
-    return out_path
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument(
-        "--run-dir",
-        type=str,
-        default=None,
-        help="Path to a specific run directory under benchmarks/results_pro/ "
-        "(defaults to latest).",
-    )
-    args, _ = ap.parse_known_args()
-
-    results_root = Path("benchmarks") / "results_pro"
-    run_dir = (
-        Path(args.run_dir).resolve() if args.run_dir else _find_latest_run(results_root)
-    )
-
-    summary = _load_summary(run_dir)
-    _print_report(summary)
-
-    lat_path = _plot_latency_per_stage(run_dir, summary)
-    met_path = _plot_metrics_overview(run_dir, summary)
-
-    print("✅ Saved plots:")
-    print(f"- {lat_path}")
-    print(f"- {met_path}")
-
-
-if __name__ == "__main__":
-    main()
+# -------------------------------------------------------------------
+# Locate latest results directory
+# -------------------------------------------------------------------
+
+ROOT = Path("benchmarks/results_pro")
+run_dirs = sorted(
+    ROOT.glob("*/summary.json"), key=lambda p: p.stat().st_mtime, reverse=True
+)
+if not run_dirs:
+    raise SystemExit("❌ No benchmark results found under benchmarks/results_pro/")
+summary_path = run_dirs[0]
+run_dir = summary_path.parent
+print(f"📂 Using latest run: {run_dir.name}")
+
+# -------------------------------------------------------------------
+# Load summary
+# -------------------------------------------------------------------
+with summary_path.open(encoding="utf-8") as f:
+    summary = json.load(f)
+
+# -------------------------------------------------------------------
+# Derive per-stage averages if not present
+# -------------------------------------------------------------------
+STAGES = ["detector", "planner", "generator", "safety", "executor", "verifier"]
+stage_means = {s: summary.get(f"{s}_avg_ms") for s in STAGES}
+need_fallback = any(v is None for v in stage_means.values())
+
+if need_fallback:
+    eval_path = run_dir / "eval.jsonl"
+    totals = {s: 0.0 for s in STAGES}
+    counts = {s: 0 for s in STAGES}
+    if eval_path.exists():
+        with eval_path.open(encoding="utf-8") as f:
+            for line in f:
+                rec = json.loads(line)
+                for t in rec.get("trace", []) or []:
+                    s = t.get("stage")
+                    ms = t.get("ms", t.get("duration_ms", 0.0))
+                    if s in totals:
+                        totals[s] += float(ms)
+                        counts[s] += 1
+    stage_means = {
+        s: round(totals[s] / max(counts[s], 1), 2) if counts[s] else 0.0 for s in STAGES
+    }
+
+latencies = [stage_means[s] for s in STAGES]
+
+# -------------------------------------------------------------------
+# Plot average latency per stage
+# -------------------------------------------------------------------
+plt.figure(figsize=(7, 5))
+plt.bar(STAGES, latencies, color="#6fa8dc")
+plt.title("Average Latency per Stage (ms)")
+plt.xlabel("Stage")
+plt.ylabel("Latency (ms)")
+plt.tight_layout()
+plt.savefig(run_dir / "latency_per_stage.png")
+print(f"📊 Saved latency chart → {run_dir / 'latency_per_stage.png'}")
+
+# -------------------------------------------------------------------
+# Plot EM / SM / ExecAcc metrics
+# -------------------------------------------------------------------
+metrics = ["EM", "SM", "ExecAcc"]
+scores = [summary.get(k, 0.0) for k in metrics]
+
+plt.figure(figsize=(7, 5))
+plt.bar(metrics, scores, color="#93c47d")
+plt.title("EM / SM / ExecAcc")
+plt.xlabel("Metric")
+plt.ylabel("Score")
+plt.ylim(0, 1)
+plt.tight_layout()
+plt.savefig(run_dir / "metrics_overview.png")
+print(f"📊 Saved metrics chart → {run_dir / 'metrics_overview.png'}")
+
+# -------------------------------------------------------------------
+# Quick textual summary
+# -------------------------------------------------------------------
+print(
+    f"\n✅ Summary for {run_dir.name}\n"
+    f"Avg latency: {summary.get('avg_latency_ms', 'n/a')} ms\n"
+    f"Success rate: {summary.get('success_rate', 0.0):.0%}\n"
+    f"EM: {summary.get('EM', 0.0):.3f} | SM: {summary.get('SM', 0.0):.3f} | ExecAcc: {summary.get('ExecAcc', 0.0):.3f}\n"
+)
+time.sleep(0.2)

benchmarks/results_pro/20251108-125829/eval.jsonl

@@ -0,0 +1,5 @@
+{"source": "demo", "db_id": "demo", "query": "list all customers", "ok": false, "latency_ms": 6652, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 2554}, {"stage": "generator", "ms": 1370}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 1}, {"stage": "repair", "ms": 1295}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "repair", "ms": 1426}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
+{"source": "demo", "db_id": "demo", "query": "show total invoices per country", "ok": true, "latency_ms": 7375, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 3866}, {"stage": "generator", "ms": 1265}, {"stage": "safety", "ms": 4}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "repair", "ms": 1126}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "repair", "ms": 1106}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
+{"source": "demo", "db_id": "demo", "query": "top 3 albums by total sales", "ok": true, "latency_ms": 1, "trace": [{"stage": "detector", "ms": 0}], "error": null}
+{"source": "demo", "db_id": "demo", "query": "artists with more than 3 albums", "ok": false, "latency_ms": 8629, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 4110}, {"stage": "generator", "ms": 1969}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "repair", "ms": 1296}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1244}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
+{"source": "demo", "db_id": "demo", "query": "number of employees per city", "ok": true, "latency_ms": 5630, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 2602}, {"stage": "generator", "ms": 1097}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 0}, {"stage": "verifier", "ms": 0}, {"stage": "repair", "ms": 1018}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "repair", "ms": 906}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "verifier", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}

benchmarks/results_pro/20251108-125829/results.csv

@@ -0,0 +1,6 @@
+source,db_id,query,em,sm,exec_acc,ok,latency_ms
+demo,demo,list all customers,,,,❌,6652
+demo,demo,show total invoices per country,,,,✅,7375
+demo,demo,top 3 albums by total sales,,,,✅,1
+demo,demo,artists with more than 3 albums,,,,❌,8629
+demo,demo,number of employees per city,,,,✅,5630

benchmarks/results_pro/20251108-125829/summary.json

@@ -0,0 +1,13 @@
+{
+  "mode": "single-db",
+  "db_path": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/demo.db",
+  "config": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/configs/sqlite_pipeline.yaml",
+  "provider_hint": "REAL",
+  "total": 5,
+  "EM": 0.0,
+  "SM": 0.0,
+  "ExecAcc": 0.0,
+  "success_rate": 0.6,
+  "avg_latency_ms": 5657.4,
+  "timestamp": "2025-11-08 12:58:58"
+}