scripts/run_benchmarks.py

#!/usr/bin/env python3
"""Generate the quantitative benchmark table from real simulation runs.

Every figure in the submission's results table comes from here. Nothing is
entered by hand.

Run:  python scripts/run_benchmarks.py [--seeds 8] [--scenario circuit_alpha_post_race]
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "backend"))

from flowtwin.benchmarks.runner import format_table, run_benchmark   # noqa: E402
from flowtwin.config import BENCHMARK_DIR, SETTINGS                  # noqa: E402

DEFAULT_SEEDS = [42193, 1177, 90210, 31337, 8080, 5150, 771, 24601,
                 60606, 13013, 4242, 909]


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--scenario", action="append", default=None,
                    help="scenario id; repeatable. Defaults to both showcase runs.")
    ap.add_argument("--seeds", type=int, default=8, help="number of seeds per arm")
    ap.add_argument("--out", default=str(BENCHMARK_DIR))
    ap.add_argument("--review", type=float, default=180.0,
                    help="seconds between FlowTwin strategy reviews")
    ap.add_argument("--horizon", type=float, default=240.0,
                    help="counterfactual roll-out horizon")
    args = ap.parse_args()

    scenarios = args.scenario or ["circuit_alpha_post_race", "barcelona_2022_egress"]
    seeds = DEFAULT_SEEDS[: args.seeds]
    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)

    out_path = out_dir / "benchmark_results.json"
    all_payloads = {}
    if out_path.exists():
        # Keep results for scenarios this invocation is not re-running, so a
        # long benchmark can be built up (or resumed) one scenario at a time.
        try:
            existing = json.loads(out_path.read_text(encoding="utf-8"))
            all_payloads.update({k: v for k, v in existing.get("scenarios", {}).items()
                                 if k not in scenarios})
        except Exception:
            pass

    for scenario_id in scenarios:
        print(f"\n=== {scenario_id} · {len(seeds)} seeds × 3 arms ===")

        def progress(done, total, result):
            m = result.metrics
            print(f"  [{done:>3}/{total}] {result.arm:<18} seed={result.seed:<9} "
                  f"peakD={m['peak_density']:5.2f}  critS={m['critical_edge_seconds']:7.0f}  "
                  f"avgTT={m['avg_travel_time_s']:6.0f}s  maxQ={m['max_queue']:6.0f}  "
                  f"({result.wall_s:.1f}s)")

        payload = run_benchmark(scenario_id, seeds, SETTINGS, progress=progress,
                                review_interval_s=args.review, horizon_s=args.horizon)
        all_payloads[scenario_id] = payload
        # Write after every scenario: a long run that is interrupted should not
        # lose the scenarios that already finished.
        out_path.write_text(json.dumps(
            {"scenarios": all_payloads, "default_scenario": scenarios[0],
             "seed_count": len(seeds)}, indent=2), encoding="utf-8")

        print()
        print(format_table(payload))
        deltas = payload["deltas_vs_shortest_path_pct"].get("flowtwin", {})
        if deltas:
            print("\nFlowTwin vs shortest path:")
            for spec in payload["metrics"]:
                key = spec["key"]
                if key in deltas:
                    print(f"  {spec['label']:<32} {deltas[key]:+7.1f}%")

    path = out_path
    md = ["# FlowTwin benchmark results", "",
          "Generated by `scripts/run_benchmarks.py`. Every value is the mean ± "
          "standard deviation over independent random seeds of the full "
          "simulation. No value is entered by hand.", ""]
    for scenario_id, payload in all_payloads.items():
        md += [f"## {payload['scenario_name']}", "",
               f"Venue `{payload['venue_id']}` · crowd {payload['crowd_size']:,} · "
               f"{len(payload['seeds'])} seeds · generated {payload['generated_utc']}", "",
               format_table(payload), ""]
        for arm in payload["arms"]:
            md.append(f"- **{arm['label']}** — {arm['description']}")
        md.append("")
    (out_dir / "BENCHMARKS.md").write_text("\n".join(md), encoding="utf-8")

    print(f"\nSaved {path}")
    print(f"Saved {out_dir / 'BENCHMARKS.md'}")


if __name__ == "__main__":
    main()