Spaces:
Runtime error
Runtime error
File size: 4,442 Bytes
e7a9f02 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #!/usr/bin/env python3
"""Generate the quantitative benchmark table from real simulation runs.
Every figure in the submission's results table comes from here. Nothing is
entered by hand.
Run: python scripts/run_benchmarks.py [--seeds 8] [--scenario circuit_alpha_post_race]
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "backend"))
from flowtwin.benchmarks.runner import format_table, run_benchmark # noqa: E402
from flowtwin.config import BENCHMARK_DIR, SETTINGS # noqa: E402
DEFAULT_SEEDS = [42193, 1177, 90210, 31337, 8080, 5150, 771, 24601,
60606, 13013, 4242, 909]
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--scenario", action="append", default=None,
help="scenario id; repeatable. Defaults to both showcase runs.")
ap.add_argument("--seeds", type=int, default=8, help="number of seeds per arm")
ap.add_argument("--out", default=str(BENCHMARK_DIR))
ap.add_argument("--review", type=float, default=180.0,
help="seconds between FlowTwin strategy reviews")
ap.add_argument("--horizon", type=float, default=240.0,
help="counterfactual roll-out horizon")
args = ap.parse_args()
scenarios = args.scenario or ["circuit_alpha_post_race", "barcelona_2022_egress"]
seeds = DEFAULT_SEEDS[: args.seeds]
out_dir = Path(args.out)
out_dir.mkdir(parents=True, exist_ok=True)
out_path = out_dir / "benchmark_results.json"
all_payloads = {}
if out_path.exists():
# Keep results for scenarios this invocation is not re-running, so a
# long benchmark can be built up (or resumed) one scenario at a time.
try:
existing = json.loads(out_path.read_text(encoding="utf-8"))
all_payloads.update({k: v for k, v in existing.get("scenarios", {}).items()
if k not in scenarios})
except Exception:
pass
for scenario_id in scenarios:
print(f"\n=== {scenario_id} · {len(seeds)} seeds × 3 arms ===")
def progress(done, total, result):
m = result.metrics
print(f" [{done:>3}/{total}] {result.arm:<18} seed={result.seed:<9} "
f"peakD={m['peak_density']:5.2f} critS={m['critical_edge_seconds']:7.0f} "
f"avgTT={m['avg_travel_time_s']:6.0f}s maxQ={m['max_queue']:6.0f} "
f"({result.wall_s:.1f}s)")
payload = run_benchmark(scenario_id, seeds, SETTINGS, progress=progress,
review_interval_s=args.review, horizon_s=args.horizon)
all_payloads[scenario_id] = payload
# Write after every scenario: a long run that is interrupted should not
# lose the scenarios that already finished.
out_path.write_text(json.dumps(
{"scenarios": all_payloads, "default_scenario": scenarios[0],
"seed_count": len(seeds)}, indent=2), encoding="utf-8")
print()
print(format_table(payload))
deltas = payload["deltas_vs_shortest_path_pct"].get("flowtwin", {})
if deltas:
print("\nFlowTwin vs shortest path:")
for spec in payload["metrics"]:
key = spec["key"]
if key in deltas:
print(f" {spec['label']:<32} {deltas[key]:+7.1f}%")
path = out_path
md = ["# FlowTwin benchmark results", "",
"Generated by `scripts/run_benchmarks.py`. Every value is the mean ± "
"standard deviation over independent random seeds of the full "
"simulation. No value is entered by hand.", ""]
for scenario_id, payload in all_payloads.items():
md += [f"## {payload['scenario_name']}", "",
f"Venue `{payload['venue_id']}` · crowd {payload['crowd_size']:,} · "
f"{len(payload['seeds'])} seeds · generated {payload['generated_utc']}", "",
format_table(payload), ""]
for arm in payload["arms"]:
md.append(f"- **{arm['label']}** — {arm['description']}")
md.append("")
(out_dir / "BENCHMARKS.md").write_text("\n".join(md), encoding="utf-8")
print(f"\nSaved {path}")
print(f"Saved {out_dir / 'BENCHMARKS.md'}")
if __name__ == "__main__":
main()
|