Spaces:
Sleeping
Sleeping
File size: 2,582 Bytes
cb330aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | #!/usr/bin/env python3
"""
Benchmark SYNAPSE-X agents across all task difficulties.
"""
import json
import random
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from agents.baseline import run_episode as run_baseline_episode
from agents.random_agent import run_episode as run_random_episode
from env.environment import SynapseXEnvironment
from env.grader import TASK_REGISTRY, TASK_SEEDS, grade, grade_with_variance
def evaluate_agent(agent_name: str, runner):
results = {}
variance = {}
for task_name in TASK_REGISTRY:
task_env = SynapseXEnvironment(task_config=TASK_REGISTRY[task_name], seed=TASK_SEEDS[task_name])
actions = runner(task_env)
results[task_name] = grade(task_name, actions).model_dump()
variance[task_name] = grade_with_variance(
task_name,
lambda _task_name, _obs, env: runner(env),
).model_dump()
return {"agent": agent_name, "results": results, "variance": variance}
def main():
baseline = evaluate_agent("baseline", run_baseline_episode)
random_agent = evaluate_agent(
"random",
lambda env: run_random_episode(env, rng=random.Random(42)),
)
summary = {"baseline": baseline["results"], "random": random_agent["results"]}
variance_summary = {"baseline": baseline["variance"], "random": random_agent["variance"]}
task_names = list(TASK_REGISTRY.keys())
print("Agent " + " ".join(f"{task_name:>7}" for task_name in task_names))
print("-" * (11 + 9 * len(task_names)))
print(
"Random "
+ " ".join(f"{summary['random'][task_name]['score']:.4f}" for task_name in task_names)
)
print(
"Baseline "
+ " ".join(f"{summary['baseline'][task_name]['score']:.4f}" for task_name in task_names)
)
print()
print("Variance mean scores")
print(
"Random "
+ " ".join(f"{variance_summary['random'][task_name]['score']:.4f}" for task_name in task_names)
)
print(
"Baseline "
+ " ".join(f"{variance_summary['baseline'][task_name]['score']:.4f}" for task_name in task_names)
)
print()
print(json.dumps({"canonical": summary, "variance_mean": variance_summary}, indent=2))
with open(PROJECT_ROOT / "benchmark_results.json", "w", encoding="utf-8") as handle:
json.dump({"canonical": summary, "variance_mean": variance_summary}, handle, indent=2)
if __name__ == "__main__":
main()
|