Spaces:
Running
Running
File size: 961 Bytes
2414d31 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | """Generate frozen held-out eval scenarios (seeds 10000-10099)."""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from server.scenarios import DIFFICULTIES, generate
def main() -> None:
out_dir = ROOT / "scenarios"
out_dir.mkdir(exist_ok=True)
scenarios = []
for seed in range(10000, 10100):
for diff in DIFFICULTIES:
sc = generate(seed=seed, task_id=diff)
sc["seed"] = seed
scenarios.append(sc)
out_path = out_dir / "eval_held_out.json"
with open(out_path, "w") as f:
json.dump(scenarios, f, indent=2)
print(f"Generated {len(scenarios)} scenarios → {out_path}")
families = {}
for sc in scenarios:
families[sc["family"]] = families.get(sc["family"], 0) + 1
print(f"Family distribution: {families}")
if __name__ == "__main__":
main()
|