| """Spawn evals of effort-trained blends at each reasoning_effort, on the deployed beam-dory-eval app. |
| |
| MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py smoke # first 15 problems per benchmark, 2 loops |
| MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py full MODEL LOOPS EFFORTS SHARDS |
| e.g. full merges/rE-t030 2 low,default,high 2 |
| |
| "default" means no reasoning_effort (the untagged prompt). Prints one JSON line per spawned call. |
| """ |
| import json |
| import sys |
|
|
| import modal |
|
|
| fn = modal.Function.from_name("beam-dory-eval", "evaluate", environment_name="main") |
|
|
|
|
| def spawn(model, loops, effort, limit=0, shard=0, n=1): |
| eff = "" if effort == "default" else effort |
| call = fn.spawn(model, loops, limit, 65536, False, "", shard, n, eff) |
| print(json.dumps({"model": model, "loops": loops, "effort": effort, "limit": limit, "shard": shard, |
| "n": n, "call": call.object_id}), flush=True) |
|
|
|
|
| if sys.argv[1] == "smoke": |
| for model in ("merges/rE-t050", "merges/rE-t030"): |
| for effort in ("low", "default", "high"): |
| spawn(model, 2, effort, limit=15) |
| else: |
| _, _, model, loops, efforts, shards = sys.argv |
| for effort in efforts.split(","): |
| for i in range(int(shards)): |
| spawn(model, int(loops), effort, shard=i, n=int(shards)) |
|
|
