File size: 1,317 Bytes
0052e99 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | """Spawn evals of effort-trained blends at each reasoning_effort, on the deployed beam-dory-eval app.
MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py smoke # first 15 problems per benchmark, 2 loops
MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py full MODEL LOOPS EFFORTS SHARDS
e.g. full merges/rE-t030 2 low,default,high 2
"default" means no reasoning_effort (the untagged prompt). Prints one JSON line per spawned call.
"""
import json
import sys
import modal
fn = modal.Function.from_name("beam-dory-eval", "evaluate", environment_name="main")
def spawn(model, loops, effort, limit=0, shard=0, n=1):
eff = "" if effort == "default" else effort
call = fn.spawn(model, loops, limit, 65536, False, "", shard, n, eff)
print(json.dumps({"model": model, "loops": loops, "effort": effort, "limit": limit, "shard": shard,
"n": n, "call": call.object_id}), flush=True)
if sys.argv[1] == "smoke":
for model in ("merges/rE-t050", "merges/rE-t030"):
for effort in ("low", "default", "high"):
spawn(model, 2, effort, limit=15)
else:
_, _, model, loops, efforts, shards = sys.argv
for effort in efforts.split(","):
for i in range(int(shards)):
spawn(model, int(loops), effort, shard=i, n=int(shards))
|