"""Spawn evals of effort-trained blends at each reasoning_effort, on the deployed beam-dory-eval app.

  MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py smoke   # first 15 problems per benchmark, 2 loops
  MODAL_PROFILE=aside python3 scripts/spawn_effort_evals.py full MODEL LOOPS EFFORTS SHARDS
      e.g. full merges/rE-t030 2 low,default,high 2

"default" means no reasoning_effort (the untagged prompt). Prints one JSON line per spawned call.
"""
import json
import sys

import modal

fn = modal.Function.from_name("beam-dory-eval", "evaluate", environment_name="main")


def spawn(model, loops, effort, limit=0, shard=0, n=1):
    eff = "" if effort == "default" else effort
    call = fn.spawn(model, loops, limit, 65536, False, "", shard, n, eff)
    print(json.dumps({"model": model, "loops": loops, "effort": effort, "limit": limit, "shard": shard,
                      "n": n, "call": call.object_id}), flush=True)


if sys.argv[1] == "smoke":
    for model in ("merges/rE-t050", "merges/rE-t030"):
        for effort in ("low", "default", "high"):
            spawn(model, 2, effort, limit=15)
else:
    _, _, model, loops, efforts, shards = sys.argv
    for effort in efforts.split(","):
        for i in range(int(shards)):
            spawn(model, int(loops), effort, shard=i, n=int(shards))