File size: 821 Bytes
9a57d8a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | """Autoregressive baselines on the same lm-eval tasks, zero-shot, one task at a time."""
import os, json, lm_eval
tasks = os.environ.get("EV_TASKS", "hellaswag,arc_easy,arc_challenge,piqa,winogrande,openbookqa,sciq,lambada_openai,mmlu").split(",")
name = os.environ["EV_MODEL"]
out = os.environ.get("EV_OUT", "/tmp/bench_ar.json")
res = {}
for t in tasks:
try:
r = lm_eval.simple_evaluate(model="hf", model_args="pretrained=%s,dtype=bfloat16" % name, tasks=[t],
num_fewshot=0, batch_size="auto", bootstrap_iters=0)
res.update(r["results"])
print("RESULT", t, json.dumps(r["results"].get(t, {})), flush=True)
except Exception as e:
print("TASKFAIL", t, str(e)[:200], flush=True)
json.dump(res, open(out, "w"))
print("BENCH DONE", flush=True)
|