# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "lighteval[accelerate]>=0.6.0",
#     "torch>=2.0.0",
#     "transformers>=4.40.0",
#     "accelerate>=0.30.0",
#     "peft>=0.7.0",
# ]
# ///

"""Evaluate fine-tuned wheattoast11/agent-zero-lfm-1.2b-v1 using accelerate backend."""

import subprocess
import sys

model_args = "model_name=wheattoast11/agent-zero-lfm-1.2b-v1,trust_remote_code=True,dtype=bfloat16"
tasks = "leaderboard|mmlu|5,leaderboard|arc:challenge|25,leaderboard|truthfulqa:mc|0"

cmd = ["lighteval", "accelerate", model_args, tasks, "--output-dir", "./eval_finetuned"]
print(f"Running: {' '.join(cmd)}")
result = subprocess.run(cmd)
sys.exit(result.returncode)