File size: 746 Bytes
322750b 59141be 322750b 126bf9d 9abfaea 322750b 59141be 9c86ea0 322750b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "lighteval>=0.6.0",
# "torch>=2.0.0",
# "transformers>=4.40.0",
# "accelerate>=0.30.0",
# "peft>=0.7.0",
# ]
# ///
"""Evaluate fine-tuned wheattoast11/agent-zero-lfm-1.2b-v1 on standard benchmarks."""
import subprocess
import sys
model_args = "model_name=wheattoast11/agent-zero-lfm-1.2b-v1,trust_remote_code=True"
tasks = "leaderboard|mmlu|5|0,leaderboard|arc:challenge|0|0,leaderboard|truthfulqa:mc|0|0"
cmd = [
sys.executable, "-m", "lighteval", "accelerate",
model_args,
tasks,
"--output-dir", "./eval_results_finetuned",
]
print(f"Running: {' '.join(cmd)}")
result = subprocess.run(cmd, capture_output=False)
sys.exit(result.returncode)
|