File size: 662 Bytes
e33c018 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "lighteval[accelerate]>=0.6.0",
# "torch>=2.0.0",
# "transformers>=4.40.0",
# "accelerate>=0.30.0",
# ]
# ///
"""Evaluate baseline LiquidAI/LFM2.5-1.2B-Instruct using accelerate backend."""
import subprocess
import sys
model_args = "model_name=LiquidAI/LFM2.5-1.2B-Instruct,trust_remote_code=True,dtype=bfloat16"
tasks = "leaderboard|mmlu|5,leaderboard|arc:challenge|25,leaderboard|truthfulqa:mc|0"
cmd = ["lighteval", "accelerate", model_args, tasks, "--output-dir", "./eval_baseline"]
print(f"Running: {' '.join(cmd)}")
result = subprocess.run(cmd)
sys.exit(result.returncode)
|