| # /// script | |
| # requires-python = ">=3.10" | |
| # dependencies = [ | |
| # "lighteval[accelerate]>=0.6.0", | |
| # "torch>=2.0.0", | |
| # "transformers>=4.40.0", | |
| # "accelerate>=0.30.0", | |
| # ] | |
| # /// | |
| """Evaluate baseline LiquidAI/LFM2.5-1.2B-Instruct using accelerate backend.""" | |
| import subprocess | |
| import sys | |
| model_args = "model_name=LiquidAI/LFM2.5-1.2B-Instruct,trust_remote_code=True,dtype=bfloat16" | |
| tasks = "leaderboard|mmlu|5,leaderboard|arc:challenge|25,leaderboard|truthfulqa:mc|0" | |
| cmd = ["lighteval", "accelerate", model_args, tasks, "--output-dir", "./eval_baseline"] | |
| print(f"Running: {' '.join(cmd)}") | |
| result = subprocess.run(cmd) | |
| sys.exit(result.returncode) | |