import os
import subprocess
# Define the tasks you want to evaluate
tasks = "mmlu,gsm8k,winogrande,hellaswag"

# Command to run evaluation using the HF backend
# We use device_map="auto" and load_in_4bit to fit in 16GB VRAM
eval_cmd = [
    "lm_eval",
    "--model", "hf",
    "--model_args", "pretrained=Girikannan/sarvam-eval,load_in_4bit=True,device_map=auto,trust_remote_code=True",
    "--tasks", tasks,
    "--batch_size", "1",
    "--output_path", "./eval_results/results.json"
]

print("Starting evaluation...")
subprocess.run(eval_cmd)