| import os | |
| import subprocess | |
| # Define the tasks you want to evaluate | |
| tasks = "mmlu,gsm8k,winogrande,hellaswag" | |
| # Command to run evaluation using the HF backend | |
| # We use device_map="auto" and load_in_4bit to fit in 16GB VRAM | |
| eval_cmd = [ | |
| "lm_eval", | |
| "--model", "hf", | |
| "--model_args", "pretrained=Girikannan/sarvam-eval,load_in_4bit=True,device_map=auto,trust_remote_code=True", | |
| "--tasks", tasks, | |
| "--batch_size", "1", | |
| "--output_path", "./eval_results/results.json" | |
| ] | |
| print("Starting evaluation...") | |
| subprocess.run(eval_cmd) |