Upload eval_v2_finetuned_all.py with huggingface_hub

eval_v2_finetuned_all.py

@@ -0,0 +1,74 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "lighteval>=0.6.0",
+#     "torch>=2.0.0",
+#     "transformers>=4.40.0",
+#     "accelerate>=0.30.0",
+#     "peft>=0.10.0",
+# ]
+# ///
+"""
+v2 Finetuned: All 6 benchmarks (MMLU, GSM8K, ARC-C, Winogrande, TruthfulQA, HellaSwag).
+Merges LoRA adapter before evaluation.
+"""
+
+import gc
+import glob
+import os
+import subprocess
+
+def main():
+    hf_token = os.getenv("HF_TOKEN")
+    if hf_token:
+        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
+        os.environ.setdefault("HF_HUB_TOKEN", hf_token)
+    os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    from peft import PeftModel
+    import torch
+
+    print("Merging v2 adapter...")
+    model = AutoModelForCausalLM.from_pretrained(
+        "LiquidAI/LFM2.5-1.2B-Instruct",
+        trust_remote_code=True,
+        torch_dtype=torch.float16,
+        device_map="cpu",
+    )
+    model = PeftModel.from_pretrained(model, "wheattoast11/agent-zero-lfm-1.2b-v2")
+    model = model.merge_and_unload()
+
+    merged_path = "/tmp/merged_model_v2"
+    model.save_pretrained(merged_path)
+    tokenizer = AutoTokenizer.from_pretrained(
+        "wheattoast11/agent-zero-lfm-1.2b-v2",
+        trust_remote_code=True,
+    )
+    tokenizer.save_pretrained(merged_path)
+    del model, tokenizer
+    gc.collect()
+    print("Adapter merged.")
+
+    model_args = f"model_name={merged_path},trust_remote_code=True,dtype=float16,max_length=2048"
+
+    # Run in two batches to manage memory
+    batches = [
+        "leaderboard|mmlu:abstract_algebra|5,leaderboard|mmlu:anatomy|5,leaderboard|mmlu:astronomy|5,leaderboard|mmlu:business_ethics|5,leaderboard|mmlu:clinical_knowledge|5,leaderboard|gsm8k|5",
+        "leaderboard|hellaswag|0,leaderboard|arc:challenge|25,leaderboard|truthfulqa:mc|0,leaderboard|winogrande|5",
+    ]
+
+    for i, tasks in enumerate(batches):
+        out_dir = f"/tmp/results_v2_batch{i}"
+        cmd = ["lighteval", "accelerate", model_args, tasks, "--output-dir", out_dir]
+        print(f"\nBatch {i}: {' '.join(cmd)}")
+        subprocess.run(cmd, check=True)
+
+    print("\n=== ALL RESULTS ===")
+    for f in sorted(glob.glob("/tmp/results_v2_*/**/*.json", recursive=True)):
+        print(f"\n=== {f} ===")
+        with open(f) as fh:
+            print(fh.read()[:10000])
+
+if __name__ == "__main__":
+    main()