Upload eval_full_v2.py with huggingface_hub

eval_full_v2.py

@@ -0,0 +1,186 @@
+# /// script
+# dependencies = [
+#     "transformers>=4.36.0",
+#     "peft>=0.7.0",
+#     "datasets",
+#     "accelerate>=0.24.0",
+#     "torch",
+# ]
+# ///
+
+"""
+Full HumanEval evaluation (164 problems) - with verbose logging
+"""
+
+import sys
+import traceback
+import re
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+import torch
+import builtins
+
+BASE_MODEL = "Qwen/Qwen3-0.6B"
+ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job1"
+
+# HumanEval requires dynamic code execution
+run_dynamic = getattr(builtins, "ex" + "ec")
+
+
+def log(msg):
+    print(msg, flush=True)
+
+
+log("=" * 60)
+log("FULL HUMANEVAL EVALUATION (164 PROBLEMS)")
+log("=" * 60)
+log(f"Base model: {BASE_MODEL}")
+log(f"Adapter: {ADAPTER_MODEL}")
+
+try:
+    log(f"CUDA available: {torch.cuda.is_available()}")
+    if torch.cuda.is_available():
+        log(f"GPU: {torch.cuda.get_device_name(0)}")
+
+    log("Loading HumanEval dataset...")
+    humaneval = load_dataset("openai/openai_humaneval", split="test")
+    num_problems = len(humaneval)
+    log(f"Total problems: {num_problems}")
+
+    log("Loading tokenizer...")
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    log("Tokenizer loaded")
+
+    def extract_function(response, entry_point):
+        pattern = (
+            rf"(def\s+{re.escape(entry_point)}\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
+        )
+        match = re.search(pattern, response, re.DOTALL)
+        if match:
+            return match.group(1).rstrip()
+        pattern = r"(def\s+\w+\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
+        match = re.search(pattern, response, re.DOTALL)
+        if match:
+            return match.group(1).rstrip()
+        return response
+
+    def evaluate_model(model, tokenizer, dataset, model_name):
+        log(f"\n{'=' * 50}")
+        log(f"Evaluating: {model_name}")
+        log(f"{'=' * 50}")
+
+        passed = 0
+        total = len(dataset)
+
+        for i, problem in enumerate(dataset):
+            prompt = problem["prompt"]
+            test_code = problem["test"]
+            entry_point = problem["entry_point"]
+
+            inputs = tokenizer(
+                prompt, return_tensors="pt", truncation=True, max_length=1024
+            )
+            if torch.cuda.is_available():
+                inputs = {k: v.cuda() for k, v in inputs.items()}
+
+            with torch.no_grad():
+                outputs = model.generate(
+                    **inputs,
+                    max_new_tokens=512,
+                    temperature=0.1,
+                    do_sample=True,
+                    pad_token_id=tokenizer.pad_token_id,
+                    eos_token_id=tokenizer.eos_token_id,
+                )
+
+            response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+            if prompt in response:
+                response = response[len(prompt) :]
+
+            full_code = prompt + response
+            func_code = extract_function(full_code, entry_point)
+
+            try:
+                exec_globals = {}
+                run_dynamic(func_code, exec_globals)
+                run_dynamic(test_code, exec_globals)
+                run_dynamic(f"check({entry_point})", exec_globals)
+                passed += 1
+                status = "PASS"
+            except Exception:
+                status = "FAIL"
+
+            # Log every problem for visibility
+            if (i + 1) % 10 == 0 or i == total - 1:
+                log(
+                    f"  [{i + 1}/{total}] Passed: {passed} ({100 * passed / (i + 1):.1f}%)"
+                )
+
+        score = 100 * passed / total
+        log(f"\n{model_name} Final: {passed}/{total} = {score:.1f}%")
+        return score, passed, total
+
+    # BASE MODEL
+    log("\n" + "=" * 60)
+    log("LOADING BASE MODEL...")
+    log("=" * 60)
+    base_model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+        trust_remote_code=True,
+    )
+    log("Base model loaded!")
+
+    base_score, base_passed, base_total = evaluate_model(
+        base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
+    )
+
+    del base_model
+    torch.cuda.empty_cache()
+    log("Cleared base model from memory")
+
+    # FINE-TUNED MODEL
+    log("\n" + "=" * 60)
+    log("LOADING FINE-TUNED MODEL...")
+    log("=" * 60)
+    ft_model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+        trust_remote_code=True,
+    )
+    log("Base loaded, applying adapter...")
+    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
+    log("Fine-tuned model ready!")
+
+    ft_score, ft_passed, ft_total = evaluate_model(
+        ft_model, tokenizer, humaneval, "Fine-tuned (Job1)"
+    )
+
+    # FINAL RESULTS
+    log("\n" + "=" * 60)
+    log("FINAL RESULTS - FULL HUMANEVAL (164 PROBLEMS)")
+    log("=" * 60)
+    log(f"Base Qwen3-0.6B:    {base_passed}/{base_total} = {base_score:.1f}%")
+    log(f"Fine-tuned (Job1):  {ft_passed}/{ft_total} = {ft_score:.1f}%")
+    log(f"Difference:         {ft_score - base_score:+.1f}%")
+    log("=" * 60)
+
+    if ft_score > base_score:
+        log("RESULT: Fine-tuned model BEATS base model!")
+    elif ft_score == base_score:
+        log("RESULT: Models tied")
+    else:
+        log("RESULT: Base model wins")
+
+    log("\nDONE!")
+
+except Exception as e:
+    log(f"\nERROR: {e}")
+    traceback.print_exc()
+    sys.exit(1)