Upload eval_job1.py with huggingface_hub

eval_job1.py

@@ -0,0 +1,180 @@
+# /// script
+# dependencies = [
+#     "transformers>=4.36.0",
+#     "peft>=0.7.0",
+#     "accelerate>=0.24.0",
+#     "torch",
+#     "datasets",
+#     "tqdm",
+# ]
+# ///
+
+"""
+Evaluate models on HumanEval with proper pass@1 execution.
+Compares base model vs fine-tuned adapter.
+"""
+
+import subprocess
+import tempfile
+import os
+import sys
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+from tqdm import tqdm
+
+# Configuration
+BASE_MODEL = "Qwen/Qwen3-0.6B"
+ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job1"
+NUM_PROBLEMS = 50  # Use 50 for faster eval, 164 for full
+
+print(f"Base model: {BASE_MODEL}")
+print(f"Adapter: {ADAPTER_MODEL}")
+print(f"Problems: {NUM_PROBLEMS}")
+
+# Load HumanEval
+print("\nLoading HumanEval dataset...")
+humaneval = load_dataset("openai/openai_humaneval", split="test")
+if NUM_PROBLEMS < 164:
+    humaneval = humaneval.select(range(NUM_PROBLEMS))
+print(f"Using {len(humaneval)} problems")
+
+
+def extract_function(text, entry_point):
+    """Extract function body from generated text."""
+    lines = text.split("\n")
+    result = []
+    in_func = False
+    base_indent = None
+
+    for line in lines:
+        stripped = line.lstrip()
+        if stripped.startswith(f"def {entry_point}"):
+            in_func = True
+            result.append(line)
+            base_indent = len(line) - len(stripped)
+        elif in_func:
+            current_indent = (
+                len(line) - len(line.lstrip()) if line.strip() else base_indent + 4
+            )
+            if line.strip() == "":
+                result.append("")
+            elif current_indent > base_indent or not line.strip():
+                result.append(line)
+            elif stripped.startswith("def ") or stripped.startswith("class "):
+                break
+            else:
+                # Check if it's a continuation
+                if current_indent > base_indent:
+                    result.append(line)
+                else:
+                    break
+
+    return "\n".join(result)
+
+
+def run_test(code, test, timeout=5):
+    """Execute code with test cases."""
+    full_code = code + "\n\n" + test
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+        f.write(full_code)
+        tmp_path = f.name
+
+    try:
+        result = subprocess.run(
+            [sys.executable, tmp_path], capture_output=True, timeout=timeout, text=True
+        )
+        return result.returncode == 0
+    except (subprocess.TimeoutExpired, Exception):
+        return False
+    finally:
+        try:
+            os.unlink(tmp_path)
+        except:
+            pass
+
+
+def evaluate_model(model, tokenizer, problems, model_name):
+    """Evaluate a model on HumanEval problems."""
+    results = []
+
+    print(f"\nEvaluating: {model_name}")
+    for problem in tqdm(problems, desc=model_name):
+        prompt = problem["prompt"]
+        entry_point = problem["entry_point"]
+        test = problem["test"]
+
+        # Generate
+        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+        with torch.no_grad():
+            outputs = model.generate(
+                **inputs,
+                max_new_tokens=512,
+                temperature=0.2,
+                top_p=0.95,
+                do_sample=True,
+                pad_token_id=tokenizer.eos_token_id,
+            )
+
+        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        code = extract_function(generated, entry_point)
+
+        # Test
+        passed = run_test(code, test)
+        results.append(passed)
+
+    score = sum(results) / len(results) * 100
+    return score, sum(results), len(results)
+
+
+# Load tokenizer
+print("\nLoading tokenizer...")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+# Evaluate BASE model
+print("\nLoading base model...")
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
+)
+
+base_score, base_passed, base_total = evaluate_model(
+    base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
+)
+
+# Clear memory
+del base_model
+torch.cuda.empty_cache()
+
+# Evaluate FINE-TUNED model
+print(f"\nLoading fine-tuned model from {ADAPTER_MODEL}...")
+try:
+    ft_model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
+    )
+    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
+
+    ft_score, ft_passed, ft_total = evaluate_model(
+        ft_model, tokenizer, humaneval, "Fine-tuned"
+    )
+except Exception as e:
+    print(f"Error loading adapter: {e}")
+    ft_score, ft_passed, ft_total = 0, 0, NUM_PROBLEMS
+
+# Results
+print("\n" + "=" * 60)
+print("HUMANEVAL RESULTS")
+print("=" * 60)
+print(f"Base Qwen3-0.6B:  {base_score:.1f}% ({base_passed}/{base_total})")
+print(f"Fine-tuned:       {ft_score:.1f}% ({ft_passed}/{ft_total})")
+print(f"Difference:       {ft_score - base_score:+.1f}%")
+print("=" * 60)
+
+if ft_score > base_score:
+    print("SUCCESS! Fine-tuned model beats base model!")
+else:
+    print("Fine-tuned model did not beat base model.")