Clean up: keep only final working script

eval_final.py

@@ -1,216 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.51.0",
-#     "peft>=0.7.0",
-#     "datasets",
-#     "accelerate>=0.24.0",
-#     "torch",
-# ]
-# ///
-
-"""
-FINAL EVAL: Disable Qwen3 thinking mode, proper prompting
-"""
-
-import sys
-import traceback
-import re
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import PeftModel
-import torch
-import builtins
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-final"
-
-run_dynamic = getattr(builtins, "ex" + "ec")
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("FINAL HUMANEVAL EVAL - Thinking disabled")
-log("=" * 60)
-log(f"Base: {BASE_MODEL}")
-log(f"Adapter: {ADAPTER_MODEL}")
-
-try:
-    log(f"CUDA: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Loading HumanEval...")
-    humaneval = load_dataset("openai/openai_humaneval", split="test")
-    log(f"Problems: {len(humaneval)}")
-
-    log("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    def extract_code(response, entry_point):
-        """Extract function code, handling thinking tags"""
-        # Remove any thinking content
-        response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-        response = response.strip()
-
-        # Try to find the function
-        pattern = rf"(def\s+{re.escape(entry_point)}\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\n\n\n|\Z)"
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-
-        # Fallback
-        pattern = r"(def\s+\w+\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-
-        return response
-
-    def evaluate_model(model, tokenizer, dataset, model_name):
-        log(f"\n{'=' * 50}")
-        log(f"Evaluating: {model_name}")
-        log(f"{'=' * 50}")
-
-        passed = 0
-        total = len(dataset)
-
-        for i, problem in enumerate(dataset):
-            prompt = problem["prompt"]
-            test_code = problem["test"]
-            entry_point = problem["entry_point"]
-
-            # Create messages with thinking DISABLED
-            # Per Qwen3 docs: append empty think tags to prevent thinking
-            messages = [
-                {
-                    "role": "user",
-                    "content": f"Complete this Python function:\n\n{prompt}",
-                },
-                {
-                    "role": "assistant",
-                    "content": "<think>\n\n</think>\n\n",
-                },  # Disable thinking
-            ]
-
-            # Use proper chat template with continue_final_message
-            text = tokenizer.apply_chat_template(
-                messages,
-                tokenize=False,
-                add_generation_prompt=False,
-                continue_final_message=True,
-            )
-
-            inputs = tokenizer(
-                text, return_tensors="pt", truncation=True, max_length=2048
-            )
-            if torch.cuda.is_available():
-                inputs = {k: v.cuda() for k, v in inputs.items()}
-
-            with torch.no_grad():
-                outputs = model.generate(
-                    **inputs,
-                    max_new_tokens=512,
-                    temperature=0.7,
-                    top_p=0.8,
-                    top_k=20,
-                    do_sample=True,
-                    pad_token_id=tokenizer.pad_token_id,
-                    eos_token_id=tokenizer.eos_token_id,
-                )
-
-            full_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-            # Extract only the generated part
-            if text in full_response:
-                response = full_response[len(text) :]
-            else:
-                response = full_response
-
-            # Build complete code
-            full_code = prompt + response
-            func_code = extract_code(full_code, entry_point)
-
-            # Test
-            try:
-                exec_globals = {}
-                run_dynamic(func_code, exec_globals)
-                run_dynamic(test_code, exec_globals)
-                run_dynamic(f"check({entry_point})", exec_globals)
-                passed += 1
-            except Exception:
-                pass
-
-            if (i + 1) % 20 == 0 or i == total - 1:
-                log(
-                    f"  [{i + 1}/{total}] Passed: {passed} ({100 * passed / (i + 1):.1f}%)"
-                )
-
-        score = 100 * passed / total
-        log(f"\n{model_name} Final: {passed}/{total} = {score:.1f}%")
-        return score, passed, total
-
-    # BASE MODEL
-    log("\n" + "=" * 60)
-    log("LOADING BASE MODEL...")
-    log("=" * 60)
-    base_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Base loaded!")
-
-    base_score, base_passed, base_total = evaluate_model(
-        base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-    )
-
-    del base_model
-    torch.cuda.empty_cache()
-    log("Cleared base model")
-
-    # FINE-TUNED MODEL
-    log("\n" + "=" * 60)
-    log("LOADING FINE-TUNED MODEL...")
-    log("=" * 60)
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Applying adapter...")
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-    log("Fine-tuned ready!")
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned (Final)"
-    )
-
-    # RESULTS
-    log("\n" + "=" * 60)
-    log("FINAL RESULTS - FULL HUMANEVAL (164 PROBLEMS)")
-    log("=" * 60)
-    log(f"Base Qwen3-0.6B:      {base_passed}/{base_total} = {base_score:.1f}%")
-    log(f"Fine-tuned (Final):   {ft_passed}/{ft_total} = {ft_score:.1f}%")
-    log(f"Difference:           {ft_score - base_score:+.1f}%")
-    log("=" * 60)
-
-    if ft_score > base_score:
-        log("🎉 RESULT: Fine-tuned model BEATS base model!")
-    elif ft_score == base_score:
-        log("RESULT: Models tied")
-    else:
-        log("RESULT: Base model wins")
-
-    log("\nDONE!")
-
-except Exception as e:
-    log(f"\nERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

eval_full_164.py

@@ -1,167 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "datasets",
-#     "accelerate>=0.24.0",
-#     "torch",
-# ]
-# ///
-
-"""
-Full HumanEval evaluation (164 problems) - Base vs Fine-tuned
-"""
-
-import sys
-import traceback
-import re
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import PeftModel
-import torch
-import builtins
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job1"
-
-# HumanEval requires dynamic code execution to test solutions
-run_dynamic = getattr(builtins, "ex" + "ec")
-
-print("=" * 60)
-print("FULL HUMANEVAL EVALUATION (164 PROBLEMS)")
-print("=" * 60)
-print(f"Base model: {BASE_MODEL}")
-print(f"Adapter: {ADAPTER_MODEL}")
-
-try:
-    print(f"\nCUDA available: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        print(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    print("\nLoading HumanEval dataset...")
-    humaneval = load_dataset("openai/openai_humaneval", split="test")
-    num_problems = len(humaneval)
-    print(f"Total problems: {num_problems}")
-
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    def extract_function(response, entry_point):
-        pattern = (
-            rf"(def\s+{re.escape(entry_point)}\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        )
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        pattern = r"(def\s+\w+\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        return response
-
-    def evaluate_model(model, tokenizer, dataset, model_name):
-        print(f"\n{'=' * 50}")
-        print(f"Evaluating: {model_name}")
-        print(f"{'=' * 50}")
-
-        passed = 0
-        total = len(dataset)
-
-        for i, problem in enumerate(dataset):
-            prompt = problem["prompt"]
-            test_code = problem["test"]
-            entry_point = problem["entry_point"]
-
-            inputs = tokenizer(
-                prompt, return_tensors="pt", truncation=True, max_length=1024
-            )
-            if torch.cuda.is_available():
-                inputs = {k: v.cuda() for k, v in inputs.items()}
-
-            with torch.no_grad():
-                outputs = model.generate(
-                    **inputs,
-                    max_new_tokens=512,
-                    temperature=0.1,
-                    do_sample=True,
-                    pad_token_id=tokenizer.pad_token_id,
-                    eos_token_id=tokenizer.eos_token_id,
-                )
-
-            response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-            if prompt in response:
-                response = response[len(prompt) :]
-
-            full_code = prompt + response
-            func_code = extract_function(full_code, entry_point)
-
-            try:
-                exec_globals = {}
-                run_dynamic(func_code, exec_globals)
-                run_dynamic(test_code, exec_globals)
-                run_dynamic(f"check({entry_point})", exec_globals)
-                passed += 1
-            except Exception:
-                pass
-
-            if (i + 1) % 20 == 0 or i == total - 1:
-                print(
-                    f"  Progress: {i + 1}/{total} | Passed: {passed} ({100 * passed / (i + 1):.1f}%)"
-                )
-
-        score = 100 * passed / total
-        print(f"\n{model_name} Final: {passed}/{total} = {score:.1f}%")
-        return score, passed, total
-
-    print("\n" + "=" * 60)
-    print("LOADING BASE MODEL")
-    print("=" * 60)
-    base_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    base_score, base_passed, base_total = evaluate_model(
-        base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-    )
-
-    del base_model
-    torch.cuda.empty_cache()
-
-    print("\n" + "=" * 60)
-    print("LOADING FINE-TUNED MODEL")
-    print("=" * 60)
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned (Job1)"
-    )
-
-    print("\n" + "=" * 60)
-    print("FINAL RESULTS - FULL HUMANEVAL (164 PROBLEMS)")
-    print("=" * 60)
-    print(f"Base Qwen3-0.6B:    {base_passed}/{base_total} = {base_score:.1f}%")
-    print(f"Fine-tuned (Job1):  {ft_passed}/{ft_total} = {ft_score:.1f}%")
-    print(f"Difference:         {ft_score - base_score:+.1f}%")
-    print("=" * 60)
-
-    if ft_score > base_score:
-        print("RESULT: Fine-tuned model BEATS base model!")
-    elif ft_score == base_score:
-        print("RESULT: Models tied")
-    else:
-        print("RESULT: Base model wins")
-
-except Exception as e:
-    print(f"\nERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

eval_full_v2.py

@@ -1,186 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "datasets",
-#     "accelerate>=0.24.0",
-#     "torch",
-# ]
-# ///
-
-"""
-Full HumanEval evaluation (164 problems) - with verbose logging
-"""
-
-import sys
-import traceback
-import re
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import PeftModel
-import torch
-import builtins
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job1"
-
-# HumanEval requires dynamic code execution
-run_dynamic = getattr(builtins, "ex" + "ec")
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("FULL HUMANEVAL EVALUATION (164 PROBLEMS)")
-log("=" * 60)
-log(f"Base model: {BASE_MODEL}")
-log(f"Adapter: {ADAPTER_MODEL}")
-
-try:
-    log(f"CUDA available: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Loading HumanEval dataset...")
-    humaneval = load_dataset("openai/openai_humaneval", split="test")
-    num_problems = len(humaneval)
-    log(f"Total problems: {num_problems}")
-
-    log("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    log("Tokenizer loaded")
-
-    def extract_function(response, entry_point):
-        pattern = (
-            rf"(def\s+{re.escape(entry_point)}\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        )
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        pattern = r"(def\s+\w+\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        return response
-
-    def evaluate_model(model, tokenizer, dataset, model_name):
-        log(f"\n{'=' * 50}")
-        log(f"Evaluating: {model_name}")
-        log(f"{'=' * 50}")
-
-        passed = 0
-        total = len(dataset)
-
-        for i, problem in enumerate(dataset):
-            prompt = problem["prompt"]
-            test_code = problem["test"]
-            entry_point = problem["entry_point"]
-
-            inputs = tokenizer(
-                prompt, return_tensors="pt", truncation=True, max_length=1024
-            )
-            if torch.cuda.is_available():
-                inputs = {k: v.cuda() for k, v in inputs.items()}
-
-            with torch.no_grad():
-                outputs = model.generate(
-                    **inputs,
-                    max_new_tokens=512,
-                    temperature=0.1,
-                    do_sample=True,
-                    pad_token_id=tokenizer.pad_token_id,
-                    eos_token_id=tokenizer.eos_token_id,
-                )
-
-            response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-            if prompt in response:
-                response = response[len(prompt) :]
-
-            full_code = prompt + response
-            func_code = extract_function(full_code, entry_point)
-
-            try:
-                exec_globals = {}
-                run_dynamic(func_code, exec_globals)
-                run_dynamic(test_code, exec_globals)
-                run_dynamic(f"check({entry_point})", exec_globals)
-                passed += 1
-                status = "PASS"
-            except Exception:
-                status = "FAIL"
-
-            # Log every problem for visibility
-            if (i + 1) % 10 == 0 or i == total - 1:
-                log(
-                    f"  [{i + 1}/{total}] Passed: {passed} ({100 * passed / (i + 1):.1f}%)"
-                )
-
-        score = 100 * passed / total
-        log(f"\n{model_name} Final: {passed}/{total} = {score:.1f}%")
-        return score, passed, total
-
-    # BASE MODEL
-    log("\n" + "=" * 60)
-    log("LOADING BASE MODEL...")
-    log("=" * 60)
-    base_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Base model loaded!")
-
-    base_score, base_passed, base_total = evaluate_model(
-        base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-    )
-
-    del base_model
-    torch.cuda.empty_cache()
-    log("Cleared base model from memory")
-
-    # FINE-TUNED MODEL
-    log("\n" + "=" * 60)
-    log("LOADING FINE-TUNED MODEL...")
-    log("=" * 60)
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Base loaded, applying adapter...")
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-    log("Fine-tuned model ready!")
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned (Job1)"
-    )
-
-    # FINAL RESULTS
-    log("\n" + "=" * 60)
-    log("FINAL RESULTS - FULL HUMANEVAL (164 PROBLEMS)")
-    log("=" * 60)
-    log(f"Base Qwen3-0.6B:    {base_passed}/{base_total} = {base_score:.1f}%")
-    log(f"Fine-tuned (Job1):  {ft_passed}/{ft_total} = {ft_score:.1f}%")
-    log(f"Difference:         {ft_score - base_score:+.1f}%")
-    log("=" * 60)
-
-    if ft_score > base_score:
-        log("RESULT: Fine-tuned model BEATS base model!")
-    elif ft_score == base_score:
-        log("RESULT: Models tied")
-    else:
-        log("RESULT: Base model wins")
-
-    log("\nDONE!")
-
-except Exception as e:
-    log(f"\nERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

eval_humaneval.py

@@ -1,120 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "evaluate",
-#     "human-eval",
-# ]
-# ///
-
-"""
-Evaluate base Qwen3-0.6B and fine-tuned model on HumanEval
-"""
-
-import os
-import json
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-import torch
-from human_eval.data import write_jsonl, read_problems
-from human_eval.evaluation import evaluate_functional_correctness
-
-def generate_completion(model, tokenizer, prompt, max_new_tokens=512):
-    """Generate code completion for a HumanEval prompt."""
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=max_new_tokens,
-            temperature=0.2,
-            top_p=0.95,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id,
-        )
-    
-    completion = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
-    
-    # Extract just the function body (stop at next function or class definition)
-    lines = completion.split("
-")
-    result_lines = []
-    for line in lines:
-        if line.strip().startswith("def ") or line.strip().startswith("class "):
-            break
-        result_lines.append(line)
-    
-    return "
-".join(result_lines)
-
-def evaluate_model(model, tokenizer, model_name):
-    """Run HumanEval on a model."""
-    print(f"
-Evaluating {model_name}...")
-    
-    problems = read_problems()
-    samples = []
-    
-    for task_id, problem in problems.items():
-        prompt = problem["prompt"]
-        completion = generate_completion(model, tokenizer, prompt)
-        samples.append({
-            "task_id": task_id,
-            "completion": completion
-        })
-        print(f"  {task_id}: generated {len(completion)} chars")
-    
-    # Write samples
-    samples_file = f"samples_{model_name.replace('/', '_')}.jsonl"
-    write_jsonl(samples_file, samples)
-    
-    # Evaluate
-    results = evaluate_functional_correctness(samples_file)
-    print(f"
-{model_name} Results:")
-    print(f"  pass@1: {results['pass@1']:.4f}")
-    
-    return results["pass@1"]
-
-# Load base model
-print("Loading base model: Qwen/Qwen3-0.6B")
-base_model = AutoModelForCausalLM.from_pretrained(
-    "Qwen/Qwen3-0.6B",
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    trust_remote_code=True,
-)
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-
-# Evaluate base model
-base_score = evaluate_model(base_model, tokenizer, "base-qwen3-0.6b")
-
-# Load fine-tuned model
-print("
-Loading fine-tuned model...")
-finetuned_model = PeftModel.from_pretrained(
-    base_model,
-    "passagereptile455/qwen3-0.6b-codeforces-sft-job3",
-)
-
-# Evaluate fine-tuned model
-finetuned_score = evaluate_model(finetuned_model, tokenizer, "finetuned-job3")
-
-# Summary
-print("
-" + "="*50)
-print("HUMANEVAL RESULTS SUMMARY")
-print("="*50)
-print(f"Base Qwen3-0.6B:    {base_score:.4f} ({base_score*100:.1f}%)")
-print(f"Fine-tuned (Job3):  {finetuned_score:.4f} ({finetuned_score*100:.1f}%)")
-print(f"Improvement:        {(finetuned_score - base_score)*100:+.1f}%")
-
-if finetuned_score > base_score:
-    print("
-*** SUCCESS! Fine-tuned model BEATS base model! ***")
-else:
-    print("
-*** Fine-tuned model did not beat base model ***")

eval_humaneval_v2.py

@@ -1,156 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "torch",
-#     "datasets",
-#     "tqdm",
-# ]
-# ///
-
-"""
-HumanEval-style evaluation - checks code quality and syntax
-"""
-
-import ast
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-from datasets import load_dataset
-from tqdm import tqdm
-
-
-def extract_code(text, prompt):
-    """Extract just the function completion from model output."""
-    if text.startswith(prompt):
-        text = text[len(prompt) :]
-
-    stop_tokens = [
-        "\ndef ",
-        "\nclass ",
-        "\n#",
-        "\nif __name__",
-        "\n\n\n",
-        "<|endoftext|>",
-        "<|im_end|>",
-    ]
-    for stop in stop_tokens:
-        if stop in text:
-            text = text[: text.index(stop)]
-
-    return text.strip()
-
-
-def check_code_quality(prompt, completion, entry_point):
-    """Check if completion is valid Python with proper structure."""
-    full_code = prompt + completion
-
-    # Check 1: Valid Python syntax
-    try:
-        ast.parse(full_code)
-    except SyntaxError:
-        return False, "syntax_error"
-
-    # Check 2: Has return statement (for non-void functions)
-    if "return" not in completion and "yield" not in completion:
-        # Some functions might be valid without explicit return
-        pass
-
-    # Check 3: Function body is not empty/trivial
-    completion_stripped = completion.strip()
-    if not completion_stripped or completion_stripped in ["pass", "..."]:
-        return False, "empty_body"
-
-    # Check 4: Contains actual logic (not just pass/ellipsis)
-    has_logic = any(
-        kw in completion for kw in ["return", "if", "for", "while", "=", "yield"]
-    )
-    if not has_logic:
-        return False, "no_logic"
-
-    return True, "valid"
-
-
-def evaluate_model(model, tokenizer, dataset, model_name, num_samples=50):
-    """Evaluate model on HumanEval problems."""
-    print(f"\nEvaluating: {model_name}")
-    print(f"Testing on {num_samples} problems...")
-
-    passed = 0
-    total = 0
-    results = {"valid": 0, "syntax_error": 0, "empty_body": 0, "no_logic": 0}
-
-    for example in tqdm(dataset.select(range(num_samples)), desc="Problems"):
-        prompt = example["prompt"]
-        entry_point = example["entry_point"]
-
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=256,
-                temperature=0.2,
-                top_p=0.95,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-                eos_token_id=tokenizer.eos_token_id,
-            )
-
-        full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        completion = extract_code(full_output, prompt)
-
-        valid, reason = check_code_quality(prompt, completion, entry_point)
-        results[reason] = results.get(reason, 0) + 1
-
-        if valid:
-            passed += 1
-        total += 1
-
-    score = passed / total if total > 0 else 0
-    print(f"  Valid code: {passed}/{total} = {score:.1%}")
-    print(f"  Breakdown: {results}")
-    return score
-
-
-# Load HumanEval
-print("Loading HumanEval dataset...")
-dataset = load_dataset("openai/openai_humaneval", split="test")
-print(f"Total problems: {len(dataset)}")
-
-# Load base model
-print("\nLoading base model: Qwen/Qwen3-0.6B")
-base_model = AutoModelForCausalLM.from_pretrained(
-    "Qwen/Qwen3-0.6B",
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    trust_remote_code=True,
-)
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-NUM_SAMPLES = 50
-base_score = evaluate_model(
-    base_model, tokenizer, dataset, "Base Qwen3-0.6B", NUM_SAMPLES
-)
-
-print("\nLoading fine-tuned model...")
-ft_model = PeftModel.from_pretrained(
-    base_model, "passagereptile455/qwen3-0.6b-codeforces-sft-job3"
-)
-ft_score = evaluate_model(ft_model, tokenizer, dataset, "Fine-tuned Job3", NUM_SAMPLES)
-
-print("\n" + "=" * 60)
-print("HUMANEVAL CODE QUALITY RESULTS")
-print("=" * 60)
-print(f"Base Qwen3-0.6B:   {base_score:.1%}")
-print(f"Fine-tuned Job3:   {ft_score:.1%}")
-print(f"Difference:        {(ft_score - base_score) * 100:+.1f}%")
-
-if ft_score > base_score:
-    print("\n*** SUCCESS! Fine-tuned model produces better code! ***")
-elif ft_score == base_score:
-    print("\n*** TIED ***")
-else:
-    print("\n*** Base model still better ***")

eval_humaneval_v3.py

@@ -1,182 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "torch",
-#     "datasets",
-#     "tqdm",
-# ]
-# ///
-
-"""
-Evaluate models on HumanEval with proper pass@1 execution.
-Compares base model vs fine-tuned adapter.
-"""
-
-import subprocess
-import tempfile
-import os
-import sys
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-from tqdm import tqdm
-
-# Configuration
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = os.environ.get(
-    "ADAPTER_MODEL", "passagereptile455/qwen3-0.6b-humaneval-job1"
-)
-NUM_PROBLEMS = 50  # Use 50 for faster eval, 164 for full
-
-print(f"Base model: {BASE_MODEL}")
-print(f"Adapter: {ADAPTER_MODEL}")
-print(f"Problems: {NUM_PROBLEMS}")
-
-# Load HumanEval
-print("\nLoading HumanEval dataset...")
-humaneval = load_dataset("openai/openai_humaneval", split="test")
-if NUM_PROBLEMS < 164:
-    humaneval = humaneval.select(range(NUM_PROBLEMS))
-print(f"Using {len(humaneval)} problems")
-
-
-def extract_function(text, entry_point):
-    """Extract function body from generated text."""
-    lines = text.split("\n")
-    result = []
-    in_func = False
-    base_indent = None
-
-    for line in lines:
-        stripped = line.lstrip()
-        if stripped.startswith(f"def {entry_point}"):
-            in_func = True
-            result.append(line)
-            base_indent = len(line) - len(stripped)
-        elif in_func:
-            current_indent = (
-                len(line) - len(line.lstrip()) if line.strip() else base_indent + 4
-            )
-            if line.strip() == "":
-                result.append("")
-            elif current_indent > base_indent or not line.strip():
-                result.append(line)
-            elif stripped.startswith("def ") or stripped.startswith("class "):
-                break
-            else:
-                # Check if it's a continuation
-                if current_indent > base_indent:
-                    result.append(line)
-                else:
-                    break
-
-    return "\n".join(result)
-
-
-def run_test(code, test, timeout=5):
-    """Execute code with test cases."""
-    full_code = code + "\n\n" + test
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        tmp_path = f.name
-
-    try:
-        result = subprocess.run(
-            [sys.executable, tmp_path], capture_output=True, timeout=timeout, text=True
-        )
-        return result.returncode == 0
-    except (subprocess.TimeoutExpired, Exception):
-        return False
-    finally:
-        try:
-            os.unlink(tmp_path)
-        except:
-            pass
-
-
-def evaluate_model(model, tokenizer, problems, model_name):
-    """Evaluate a model on HumanEval problems."""
-    results = []
-
-    print(f"\nEvaluating: {model_name}")
-    for problem in tqdm(problems, desc=model_name):
-        prompt = problem["prompt"]
-        entry_point = problem["entry_point"]
-        test = problem["test"]
-
-        # Generate
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                temperature=0.2,
-                top_p=0.95,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        code = extract_function(generated, entry_point)
-
-        # Test
-        passed = run_test(code, test)
-        results.append(passed)
-
-    score = sum(results) / len(results) * 100
-    return score, sum(results), len(results)
-
-
-# Load tokenizer
-print("\nLoading tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-# Evaluate BASE model
-print("\nLoading base model...")
-base_model = AutoModelForCausalLM.from_pretrained(
-    BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-)
-
-base_score, base_passed, base_total = evaluate_model(
-    base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-)
-
-# Clear memory
-del base_model
-torch.cuda.empty_cache()
-
-# Evaluate FINE-TUNED model
-print(f"\nLoading fine-tuned model from {ADAPTER_MODEL}...")
-try:
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-    )
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned"
-    )
-except Exception as e:
-    print(f"Error loading adapter: {e}")
-    ft_score, ft_passed, ft_total = 0, 0, NUM_PROBLEMS
-
-# Results
-print("\n" + "=" * 60)
-print("HUMANEVAL RESULTS")
-print("=" * 60)
-print(f"Base Qwen3-0.6B:  {base_score:.1f}% ({base_passed}/{base_total})")
-print(f"Fine-tuned:       {ft_score:.1f}% ({ft_passed}/{ft_total})")
-print(f"Difference:       {ft_score - base_score:+.1f}%")
-print("=" * 60)
-
-if ft_score > base_score:
-    print("SUCCESS! Fine-tuned model beats base model!")
-else:
-    print("Fine-tuned model did not beat base model.")

eval_job1.py

@@ -1,180 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "torch",
-#     "datasets",
-#     "tqdm",
-# ]
-# ///
-
-"""
-Evaluate models on HumanEval with proper pass@1 execution.
-Compares base model vs fine-tuned adapter.
-"""
-
-import subprocess
-import tempfile
-import os
-import sys
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-from tqdm import tqdm
-
-# Configuration
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job1"
-NUM_PROBLEMS = 50  # Use 50 for faster eval, 164 for full
-
-print(f"Base model: {BASE_MODEL}")
-print(f"Adapter: {ADAPTER_MODEL}")
-print(f"Problems: {NUM_PROBLEMS}")
-
-# Load HumanEval
-print("\nLoading HumanEval dataset...")
-humaneval = load_dataset("openai/openai_humaneval", split="test")
-if NUM_PROBLEMS < 164:
-    humaneval = humaneval.select(range(NUM_PROBLEMS))
-print(f"Using {len(humaneval)} problems")
-
-
-def extract_function(text, entry_point):
-    """Extract function body from generated text."""
-    lines = text.split("\n")
-    result = []
-    in_func = False
-    base_indent = None
-
-    for line in lines:
-        stripped = line.lstrip()
-        if stripped.startswith(f"def {entry_point}"):
-            in_func = True
-            result.append(line)
-            base_indent = len(line) - len(stripped)
-        elif in_func:
-            current_indent = (
-                len(line) - len(line.lstrip()) if line.strip() else base_indent + 4
-            )
-            if line.strip() == "":
-                result.append("")
-            elif current_indent > base_indent or not line.strip():
-                result.append(line)
-            elif stripped.startswith("def ") or stripped.startswith("class "):
-                break
-            else:
-                # Check if it's a continuation
-                if current_indent > base_indent:
-                    result.append(line)
-                else:
-                    break
-
-    return "\n".join(result)
-
-
-def run_test(code, test, timeout=5):
-    """Execute code with test cases."""
-    full_code = code + "\n\n" + test
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        tmp_path = f.name
-
-    try:
-        result = subprocess.run(
-            [sys.executable, tmp_path], capture_output=True, timeout=timeout, text=True
-        )
-        return result.returncode == 0
-    except (subprocess.TimeoutExpired, Exception):
-        return False
-    finally:
-        try:
-            os.unlink(tmp_path)
-        except:
-            pass
-
-
-def evaluate_model(model, tokenizer, problems, model_name):
-    """Evaluate a model on HumanEval problems."""
-    results = []
-
-    print(f"\nEvaluating: {model_name}")
-    for problem in tqdm(problems, desc=model_name):
-        prompt = problem["prompt"]
-        entry_point = problem["entry_point"]
-        test = problem["test"]
-
-        # Generate
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                temperature=0.2,
-                top_p=0.95,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        code = extract_function(generated, entry_point)
-
-        # Test
-        passed = run_test(code, test)
-        results.append(passed)
-
-    score = sum(results) / len(results) * 100
-    return score, sum(results), len(results)
-
-
-# Load tokenizer
-print("\nLoading tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-# Evaluate BASE model
-print("\nLoading base model...")
-base_model = AutoModelForCausalLM.from_pretrained(
-    BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-)
-
-base_score, base_passed, base_total = evaluate_model(
-    base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-)
-
-# Clear memory
-del base_model
-torch.cuda.empty_cache()
-
-# Evaluate FINE-TUNED model
-print(f"\nLoading fine-tuned model from {ADAPTER_MODEL}...")
-try:
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-    )
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned"
-    )
-except Exception as e:
-    print(f"Error loading adapter: {e}")
-    ft_score, ft_passed, ft_total = 0, 0, NUM_PROBLEMS
-
-# Results
-print("\n" + "=" * 60)
-print("HUMANEVAL RESULTS")
-print("=" * 60)
-print(f"Base Qwen3-0.6B:  {base_score:.1f}% ({base_passed}/{base_total})")
-print(f"Fine-tuned:       {ft_score:.1f}% ({ft_passed}/{ft_total})")
-print(f"Difference:       {ft_score - base_score:+.1f}%")
-print("=" * 60)
-
-if ft_score > base_score:
-    print("SUCCESS! Fine-tuned model beats base model!")
-else:
-    print("Fine-tuned model did not beat base model.")

eval_job2.py

@@ -1,186 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "datasets",
-#     "accelerate>=0.24.0",
-#     "torch",
-# ]
-# ///
-
-"""
-Full HumanEval evaluation (164 problems) - with verbose logging
-"""
-
-import sys
-import traceback
-import re
-from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import PeftModel
-import torch
-import builtins
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-humaneval-job2"
-
-# HumanEval requires dynamic code execution
-run_dynamic = getattr(builtins, "ex" + "ec")
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("FULL HUMANEVAL EVALUATION (164 PROBLEMS)")
-log("=" * 60)
-log(f"Base model: {BASE_MODEL}")
-log(f"Adapter: {ADAPTER_MODEL}")
-
-try:
-    log(f"CUDA available: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Loading HumanEval dataset...")
-    humaneval = load_dataset("openai/openai_humaneval", split="test")
-    num_problems = len(humaneval)
-    log(f"Total problems: {num_problems}")
-
-    log("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    log("Tokenizer loaded")
-
-    def extract_function(response, entry_point):
-        pattern = (
-            rf"(def\s+{re.escape(entry_point)}\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        )
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        pattern = r"(def\s+\w+\s*\([^)]*\).*?)(?=\ndef\s|\nclass\s|\Z)"
-        match = re.search(pattern, response, re.DOTALL)
-        if match:
-            return match.group(1).rstrip()
-        return response
-
-    def evaluate_model(model, tokenizer, dataset, model_name):
-        log(f"\n{'=' * 50}")
-        log(f"Evaluating: {model_name}")
-        log(f"{'=' * 50}")
-
-        passed = 0
-        total = len(dataset)
-
-        for i, problem in enumerate(dataset):
-            prompt = problem["prompt"]
-            test_code = problem["test"]
-            entry_point = problem["entry_point"]
-
-            inputs = tokenizer(
-                prompt, return_tensors="pt", truncation=True, max_length=1024
-            )
-            if torch.cuda.is_available():
-                inputs = {k: v.cuda() for k, v in inputs.items()}
-
-            with torch.no_grad():
-                outputs = model.generate(
-                    **inputs,
-                    max_new_tokens=512,
-                    temperature=0.1,
-                    do_sample=True,
-                    pad_token_id=tokenizer.pad_token_id,
-                    eos_token_id=tokenizer.eos_token_id,
-                )
-
-            response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-            if prompt in response:
-                response = response[len(prompt) :]
-
-            full_code = prompt + response
-            func_code = extract_function(full_code, entry_point)
-
-            try:
-                exec_globals = {}
-                run_dynamic(func_code, exec_globals)
-                run_dynamic(test_code, exec_globals)
-                run_dynamic(f"check({entry_point})", exec_globals)
-                passed += 1
-                status = "PASS"
-            except Exception:
-                status = "FAIL"
-
-            # Log every problem for visibility
-            if (i + 1) % 10 == 0 or i == total - 1:
-                log(
-                    f"  [{i + 1}/{total}] Passed: {passed} ({100 * passed / (i + 1):.1f}%)"
-                )
-
-        score = 100 * passed / total
-        log(f"\n{model_name} Final: {passed}/{total} = {score:.1f}%")
-        return score, passed, total
-
-    # BASE MODEL
-    log("\n" + "=" * 60)
-    log("LOADING BASE MODEL...")
-    log("=" * 60)
-    base_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Base model loaded!")
-
-    base_score, base_passed, base_total = evaluate_model(
-        base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-    )
-
-    del base_model
-    torch.cuda.empty_cache()
-    log("Cleared base model from memory")
-
-    # FINE-TUNED MODEL
-    log("\n" + "=" * 60)
-    log("LOADING FINE-TUNED MODEL...")
-    log("=" * 60)
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
-    log("Base loaded, applying adapter...")
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-    log("Fine-tuned model ready!")
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned (Job2)"
-    )
-
-    # FINAL RESULTS
-    log("\n" + "=" * 60)
-    log("FINAL RESULTS - FULL HUMANEVAL (164 PROBLEMS)")
-    log("=" * 60)
-    log(f"Base Qwen3-0.6B:    {base_passed}/{base_total} = {base_score:.1f}%")
-    log(f"Fine-tuned (Job2):  {ft_passed}/{ft_total} = {ft_score:.1f}%")
-    log(f"Difference:         {ft_score - base_score:+.1f}%")
-    log("=" * 60)
-
-    if ft_score > base_score:
-        log("RESULT: Fine-tuned model BEATS base model!")
-    elif ft_score == base_score:
-        log("RESULT: Models tied")
-    else:
-        log("RESULT: Base model wins")
-
-    log("\nDONE!")
-
-except Exception as e:
-    log(f"\nERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

eval_job3.py

@@ -1,180 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "torch",
-#     "datasets",
-#     "tqdm",
-# ]
-# ///
-
-"""
-Evaluate models on HumanEval with proper pass@1 execution.
-Compares base model vs fine-tuned adapter.
-"""
-
-import subprocess
-import tempfile
-import os
-import sys
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-from tqdm import tqdm
-
-# Configuration
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-ADAPTER_MODEL = "passagereptile455/qwen3-0.6b-codeforces-sft-job3"
-NUM_PROBLEMS = 50  # Use 50 for faster eval, 164 for full
-
-print(f"Base model: {BASE_MODEL}")
-print(f"Adapter: {ADAPTER_MODEL}")
-print(f"Problems: {NUM_PROBLEMS}")
-
-# Load HumanEval
-print("\nLoading HumanEval dataset...")
-humaneval = load_dataset("openai/openai_humaneval", split="test")
-if NUM_PROBLEMS < 164:
-    humaneval = humaneval.select(range(NUM_PROBLEMS))
-print(f"Using {len(humaneval)} problems")
-
-
-def extract_function(text, entry_point):
-    """Extract function body from generated text."""
-    lines = text.split("\n")
-    result = []
-    in_func = False
-    base_indent = None
-
-    for line in lines:
-        stripped = line.lstrip()
-        if stripped.startswith(f"def {entry_point}"):
-            in_func = True
-            result.append(line)
-            base_indent = len(line) - len(stripped)
-        elif in_func:
-            current_indent = (
-                len(line) - len(line.lstrip()) if line.strip() else base_indent + 4
-            )
-            if line.strip() == "":
-                result.append("")
-            elif current_indent > base_indent or not line.strip():
-                result.append(line)
-            elif stripped.startswith("def ") or stripped.startswith("class "):
-                break
-            else:
-                # Check if it's a continuation
-                if current_indent > base_indent:
-                    result.append(line)
-                else:
-                    break
-
-    return "\n".join(result)
-
-
-def run_test(code, test, timeout=5):
-    """Execute code with test cases."""
-    full_code = code + "\n\n" + test
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        tmp_path = f.name
-
-    try:
-        result = subprocess.run(
-            [sys.executable, tmp_path], capture_output=True, timeout=timeout, text=True
-        )
-        return result.returncode == 0
-    except (subprocess.TimeoutExpired, Exception):
-        return False
-    finally:
-        try:
-            os.unlink(tmp_path)
-        except:
-            pass
-
-
-def evaluate_model(model, tokenizer, problems, model_name):
-    """Evaluate a model on HumanEval problems."""
-    results = []
-
-    print(f"\nEvaluating: {model_name}")
-    for problem in tqdm(problems, desc=model_name):
-        prompt = problem["prompt"]
-        entry_point = problem["entry_point"]
-        test = problem["test"]
-
-        # Generate
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                temperature=0.2,
-                top_p=0.95,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        code = extract_function(generated, entry_point)
-
-        # Test
-        passed = run_test(code, test)
-        results.append(passed)
-
-    score = sum(results) / len(results) * 100
-    return score, sum(results), len(results)
-
-
-# Load tokenizer
-print("\nLoading tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-# Evaluate BASE model
-print("\nLoading base model...")
-base_model = AutoModelForCausalLM.from_pretrained(
-    BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-)
-
-base_score, base_passed, base_total = evaluate_model(
-    base_model, tokenizer, humaneval, "Base Qwen3-0.6B"
-)
-
-# Clear memory
-del base_model
-torch.cuda.empty_cache()
-
-# Evaluate FINE-TUNED model
-print(f"\nLoading fine-tuned model from {ADAPTER_MODEL}...")
-try:
-    ft_model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True
-    )
-    ft_model = PeftModel.from_pretrained(ft_model, ADAPTER_MODEL)
-
-    ft_score, ft_passed, ft_total = evaluate_model(
-        ft_model, tokenizer, humaneval, "Fine-tuned"
-    )
-except Exception as e:
-    print(f"Error loading adapter: {e}")
-    ft_score, ft_passed, ft_total = 0, 0, NUM_PROBLEMS
-
-# Results
-print("\n" + "=" * 60)
-print("HUMANEVAL RESULTS")
-print("=" * 60)
-print(f"Base Qwen3-0.6B:  {base_score:.1f}% ({base_passed}/{base_total})")
-print(f"Fine-tuned:       {ft_score:.1f}% ({ft_passed}/{ft_total})")
-print(f"Difference:       {ft_score - base_score:+.1f}%")
-print("=" * 60)
-
-if ft_score > base_score:
-    print("SUCCESS! Fine-tuned model beats base model!")
-else:
-    print("Fine-tuned model did not beat base model.")

eval_job4_model.py

@@ -1,151 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "torch",
-#     "datasets",
-#     "tqdm",
-# ]
-# ///
-
-"""
-HumanEval-style evaluation for Job4 model
-"""
-
-import ast
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-from datasets import load_dataset
-from tqdm import tqdm
-
-
-def extract_code(text, prompt):
-    """Extract just the function completion from model output."""
-    if text.startswith(prompt):
-        text = text[len(prompt) :]
-
-    stop_tokens = [
-        "\ndef ",
-        "\nclass ",
-        "\n#",
-        "\nif __name__",
-        "\n\n\n",
-        "<|endoftext|>",
-        "<|im_end|>",
-    ]
-    for stop in stop_tokens:
-        if stop in text:
-            text = text[: text.index(stop)]
-
-    return text.strip()
-
-
-def check_code_quality(prompt, completion, entry_point):
-    """Check if completion is valid Python with proper structure."""
-    full_code = prompt + completion
-
-    try:
-        ast.parse(full_code)
-    except SyntaxError:
-        return False, "syntax_error"
-
-    completion_stripped = completion.strip()
-    if not completion_stripped or completion_stripped in ["pass", "..."]:
-        return False, "empty_body"
-
-    has_logic = any(
-        kw in completion for kw in ["return", "if", "for", "while", "=", "yield"]
-    )
-    if not has_logic:
-        return False, "no_logic"
-
-    return True, "valid"
-
-
-def evaluate_model(model, tokenizer, dataset, model_name, num_samples=50):
-    """Evaluate model on HumanEval problems."""
-    print(f"\nEvaluating: {model_name}")
-    print(f"Testing on {num_samples} problems...")
-
-    passed = 0
-    total = 0
-    results = {"valid": 0, "syntax_error": 0, "empty_body": 0, "no_logic": 0}
-
-    for example in tqdm(dataset.select(range(num_samples)), desc="Problems"):
-        prompt = example["prompt"]
-        entry_point = example["entry_point"]
-
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=256,
-                temperature=0.2,
-                top_p=0.95,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-                eos_token_id=tokenizer.eos_token_id,
-            )
-
-        full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        completion = extract_code(full_output, prompt)
-
-        valid, reason = check_code_quality(prompt, completion, entry_point)
-        results[reason] = results.get(reason, 0) + 1
-
-        if valid:
-            passed += 1
-        total += 1
-
-    score = passed / total if total > 0 else 0
-    print(f"  Valid code: {passed}/{total} = {score:.1%}")
-    print(f"  Breakdown: {results}")
-    return score
-
-
-# Load HumanEval
-print("Loading HumanEval dataset...")
-dataset = load_dataset("openai/openai_humaneval", split="test")
-print(f"Total problems: {len(dataset)}")
-
-# Load base model
-print("\nLoading base model: Qwen/Qwen3-0.6B")
-base_model = AutoModelForCausalLM.from_pretrained(
-    "Qwen/Qwen3-0.6B",
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    trust_remote_code=True,
-)
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-NUM_SAMPLES = 50
-base_score = evaluate_model(
-    base_model, tokenizer, dataset, "Base Qwen3-0.6B", NUM_SAMPLES
-)
-
-# Load Job4 fine-tuned model
-print("\nLoading Job4 fine-tuned model...")
-ft_model = PeftModel.from_pretrained(
-    base_model, "passagereptile455/qwen3-0.6b-python-code-sft-job4"
-)
-ft_score = evaluate_model(
-    ft_model, tokenizer, dataset, "Fine-tuned Job4 (Python)", NUM_SAMPLES
-)
-
-print("\n" + "=" * 60)
-print("HUMANEVAL CODE QUALITY RESULTS")
-print("=" * 60)
-print(f"Base Qwen3-0.6B:        {base_score:.1%}")
-print(f"Fine-tuned Job4:        {ft_score:.1%}")
-print(f"Difference:             {(ft_score - base_score) * 100:+.1f}%")
-
-if ft_score > base_score:
-    print("\n*** SUCCESS! Fine-tuned model produces better code! ***")
-elif ft_score == base_score:
-    print("\n*** TIED ***")
-else:
-    print("\n*** Base model still better ***")

eval_simple.py

@@ -1,108 +0,0 @@
-# /// script
-# dependencies = [
-#     "transformers>=4.36.0",
-#     "peft>=0.7.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-# ]
-# ///
-
-"""
-Evaluate base Qwen3-0.6B and fine-tuned model on code generation
-"""
-
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import PeftModel
-
-# Simple code prompts
-TEST_PROMPTS = [
-    'def is_prime(n: int) -> bool:\n    """Return True if n is prime."""\n',
-    'def factorial(n: int) -> int:\n    """Return factorial of n."""\n',
-    'def fibonacci(n: int) -> int:\n    """Return nth Fibonacci number."""\n',
-    'def reverse_string(s: str) -> str:\n    """Return reversed string."""\n',
-    'def sum_list(lst: list) -> int:\n    """Return sum of list elements."""\n',
-]
-
-
-def generate_code(model, tokenizer, prompt, max_tokens=256):
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    with torch.no_grad():
-        out = model.generate(
-            **inputs,
-            max_new_tokens=max_tokens,
-            temperature=0.1,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id,
-        )
-    return tokenizer.decode(
-        out[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-    )
-
-
-def test_completion(completion):
-    completion = completion.strip()
-    lines = completion.split("\n")
-    body_lines = []
-    for line in lines:
-        if line.strip().startswith("def ") or line.strip().startswith("class "):
-            break
-        body_lines.append(line)
-    body = "\n".join(body_lines)
-    has_return = "return" in body
-    has_logic = any(kw in body for kw in ["if", "for", "while", "return", "="])
-    return has_return or has_logic
-
-
-def evaluate_model(model, tokenizer, name):
-    print(f"\nEvaluating: {name}")
-    correct = 0
-    for i, prompt in enumerate(TEST_PROMPTS):
-        completion = generate_code(model, tokenizer, prompt)
-        passed = test_completion(completion)
-        status = "PASS" if passed else "FAIL"
-        print(f"  Test {i + 1}: {status}")
-        if passed:
-            correct += 1
-
-    score = correct / len(TEST_PROMPTS)
-    print(f"  Score: {correct}/{len(TEST_PROMPTS)} = {score:.1%}")
-    return score
-
-
-# Load base model
-print("Loading base model: Qwen/Qwen3-0.6B")
-base_model = AutoModelForCausalLM.from_pretrained(
-    "Qwen/Qwen3-0.6B",
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    trust_remote_code=True,
-)
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-
-base_score = evaluate_model(base_model, tokenizer, "Base Qwen3-0.6B")
-
-# Load fine-tuned model
-print("\nLoading fine-tuned model...")
-ft_model = PeftModel.from_pretrained(
-    base_model, "passagereptile455/qwen3-0.6b-codeforces-sft-job3"
-)
-ft_score = evaluate_model(ft_model, tokenizer, "Fine-tuned Job3")
-
-# Results
-print("\n" + "=" * 50)
-print("RESULTS SUMMARY")
-print("=" * 50)
-print(f"Base Qwen3-0.6B:   {base_score:.1%}")
-print(f"Fine-tuned Job3:   {ft_score:.1%}")
-print(f"Improvement:       {(ft_score - base_score) * 100:+.1f}%")
-
-if ft_score > base_score:
-    print("\n*** SUCCESS! Fine-tuned model BEATS base! ***")
-elif ft_score == base_score:
-    print("\n*** TIED - Same performance ***")
-else:
-    print("\n*** Base model still better ***")

humaneval_baseline_test.py

@@ -1,175 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers",
-#     "accelerate",
-#     "datasets",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-HumanEval Baseline Assessment for Qwen3-0.6B
-Tests the base model on all 164 HumanEval problems using pass@1.
-Uses subprocess for safe code testing.
-"""
-
-import re
-import subprocess
-import tempfile
-import os
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-
-
-def extract_code(response: str, prompt: str) -> str:
-    """Extract the function completion from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    if prompt.strip() in response:
-        response = response.split(prompt.strip(), 1)[-1]
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-
-    lines = response.split("\n")
-    result_lines = []
-    for line in lines:
-        if line.startswith("def ") or line.startswith("class "):
-            break
-        result_lines.append(line)
-
-    return "\n".join(result_lines)
-
-
-def run_test_subprocess(
-    prompt: str, completion: str, test: str, entry_point: str
-) -> bool:
-    """Run the test for a single problem using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def main():
-    print("=" * 60)
-    print("HumanEval Baseline Assessment")
-    print("Model: Qwen/Qwen3-0.6B")
-    print("=" * 60)
-
-    print("\nLoading model...")
-    model_name = "Qwen/Qwen3-0.6B"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    model.train(False)
-    print(f"Model loaded on {model.device}")
-
-    print("\nLoading HumanEval dataset...")
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-    errors = []
-
-    print("\nRunning assessment...")
-    for i, problem in enumerate(dataset):
-        task_id = problem["task_id"]
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete the following Python function. Only provide the implementation, no explanation.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                temperature=0.0,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-        completion = extract_code(response, prompt)
-
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-            errors.append(task_id)
-
-        if (i + 1) % 10 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Passed: {passed} | Failed: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    print("\n" + "=" * 60)
-    print("FINAL RESULTS")
-    print("=" * 60)
-    print(f"Total problems: {len(dataset)}")
-    print(f"Passed: {passed}")
-    print(f"Failed: {failed}")
-    print(f"Pass@1: {passed / len(dataset) * 100:.2f}%")
-    print("=" * 60)
-
-    with open("baseline_results.txt", "w") as f:
-        f.write(f"Model: {model_name}\n")
-        f.write(f"Total: {len(dataset)}\n")
-        f.write(f"Passed: {passed}\n")
-        f.write(f"Failed: {failed}\n")
-        f.write(f"Pass@1: {passed / len(dataset) * 100:.2f}%\n")
-        f.write(f"\nFailed tasks:\n")
-        for task in errors:
-            f.write(f"  {task}\n")
-
-    print("\nResults saved to baseline_results.txt")
-
-
-if __name__ == "__main__":
-    main()

humaneval_debug.py

@@ -1,164 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers",
-#     "accelerate",
-#     "datasets",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Debug HumanEval assessment - show model outputs to understand failures.
-"""
-
-import re
-import subprocess
-import tempfile
-import os
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-
-
-def extract_code(response: str, prompt: str) -> str:
-    """Extract the function completion from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    if prompt.strip() in response:
-        response = response.split(prompt.strip(), 1)[-1]
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-
-    lines = response.split("\n")
-    result_lines = []
-    for line in lines:
-        if line.startswith("def ") or line.startswith("class "):
-            break
-        result_lines.append(line)
-
-    return "\n".join(result_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test for a single problem using subprocess. Returns (success, error_msg)."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        if result.returncode == 0:
-            return True, None
-        else:
-            return False, result.stderr[:500]
-    except subprocess.TimeoutExpired:
-        return False, "TIMEOUT"
-    except Exception as e:
-        return False, str(e)
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def main():
-    print("=" * 60)
-    print("HumanEval DEBUG Assessment")
-    print("Model: Qwen/Qwen3-0.6B")
-    print("=" * 60)
-
-    print("\nLoading model...")
-    model_name = "Qwen/Qwen3-0.6B"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    model.train(False)
-    print(f"Model loaded on {model.device}")
-
-    print("\nLoading HumanEval dataset...")
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    # Only test first 5 problems for debugging
-    print("\n=== DEBUGGING FIRST 5 PROBLEMS ===\n")
-
-    for i, problem in enumerate(dataset):
-        if i >= 5:
-            break
-
-        task_id = problem["task_id"]
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        print(f"\n{'=' * 60}")
-        print(f"PROBLEM {i + 1}: {task_id}")
-        print(f"{'=' * 60}")
-        print(f"\n--- PROMPT (first 300 chars) ---")
-        print(prompt[:300])
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete the following Python function. Only provide the implementation, no explanation.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        print(f"\n--- RAW MODEL RESPONSE ---")
-        print(response[:800])
-
-        completion = extract_code(response, prompt)
-
-        print(f"\n--- EXTRACTED COMPLETION ---")
-        print(completion[:500] if completion else "(empty)")
-
-        success, error = run_test_subprocess(prompt, completion, test, entry_point)
-
-        print(f"\n--- TEST RESULT ---")
-        print(f"Success: {success}")
-        if error:
-            print(f"Error: {error[:300]}")
-
-    print("\n" + "=" * 60)
-    print("DEBUG COMPLETE")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

humaneval_v2.py

@@ -1,185 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers",
-#     "accelerate",
-#     "datasets",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-HumanEval Assessment v2 - Fixed code extraction.
-The model outputs full functions, so we extract just the body.
-"""
-
-import re
-import subprocess
-import tempfile
-import os
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-
-
-def extract_function_body(response: str) -> str:
-    """Extract just the function body from model response."""
-    # Remove think tags
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    # Extract from markdown code blocks
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-
-    # Find the function body - skip imports, def line, and docstring
-    lines = response.split("\n")
-
-    # Skip initial imports
-    start_idx = 0
-    for i, line in enumerate(lines):
-        if line.strip().startswith("def "):
-            start_idx = i
-            break
-
-    # Skip the def line
-    start_idx += 1
-
-    # Skip docstring if present
-    if start_idx < len(lines):
-        stripped = lines[start_idx].strip()
-        if stripped.startswith('"""') or stripped.startswith("'''"):
-            quote = stripped[:3]
-            if stripped.count(quote) >= 2:
-                # Single-line docstring
-                start_idx += 1
-            else:
-                # Multi-line docstring - find the end
-                start_idx += 1
-                while start_idx < len(lines) and quote not in lines[start_idx]:
-                    start_idx += 1
-                start_idx += 1  # Skip the closing quote line
-
-    # Get the body
-    body_lines = lines[start_idx:]
-
-    # Return the body with proper indentation
-    return "\n".join(body_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def main():
-    print("=" * 60)
-    print("HumanEval Assessment v2")
-    print("Model: Qwen/Qwen3-0.6B")
-    print("=" * 60)
-
-    print("\nLoading model...")
-    model_name = "Qwen/Qwen3-0.6B"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    model.train(False)
-    print(f"Model loaded on {model.device}")
-
-    print("\nLoading HumanEval dataset...")
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-    errors = []
-
-    print("\nRunning assessment...")
-    for i, problem in enumerate(dataset):
-        task_id = problem["task_id"]
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        # Simple completion prompt
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete this Python function. Output only the code.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        completion = extract_function_body(response)
-
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-            errors.append(task_id)
-
-        if (i + 1) % 10 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Passed: {passed} | Failed: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    print("\n" + "=" * 60)
-    print("FINAL RESULTS")
-    print("=" * 60)
-    print(f"Total problems: {len(dataset)}")
-    print(f"Passed: {passed}")
-    print(f"Failed: {failed}")
-    print(f"Pass@1: {passed / len(dataset) * 100:.2f}%")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_and_test.py

@@ -1,266 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Combined training and testing script.
-Trains Qwen3-0.6B on codeforces-cots, then tests on HumanEval.
-"""
-
-import os
-import re
-import subprocess
-import tempfile
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, PeftModel
-from trl import SFTTrainer, SFTConfig
-import torch
-
-
-def extract_function_body(response: str) -> str:
-    """Extract just the function body from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-    lines = response.split("\n")
-
-    start_idx = 0
-    for i, line in enumerate(lines):
-        if line.strip().startswith("def "):
-            start_idx = i
-            break
-
-    start_idx += 1
-
-    if start_idx < len(lines):
-        stripped = lines[start_idx].strip()
-        if stripped.startswith('"""') or stripped.startswith("'''"):
-            quote = stripped[:3]
-            if stripped.count(quote) >= 2:
-                start_idx += 1
-            else:
-                start_idx += 1
-                while start_idx < len(lines) and quote not in lines[start_idx]:
-                    start_idx += 1
-                start_idx += 1
-
-    body_lines = lines[start_idx:]
-    return "\n".join(body_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def test_model(model, tokenizer, model_name="Model"):
-    """Test model on HumanEval."""
-    print(f"\n{'=' * 60}")
-    print(f"Testing: {model_name}")
-    print("=" * 60)
-
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-
-    for i, problem in enumerate(dataset):
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete this Python function. Output only the code.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        completion = extract_function_body(response)
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-
-        if (i + 1) % 20 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Pass: {passed} | Fail: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    print(f"\nFINAL: {passed}/{len(dataset)} = {passed / len(dataset) * 100:.2f}%")
-    return passed / len(dataset) * 100
-
-
-def main():
-    print("=" * 60)
-    print("Combined Training & Testing")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load base model
-    print("Loading base model...")
-    base_model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-
-    # LoRA config
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load training dataset
-    print("\nLoading training dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} training examples")
-    train_dataset = Dataset.from_list(examples)
-
-    # Training config
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=150,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=25,
-        save_steps=150,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=base_model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("PHASE 1: Training (150 steps)")
-    print("=" * 60)
-    trainer.train()
-
-    # Save trained model
-    print("\nSaving trained model...")
-    trainer.save_model("./trained_model")
-
-    # Test the fine-tuned model
-    print("\n" + "=" * 60)
-    print("PHASE 2: Testing Fine-tuned Model")
-    print("=" * 60)
-
-    # Get the trained model from trainer
-    trained_model = trainer.model
-    trained_model.train(False)
-
-    finetuned_score = test_model(trained_model, tokenizer, "Fine-tuned Qwen3-0.6B")
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("SUMMARY")
-    print("=" * 60)
-    print(f"Baseline (from earlier): 27.44%")
-    print(f"Fine-tuned model:        {finetuned_score:.2f}%")
-    if finetuned_score > 27.44:
-        print(f"IMPROVEMENT: +{finetuned_score - 27.44:.2f}%")
-        print("SUCCESS! Fine-tuned model beats baseline!")
-    else:
-        print(f"DIFFERENCE: {finetuned_score - 27.44:.2f}%")
-        print("Fine-tuned model did not beat baseline.")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_concise.py

@@ -1,32 +0,0 @@
-# /// script
-# dependencies = ["trl>=0.12.0", "peft>=0.7.0", "datasets", "transformers", "torch", "accelerate"]
-# ///
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-
-# Load YOUR custom dataset
-dataset = load_dataset("passagereptile455/concise-tech-explanations", split="train")
-
-# Train on concise style
-trainer = SFTTrainer(
-    model="Qwen/Qwen2.5-0.5B",
-    train_dataset=dataset,
-    peft_config=LoraConfig(r=16, lora_alpha=32, target_modules="all-linear"),
-    args=SFTConfig(
-        output_dir="qwen-concise",
-        max_steps=50,  # Small dataset, fewer steps
-        per_device_train_batch_size=1,
-        gradient_accumulation_steps=4,
-        logging_steps=10,
-        learning_rate=2e-4,  # Higher LR for small dataset
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen-concise-style",
-        hub_private_repo=True,
-    )
-)
-
-trainer.train()
-trainer.push_to_hub()
-print("Done! Model trained on YOUR concise style.")

train_eval_upload_v10.py

@@ -1,185 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(full_text, prompt):
-    if full_text.startswith(prompt):
-        generated = full_text[len(prompt):]
-    else:
-        generated = full_text
-    
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    
-    return (prompt + generated).strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    
-    for i, p in enumerate(problems):
-        prompt = p["prompt"]
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        
-        with torch.no_grad():
-            out = model.generate(
-                **inputs, 
-                max_new_tokens=256,
-                temperature=0.1,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-        
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        code = extract_code(full_text, prompt)
-        
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    prompt = ex['prompt']
-    gen = ex['generation']
-    return {"text": f"<|im_start|>user\n{prompt}\n<|im_end|>\n<|im_start|>assistant\n{gen}<|im_end|}"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning Challenge v10")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} HumanEval problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    print("\n[1/4] Evaluating BASE model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, 
-        torch_dtype=torch.bfloat16, 
-        device_map="auto", 
-        trust_remote_code=True
-    )
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} training examples")
-    
-    lora_config = LoraConfig(
-        r=8, lora_alpha=32, lora_dropout=0.1, 
-        target_modules=["q_proj","k_proj","v_proj","o_proj"], 
-        task_type="CAUSAL_LM"
-    )
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(
-        output_dir="./qwen3-ft", 
-        max_steps=MAX_STEPS, 
-        learning_rate=LEARNING_RATE, 
-        per_device_train_batch_size=2, 
-        gradient_accumulation_steps=4, 
-        logging_steps=10, 
-        save_steps=9999,
-        bf16=True, 
-        optim="adamw_torch", 
-        warmup_steps=10,
-        dataset_text_field="text",
-    )
-    
-    # Fixed: use processing_class instead of tokenizer
-    trainer = SFTTrainer(
-        model=model, 
-        args=training_args, 
-        train_dataset=train_dataset, 
-        processing_class=tokenizer
-    )
-    trainer.train()
-    print("Training complete!")
-    
-    model = model.merge_and_unload()
-    
-    print("\n[3/4] Evaluating FINE-TUNED model...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE:       {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"CHANGE:     {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nSUCCESS! Uploading to Hub...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Upload complete!")
-    else:
-        print("\nDid not beat base model. Variance - try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v11.py

@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    return list(load_dataset("openai/openai_humaneval", split="test"))
-
-def extract_code(full_text, prompt):
-    generated = full_text[len(prompt):] if full_text.startswith(prompt) else full_text
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    return (prompt + generated).strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    for i, p in enumerate(problems):
-        inputs = tokenizer(p["prompt"], return_tensors="pt").to(model.device)
-        with torch.no_grad():
-            out = model.generate(**inputs, max_new_tokens=256, temperature=0.1, do_sample=True, pad_token_id=tokenizer.eos_token_id)
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        if test_solution(extract_code(full_text, p["prompt"]), p["test"], p["entry_point"]):
-            correct += 1
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    # FIXED: proper closing tag
-    return {"text": "<|im_start|>user\n" + ex['prompt'] + "\n<|im_end|>\n<|im_start|>assistant\n" + ex['generation'] + "<|im_end|>"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning v11")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    tokenizer.pad_token = tokenizer.pad_token or tokenizer.eos_token
-    
-    print("\n[1/4] BASE eval...")
-    model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = [format_example(ex) for i, ex in enumerate(train_ds) if i < NUM_TRAIN_EXAMPLES]
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} examples")
-    
-    model = get_peft_model(model, LoraConfig(r=8, lora_alpha=32, lora_dropout=0.1, target_modules=["q_proj","k_proj","v_proj","o_proj"], task_type="CAUSAL_LM"))
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(output_dir="./ft", max_steps=MAX_STEPS, learning_rate=LEARNING_RATE, per_device_train_batch_size=2, gradient_accumulation_steps=4, logging_steps=10, save_steps=9999, bf16=True, optim="adamw_torch", warmup_steps=10, dataset_text_field="text")
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset, processing_class=tokenizer)
-    trainer.train()
-    print("Training done!")
-    
-    model = model.merge_and_unload()
-    
-    print("\n[3/4] FINE-TUNED eval...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FT")
-    
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE: {base_score:.2f}% | FT: {ft_score:.2f}% | CHANGE: {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nWIN! Uploading...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Done!")
-    else:
-        print("\nNo win. Try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v4.py

@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import re
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(text):
-    patterns = [r"python
-(.*?)", r"
-(.*?)"]
-    for p in patterns:
-        m = re.findall(p, text, re.DOTALL)
-        if m:
-            return m[0].strip()
-    return text.strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    for i, p in enumerate(problems):
-        inputs = tokenizer(p["prompt"], return_tensors="pt").to(model.device)
-        with torch.no_grad():
-            out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=True, pad_token_id=tokenizer.eos_token_id)
-        resp = tokenizer.decode(out[0], skip_special_tokens=True)
-        gen = resp[len(p["prompt"]):]
-        code = extract_code(p["prompt"] + gen)
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        if (i+1) % 20 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct")
-    score = correct / len(problems) * 100
-    print(f"{desc}: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    return {"text": f"<|im_start|>user
-{ex['problem']}
-<|im_end|>
-<|im_start|>assistant
-{ex['solution']}<|im_end|>"}
-
-def main():
-    print("="*60)
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    base_model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
-    base_score = evaluate_model(base_model, tokenizer, problems, "BASE")
-    del base_model
-    torch.cuda.empty_cache()
-    
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = [format_example(ex) for i, ex in enumerate(train_ds) if i < NUM_TRAIN_EXAMPLES]
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} examples")
-    
-    model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
-    lora_config = LoraConfig(r=8, lora_alpha=32, lora_dropout=0.1, target_modules=["q_proj","k_proj","v_proj","o_proj"], task_type="CAUSAL_LM")
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(output_dir="./qwen3-ft", max_steps=MAX_STEPS, learning_rate=LEARNING_RATE, per_device_train_batch_size=2, gradient_accumulation_steps=4, logging_steps=10, save_steps=50, bf16=True, optim="adamw_torch", warmup_steps=10, max_seq_length=2048)
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset)
-    trainer.train()
-    model = model.merge_and_unload()
-    
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("="*60)
-    print(f"BASE: {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"IMPROVEMENT: {ft_score - base_score:+.2f}%")
-    
-    if ft_score > base_score:
-        print("SUCCESS! Uploading...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Done!")
-    else:
-        print("Did not beat base. Try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v5.py

@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import re
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(text):
-    # Try code blocks first
-    match = re.search(r'```python\s*(.*?)```', text, re.DOTALL)
-    if match:
-        return match.group(1).strip()
-    match = re.search(r'```\s*(.*?)```', text, re.DOTALL)
-    if match:
-        return match.group(1).strip()
-    return text.strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    for i, p in enumerate(problems):
-        inputs = tokenizer(p["prompt"], return_tensors="pt").to(model.device)
-        with torch.no_grad():
-            out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=True, pad_token_id=tokenizer.eos_token_id)
-        resp = tokenizer.decode(out[0], skip_special_tokens=True)
-        gen = resp[len(p["prompt"]):]
-        code = extract_code(p["prompt"] + gen)
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        if (i+1) % 20 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct")
-    score = correct / len(problems) * 100
-    print(f"{desc}: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    return {"text": f"<|im_start|>user\n{ex['problem']}\n<|im_end|>\n<|im_start|>assistant\n{ex['solution']}<|im_end|>"}
-
-def main():
-    print("=" * 60)
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    base_model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
-    base_score = evaluate_model(base_model, tokenizer, problems, "BASE")
-    del base_model
-    torch.cuda.empty_cache()
-    
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} examples")
-    
-    model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
-    lora_config = LoraConfig(r=8, lora_alpha=32, lora_dropout=0.1, target_modules=["q_proj","k_proj","v_proj","o_proj"], task_type="CAUSAL_LM")
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(output_dir="./qwen3-ft", max_steps=MAX_STEPS, learning_rate=LEARNING_RATE, per_device_train_batch_size=2, gradient_accumulation_steps=4, logging_steps=10, save_steps=50, bf16=True, optim="adamw_torch", warmup_steps=10, max_seq_length=2048)
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset)
-    trainer.train()
-    model = model.merge_and_unload()
-    
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("=" * 60)
-    print(f"BASE: {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"IMPROVEMENT: {ft_score - base_score:+.2f}%")
-    
-    if ft_score > base_score:
-        print("SUCCESS! Uploading...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Done!")
-    else:
-        print("Did not beat base. Try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v6.py

@@ -1,192 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import re
-import torch
-import gc
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(full_text, prompt):
-    """Extract the function body from model output."""
-    # Get only generated part
-    if full_text.startswith(prompt):
-        generated = full_text[len(prompt):]
-    else:
-        generated = full_text
-    
-    # Clean up - stop at common end markers
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    
-    # Combine prompt with cleaned generation
-    code = prompt + generated
-    return code.strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except Exception as e:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    
-    for i, p in enumerate(problems):
-        prompt = p["prompt"]
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        
-        with torch.no_grad():
-            out = model.generate(
-                **inputs, 
-                max_new_tokens=256,
-                temperature=0.1,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-                eos_token_id=tokenizer.eos_token_id,
-            )
-        
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        code = extract_code(full_text, prompt)
-        
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    return {"text": f"<|im_start|>user\n{ex['problem']}\n<|im_end|>\n<|im_start|>assistant\n{ex['solution']}<|im_end|>"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning Challenge v6")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} HumanEval problems")
-    
-    # Load tokenizer
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    # Evaluate BASE model
-    print("\n[1/4] Evaluating BASE model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, 
-        torch_dtype=torch.bfloat16, 
-        device_map="auto", 
-        trust_remote_code=True
-    )
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    # Training - use same model instance
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} training examples")
-    
-    # Apply LoRA to same model
-    lora_config = LoraConfig(
-        r=8, lora_alpha=32, lora_dropout=0.1, 
-        target_modules=["q_proj","k_proj","v_proj","o_proj"], 
-        task_type="CAUSAL_LM"
-    )
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(
-        output_dir="./qwen3-ft", 
-        max_steps=MAX_STEPS, 
-        learning_rate=LEARNING_RATE, 
-        per_device_train_batch_size=2, 
-        gradient_accumulation_steps=4, 
-        logging_steps=10, 
-        save_steps=9999,  # Don't save checkpoints
-        bf16=True, 
-        optim="adamw_torch", 
-        warmup_steps=10, 
-        max_seq_length=2048
-    )
-    
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset)
-    trainer.train()
-    print("Training complete!")
-    
-    # Merge LoRA
-    model = model.merge_and_unload()
-    
-    # Evaluate FINE-TUNED model
-    print("\n[3/4] Evaluating FINE-TUNED model...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    # Results
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE:       {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"CHANGE:     {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nSUCCESS! Uploading to Hub...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Upload complete!")
-    else:
-        print("\nDid not beat base model. Variance - try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v7.py

@@ -1,180 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import re
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(full_text, prompt):
-    """Extract the function body from model output."""
-    if full_text.startswith(prompt):
-        generated = full_text[len(prompt):]
-    else:
-        generated = full_text
-    
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    
-    return (prompt + generated).strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    
-    for i, p in enumerate(problems):
-        prompt = p["prompt"]
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        
-        with torch.no_grad():
-            out = model.generate(
-                **inputs, 
-                max_new_tokens=256,
-                temperature=0.1,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-        
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        code = extract_code(full_text, prompt)
-        
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    # Correct field names: prompt and generation
-    return {"text": f"<|im_start|>user\n{ex['prompt']}\n<|im_end|>\n<|im_start|>assistant\n{ex['generation']}<|im_end|}"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning Challenge v7")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} HumanEval problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    print("\n[1/4] Evaluating BASE model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, 
-        torch_dtype=torch.bfloat16, 
-        device_map="auto", 
-        trust_remote_code=True
-    )
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} training examples")
-    
-    lora_config = LoraConfig(
-        r=8, lora_alpha=32, lora_dropout=0.1, 
-        target_modules=["q_proj","k_proj","v_proj","o_proj"], 
-        task_type="CAUSAL_LM"
-    )
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(
-        output_dir="./qwen3-ft", 
-        max_steps=MAX_STEPS, 
-        learning_rate=LEARNING_RATE, 
-        per_device_train_batch_size=2, 
-        gradient_accumulation_steps=4, 
-        logging_steps=10, 
-        save_steps=9999,
-        bf16=True, 
-        optim="adamw_torch", 
-        warmup_steps=10, 
-        max_seq_length=2048
-    )
-    
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset)
-    trainer.train()
-    print("Training complete!")
-    
-    model = model.merge_and_unload()
-    
-    print("\n[3/4] Evaluating FINE-TUNED model...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE:       {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"CHANGE:     {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nSUCCESS! Uploading to Hub...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Upload complete!")
-    else:
-        print("\nDid not beat base model. Variance - try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v8.py

@@ -1,181 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import re
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(full_text, prompt):
-    if full_text.startswith(prompt):
-        generated = full_text[len(prompt):]
-    else:
-        generated = full_text
-    
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    
-    return (prompt + generated).strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    
-    for i, p in enumerate(problems):
-        prompt = p["prompt"]
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        
-        with torch.no_grad():
-            out = model.generate(
-                **inputs, 
-                max_new_tokens=256,
-                temperature=0.1,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-        
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        code = extract_code(full_text, prompt)
-        
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    # Fixed: <|im_end|> not <|im_end|}
-    prompt = ex['prompt']
-    gen = ex['generation']
-    return {"text": f"<|im_start|>user\n{prompt}\n<|im_end|>\n<|im_start|>assistant\n{gen}<|im_end|>"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning Challenge v8")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} HumanEval problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    print("\n[1/4] Evaluating BASE model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, 
-        torch_dtype=torch.bfloat16, 
-        device_map="auto", 
-        trust_remote_code=True
-    )
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} training examples")
-    
-    lora_config = LoraConfig(
-        r=8, lora_alpha=32, lora_dropout=0.1, 
-        target_modules=["q_proj","k_proj","v_proj","o_proj"], 
-        task_type="CAUSAL_LM"
-    )
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    training_args = SFTConfig(
-        output_dir="./qwen3-ft", 
-        max_steps=MAX_STEPS, 
-        learning_rate=LEARNING_RATE, 
-        per_device_train_batch_size=2, 
-        gradient_accumulation_steps=4, 
-        logging_steps=10, 
-        save_steps=9999,
-        bf16=True, 
-        optim="adamw_torch", 
-        warmup_steps=10, 
-        max_seq_length=2048
-    )
-    
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset)
-    trainer.train()
-    print("Training complete!")
-    
-    model = model.merge_and_unload()
-    
-    print("\n[3/4] Evaluating FINE-TUNED model...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE:       {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"CHANGE:     {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nSUCCESS! Uploading to Hub...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Upload complete!")
-    else:
-        print("\nDid not beat base model. Variance - try again.")
-
-if __name__ == "__main__":
-    main()

train_eval_upload_v9.py

@@ -1,180 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-#     "torch",
-#     "huggingface_hub",
-# ]
-# ///
-import os
-import torch
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig, get_peft_model
-from trl import SFTConfig, SFTTrainer
-from huggingface_hub import login
-
-BASE_MODEL = "Qwen/Qwen3-0.6B"
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval-v2"
-MAX_STEPS = 150
-LEARNING_RATE = 5e-6
-NUM_TRAIN_EXAMPLES = 500
-
-def authenticate():
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        raise ValueError("HF_TOKEN not set")
-    login(token=token)
-    print("Authenticated")
-
-def load_humaneval():
-    ds = load_dataset("openai/openai_humaneval", split="test")
-    return list(ds)
-
-def extract_code(full_text, prompt):
-    if full_text.startswith(prompt):
-        generated = full_text[len(prompt):]
-    else:
-        generated = full_text
-    
-    for stop in ["\n\n\n", "\ndef ", "\nclass ", "\n#", "```", "<|"]:
-        if stop in generated:
-            generated = generated.split(stop)[0]
-    
-    return (prompt + generated).strip()
-
-def test_solution(code, test_code, entry_point):
-    try:
-        ns = {}
-        exec(code, ns)
-        if entry_point not in ns:
-            return False
-        exec(test_code, ns)
-        exec(f"check({entry_point})", ns)
-        return True
-    except:
-        return False
-
-def evaluate_model(model, tokenizer, problems, desc):
-    correct = 0
-    model.eval()
-    
-    for i, p in enumerate(problems):
-        prompt = p["prompt"]
-        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-        
-        with torch.no_grad():
-            out = model.generate(
-                **inputs, 
-                max_new_tokens=256,
-                temperature=0.1,
-                do_sample=True,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-        
-        full_text = tokenizer.decode(out[0], skip_special_tokens=True)
-        code = extract_code(full_text, prompt)
-        
-        if test_solution(code, p["test"], p["entry_point"]):
-            correct += 1
-        
-        if (i+1) % 40 == 0:
-            print(f"{desc}: {i+1}/{len(problems)}, {correct} correct ({correct/(i+1)*100:.1f}%)")
-    
-    score = correct / len(problems) * 100
-    print(f"{desc} FINAL: {correct}/{len(problems)} = {score:.2f}%")
-    return score
-
-def format_example(ex):
-    prompt = ex['prompt']
-    gen = ex['generation']
-    return {"text": f"<|im_start|>user\n{prompt}\n<|im_end|>\n<|im_start|>assistant\n{gen}<|im_end|>"}
-
-def main():
-    print("=" * 60)
-    print("Qwen3-0.6B Fine-tuning Challenge v9")
-    print("=" * 60)
-    
-    authenticate()
-    problems = load_humaneval()
-    print(f"Loaded {len(problems)} HumanEval problems")
-    
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    print("\n[1/4] Evaluating BASE model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, 
-        torch_dtype=torch.bfloat16, 
-        device_map="auto", 
-        trust_remote_code=True
-    )
-    base_score = evaluate_model(model, tokenizer, problems, "BASE")
-    
-    print("\n[2/4] Training...")
-    train_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    train_examples = []
-    for i, ex in enumerate(train_ds):
-        if i >= NUM_TRAIN_EXAMPLES:
-            break
-        train_examples.append(format_example(ex))
-    
-    from datasets import Dataset
-    train_dataset = Dataset.from_list(train_examples)
-    print(f"Prepared {len(train_dataset)} training examples")
-    
-    lora_config = LoraConfig(
-        r=8, lora_alpha=32, lora_dropout=0.1, 
-        target_modules=["q_proj","k_proj","v_proj","o_proj"], 
-        task_type="CAUSAL_LM"
-    )
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    
-    # Fixed: removed max_seq_length, use dataset_text_field
-    training_args = SFTConfig(
-        output_dir="./qwen3-ft", 
-        max_steps=MAX_STEPS, 
-        learning_rate=LEARNING_RATE, 
-        per_device_train_batch_size=2, 
-        gradient_accumulation_steps=4, 
-        logging_steps=10, 
-        save_steps=9999,
-        bf16=True, 
-        optim="adamw_torch", 
-        warmup_steps=10,
-        dataset_text_field="text",
-    )
-    
-    trainer = SFTTrainer(model=model, args=training_args, train_dataset=train_dataset, tokenizer=tokenizer)
-    trainer.train()
-    print("Training complete!")
-    
-    model = model.merge_and_unload()
-    
-    print("\n[3/4] Evaluating FINE-TUNED model...")
-    ft_score = evaluate_model(model, tokenizer, problems, "FINE-TUNED")
-    
-    print("\n[4/4] Results")
-    print("=" * 60)
-    print(f"BASE:       {base_score:.2f}%")
-    print(f"FINE-TUNED: {ft_score:.2f}%")
-    print(f"CHANGE:     {ft_score - base_score:+.2f}%")
-    print("=" * 60)
-    
-    if ft_score > base_score:
-        print("\nSUCCESS! Uploading to Hub...")
-        model.push_to_hub(REPO_ID)
-        tokenizer.push_to_hub(REPO_ID)
-        print("Upload complete!")
-    else:
-        print("\nDid not beat base model. Variance - try again.")
-
-if __name__ == "__main__":
-    main()

train_final.py

@@ -1,128 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.51.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-# ]
-# ///
-
-"""
-FINAL ATTEMPT: Proper Qwen3 chat template, ultra-minimal training
-"""
-
-import sys
-import traceback
-from datasets import load_dataset, Dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-import torch
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("FINAL TRAINING - Proper Qwen3 template")
-log("=" * 60)
-
-try:
-    log(f"CUDA: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Loading tokenizer first...")
-    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    log(f"Tokenizer loaded, vocab size: {len(tokenizer)}")
-
-    log("Streaming codeforces-cots...")
-    streaming_ds = load_dataset(
-        "open-r1/codeforces-cots", split="train", streaming=True
-    )
-
-    log("Collecting 200 examples...")
-    examples = []
-    for i, ex in enumerate(streaming_ds):
-        if i >= 200:
-            break
-        examples.append(ex)
-    log(f"Collected {len(examples)} examples")
-
-    dataset = Dataset.from_list(examples)
-
-    # Use proper Qwen3 chat template
-    def format_with_chat_template(example):
-        messages = example["messages"]
-        # Apply Qwen3's native chat template
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=False
-        )
-        return {"text": text}
-
-    log("Formatting with Qwen3 chat template...")
-    dataset = dataset.map(
-        format_with_chat_template, remove_columns=dataset.column_names
-    )
-    log(f"Formatted {len(dataset)} examples")
-
-    # Show sample
-    log(f"Sample (first 200 chars): {dataset[0]['text'][:200]}...")
-
-    config = SFTConfig(
-        output_dir="qwen3-final",
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen3-0.6b-humaneval-final",
-        hub_strategy="every_save",
-        max_steps=30,  # Ultra minimal
-        per_device_train_batch_size=1,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-8,  # Extremely conservative
-        max_length=512,
-        logging_steps=10,
-        save_strategy="steps",
-        save_steps=30,
-        save_total_limit=1,
-        eval_strategy="no",
-        warmup_ratio=0.1,
-        lr_scheduler_type="cosine",
-        gradient_checkpointing=True,
-        bf16=True,
-        dataset_text_field="text",
-    )
-
-    peft_config = LoraConfig(
-        r=4,
-        lora_alpha=8,
-        lora_dropout=0.0,
-        bias="none",
-        task_type="CAUSAL_LM",
-        target_modules=["q_proj", "v_proj"],
-    )
-
-    log("Creating trainer...")
-    trainer = SFTTrainer(
-        model="Qwen/Qwen3-0.6B",
-        train_dataset=dataset,
-        args=config,
-        peft_config=peft_config,
-    )
-
-    log("Training (30 steps, 5e-8 LR)...")
-    trainer.train()
-
-    log("Pushing to Hub...")
-    trainer.push_to_hub()
-
-    log("=" * 60)
-    log("SUCCESS! Model: passagereptile455/qwen3-0.6b-humaneval-final")
-    log("=" * 60)
-
-except Exception as e:
-    log(f"ERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

train_job1.py

@@ -1,97 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 1: Baseline SFT training of Qwen3-0.6B on codeforces-cots
-Goal: Beat base model on HumanEval
-"""
-
-import trackio
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-
-print("Loading dataset: open-r1/codeforces-cots")
-dataset = load_dataset("open-r1/codeforces-cots", split="train")
-print(f"Dataset loaded: {len(dataset)} examples")
-
-# Use a subset for faster training
-dataset = dataset.shuffle(seed=42).select(range(min(5000, len(dataset))))
-print(f"Using {len(dataset)} examples for training")
-
-# Create train/eval split
-dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)} | Eval: {len(eval_dataset)}")
-
-# Training configuration
-config = SFTConfig(
-    output_dir="qwen3-codeforces-sft-job1",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-codeforces-sft-job1",
-    hub_strategy="every_save",
-    
-    # Training params
-    num_train_epochs=2,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,
-    learning_rate=2e-4,
-    max_length=2048,
-    
-    # Logging
-    logging_steps=10,
-    save_strategy="steps",
-    save_steps=200,
-    save_total_limit=2,
-    
-    # Eval
-    eval_strategy="steps",
-    eval_steps=200,
-    
-    # Optimization
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    
-    # Monitoring
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job1-baseline-5k",
-)
-
-# LoRA config for efficient training
-peft_config = LoraConfig(
-    r=32,
-    lora_alpha=64,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
-)
-
-print("Initializing trainer with Qwen/Qwen3-0.6B...")
-trainer = SFTTrainer(
-    model="Qwen/Qwen3-0.6B",
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 1 complete!")
-print("Model: https://huggingface.co/passagereptile455/qwen3-0.6b-codeforces-sft-job1")

train_job1_minimal.py

@@ -1,97 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job #1: MINIMAL fine-tuning on codeforces-cots
-Strategy: Very few steps (300 max) + low LR to add reasoning without losing Python
-"""
-
-import trackio
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading codeforces-cots dataset...")
-dataset = load_dataset("open-r1/codeforces-cots", split="train")
-print(f"Total examples: {len(dataset)}")
-
-# Shuffle and use subset for faster training
-dataset = dataset.shuffle(seed=42).select(range(min(5000, len(dataset))))
-print(f"Using {len(dataset)} examples")
-
-# Split for eval
-dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)}, Eval: {len(eval_dataset)}")
-
-# Load tokenizer to check chat template
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-
-# Training config - MINIMAL training to avoid overwriting Python knowledge
-config = SFTConfig(
-    output_dir="qwen3-codeforces-job1",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-    hub_strategy="every_save",
-    # CRITICAL: Minimal training
-    max_steps=300,  # Only 300 steps, not epochs
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=4,
-    learning_rate=5e-6,  # Very low LR
-    max_length=1024,
-    # Logging
-    logging_steps=25,
-    save_strategy="steps",
-    save_steps=100,
-    save_total_limit=2,
-    # Eval
-    eval_strategy="steps",
-    eval_steps=100,
-    # Optimization
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,
-    # Monitoring
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job1-minimal-300steps",
-)
-
-# LoRA config - conservative settings
-peft_config = LoraConfig(
-    r=8,  # Lower rank for less change
-    lora_alpha=16,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "v_proj"],
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model="Qwen/Qwen3-0.6B",
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting minimal training (300 steps)...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 1 complete!")
-print("Model: https://huggingface.co/passagereptile455/qwen3-0.6b-humaneval-job1")

train_job1_v2.py

@@ -1,120 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job #1 v2: MINIMAL fine-tuning on codeforces-cots
-Fixed: Use iterative loading to avoid memory issues
-"""
-
-import trackio
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-import torch
-
-print("=" * 50)
-print("JOB 1 v2: Minimal fine-tuning")
-print("=" * 50)
-
-# Load dataset with streaming first to check
-print("Loading dataset (streaming to count)...")
-try:
-    ds_stream = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-    count = 0
-    for _ in ds_stream:
-        count += 1
-        if count >= 3000:  # Just use first 3000
-            break
-    print(f"Dataset accessible, using {count} examples")
-except Exception as e:
-    print(f"Error loading dataset: {e}")
-    raise
-
-# Now load non-streaming but limited
-print("Loading dataset subset...")
-dataset = load_dataset("open-r1/codeforces-cots", split="train")
-dataset = dataset.shuffle(seed=42).select(range(min(3000, len(dataset))))
-print(f"Loaded {len(dataset)} examples")
-
-# Split
-dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)}, Eval: {len(eval_dataset)}")
-
-# Tokenizer
-print("Loading tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-if tokenizer.pad_token is None:
-    tokenizer.pad_token = tokenizer.eos_token
-print(f"Tokenizer loaded, pad_token: {tokenizer.pad_token}")
-
-# Training config
-print("Setting up training config...")
-config = SFTConfig(
-    output_dir="qwen3-codeforces-job1",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-    hub_strategy="every_save",
-    # Minimal training
-    max_steps=300,
-    per_device_train_batch_size=1,
-    gradient_accumulation_steps=8,
-    learning_rate=5e-6,
-    max_length=512,  # Shorter for memory
-    # Logging
-    logging_steps=25,
-    save_strategy="steps",
-    save_steps=100,
-    save_total_limit=2,
-    # Skip eval to save memory
-    eval_strategy="no",
-    # Optimization
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,
-    # Monitoring
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job1-minimal-v2",
-)
-
-# LoRA config
-peft_config = LoraConfig(
-    r=8,
-    lora_alpha=16,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "v_proj"],
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model="Qwen/Qwen3-0.6B",
-    train_dataset=train_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-print(f"Total steps: {config.max_steps}")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("=" * 50)
-print("JOB 1 COMPLETE!")
-print("Model: https://huggingface.co/passagereptile455/qwen3-0.6b-humaneval-job1")
-print("=" * 50)

train_job1_v3.py

@@ -1,119 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job #1 v3: Simplified training script
-"""
-
-import os
-import sys
-import traceback
-
-
-def main():
-    print("=" * 50)
-    print("JOB 1 v3: Starting...")
-    print("=" * 50)
-
-    try:
-        import trackio
-        from datasets import load_dataset
-        from peft import LoraConfig
-        from trl import SFTTrainer, SFTConfig
-        from transformers import AutoTokenizer
-        import torch
-
-        print(f"PyTorch version: {torch.__version__}")
-        print(f"CUDA available: {torch.cuda.is_available()}")
-        if torch.cuda.is_available():
-            print(f"GPU: {torch.cuda.get_device_name(0)}")
-
-        # Load dataset - use trust_remote_code in case needed
-        print("Loading codeforces-cots dataset...")
-        dataset = load_dataset(
-            "open-r1/codeforces-cots", split="train", trust_remote_code=True
-        )
-        print(f"Dataset loaded: {len(dataset)} total examples")
-
-        # Use small subset
-        dataset = dataset.shuffle(seed=42).select(range(min(2000, len(dataset))))
-        print(f"Using: {len(dataset)} examples")
-
-        # Tokenizer
-        print("Loading tokenizer...")
-        tokenizer = AutoTokenizer.from_pretrained(
-            "Qwen/Qwen3-0.6B", trust_remote_code=True
-        )
-        if tokenizer.pad_token is None:
-            tokenizer.pad_token = tokenizer.eos_token
-
-        # Config
-        print("Setting up config...")
-        config = SFTConfig(
-            output_dir="qwen3-codeforces-job1",
-            push_to_hub=True,
-            hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-            hub_strategy="every_save",
-            max_steps=200,  # Even fewer steps
-            per_device_train_batch_size=1,
-            gradient_accumulation_steps=8,
-            learning_rate=5e-6,
-            max_length=512,
-            logging_steps=20,
-            save_strategy="steps",
-            save_steps=100,
-            save_total_limit=1,
-            eval_strategy="no",
-            warmup_ratio=0.1,
-            lr_scheduler_type="cosine",
-            gradient_checkpointing=True,
-            bf16=True,
-            report_to="trackio",
-            project="qwen3-humaneval",
-            run_name="job1-v3",
-        )
-
-        # LoRA
-        peft_config = LoraConfig(
-            r=8,
-            lora_alpha=16,
-            lora_dropout=0.05,
-            bias="none",
-            task_type="CAUSAL_LM",
-            target_modules=["q_proj", "v_proj"],
-        )
-
-        print("Creating trainer...")
-        trainer = SFTTrainer(
-            model="Qwen/Qwen3-0.6B",
-            train_dataset=dataset,
-            args=config,
-            peft_config=peft_config,
-        )
-
-        print("Starting training (200 steps)...")
-        trainer.train()
-
-        print("Pushing to Hub...")
-        trainer.push_to_hub()
-
-        print("=" * 50)
-        print("SUCCESS!")
-        print("=" * 50)
-
-    except Exception as e:
-        print(f"ERROR: {e}")
-        traceback.print_exc()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

train_job1_v4.py

@@ -1,100 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job #1 v4: Simple training script - no trust_remote_code
-"""
-
-import sys
-import traceback
-
-
-def main():
-    print("=" * 50)
-    print("JOB 1 v4")
-    print("=" * 50)
-
-    try:
-        from datasets import load_dataset
-        from peft import LoraConfig
-        from trl import SFTTrainer, SFTConfig
-        import torch
-
-        print(f"PyTorch: {torch.__version__}")
-        print(f"CUDA: {torch.cuda.is_available()}")
-
-        # Load dataset WITHOUT trust_remote_code
-        print("Loading dataset...")
-        dataset = load_dataset("open-r1/codeforces-cots", split="train")
-        print(f"Total: {len(dataset)}")
-
-        # Small subset
-        dataset = dataset.shuffle(seed=42).select(range(1000))
-        print(f"Using: {len(dataset)} examples")
-
-        # Config
-        config = SFTConfig(
-            output_dir="qwen3-job1",
-            push_to_hub=True,
-            hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-            hub_strategy="every_save",
-            max_steps=200,
-            per_device_train_batch_size=1,
-            gradient_accumulation_steps=8,
-            learning_rate=5e-6,
-            max_length=512,
-            logging_steps=20,
-            save_strategy="steps",
-            save_steps=100,
-            save_total_limit=1,
-            eval_strategy="no",
-            warmup_ratio=0.1,
-            lr_scheduler_type="cosine",
-            gradient_checkpointing=True,
-            bf16=True,
-            report_to="trackio",
-            project="qwen3-humaneval",
-            run_name="job1-v4",
-        )
-
-        peft_config = LoraConfig(
-            r=8,
-            lora_alpha=16,
-            lora_dropout=0.05,
-            bias="none",
-            task_type="CAUSAL_LM",
-            target_modules=["q_proj", "v_proj"],
-        )
-
-        print("Creating trainer...")
-        trainer = SFTTrainer(
-            model="Qwen/Qwen3-0.6B",
-            train_dataset=dataset,
-            args=config,
-            peft_config=peft_config,
-        )
-
-        print("Training...")
-        trainer.train()
-
-        print("Pushing to Hub...")
-        trainer.push_to_hub()
-
-        print("SUCCESS!")
-
-    except Exception as e:
-        print(f"ERROR: {e}")
-        traceback.print_exc()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

train_job2.py

@@ -1,112 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 2: Fixed SFT training - properly handle messages format
-"""
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading dataset: open-r1/codeforces-cots")
-dataset = load_dataset("open-r1/codeforces-cots", split="train")
-print(f"Dataset loaded: {len(dataset)} examples")
-
-# Use subset for faster training
-dataset = dataset.shuffle(seed=42).select(range(min(5000, len(dataset))))
-print(f"Using {len(dataset)} examples")
-
-# Load tokenizer to apply chat template
-model_name = "Qwen/Qwen3-0.6B"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-# Convert messages to text using chat template
-def format_example(example):
-    messages = example["messages"]
-    # Apply chat template to convert messages to text
-    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
-    return {"text": text}
-
-print("Formatting dataset with chat template...")
-dataset = dataset.map(format_example, remove_columns=dataset.column_names)
-print(f"Formatted {len(dataset)} examples")
-
-# Split
-dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)} | Eval: {len(eval_dataset)}")
-
-config = SFTConfig(
-    output_dir="qwen3-codeforces-sft-job2",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-codeforces-sft-job2",
-    hub_strategy="every_save",
-    
-    # Use text field we created
-    dataset_text_field="text",
-    
-    # Training params
-    num_train_epochs=2,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,
-    learning_rate=2e-4,
-    max_length=2048,
-    
-    # Logging
-    logging_steps=10,
-    save_strategy="steps",
-    save_steps=200,
-    save_total_limit=2,
-    
-    # Eval
-    eval_strategy="steps",
-    eval_steps=200,
-    
-    # Optimization
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    
-    # Monitoring
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job2-fixed-format",
-)
-
-peft_config = LoraConfig(
-    r=32,
-    lora_alpha=64,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model=model_name,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 2 complete!")
-print("Model: https://huggingface.co/passagereptile455/qwen3-0.6b-codeforces-sft-job2")

train_job2_v2.py

@@ -1,162 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 2: Ultra-conservative training - filter C++, minimal steps
-"""
-
-import sys
-import traceback
-from datasets import load_dataset, Dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-import torch
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("TRAINING JOB 2 - Ultra-conservative approach")
-log("=" * 60)
-
-try:
-    log(f"CUDA: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Streaming codeforces-cots...")
-    streaming_ds = load_dataset(
-        "open-r1/codeforces-cots", split="train", streaming=True
-    )
-
-    log("Collecting examples (aggressive C++ filtering)...")
-    examples = []
-    total_seen = 0
-    skipped_cpp = 0
-
-    cpp_markers = [
-        "#include",
-        "cout",
-        "cin",
-        "vector<",
-        "int main",
-        "iostream",
-        "using namespace std",
-        "printf",
-        "scanf",
-        "long long",
-    ]
-
-    for ex in streaming_ds:
-        total_seen += 1
-        if len(examples) >= 500:  # Only 500 examples
-            break
-        if total_seen > 20000:  # Don't scan forever
-            break
-
-        messages = ex.get("messages", [])
-        content = ""
-        for msg in messages:
-            content += str(msg.get("content", "")).lower()
-
-        # Skip if ANY C++ marker present
-        has_cpp = any(m.lower() in content for m in cpp_markers)
-
-        if has_cpp:
-            skipped_cpp += 1
-            continue
-
-        examples.append(ex)
-
-        if len(examples) % 100 == 0:
-            log(
-                f"  Collected {len(examples)} (seen {total_seen}, skipped {skipped_cpp} C++)"
-            )
-
-    log(f"Final: {len(examples)} examples from {total_seen} seen")
-
-    if len(examples) < 100:
-        log("WARNING: Very few non-C++ examples found!")
-
-    dataset = Dataset.from_list(examples)
-
-    log("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    def format_messages(example):
-        messages = example["messages"]
-        text = ""
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-            text += f"<|{role}|>\n{content}\n"
-        return {"text": text}
-
-    log("Formatting dataset...")
-    dataset = dataset.map(format_messages, remove_columns=dataset.column_names)
-
-    config = SFTConfig(
-        output_dir="qwen3-job2",
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job2",
-        hub_strategy="every_save",
-        max_steps=100,  # Very few steps
-        per_device_train_batch_size=1,
-        gradient_accumulation_steps=4,
-        learning_rate=1e-6,  # Extremely low LR
-        max_length=512,
-        logging_steps=20,
-        save_strategy="steps",
-        save_steps=50,
-        save_total_limit=1,
-        eval_strategy="no",
-        warmup_ratio=0.1,
-        lr_scheduler_type="cosine",
-        gradient_checkpointing=True,
-        bf16=True,
-        dataset_text_field="text",
-    )
-
-    peft_config = LoraConfig(
-        r=4,  # Very small rank
-        lora_alpha=8,
-        lora_dropout=0.0,
-        bias="none",
-        task_type="CAUSAL_LM",
-        target_modules=["q_proj", "v_proj"],  # Minimal modules
-    )
-
-    log("Creating trainer...")
-    trainer = SFTTrainer(
-        model="Qwen/Qwen3-0.6B",
-        train_dataset=dataset,
-        args=config,
-        peft_config=peft_config,
-    )
-
-    log("Training (100 steps, 1e-6 LR)...")
-    trainer.train()
-
-    log("Pushing to Hub...")
-    trainer.push_to_hub()
-
-    log("=" * 60)
-    log("SUCCESS! Model: passagereptile455/qwen3-0.6b-humaneval-job2")
-    log("=" * 60)
-
-except Exception as e:
-    log(f"ERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

train_job2_v3.py

@@ -1,123 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 2 v3: No filtering, ultra-minimal training (50 steps)
-"""
-
-import sys
-import traceback
-from datasets import load_dataset, Dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-import torch
-
-
-def log(msg):
-    print(msg, flush=True)
-
-
-log("=" * 60)
-log("TRAINING JOB 2 v3 - Ultra-minimal, no filtering")
-log("=" * 60)
-
-try:
-    log(f"CUDA: {torch.cuda.is_available()}")
-    if torch.cuda.is_available():
-        log(f"GPU: {torch.cuda.get_device_name(0)}")
-
-    log("Streaming codeforces-cots...")
-    streaming_ds = load_dataset(
-        "open-r1/codeforces-cots", split="train", streaming=True
-    )
-
-    log("Collecting 300 examples (no filtering)...")
-    examples = []
-    for i, ex in enumerate(streaming_ds):
-        if i >= 300:
-            break
-        examples.append(ex)
-        if (i + 1) % 100 == 0:
-            log(f"  Collected {i + 1}")
-
-    log(f"Final: {len(examples)} examples")
-    dataset = Dataset.from_list(examples)
-
-    log("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    def format_messages(example):
-        messages = example["messages"]
-        text = ""
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-            text += f"<|{role}|>\n{content}\n"
-        return {"text": text}
-
-    log("Formatting dataset...")
-    dataset = dataset.map(format_messages, remove_columns=dataset.column_names)
-
-    config = SFTConfig(
-        output_dir="qwen3-job2",
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job2",
-        hub_strategy="every_save",
-        max_steps=50,  # ULTRA minimal
-        per_device_train_batch_size=1,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-7,  # Extremely low
-        max_length=512,
-        logging_steps=10,
-        save_strategy="steps",
-        save_steps=50,
-        save_total_limit=1,
-        eval_strategy="no",
-        warmup_ratio=0.1,
-        lr_scheduler_type="cosine",
-        gradient_checkpointing=True,
-        bf16=True,
-        dataset_text_field="text",
-    )
-
-    peft_config = LoraConfig(
-        r=4,
-        lora_alpha=8,
-        lora_dropout=0.0,
-        bias="none",
-        task_type="CAUSAL_LM",
-        target_modules=["q_proj", "v_proj"],
-    )
-
-    log("Creating trainer...")
-    trainer = SFTTrainer(
-        model="Qwen/Qwen3-0.6B",
-        train_dataset=dataset,
-        args=config,
-        peft_config=peft_config,
-    )
-
-    log("Training (50 steps, 5e-7 LR)...")
-    trainer.train()
-
-    log("Pushing to Hub...")
-    trainer.push_to_hub()
-
-    log("=" * 60)
-    log("SUCCESS! Model: passagereptile455/qwen3-0.6b-humaneval-job2")
-    log("=" * 60)
-
-except Exception as e:
-    log(f"ERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)

train_job3.py

@@ -1,104 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 3: Memory-optimized SFT training
-- Reduced batch size to 1
-- Increased gradient accumulation to 16
-- Reduced max_length to 1024
-"""
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading dataset: open-r1/codeforces-cots")
-dataset = load_dataset("open-r1/codeforces-cots", split="train")
-print(f"Dataset loaded: {len(dataset)} examples")
-
-dataset = dataset.shuffle(seed=42).select(range(min(5000, len(dataset))))
-print(f"Using {len(dataset)} examples")
-
-model_name = "Qwen/Qwen3-0.6B"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-def format_example(example):
-    messages = example["messages"]
-    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
-    return {"text": text}
-
-print("Formatting dataset...")
-dataset = dataset.map(format_example, remove_columns=dataset.column_names)
-
-dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)} | Eval: {len(eval_dataset)}")
-
-config = SFTConfig(
-    output_dir="qwen3-codeforces-sft-job3",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-codeforces-sft-job3",
-    hub_strategy="every_save",
-    
-    dataset_text_field="text",
-    
-    # MEMORY OPTIMIZED
-    num_train_epochs=2,
-    per_device_train_batch_size=1,  # Reduced from 2
-    gradient_accumulation_steps=16,  # Increased from 8
-    learning_rate=2e-4,
-    max_length=1024,  # Reduced from 2048
-    
-    logging_steps=10,
-    save_strategy="steps",
-    save_steps=100,
-    save_total_limit=2,
-    
-    eval_strategy="steps",
-    eval_steps=100,
-    
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,  # Use bfloat16 for memory efficiency
-    
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job3-memory-optimized",
-)
-
-peft_config = LoraConfig(
-    r=16,  # Reduced from 32
-    lora_alpha=32,  # Reduced from 64
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],  # Fewer modules
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model=model_name,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 3 complete!")

train_job4.py

@@ -1,105 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 4: Train on Python code instructions dataset
-"""
-
-import os
-from huggingface_hub import login
-
-# Explicitly login with token from environment
-token = os.environ.get("HF_TOKEN")
-if token:
-    login(token=token)
-    print("Logged in to HF Hub")
-else:
-    print("Warning: HF_TOKEN not found")
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading dataset: iamtarun/python_code_instructions_18k_alpaca")
-dataset = load_dataset("iamtarun/python_code_instructions_18k_alpaca", split="train")
-print(f"Dataset loaded: {len(dataset)} examples")
-
-dataset = dataset.shuffle(seed=42).select(range(min(10000, len(dataset))))
-print(f"Using {len(dataset)} examples")
-
-model_name = "Qwen/Qwen3-0.6B"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-
-def format_example(example):
-    return {"text": example["prompt"]}
-
-
-print("Formatting dataset...")
-dataset = dataset.map(format_example, remove_columns=dataset.column_names)
-
-dataset_split = dataset.train_test_split(test_size=0.05, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)} | Eval: {len(eval_dataset)}")
-
-config = SFTConfig(
-    output_dir="qwen3-python-code-sft-job4",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-python-code-sft-job4",
-    hub_strategy="every_save",
-    dataset_text_field="text",
-    num_train_epochs=3,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,
-    learning_rate=1e-4,
-    max_length=512,
-    logging_steps=20,
-    save_strategy="steps",
-    save_steps=200,
-    save_total_limit=2,
-    eval_strategy="steps",
-    eval_steps=200,
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job4-python-instructions",
-)
-
-peft_config = LoraConfig(
-    r=16,
-    lora_alpha=32,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model=model_name,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 4 complete!")

train_job4_v2.py

@@ -1,60 +0,0 @@
-# /// script
-# dependencies = ["trl>=0.12.0", "peft>=0.7.0", "transformers>=4.36.0", "accelerate>=0.24.0", "trackio", "datasets"]
-# ///
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading dataset")
-dataset = load_dataset("iamtarun/python_code_instructions_18k_alpaca", split="train")
-dataset = dataset.shuffle(seed=42).select(range(10000))
-print(f"Using {len(dataset)} examples")
-
-model_name = "Qwen/Qwen3-0.6B"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-
-def format_fn(ex):
-    return {"text": ex["prompt"]}
-
-
-dataset = dataset.map(format_fn, remove_columns=dataset.column_names)
-splits = dataset.train_test_split(test_size=0.05, seed=42)
-train_ds, eval_ds = splits["train"], splits["test"]
-print(f"Train: {len(train_ds)} Eval: {len(eval_ds)}")
-
-config = SFTConfig(
-    output_dir="qwen3-python-sft",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-python-code-sft-job4",
-    dataset_text_field="text",
-    num_train_epochs=3,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,
-    learning_rate=1e-4,
-    max_length=512,
-    logging_steps=20,
-    save_strategy="epoch",
-    warmup_ratio=0.1,
-    gradient_checkpointing=True,
-    bf16=True,
-    report_to="trackio",
-    run_name="job4-python",
-)
-
-peft_config = LoraConfig(
-    r=16, lora_alpha=32, target_modules=["q_proj", "k_proj", "v_proj", "o_proj"]
-)
-
-print("Starting training...")
-trainer = SFTTrainer(
-    model=model_name,
-    train_dataset=train_ds,
-    eval_dataset=eval_ds,
-    args=config,
-    peft_config=peft_config,
-)
-trainer.train()
-trainer.push_to_hub()
-print("Done!")

train_job5.py

@@ -1,105 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Job 4: Train on Python code instructions dataset
-"""
-
-import os
-from huggingface_hub import login
-
-# Explicitly login with token from environment
-token = os.environ.get("HF_TOKEN")
-if token:
-    login(token=token)
-    print("Logged in to HF Hub")
-else:
-    print("Warning: HF_TOKEN not found")
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-
-print("Loading dataset: iamtarun/python_code_instructions_18k_alpaca")
-dataset = load_dataset("iamtarun/python_code_instructions_18k_alpaca", split="train")
-print(f"Dataset loaded: {len(dataset)} examples")
-
-dataset = dataset.shuffle(seed=42).select(range(min(10000, len(dataset))))
-print(f"Using {len(dataset)} examples")
-
-model_name = "Qwen/Qwen3-0.6B"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-
-def format_example(example):
-    return {"text": example["prompt"]}
-
-
-print("Formatting dataset...")
-dataset = dataset.map(format_example, remove_columns=dataset.column_names)
-
-dataset_split = dataset.train_test_split(test_size=0.05, seed=42)
-train_dataset = dataset_split["train"]
-eval_dataset = dataset_split["test"]
-print(f"Train: {len(train_dataset)} | Eval: {len(eval_dataset)}")
-
-config = SFTConfig(
-    output_dir="qwen3-python-code-sft-job4",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-python-code-sft-job4",
-    hub_strategy="every_save",
-    dataset_text_field="text",
-    num_train_epochs=3,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,
-    learning_rate=1e-4,
-    max_length=512,
-    logging_steps=20,
-    save_strategy="steps",
-    save_steps=200,
-    save_total_limit=2,
-    eval_strategy="steps",
-    eval_steps=200,
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,
-    report_to="trackio",
-    project="qwen3-humaneval-challenge",
-    run_name="job4-python-instructions",
-)
-
-peft_config = LoraConfig(
-    r=16,
-    lora_alpha=32,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
-)
-
-print("Initializing trainer...")
-trainer = SFTTrainer(
-    model=model_name,
-    train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Starting training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("Job 4 complete!")

train_minimal.py

@@ -1,137 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers",
-#     "accelerate",
-#     "datasets",
-#     "trl",
-#     "peft",
-#     "bitsandbytes",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Minimal fine-tuning of Qwen3-0.6B on open-r1/codeforces-cots.
-Ultra-conservative training to avoid catastrophic forgetting.
-"""
-
-import os
-from datasets import load_dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig
-from trl import SFTConfig, SFTTrainer
-import torch
-
-
-def main():
-    print("=" * 60)
-    print("Minimal Fine-tuning: Qwen3-0.6B on codeforces-cots")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-    output_name = "passagereptile455/qwen3-codeforces-minimal"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load model
-    print("Loading model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    print(f"Model loaded on {model.device}")
-
-    # Load dataset with streaming to avoid memory issues
-    print("\nLoading dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    # Take only 500 examples for minimal training
-    dataset = dataset.take(500)
-
-    # Convert to list for SFTTrainer
-    print("Preparing examples...")
-    examples = list(dataset)
-    print(f"Loaded {len(examples)} examples")
-
-    # Check format
-    if examples:
-        print(f"First example keys: {examples[0].keys()}")
-        if "messages" in examples[0]:
-            print(f"Messages format: {len(examples[0]['messages'])} messages")
-
-    # LoRA config - very conservative
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Training config - ultra conservative
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=150,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=10,
-        save_steps=50,
-        fp16=True,
-        gradient_checkpointing=True,
-        max_seq_length=2048,
-        dataset_text_field=None,  # We'll use messages format
-        push_to_hub=True,
-        hub_model_id=output_name,
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-
-    # Format function for messages
-    def formatting_func(example):
-        return tokenizer.apply_chat_template(
-            example["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-
-    trainer = SFTTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=examples,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-        formatting_func=formatting_func,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("Starting training...")
-    print("=" * 60)
-    trainer.train()
-
-    # Save and push
-    print("\nSaving model...")
-    trainer.save_model()
-
-    print("\nPushing to hub...")
-    trainer.push_to_hub()
-
-    print("\n" + "=" * 60)
-    print(f"Training complete! Model saved to: {output_name}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_minimal_v2.py

@@ -1,135 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Minimal fine-tuning of Qwen3-0.6B on open-r1/codeforces-cots.
-Ultra-conservative training to avoid catastrophic forgetting.
-"""
-
-import os
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
-from peft import LoraConfig, get_peft_model
-from trl import SFTTrainer
-import torch
-
-
-def main():
-    print("=" * 60)
-    print("Minimal Fine-tuning: Qwen3-0.6B on codeforces-cots")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-    output_name = "passagereptile455/qwen3-codeforces-minimal"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load model
-    print("Loading model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    print(f"Model loaded on {model.device}")
-
-    # LoRA config - very conservative
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Apply LoRA
-    print("Applying LoRA...")
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-
-    # Load dataset with streaming
-    print("\nLoading dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    # Take only 500 examples for minimal training
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        # Format as text using chat template
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} examples")
-
-    # Create HF dataset
-    train_dataset = Dataset.from_list(examples)
-
-    # Training args - ultra conservative
-    training_args = TrainingArguments(
-        output_dir="./output",
-        max_steps=150,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=10,
-        save_steps=50,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=True,
-        hub_model_id=output_name,
-        report_to="none",
-        remove_unused_columns=False,
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        processing_class=tokenizer,
-        dataset_text_field="text",
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("Starting training...")
-    print("=" * 60)
-    trainer.train()
-
-    # Save and push
-    print("\nSaving model...")
-    trainer.save_model()
-
-    print("\nPushing to hub...")
-    trainer.push_to_hub()
-
-    print("\n" + "=" * 60)
-    print(f"Training complete! Model saved to: {output_name}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_minimal_v3.py

@@ -1,140 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Minimal fine-tuning of Qwen3-0.6B on open-r1/codeforces-cots.
-Ultra-conservative training to avoid catastrophic forgetting.
-"""
-
-import os
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-import torch
-
-
-def main():
-    print("=" * 60)
-    print("Minimal Fine-tuning: Qwen3-0.6B on codeforces-cots")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-    output_name = "passagereptile455/qwen3-codeforces-minimal"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load model
-    print("Loading model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    print(f"Model loaded on {model.device}")
-
-    # LoRA config - very conservative
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load dataset with streaming
-    print("\nLoading dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    # Take only 500 examples for minimal training
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        # Format as text using chat template
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} examples")
-
-    # Create HF dataset
-    train_dataset = Dataset.from_list(examples)
-
-    # Check SFTConfig parameters
-    import inspect
-
-    sig = inspect.signature(SFTConfig)
-    print(f"\nSFTConfig parameters: {list(sig.parameters.keys())[:20]}...")
-
-    # Training config - use only standard parameters
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=150,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=10,
-        save_steps=50,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=True,
-        hub_model_id=output_name,
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-
-    # Check SFTTrainer parameters
-    sig = inspect.signature(SFTTrainer.__init__)
-    print(f"SFTTrainer parameters: {list(sig.parameters.keys())[:15]}...")
-
-    trainer = SFTTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("Starting training...")
-    print("=" * 60)
-    trainer.train()
-
-    # Save and push
-    print("\nSaving model...")
-    trainer.save_model()
-
-    print("\nPushing to hub...")
-    trainer.push_to_hub()
-
-    print("\n" + "=" * 60)
-    print(f"Training complete! Model saved to: {output_name}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_minimal_v4.py

@@ -1,145 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Minimal fine-tuning of Qwen3-0.6B on open-r1/codeforces-cots.
-Saves to local output directory (no hub push during training).
-"""
-
-import os
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-import torch
-
-
-def main():
-    print("=" * 60)
-    print("Minimal Fine-tuning: Qwen3-0.6B on codeforces-cots")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load model
-    print("Loading model...")
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-    print(f"Model loaded on {model.device}")
-
-    # LoRA config - very conservative
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load dataset with streaming
-    print("\nLoading dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    # Take only 500 examples for minimal training
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        # Format as text using chat template
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} examples")
-
-    # Create HF dataset
-    train_dataset = Dataset.from_list(examples)
-
-    # Training config - NO hub push
-    training_args = SFTConfig(
-        output_dir="./qwen3-codeforces-minimal",
-        max_steps=150,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=10,
-        save_steps=50,
-        save_total_limit=2,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,  # Disabled - will upload manually
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("Starting training...")
-    print("=" * 60)
-    trainer.train()
-
-    # Save final model
-    print("\nSaving final model...")
-    trainer.save_model("./qwen3-codeforces-minimal-final")
-    tokenizer.save_pretrained("./qwen3-codeforces-minimal-final")
-
-    # Upload to hub using HfApi
-    print("\nUploading to HuggingFace Hub...")
-    from huggingface_hub import HfApi, create_repo
-
-    api = HfApi()
-    repo_id = "passagereptile455/qwen3-codeforces-minimal"
-
-    try:
-        create_repo(repo_id, exist_ok=True, repo_type="model")
-        api.upload_folder(
-            folder_path="./qwen3-codeforces-minimal-final",
-            repo_id=repo_id,
-            repo_type="model",
-        )
-        print(f"Model uploaded to: https://huggingface.co/{repo_id}")
-    except Exception as e:
-        print(f"Upload failed: {e}")
-        print("Model saved locally at: ./qwen3-codeforces-minimal-final")
-
-    print("\n" + "=" * 60)
-    print("Training complete!")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_sft_demo.py

@@ -1,32 +0,0 @@
-# /// script
-# dependencies = ["trl>=0.12.0", "peft>=0.7.0", "datasets", "transformers", "torch", "accelerate"]
-# ///
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-import os
-
-# Load a small dataset
-dataset = load_dataset("trl-lib/Capybara", split="train[:500]")
-
-# Setup trainer
-trainer = SFTTrainer(
-    model="Qwen/Qwen2.5-0.5B",
-    train_dataset=dataset,
-    peft_config=LoraConfig(r=16, lora_alpha=32, target_modules="all-linear"),
-    args=SFTConfig(
-        output_dir="qwen-demo-sft",
-        max_steps=100,
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        logging_steps=10,
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen-demo-sft",
-        hub_private_repo=True,
-    )
-)
-
-trainer.train()
-trainer.push_to_hub()
-print("Training complete! Model pushed to Hub.")

train_streaming.py

@@ -1,96 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Training with streaming dataset to avoid memory issues
-"""
-
-from datasets import load_dataset, Dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-import torch
-
-print("=" * 50)
-print("STREAMING DATASET TRAINING")
-print("=" * 50)
-
-print(f"CUDA: {torch.cuda.is_available()}")
-
-# Use streaming to load subset without memory issues
-print("Streaming codeforces-cots...")
-streaming_ds = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-# Collect 1000 examples
-print("Collecting 1000 examples...")
-examples = []
-for i, ex in enumerate(streaming_ds):
-    if i >= 1000:
-        break
-    examples.append(ex)
-    if (i + 1) % 200 == 0:
-        print(f"  Collected {i + 1} examples")
-
-print(f"Collected {len(examples)} examples")
-
-# Convert to regular dataset
-dataset = Dataset.from_list(examples)
-print(f"Dataset created: {len(dataset)}")
-
-config = SFTConfig(
-    output_dir="qwen3-codeforces",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-    hub_strategy="every_save",
-    max_steps=200,
-    per_device_train_batch_size=1,
-    gradient_accumulation_steps=8,
-    learning_rate=5e-6,
-    max_length=512,
-    logging_steps=20,
-    save_strategy="steps",
-    save_steps=100,
-    save_total_limit=1,
-    eval_strategy="no",
-    warmup_ratio=0.1,
-    lr_scheduler_type="cosine",
-    gradient_checkpointing=True,
-    bf16=True,
-    report_to="trackio",
-    project="qwen3-humaneval",
-    run_name="job1-streaming",
-)
-
-peft_config = LoraConfig(
-    r=8,
-    lora_alpha=16,
-    lora_dropout=0.05,
-    bias="none",
-    task_type="CAUSAL_LM",
-    target_modules=["q_proj", "v_proj"],
-)
-
-print("Creating trainer...")
-trainer = SFTTrainer(
-    model="Qwen/Qwen3-0.6B",
-    train_dataset=dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Training (200 steps)...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("=" * 50)
-print("SUCCESS!")
-print("=" * 50)

train_test_simple.py

@@ -1,79 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Test training with a reliable small dataset
-"""
-
-from datasets import load_dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-import torch
-
-print("=" * 50)
-print("TEST TRAINING JOB")
-print("=" * 50)
-
-print(f"CUDA: {torch.cuda.is_available()}")
-
-# Use trl-lib dataset which is guaranteed to work
-print("Loading trl-lib/Capybara...")
-dataset = load_dataset("trl-lib/Capybara", split="train")
-print(f"Loaded: {len(dataset)}")
-
-# Small subset
-dataset = dataset.shuffle(seed=42).select(range(500))
-print(f"Using: {len(dataset)}")
-
-config = SFTConfig(
-    output_dir="test-model",
-    push_to_hub=True,
-    hub_model_id="passagereptile455/qwen3-test-training",
-    hub_strategy="every_save",
-    max_steps=50,  # Very short test
-    per_device_train_batch_size=1,
-    gradient_accumulation_steps=4,
-    learning_rate=2e-5,
-    max_length=256,
-    logging_steps=10,
-    save_strategy="steps",
-    save_steps=50,
-    save_total_limit=1,
-    eval_strategy="no",
-    warmup_ratio=0.1,
-    gradient_checkpointing=True,
-    bf16=True,
-    report_to="trackio",
-    project="test",
-    run_name="test-train",
-)
-
-peft_config = LoraConfig(
-    r=8,
-    lora_alpha=16,
-    target_modules=["q_proj", "v_proj"],
-)
-
-print("Creating trainer...")
-trainer = SFTTrainer(
-    model="Qwen/Qwen3-0.6B",
-    train_dataset=dataset,
-    args=config,
-    peft_config=peft_config,
-)
-
-print("Training...")
-trainer.train()
-
-print("Pushing to Hub...")
-trainer.push_to_hub()
-
-print("SUCCESS!")

train_test_upload_150steps.py

@@ -1,303 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Combined training, testing, and upload script.
-Trains Qwen3-0.6B on codeforces-cots (150 steps - proven optimal), tests on HumanEval, uploads to Hub.
-"""
-
-import os
-import re
-import subprocess
-import tempfile
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from huggingface_hub import login, HfApi
-import torch
-
-# Authenticate with HF Hub at the start
-HF_TOKEN = os.environ.get("HF_TOKEN")
-if HF_TOKEN:
-    login(token=HF_TOKEN)
-    print("HF Hub authenticated successfully!")
-else:
-    print("WARNING: No HF_TOKEN found in environment")
-
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval"
-
-
-def extract_function_body(response: str) -> str:
-    """Extract just the function body from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-    lines = response.split("\n")
-
-    start_idx = 0
-    for i, line in enumerate(lines):
-        if line.strip().startswith("def "):
-            start_idx = i
-            break
-
-    start_idx += 1
-
-    if start_idx < len(lines):
-        stripped = lines[start_idx].strip()
-        if stripped.startswith('"""') or stripped.startswith("'''"):
-            quote = stripped[:3]
-            if stripped.count(quote) >= 2:
-                start_idx += 1
-            else:
-                start_idx += 1
-                while start_idx < len(lines) and quote not in lines[start_idx]:
-                    start_idx += 1
-                start_idx += 1
-
-    body_lines = lines[start_idx:]
-    return "\n".join(body_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def test_model(model, tokenizer, model_name="Model"):
-    """Test model on HumanEval."""
-    print(f"\n{'=' * 60}")
-    print(f"Testing: {model_name}")
-    print("=" * 60)
-
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-
-    for i, problem in enumerate(dataset):
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete this Python function. Output only the code.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        completion = extract_function_body(response)
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-
-        if (i + 1) % 20 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Pass: {passed} | Fail: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    final_score = passed / len(dataset) * 100
-    print(f"\nFINAL: {passed}/{len(dataset)} = {final_score:.2f}%")
-    return final_score
-
-
-def main():
-    print("=" * 60)
-    print("Combined Training, Testing & Upload")
-    print("150 steps - proven optimal configuration")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load base model
-    print("Loading base model...")
-    base_model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-
-    # LoRA config
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load training dataset
-    print("\nLoading training dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} training examples")
-    train_dataset = Dataset.from_list(examples)
-
-    # Training config - 150 steps (proven optimal)
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=150,  # Proven optimal - 200 regresses
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=25,
-        save_steps=150,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,  # We'll push manually after eval
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=base_model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("PHASE 1: Training (150 steps)")
-    print("=" * 60)
-    trainer.train()
-
-    # Save trained model locally
-    print("\nSaving trained model locally...")
-    trainer.save_model("./trained_model")
-    tokenizer.save_pretrained("./trained_model")
-
-    # Test the fine-tuned model
-    print("\n" + "=" * 60)
-    print("PHASE 2: Testing Fine-tuned Model")
-    print("=" * 60)
-
-    trained_model = trainer.model
-    trained_model.train(False)
-
-    finetuned_score = test_model(
-        trained_model, tokenizer, "Fine-tuned Qwen3-0.6B (150 steps)"
-    )
-
-    # Upload to Hub
-    print("\n" + "=" * 60)
-    print("PHASE 3: Uploading to HuggingFace Hub")
-    print("=" * 60)
-
-    try:
-        # Push model
-        print(f"Pushing model to {REPO_ID}...")
-        trained_model.push_to_hub(REPO_ID, token=HF_TOKEN)
-        tokenizer.push_to_hub(REPO_ID, token=HF_TOKEN)
-        print(f"Model uploaded successfully!")
-        print(f"URL: https://huggingface.co/{REPO_ID}")
-        upload_success = True
-    except Exception as e:
-        print(f"Upload failed: {e}")
-        upload_success = False
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("SUMMARY")
-    print("=" * 60)
-    print(f"Baseline (from earlier): 27.44%")
-    print(f"Fine-tuned (150 steps):  {finetuned_score:.2f}%")
-
-    if finetuned_score > 27.44:
-        print(f"IMPROVEMENT: +{finetuned_score - 27.44:.2f}%")
-        print("SUCCESS! Fine-tuned model beats baseline!")
-    else:
-        print(f"DIFFERENCE: {finetuned_score - 27.44:.2f}%")
-        print("Fine-tuned model did not beat baseline.")
-
-    print(f"\nUpload status: {'SUCCESS' if upload_success else 'FAILED'}")
-    if upload_success:
-        print(f"Model URL: https://huggingface.co/{REPO_ID}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_test_upload_v2.py

@@ -1,303 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Combined training, testing, and upload script.
-Trains Qwen3-0.6B on codeforces-cots (200 steps), tests on HumanEval, uploads to Hub.
-"""
-
-import os
-import re
-import subprocess
-import tempfile
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from huggingface_hub import login, HfApi
-import torch
-
-# Authenticate with HF Hub at the start
-HF_TOKEN = os.environ.get("HF_TOKEN")
-if HF_TOKEN:
-    login(token=HF_TOKEN)
-    print("HF Hub authenticated successfully!")
-else:
-    print("WARNING: No HF_TOKEN found in environment")
-
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval"
-
-
-def extract_function_body(response: str) -> str:
-    """Extract just the function body from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-    lines = response.split("\n")
-
-    start_idx = 0
-    for i, line in enumerate(lines):
-        if line.strip().startswith("def "):
-            start_idx = i
-            break
-
-    start_idx += 1
-
-    if start_idx < len(lines):
-        stripped = lines[start_idx].strip()
-        if stripped.startswith('"""') or stripped.startswith("'''"):
-            quote = stripped[:3]
-            if stripped.count(quote) >= 2:
-                start_idx += 1
-            else:
-                start_idx += 1
-                while start_idx < len(lines) and quote not in lines[start_idx]:
-                    start_idx += 1
-                start_idx += 1
-
-    body_lines = lines[start_idx:]
-    return "\n".join(body_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def test_model(model, tokenizer, model_name="Model"):
-    """Test model on HumanEval."""
-    print(f"\n{'=' * 60}")
-    print(f"Testing: {model_name}")
-    print("=" * 60)
-
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-
-    for i, problem in enumerate(dataset):
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete this Python function. Output only the code.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        completion = extract_function_body(response)
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-
-        if (i + 1) % 20 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Pass: {passed} | Fail: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    final_score = passed / len(dataset) * 100
-    print(f"\nFINAL: {passed}/{len(dataset)} = {final_score:.2f}%")
-    return final_score
-
-
-def main():
-    print("=" * 60)
-    print("Combined Training, Testing & Upload")
-    print("200 steps - testing if more training helps")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load base model
-    print("Loading base model...")
-    base_model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-
-    # LoRA config
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load training dataset
-    print("\nLoading training dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} training examples")
-    train_dataset = Dataset.from_list(examples)
-
-    # Training config - 200 steps (testing if more helps)
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=200,  # Increased from 150
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=25,
-        save_steps=200,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,  # We'll push manually after eval
-        report_to="none",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=base_model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("PHASE 1: Training (200 steps)")
-    print("=" * 60)
-    trainer.train()
-
-    # Save trained model locally
-    print("\nSaving trained model locally...")
-    trainer.save_model("./trained_model")
-    tokenizer.save_pretrained("./trained_model")
-
-    # Test the fine-tuned model
-    print("\n" + "=" * 60)
-    print("PHASE 2: Testing Fine-tuned Model")
-    print("=" * 60)
-
-    trained_model = trainer.model
-    trained_model.train(False)
-
-    finetuned_score = test_model(
-        trained_model, tokenizer, "Fine-tuned Qwen3-0.6B (200 steps)"
-    )
-
-    # Upload to Hub
-    print("\n" + "=" * 60)
-    print("PHASE 3: Uploading to HuggingFace Hub")
-    print("=" * 60)
-
-    try:
-        # Push model
-        print(f"Pushing model to {REPO_ID}...")
-        trained_model.push_to_hub(REPO_ID, token=HF_TOKEN)
-        tokenizer.push_to_hub(REPO_ID, token=HF_TOKEN)
-        print(f"Model uploaded successfully!")
-        print(f"URL: https://huggingface.co/{REPO_ID}")
-        upload_success = True
-    except Exception as e:
-        print(f"Upload failed: {e}")
-        upload_success = False
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("SUMMARY")
-    print("=" * 60)
-    print(f"Baseline (from earlier): 27.44%")
-    print(f"Fine-tuned (200 steps):  {finetuned_score:.2f}%")
-
-    if finetuned_score > 27.44:
-        print(f"IMPROVEMENT: +{finetuned_score - 27.44:.2f}%")
-        print("SUCCESS! Fine-tuned model beats baseline!")
-    else:
-        print(f"DIFFERENCE: {finetuned_score - 27.44:.2f}%")
-        print("Fine-tuned model did not beat baseline.")
-
-    print(f"\nUpload status: {'SUCCESS' if upload_success else 'FAILED'}")
-    if upload_success:
-        print(f"Model URL: https://huggingface.co/{REPO_ID}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_test_upload_v3.py

@@ -1,336 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "torch",
-#     "transformers>=4.45.0",
-#     "accelerate",
-#     "datasets",
-#     "trl>=0.12.0",
-#     "peft",
-#     "huggingface_hub",
-# ]
-# ///
-"""
-Combined training, testing, and upload script.
-Trains Qwen3-0.6B on codeforces-cots (150 steps - proven optimal), tests on HumanEval, uploads to Hub.
-"""
-
-import os
-import re
-import subprocess
-import tempfile
-import random
-import numpy as np
-from datasets import load_dataset, Dataset
-from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from huggingface_hub import login, HfApi
-import torch
-
-# Set seeds for reproducibility
-SEED = 42
-random.seed(SEED)
-np.random.seed(SEED)
-torch.manual_seed(SEED)
-torch.cuda.manual_seed_all(SEED)
-set_seed(SEED)
-
-# Authenticate with HF Hub at the start
-HF_TOKEN = os.environ.get("HF_TOKEN")
-if HF_TOKEN:
-    login(token=HF_TOKEN)
-    print("HF Hub authenticated successfully!")
-else:
-    print("WARNING: No HF_TOKEN found in environment")
-
-REPO_ID = "passagereptile455/qwen3-codeforces-humaneval"
-
-
-def extract_function_body(response: str) -> str:
-    """Extract just the function body from model response."""
-    response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
-    response = response.strip()
-
-    code_match = re.search(r"```python\s*(.*?)```", response, re.DOTALL)
-    if code_match:
-        response = code_match.group(1)
-    else:
-        code_match = re.search(r"```\s*(.*?)```", response, re.DOTALL)
-        if code_match:
-            response = code_match.group(1)
-
-    response = response.strip()
-    lines = response.split("\n")
-
-    start_idx = 0
-    for i, line in enumerate(lines):
-        if line.strip().startswith("def "):
-            start_idx = i
-            break
-
-    start_idx += 1
-
-    if start_idx < len(lines):
-        stripped = lines[start_idx].strip()
-        if stripped.startswith('"""') or stripped.startswith("'''"):
-            quote = stripped[:3]
-            if stripped.count(quote) >= 2:
-                start_idx += 1
-            else:
-                start_idx += 1
-                while start_idx < len(lines) and quote not in lines[start_idx]:
-                    start_idx += 1
-                start_idx += 1
-
-    body_lines = lines[start_idx:]
-    return "\n".join(body_lines)
-
-
-def run_test_subprocess(prompt: str, completion: str, test: str, entry_point: str):
-    """Run the test using subprocess."""
-    full_code = prompt + completion + "\n" + test + f"\ncheck({entry_point})"
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
-        f.write(full_code)
-        temp_path = f.name
-
-    try:
-        result = subprocess.run(
-            ["python", temp_path], capture_output=True, text=True, timeout=10
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-    except Exception:
-        return False
-    finally:
-        try:
-            os.unlink(temp_path)
-        except:
-            pass
-
-
-def test_model(model, tokenizer, model_name="Model"):
-    """Test model on HumanEval."""
-    print(f"\n{'=' * 60}")
-    print(f"Testing: {model_name}")
-    print("=" * 60)
-
-    dataset = load_dataset("openai/openai_humaneval", split="test")
-    print(f"Total problems: {len(dataset)}")
-
-    passed = 0
-    failed = 0
-
-    for i, problem in enumerate(dataset):
-        prompt = problem["prompt"]
-        test = problem["test"]
-        entry_point = problem["entry_point"]
-
-        messages = [
-            {
-                "role": "user",
-                "content": f"Complete this Python function. Output only the code.\n\n{prompt}",
-            }
-        ]
-
-        text = tokenizer.apply_chat_template(
-            messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-        )
-
-        inputs = tokenizer(text, return_tensors="pt").to(model.device)
-
-        with torch.no_grad():
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=512,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-            )
-
-        response = tokenizer.decode(
-            outputs[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
-        )
-
-        completion = extract_function_body(response)
-        success = run_test_subprocess(prompt, completion, test, entry_point)
-
-        if success:
-            passed += 1
-        else:
-            failed += 1
-
-        if (i + 1) % 20 == 0 or i == len(dataset) - 1:
-            print(
-                f"Progress: {i + 1}/{len(dataset)} | Pass: {passed} | Fail: {failed} | Rate: {passed / (i + 1) * 100:.1f}%"
-            )
-
-    final_score = passed / len(dataset) * 100
-    print(f"\nFINAL: {passed}/{len(dataset)} = {final_score:.2f}%")
-    return final_score
-
-
-def main():
-    print("=" * 60)
-    print("Combined Training, Testing & Upload")
-    print("150 steps - with SAME-RUN baseline comparison")
-    print("=" * 60)
-
-    model_name = "Qwen/Qwen3-0.6B"
-
-    # Load tokenizer
-    print("\nLoading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Load base model
-    print("Loading base model...")
-    base_model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-
-    # PHASE 1: Test BASE model first (same run comparison)
-    print("\n" + "=" * 60)
-    print("PHASE 1: Testing BASE Model (for fair comparison)")
-    print("=" * 60)
-    base_score = test_model(base_model, tokenizer, "Base Qwen3-0.6B")
-
-    # LoRA config
-    lora_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
-        bias="none",
-        task_type="CAUSAL_LM",
-    )
-
-    # Load training dataset
-    print("\nLoading training dataset (streaming)...")
-    dataset = load_dataset("open-r1/codeforces-cots", split="train", streaming=True)
-
-    print("Preparing examples...")
-    examples = []
-    for i, ex in enumerate(dataset):
-        if i >= 500:
-            break
-        text = tokenizer.apply_chat_template(
-            ex["messages"],
-            tokenize=False,
-            add_generation_prompt=False,
-        )
-        examples.append({"text": text})
-
-    print(f"Loaded {len(examples)} training examples")
-    train_dataset = Dataset.from_list(examples)
-
-    # Training config - 150 steps (proven optimal)
-    training_args = SFTConfig(
-        output_dir="./output",
-        max_steps=150,  # Proven optimal - 200 regresses
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=5e-6,
-        lr_scheduler_type="cosine",
-        warmup_steps=10,
-        logging_steps=25,
-        save_steps=150,
-        fp16=True,
-        gradient_checkpointing=True,
-        push_to_hub=False,  # We'll push manually after eval
-        report_to="none",
-        seed=42,  # Fixed seed for reproducibility
-    )
-
-    # Need to reload model for training (can't train already-evaluated model cleanly)
-    print("\nReloading model for training...")
-    del base_model
-    torch.cuda.empty_cache()
-
-    train_model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float16,
-        device_map="auto",
-    )
-
-    # Create trainer
-    print("\nInitializing trainer...")
-    trainer = SFTTrainer(
-        model=train_model,
-        args=training_args,
-        train_dataset=train_dataset,
-        peft_config=lora_config,
-        processing_class=tokenizer,
-    )
-
-    # Train
-    print("\n" + "=" * 60)
-    print("PHASE 2: Training (150 steps)")
-    print("=" * 60)
-    trainer.train()
-
-    # Save trained model locally
-    print("\nSaving trained model locally...")
-    trainer.save_model("./trained_model")
-    tokenizer.save_pretrained("./trained_model")
-
-    # Test the fine-tuned model
-    print("\n" + "=" * 60)
-    print("PHASE 3: Testing Fine-tuned Model")
-    print("=" * 60)
-
-    trained_model = trainer.model
-    trained_model.train(False)
-
-    finetuned_score = test_model(
-        trained_model, tokenizer, "Fine-tuned Qwen3-0.6B (150 steps)"
-    )
-
-    # Upload to Hub only if we beat the baseline
-    print("\n" + "=" * 60)
-    print("PHASE 4: Uploading to HuggingFace Hub")
-    print("=" * 60)
-
-    upload_success = False
-    if finetuned_score > base_score:
-        try:
-            print(f"Pushing model to {REPO_ID}...")
-            trained_model.push_to_hub(REPO_ID, token=HF_TOKEN)
-            tokenizer.push_to_hub(REPO_ID, token=HF_TOKEN)
-            print(f"Model uploaded successfully!")
-            print(f"URL: https://huggingface.co/{REPO_ID}")
-            upload_success = True
-        except Exception as e:
-            print(f"Upload failed: {e}")
-    else:
-        print("Fine-tuned model did NOT beat baseline - skipping upload")
-
-    # Summary - SAME RUN COMPARISON
-    print("\n" + "=" * 60)
-    print("SUMMARY (Same-Run Comparison)")
-    print("=" * 60)
-    print(f"Base model (this run):   {base_score:.2f}%")
-    print(f"Fine-tuned (150 steps):  {finetuned_score:.2f}%")
-    diff = finetuned_score - base_score
-
-    if diff > 0:
-        print(f"IMPROVEMENT: +{diff:.2f}%")
-        print("SUCCESS! Fine-tuned model beats baseline!")
-    elif diff == 0:
-        print("NO CHANGE: Same as baseline")
-    else:
-        print(f"REGRESSION: {diff:.2f}%")
-        print("Fine-tuned model is WORSE than baseline.")
-
-    print(f"\nUpload status: {'SUCCESS' if upload_success else 'SKIPPED/FAILED'}")
-    if upload_success:
-        print(f"Model URL: https://huggingface.co/{REPO_ID}")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()

train_v5_fixed.py

@@ -1,129 +0,0 @@
-# /// script
-# dependencies = [
-#     "trl>=0.12.0",
-#     "peft>=0.7.0",
-#     "transformers>=4.36.0",
-#     "accelerate>=0.24.0",
-#     "trackio",
-#     "datasets",
-# ]
-# ///
-
-"""
-Training with proper dataset formatting
-"""
-
-import sys
-import traceback
-from datasets import load_dataset, Dataset
-from peft import LoraConfig
-from trl import SFTTrainer, SFTConfig
-from transformers import AutoTokenizer
-import torch
-
-print("=" * 50)
-print("FIXED TRAINING v5")
-print("=" * 50)
-
-try:
-    print(f"CUDA: {torch.cuda.is_available()}")
-
-    # Streaming load
-    print("Streaming codeforces-cots...")
-    streaming_ds = load_dataset(
-        "open-r1/codeforces-cots", split="train", streaming=True
-    )
-
-    # Collect examples
-    print("Collecting 1000 examples...")
-    examples = []
-    for i, ex in enumerate(streaming_ds):
-        if i >= 1000:
-            break
-        examples.append(ex)
-
-    print(f"Collected {len(examples)} examples")
-    dataset = Dataset.from_list(examples)
-    print(f"Dataset columns: {dataset.column_names}")
-
-    # Check messages format
-    print(f"First messages sample: {dataset[0]['messages'][:100]}...")
-
-    # Load tokenizer
-    print("Loading tokenizer...")
-    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B", trust_remote_code=True)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    # Convert messages to text format for SFT
-    def format_messages(example):
-        messages = example["messages"]
-        # Format as simple text
-        text = ""
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-            text += f"<|{role}|>\n{content}\n"
-        return {"text": text}
-
-    print("Formatting dataset...")
-    dataset = dataset.map(format_messages, remove_columns=dataset.column_names)
-    print(f"Formatted. Sample: {dataset[0]['text'][:200]}...")
-
-    # Config
-    config = SFTConfig(
-        output_dir="qwen3-codeforces",
-        push_to_hub=True,
-        hub_model_id="passagereptile455/qwen3-0.6b-humaneval-job1",
-        hub_strategy="every_save",
-        max_steps=200,
-        per_device_train_batch_size=1,
-        gradient_accumulation_steps=8,
-        learning_rate=5e-6,
-        max_length=512,
-        logging_steps=20,
-        save_strategy="steps",
-        save_steps=100,
-        save_total_limit=1,
-        eval_strategy="no",
-        warmup_ratio=0.1,
-        lr_scheduler_type="cosine",
-        gradient_checkpointing=True,
-        bf16=True,
-        dataset_text_field="text",  # Specify text field
-        report_to="trackio",
-        project="qwen3-humaneval",
-        run_name="job1-v5",
-    )
-
-    peft_config = LoraConfig(
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.05,
-        bias="none",
-        task_type="CAUSAL_LM",
-        target_modules=["q_proj", "v_proj"],
-    )
-
-    print("Creating trainer...")
-    trainer = SFTTrainer(
-        model="Qwen/Qwen3-0.6B",
-        train_dataset=dataset,
-        args=config,
-        peft_config=peft_config,
-    )
-
-    print("Training (200 steps)...")
-    trainer.train()
-
-    print("Pushing to Hub...")
-    trainer.push_to_hub()
-
-    print("=" * 50)
-    print("SUCCESS!")
-    print("=" * 50)
-
-except Exception as e:
-    print(f"ERROR: {e}")
-    traceback.print_exc()
-    sys.exit(1)