comprehensive_eval.py

# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "datasets",
#     "huggingface_hub",
# ]
# ///
"""
Comprehensive SWE-Bench Pro Agent Evaluation
Tests DeepSeek-V4-Flash across categories to show performance variations.
"""
import json
import time
import re
import sys
from pathlib import Path
from datasets import load_dataset
from huggingface_hub import InferenceClient

# Configuration
MODEL = "deepseek-ai/DeepSeek-V4-Flash"
MAX_TOKENS = 3072
RATE_LIMIT_DELAY = 2.0
OUTPUT_FILE = "/tmp/comprehensive_eval_results.json"

# Sample sizes per category for statistical significance
SINGLE_FILE_N = 15
MULTI_FILE_N = 15

client = InferenceClient()

def is_valid_patch(response):
    if not response:
        return False, "No response"
    has_diff = bool(re.search(r'^(---|\+\+\+|diff --git)', response, re.MULTILINE))
    has_hunk = bool(re.search(r'^@@', response, re.MULTILINE))
    has_changes = bool(re.search(r'^[+-][^+-]', response, re.MULTILINE))
    markers = sum([has_diff, has_hunk, has_changes])
    if markers >= 2:
        return True, "Valid unified diff"
    return False, f"Insufficient diff markers ({markers}/3)"

def call_model(prompt, max_retries=3):
    for attempt in range(max_retries):
        try:
            response = client.chat.completions.create(
                model=MODEL,
                messages=[{"role": "user", "content": prompt}],
                max_tokens=MAX_TOKENS,
                temperature=0.0
            )
            return response.choices[0].message.content
        except Exception as e:
            if attempt < max_retries - 1:
                time.sleep(RATE_LIMIT_DELAY * (attempt + 1))
            else:
                return f"[ERROR: {e}]"

def create_prompt(instance):
    repo = instance.get("repo", "unknown")
    problem = instance.get("problem_statement", "")
    return f"""You are an expert software engineer. Fix the following bug in {repo}.

Issue: {problem}

Generate a unified diff patch (---/+++ format) that fixes the issue. Output ONLY the patch:"""

def main():
    print(f"Loading SWE-Bench Pro dataset...")
    ds = load_dataset("ScaleAI/SWE-bench_Pro", split="test")
    print(f"Total instances: {len(ds)}")

    # Classify tasks by patch complexity
    single_file = []
    multi_file = []
    for d in ds:
        patch = d.get("patch", "")
        files = set()
        for m in re.finditer(r'^\+\+\+ b/(\S+)', patch, re.MULTILINE):
            files.add(m.group(1))
        n_files = len(files)
        if n_files <= 1:
            single_file.append(d)
        else:
            multi_file.append(d)

    print(f"Single-file tasks: {len(single_file)}")
    print(f"Multi-file tasks: {len(multi_file)} ({len(multi_file)/len(ds)*100:.1f}%)")

    # Sample tasks from each category
    import random
    random.seed(42)

    def sample_eval(tasks, n, label):
        sampled = random.sample(tasks, min(n, len(tasks)))
        results = []
        for i, instance in enumerate(sampled):
            iid = instance.get("instance_id", f"{label}_{i}")
            repo = instance.get("repo", "")
            patch = instance.get("patch", "")
            n_gold_files = len(set(re.findall(r'^\+\+\+ b/(\S+)', patch, re.MULTILINE)))
            problem_len = len(instance.get("problem_statement", ""))
            repo_lang = instance.get("repo_language", "")

            prompt = create_prompt(instance)
            response = call_model(prompt)
            is_valid, reason = is_valid_patch(response)

            result = {
                "instance_id": iid,
                "repo": repo,
                "category": label,
                "n_gold_files": n_gold_files,
                "problem_len": problem_len,
                "repo_language": repo_lang,
                "response_len": len(response) if response else 0,
                "is_valid_patch": is_valid,
                "validation_reason": reason,
            }
            results.append(result)
            print(f"  [{i+1}/{len(sampled)}] {iid[:50]:50s} {'✓' if is_valid else '✗'} ({n_gold_files} files, {repo_lang})")
            time.sleep(RATE_LIMIT_DELAY)
        return results

    print(f"\n{'='*60}")
    print(f"Evaluating single-file tasks ({SINGLE_FILE_N})...")
    sf_results = sample_eval(single_file, SINGLE_FILE_N, "single_file")

    print(f"\n{'='*60}")
    print(f"Evaluating multi-file tasks ({MULTI_FILE_N})...")
    mf_results = sample_eval(multi_file, MULTI_FILE_N, "multi_file")

    # Combine and analyze
    all_results = sf_results + mf_results

    # Category comparison
    sf_valid = sum(1 for r in sf_results if r["is_valid_patch"])
    mf_valid = sum(1 for r in mf_results if r["is_valid_patch"])

    # By language
    lang_results = {}
    for r in all_results:
        lang = r["repo_language"]
        if lang not in lang_results:
            lang_results[lang] = {"total": 0, "valid": 0}
        lang_results[lang]["total"] += 1
        if r["is_valid_patch"]:
            lang_results[lang]["valid"] += 1

    # By repo
    repo_results = {}
    for r in all_results:
        repo = r["repo"]
        if repo not in repo_results:
            repo_results[repo] = {"total": 0, "valid": 0}
        repo_results[repo]["total"] += 1
        if r["is_valid_patch"]:
            repo_results[repo]["valid"] += 1

    summary = {
        "model": MODEL,
        "total_tested": len(all_results),
        "single_file": {
            "tested": len(sf_results),
            "valid_patches": sf_valid,
            "rate": f"{sf_valid/len(sf_results)*100:.1f}%" if sf_results else "N/A"
        },
        "multi_file": {
            "tested": len(mf_results),
            "valid_patches": mf_valid,
            "rate": f"{mf_valid/len(mf_results)*100:.1f}%" if mf_results else "N/A"
        },
        "overall_rate": f"{(sf_valid + mf_valid)/len(all_results)*100:.1f}%" if all_results else "N/A",
        "by_language": {k: f"{v['valid']}/{v['total']} ({v['valid']/v['total']*100:.1f}%)" for k, v in sorted(lang_results.items())},
        "by_repo": {k: f"{v['valid']}/{v['total']} ({v['valid']/v['total']*100:.1f}%)" for k, v in sorted(repo_results.items())},
        "findings": []
    }

    # Compute findings
    if sf_results and mf_results:
        sf_pct = sf_valid / len(sf_results) * 100
        mf_pct = mf_valid / len(mf_results) * 100
        summary["findings"].append(
            f"Single-file tasks: {sf_pct:.1f}% format compliance vs Multi-file: {mf_pct:.1f}%. "
            f"Difference: {abs(sf_pct - mf_pct):.1f}pp. "
            f"{'Performance varies by task complexity (multi-file harder)' if sf_pct > mf_pct else 'No significant variation detected'}"
        )

    if len(lang_results) > 1:
        lang_pcts = {k: v['valid']/v['total']*100 for k, v in lang_results.items()}
        best_lang = max(lang_pcts, key=lang_pcts.get)
        worst_lang = min(lang_pcts, key=lang_pcts.get)
        summary["findings"].append(
            f"Performance varies by language: {best_lang} ({lang_pcts[best_lang]:.1f}%) best, "
            f"{worst_lang} ({lang_pcts[worst_lang]:.1f}%) worst. "
            f"Gap: {lang_pcts[best_lang] - lang_pcts[worst_lang]:.1f}pp"
        )

    if len(repo_results) > 1:
        repo_pcts = {k: v['valid']/v['total']*100 for k, v in repo_results.items()}
        best_repo = max(repo_pcts, key=repo_pcts.get)
        worst_repo = min(repo_pcts, key=repo_pcts.get)
        summary["findings"].append(
            f"Performance varies by repo: {best_repo} ({repo_pcts[best_repo]:.1f}%) best, "
            f"{worst_repo} ({repo_pcts[worst_repo]:.1f}%) worst"
        )

    summary["results"] = all_results

    with open(OUTPUT_FILE, "w") as f:
        json.dump(summary, f, indent=2)

    print(f"\n{'='*60}")
    print(f"RESULTS SUMMARY")
    print(f"{'='*60}")
    print(f"Model: {MODEL}")
    print(f"Single-file: {sf_valid}/{len(sf_results)} ({summary['single_file']['rate']})")
    print(f"Multi-file:  {mf_valid}/{len(mf_results)} ({summary['multi_file']['rate']})")
    print(f"Overall:     {sf_valid + mf_valid}/{len(all_results)} ({summary['overall_rate']})")
    print(f"\nBy language:")
    for lang, rate in summary["by_language"].items():
        print(f"  {lang}: {rate}")
    print(f"\nBy repo:")
    for repo, rate in summary["by_repo"].items():
        print(f"  {repo}: {rate}")
    print(f"\nFindings:")
    for f in summary["findings"]:
        print(f"  • {f}")
    print(f"\nResults saved to: {OUTPUT_FILE}")

if __name__ == "__main__":
    main()