solve_rate_eval.py

# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "datasets",
#     "huggingface_hub",
# ]
# ///
"""
SWE-Bench Pro Agent Solve Rate - Actual Patch Quality Evaluation
Measures how closely generated patches match gold patches (proxy for correctness).
"""
import json
import time
import re
import difflib
import random
import sys
from pathlib import Path
from datasets import load_dataset
from huggingface_hub import InferenceClient

MODEL = "deepseek-ai/DeepSeek-V4-Flash"
MAX_TOKENS = 4096
RATE_LIMIT_DELAY = 2.5
OUTPUT_FILE = "/tmp/solve_rate_results.json"
N_TASKS = 50

client = InferenceClient()

def extract_files_from_patch(patch_text):
    """Extract file paths from a unified diff patch."""
    if not patch_text:
        return set()
    files = set()
    for m in re.finditer(r'^\+\+\+ b/(\S+)', patch_text, re.MULTILINE):
        files.add(m.group(1))
    return files

def file_match_score(gen_patch, gold_patch):
    """Score 0-1: how well the generated patch targets the right files."""
    gen_files = extract_files_from_patch(gen_patch)
    gold_files = extract_files_from_patch(gold_patch)
    if not gold_files:
        return 1.0 if not gen_files else 0.0
    intersection = gen_files & gold_files
    if not gen_files:
        return 0.0
    precision = len(intersection) / len(gen_files)
    recall = len(intersection) / len(gold_files)
    return 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0

def patch_similarity(gen_patch, gold_patch):
    """Compute SequenceMatcher similarity between patches (approximate)."""
    if not gen_patch or not gold_patch:
        return 0.0
    # Normalize: remove whitespace differences
    gen_norm = '\n'.join(line.rstrip() for line in gen_patch.split('\n') if line.strip())
    gold_norm = '\n'.join(line.rstrip() for line in gold_patch.split('\n') if line.strip())
    if not gen_norm or not gold_norm:
        return 0.0
    return difflib.SequenceMatcher(None, gen_norm, gold_norm).ratio()

def call_model(prompt, max_retries=3):
    for attempt in range(max_retries):
        try:
            response = client.chat.completions.create(
                model=MODEL,
                messages=[{"role": "user", "content": prompt}],
                max_tokens=MAX_TOKENS,
                temperature=0.0
            )
            return response.choices[0].message.content
        except Exception as e:
            if attempt < max_retries - 1:
                time.sleep(RATE_LIMIT_DELAY * (attempt + 1))
            else:
                return None

def eval_instance(instance):
    iid = instance["instance_id"]
    repo = instance["repo"]
    gold_patch = instance["patch"]
    problem = instance["problem_statement"]
    lang = instance.get("repo_language", "")
    n_gold_files = len(extract_files_from_patch(gold_patch))
    in_cat = "single" if n_gold_files <= 1 else "multi"
    
    prompt = f"""You are an expert software engineer. Fix the following bug in {repo}.

Issue: {problem}

Generate a unified diff patch (---/+++ format) that fixes the issue. Output ONLY the patch:"""
    
    response = call_model(prompt)
    if response is None:
        return None
    
    # Quality metrics
    has_diff = bool(re.search(r'^(---|\+\+\+|diff --git)', response, re.MULTILINE))
    has_hunk = bool(re.search(r'^@@', response, re.MULTILINE))
    gen_files = extract_files_from_patch(response)
    
    # File match score
    gold_files = extract_files_from_patch(gold_patch)
    fm_score = file_match_score(response, gold_patch)
    
    # Patch content similarity
    p_sim = patch_similarity(response, gold_patch)
    
    return {
        "instance_id": iid,
        "repo": repo,
        "language": lang,
        "n_gold_files": n_gold_files,
        "category": in_cat,
        "gold_files": list(gold_files),
        "gen_files": list(gen_files),
        "file_match_f1": round(fm_score, 4),
        "patch_similarity": round(p_sim, 4),
        "format_valid": has_diff and has_hunk,
        "gold_patch_len": len(gold_patch),
        "gen_patch_len": len(response),
    }

def main():
    print(f"Loading SWE-Bench Pro dataset...")
    ds = load_dataset("ScaleAI/SWE-bench_Pro", split="test")
    print(f"Total: {len(ds)}")
    
    # Stratified sampling: balance single/multi file
    single = [d for d in ds if len(extract_files_from_patch(d["patch"])) <= 1]
    multi = [d for d in ds if len(extract_files_from_patch(d["patch"])) > 1]
    print(f"Single-file: {len(single)}, Multi-file: {len(multi)}")
    
    random.seed(42)
    n_per = min(N_TASKS // 2, min(len(single), len(multi)))
    tasks = random.sample(single, n_per) + random.sample(multi, n_per)
    random.shuffle(tasks)
    
    results = []
    for i, instance in enumerate(tasks):
        iid = instance["instance_id"]
        print(f"[{i+1}/{len(tasks)}] {iid[:60]}...")
        result = eval_instance(instance)
        if result is None:
            print(f"  SKIP (no response)")
            continue
        results.append(result)
        fm = result["file_match_f1"]
        sim = result["patch_similarity"]
        cat = result["category"]
        print(f"  {cat:6s} | F1={fm:.3f} | sim={sim:.3f} | files={result['n_gold_files']}")
        time.sleep(RATE_LIMIT_DELAY)
    
    # Analysis
    by_cat = {}
    by_lang = {}
    by_repo = {}
    
    for r in results:
        for key, bucket in [("category", by_cat), ("language", by_lang), ("repo", by_repo)]:
            val = r[key]
            if val not in bucket:
                bucket[val] = []
            bucket[val].append(r)
    
    def agg(name, data):
        if not data:
            return {}
        fms = [d["file_match_f1"] for d in data]
        sims = [d["patch_similarity"] for d in data]
        valid = sum(1 for d in data if d["format_valid"])
        return {
            "n": len(data),
            "mean_file_match_f1": round(sum(fms)/len(fms), 4),
            "mean_patch_sim": round(sum(sims)/len(sims), 4),
            "format_valid_rate": f"{valid}/{len(data)} ({valid/len(data)*100:.1f}%)",
        }
    
    summary = {
        "model": MODEL,
        "total": len(results),
        "by_category": {k: agg(k, v) for k, v in sorted(by_cat.items())},
        "by_language": {k: agg(k, v) for k, v in sorted(by_lang.items())},
        "by_repo": {k: agg(k, v) for k, v in sorted(by_repo.items())},
        "overall": agg("overall", results),
        "findings": [],
        "results": results,
    }
    
    # Findings
    if "single" in by_cat and "multi" in by_cat:
        s_f1 = sum(r["file_match_f1"] for r in by_cat["single"]) / len(by_cat["single"])
        m_f1 = sum(r["file_match_f1"] for r in by_cat["multi"]) / len(by_cat["multi"])
        summary["findings"].append(
            f"SINGLE vs MULTI: File-match F1 = {s_f1:.3f} vs {m_f1:.3f} "
            f"(gap: {abs(s_f1-m_f1):.3f}). "
            f"{'Performance varies by task complexity' if abs(s_f1-m_f1) > 0.05 else 'Minimal variation'}"
        )
    
    lang_f1s = {k: sum(r["file_match_f1"] for r in v) / len(v) for k, v in by_lang.items()}
    if len(lang_f1s) > 1:
        best = max(lang_f1s, key=lang_f1s.get)
        worst = min(lang_f1s, key=lang_f1s.get)
        summary["findings"].append(
            f"BY LANGUAGE: {best}={lang_f1s[best]:.3f}, {worst}={lang_f1s[worst]:.3f}. "
            f"Gap: {lang_f1s[best] - lang_f1s[worst]:.3f}"
        )
    
    with open(OUTPUT_FILE, "w") as f:
        json.dump(summary, f, indent=2)
    
    print(f"\n{'='*60}")
    print("FINAL RESULTS")
    print(f"{'='*60}")
    print(f"Total: {len(results)}")
    print(f"By language:")
    for lang, data in sorted(by_lang.items()):
        agg_data = agg(lang, data)
        print(f"  {lang:10s}: F1={agg_data['mean_file_match_f1']:.3f} sim={agg_data['mean_patch_sim']:.3f} n={agg_data['n']}")
    print(f"\nBy category:")
    for cat, data in sorted(by_cat.items()):
        agg_data = agg(cat, data)
        print(f"  {cat:10s}: F1={agg_data['mean_file_match_f1']:.3f} sim={agg_data['mean_patch_sim']:.3f} n={agg_data['n']}")
    print(f"\nFindings:")
    for f in summary["findings"]:
        print(f"  • {f}")

if __name__ == "__main__":
    main()