# /// script # requires-python = ">=3.10" # dependencies = [ # "datasets", # "huggingface_hub", # ] # /// """ SWE-Bench Pro Agent Solve Rate - Actual Patch Quality Evaluation Measures how closely generated patches match gold patches (proxy for correctness). """ import json import time import re import difflib import random import sys from pathlib import Path from datasets import load_dataset from huggingface_hub import InferenceClient MODEL = "deepseek-ai/DeepSeek-V4-Flash" MAX_TOKENS = 4096 RATE_LIMIT_DELAY = 2.5 OUTPUT_FILE = "/tmp/solve_rate_results.json" N_TASKS = 50 client = InferenceClient() def extract_files_from_patch(patch_text): """Extract file paths from a unified diff patch.""" if not patch_text: return set() files = set() for m in re.finditer(r'^\+\+\+ b/(\S+)', patch_text, re.MULTILINE): files.add(m.group(1)) return files def file_match_score(gen_patch, gold_patch): """Score 0-1: how well the generated patch targets the right files.""" gen_files = extract_files_from_patch(gen_patch) gold_files = extract_files_from_patch(gold_patch) if not gold_files: return 1.0 if not gen_files else 0.0 intersection = gen_files & gold_files if not gen_files: return 0.0 precision = len(intersection) / len(gen_files) recall = len(intersection) / len(gold_files) return 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 def patch_similarity(gen_patch, gold_patch): """Compute SequenceMatcher similarity between patches (approximate).""" if not gen_patch or not gold_patch: return 0.0 # Normalize: remove whitespace differences gen_norm = '\n'.join(line.rstrip() for line in gen_patch.split('\n') if line.strip()) gold_norm = '\n'.join(line.rstrip() for line in gold_patch.split('\n') if line.strip()) if not gen_norm or not gold_norm: return 0.0 return difflib.SequenceMatcher(None, gen_norm, gold_norm).ratio() def call_model(prompt, max_retries=3): for attempt in range(max_retries): try: response = client.chat.completions.create( model=MODEL, messages=[{"role": "user", "content": prompt}], max_tokens=MAX_TOKENS, temperature=0.0 ) return response.choices[0].message.content except Exception as e: if attempt < max_retries - 1: time.sleep(RATE_LIMIT_DELAY * (attempt + 1)) else: return None def eval_instance(instance): iid = instance["instance_id"] repo = instance["repo"] gold_patch = instance["patch"] problem = instance["problem_statement"] lang = instance.get("repo_language", "") n_gold_files = len(extract_files_from_patch(gold_patch)) in_cat = "single" if n_gold_files <= 1 else "multi" prompt = f"""You are an expert software engineer. Fix the following bug in {repo}. Issue: {problem} Generate a unified diff patch (---/+++ format) that fixes the issue. Output ONLY the patch:""" response = call_model(prompt) if response is None: return None # Quality metrics has_diff = bool(re.search(r'^(---|\+\+\+|diff --git)', response, re.MULTILINE)) has_hunk = bool(re.search(r'^@@', response, re.MULTILINE)) gen_files = extract_files_from_patch(response) # File match score gold_files = extract_files_from_patch(gold_patch) fm_score = file_match_score(response, gold_patch) # Patch content similarity p_sim = patch_similarity(response, gold_patch) return { "instance_id": iid, "repo": repo, "language": lang, "n_gold_files": n_gold_files, "category": in_cat, "gold_files": list(gold_files), "gen_files": list(gen_files), "file_match_f1": round(fm_score, 4), "patch_similarity": round(p_sim, 4), "format_valid": has_diff and has_hunk, "gold_patch_len": len(gold_patch), "gen_patch_len": len(response), } def main(): print(f"Loading SWE-Bench Pro dataset...") ds = load_dataset("ScaleAI/SWE-bench_Pro", split="test") print(f"Total: {len(ds)}") # Stratified sampling: balance single/multi file single = [d for d in ds if len(extract_files_from_patch(d["patch"])) <= 1] multi = [d for d in ds if len(extract_files_from_patch(d["patch"])) > 1] print(f"Single-file: {len(single)}, Multi-file: {len(multi)}") random.seed(42) n_per = min(N_TASKS // 2, min(len(single), len(multi))) tasks = random.sample(single, n_per) + random.sample(multi, n_per) random.shuffle(tasks) results = [] for i, instance in enumerate(tasks): iid = instance["instance_id"] print(f"[{i+1}/{len(tasks)}] {iid[:60]}...") result = eval_instance(instance) if result is None: print(f" SKIP (no response)") continue results.append(result) fm = result["file_match_f1"] sim = result["patch_similarity"] cat = result["category"] print(f" {cat:6s} | F1={fm:.3f} | sim={sim:.3f} | files={result['n_gold_files']}") time.sleep(RATE_LIMIT_DELAY) # Analysis by_cat = {} by_lang = {} by_repo = {} for r in results: for key, bucket in [("category", by_cat), ("language", by_lang), ("repo", by_repo)]: val = r[key] if val not in bucket: bucket[val] = [] bucket[val].append(r) def agg(name, data): if not data: return {} fms = [d["file_match_f1"] for d in data] sims = [d["patch_similarity"] for d in data] valid = sum(1 for d in data if d["format_valid"]) return { "n": len(data), "mean_file_match_f1": round(sum(fms)/len(fms), 4), "mean_patch_sim": round(sum(sims)/len(sims), 4), "format_valid_rate": f"{valid}/{len(data)} ({valid/len(data)*100:.1f}%)", } summary = { "model": MODEL, "total": len(results), "by_category": {k: agg(k, v) for k, v in sorted(by_cat.items())}, "by_language": {k: agg(k, v) for k, v in sorted(by_lang.items())}, "by_repo": {k: agg(k, v) for k, v in sorted(by_repo.items())}, "overall": agg("overall", results), "findings": [], "results": results, } # Findings if "single" in by_cat and "multi" in by_cat: s_f1 = sum(r["file_match_f1"] for r in by_cat["single"]) / len(by_cat["single"]) m_f1 = sum(r["file_match_f1"] for r in by_cat["multi"]) / len(by_cat["multi"]) summary["findings"].append( f"SINGLE vs MULTI: File-match F1 = {s_f1:.3f} vs {m_f1:.3f} " f"(gap: {abs(s_f1-m_f1):.3f}). " f"{'Performance varies by task complexity' if abs(s_f1-m_f1) > 0.05 else 'Minimal variation'}" ) lang_f1s = {k: sum(r["file_match_f1"] for r in v) / len(v) for k, v in by_lang.items()} if len(lang_f1s) > 1: best = max(lang_f1s, key=lang_f1s.get) worst = min(lang_f1s, key=lang_f1s.get) summary["findings"].append( f"BY LANGUAGE: {best}={lang_f1s[best]:.3f}, {worst}={lang_f1s[worst]:.3f}. " f"Gap: {lang_f1s[best] - lang_f1s[worst]:.3f}" ) with open(OUTPUT_FILE, "w") as f: json.dump(summary, f, indent=2) print(f"\n{'='*60}") print("FINAL RESULTS") print(f"{'='*60}") print(f"Total: {len(results)}") print(f"By language:") for lang, data in sorted(by_lang.items()): agg_data = agg(lang, data) print(f" {lang:10s}: F1={agg_data['mean_file_match_f1']:.3f} sim={agg_data['mean_patch_sim']:.3f} n={agg_data['n']}") print(f"\nBy category:") for cat, data in sorted(by_cat.items()): agg_data = agg(cat, data) print(f" {cat:10s}: F1={agg_data['mean_file_match_f1']:.3f} sim={agg_data['mean_patch_sim']:.3f} n={agg_data['n']}") print(f"\nFindings:") for f in summary["findings"]: print(f" • {f}") if __name__ == "__main__": main()