# /// script # requires-python = ">=3.10" # dependencies = [ # "datasets", # "huggingface_hub", # ] # /// """ Comprehensive SWE-Bench Pro Agent Evaluation Tests DeepSeek-V4-Flash across categories to show performance variations. """ import json import time import re import sys from pathlib import Path from datasets import load_dataset from huggingface_hub import InferenceClient # Configuration MODEL = "deepseek-ai/DeepSeek-V4-Flash" MAX_TOKENS = 3072 RATE_LIMIT_DELAY = 2.0 OUTPUT_FILE = "/tmp/comprehensive_eval_results.json" # Sample sizes per category for statistical significance SINGLE_FILE_N = 15 MULTI_FILE_N = 15 client = InferenceClient() def is_valid_patch(response): if not response: return False, "No response" has_diff = bool(re.search(r'^(---|\+\+\+|diff --git)', response, re.MULTILINE)) has_hunk = bool(re.search(r'^@@', response, re.MULTILINE)) has_changes = bool(re.search(r'^[+-][^+-]', response, re.MULTILINE)) markers = sum([has_diff, has_hunk, has_changes]) if markers >= 2: return True, "Valid unified diff" return False, f"Insufficient diff markers ({markers}/3)" def call_model(prompt, max_retries=3): for attempt in range(max_retries): try: response = client.chat.completions.create( model=MODEL, messages=[{"role": "user", "content": prompt}], max_tokens=MAX_TOKENS, temperature=0.0 ) return response.choices[0].message.content except Exception as e: if attempt < max_retries - 1: time.sleep(RATE_LIMIT_DELAY * (attempt + 1)) else: return f"[ERROR: {e}]" def create_prompt(instance): repo = instance.get("repo", "unknown") problem = instance.get("problem_statement", "") return f"""You are an expert software engineer. Fix the following bug in {repo}. Issue: {problem} Generate a unified diff patch (---/+++ format) that fixes the issue. Output ONLY the patch:""" def main(): print(f"Loading SWE-Bench Pro dataset...") ds = load_dataset("ScaleAI/SWE-bench_Pro", split="test") print(f"Total instances: {len(ds)}") # Classify tasks by patch complexity single_file = [] multi_file = [] for d in ds: patch = d.get("patch", "") files = set() for m in re.finditer(r'^\+\+\+ b/(\S+)', patch, re.MULTILINE): files.add(m.group(1)) n_files = len(files) if n_files <= 1: single_file.append(d) else: multi_file.append(d) print(f"Single-file tasks: {len(single_file)}") print(f"Multi-file tasks: {len(multi_file)} ({len(multi_file)/len(ds)*100:.1f}%)") # Sample tasks from each category import random random.seed(42) def sample_eval(tasks, n, label): sampled = random.sample(tasks, min(n, len(tasks))) results = [] for i, instance in enumerate(sampled): iid = instance.get("instance_id", f"{label}_{i}") repo = instance.get("repo", "") patch = instance.get("patch", "") n_gold_files = len(set(re.findall(r'^\+\+\+ b/(\S+)', patch, re.MULTILINE))) problem_len = len(instance.get("problem_statement", "")) repo_lang = instance.get("repo_language", "") prompt = create_prompt(instance) response = call_model(prompt) is_valid, reason = is_valid_patch(response) result = { "instance_id": iid, "repo": repo, "category": label, "n_gold_files": n_gold_files, "problem_len": problem_len, "repo_language": repo_lang, "response_len": len(response) if response else 0, "is_valid_patch": is_valid, "validation_reason": reason, } results.append(result) print(f" [{i+1}/{len(sampled)}] {iid[:50]:50s} {'✓' if is_valid else '✗'} ({n_gold_files} files, {repo_lang})") time.sleep(RATE_LIMIT_DELAY) return results print(f"\n{'='*60}") print(f"Evaluating single-file tasks ({SINGLE_FILE_N})...") sf_results = sample_eval(single_file, SINGLE_FILE_N, "single_file") print(f"\n{'='*60}") print(f"Evaluating multi-file tasks ({MULTI_FILE_N})...") mf_results = sample_eval(multi_file, MULTI_FILE_N, "multi_file") # Combine and analyze all_results = sf_results + mf_results # Category comparison sf_valid = sum(1 for r in sf_results if r["is_valid_patch"]) mf_valid = sum(1 for r in mf_results if r["is_valid_patch"]) # By language lang_results = {} for r in all_results: lang = r["repo_language"] if lang not in lang_results: lang_results[lang] = {"total": 0, "valid": 0} lang_results[lang]["total"] += 1 if r["is_valid_patch"]: lang_results[lang]["valid"] += 1 # By repo repo_results = {} for r in all_results: repo = r["repo"] if repo not in repo_results: repo_results[repo] = {"total": 0, "valid": 0} repo_results[repo]["total"] += 1 if r["is_valid_patch"]: repo_results[repo]["valid"] += 1 summary = { "model": MODEL, "total_tested": len(all_results), "single_file": { "tested": len(sf_results), "valid_patches": sf_valid, "rate": f"{sf_valid/len(sf_results)*100:.1f}%" if sf_results else "N/A" }, "multi_file": { "tested": len(mf_results), "valid_patches": mf_valid, "rate": f"{mf_valid/len(mf_results)*100:.1f}%" if mf_results else "N/A" }, "overall_rate": f"{(sf_valid + mf_valid)/len(all_results)*100:.1f}%" if all_results else "N/A", "by_language": {k: f"{v['valid']}/{v['total']} ({v['valid']/v['total']*100:.1f}%)" for k, v in sorted(lang_results.items())}, "by_repo": {k: f"{v['valid']}/{v['total']} ({v['valid']/v['total']*100:.1f}%)" for k, v in sorted(repo_results.items())}, "findings": [] } # Compute findings if sf_results and mf_results: sf_pct = sf_valid / len(sf_results) * 100 mf_pct = mf_valid / len(mf_results) * 100 summary["findings"].append( f"Single-file tasks: {sf_pct:.1f}% format compliance vs Multi-file: {mf_pct:.1f}%. " f"Difference: {abs(sf_pct - mf_pct):.1f}pp. " f"{'Performance varies by task complexity (multi-file harder)' if sf_pct > mf_pct else 'No significant variation detected'}" ) if len(lang_results) > 1: lang_pcts = {k: v['valid']/v['total']*100 for k, v in lang_results.items()} best_lang = max(lang_pcts, key=lang_pcts.get) worst_lang = min(lang_pcts, key=lang_pcts.get) summary["findings"].append( f"Performance varies by language: {best_lang} ({lang_pcts[best_lang]:.1f}%) best, " f"{worst_lang} ({lang_pcts[worst_lang]:.1f}%) worst. " f"Gap: {lang_pcts[best_lang] - lang_pcts[worst_lang]:.1f}pp" ) if len(repo_results) > 1: repo_pcts = {k: v['valid']/v['total']*100 for k, v in repo_results.items()} best_repo = max(repo_pcts, key=repo_pcts.get) worst_repo = min(repo_pcts, key=repo_pcts.get) summary["findings"].append( f"Performance varies by repo: {best_repo} ({repo_pcts[best_repo]:.1f}%) best, " f"{worst_repo} ({repo_pcts[worst_repo]:.1f}%) worst" ) summary["results"] = all_results with open(OUTPUT_FILE, "w") as f: json.dump(summary, f, indent=2) print(f"\n{'='*60}") print(f"RESULTS SUMMARY") print(f"{'='*60}") print(f"Model: {MODEL}") print(f"Single-file: {sf_valid}/{len(sf_results)} ({summary['single_file']['rate']})") print(f"Multi-file: {mf_valid}/{len(mf_results)} ({summary['multi_file']['rate']})") print(f"Overall: {sf_valid + mf_valid}/{len(all_results)} ({summary['overall_rate']})") print(f"\nBy language:") for lang, rate in summary["by_language"].items(): print(f" {lang}: {rate}") print(f"\nBy repo:") for repo, rate in summary["by_repo"].items(): print(f" {repo}: {rate}") print(f"\nFindings:") for f in summary["findings"]: print(f" • {f}") print(f"\nResults saved to: {OUTPUT_FILE}") if __name__ == "__main__": main()