import json import os from datetime import datetime import requests LEADERBOARD_URL = "https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json" def download_leaderboard(output_dir: str = "data") -> str: response = requests.get(LEADERBOARD_URL) response.raise_for_status() data = response.json() os.makedirs(output_dir, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = f"{output_dir}/swebench_leaderboard_{timestamp}.json" with open(filename, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) print(f"Downloaded leaderboard to {filename}") print(f"Available keys: {list(data.keys()) if isinstance(data, dict) else 'list'}") return filename def get_leaderboard(leaderboard_name: str = "bash-only") -> list: response = requests.get(LEADERBOARD_URL) response.raise_for_status() data = response.json() leaderboards = data.get("leaderboards", []) for lb in leaderboards: if lb.get("name") == leaderboard_name: return lb.get("results", []) available = [lb.get("name") for lb in leaderboards] raise ValueError(f"Leaderboard '{leaderboard_name}' not found. Available: {available}") def main(): import argparse parser = argparse.ArgumentParser(description="Download SWE-bench leaderboard data") parser.add_argument( "--output-dir", default="data", help="Output directory for downloaded data", ) parser.add_argument( "--show-structure", action="store_true", help="Print the structure of the JSON data", ) args = parser.parse_args() filename = download_leaderboard(output_dir=args.output_dir) if args.show_structure: with open(filename, "r", encoding="utf-8") as f: data = json.load(f) if isinstance(data, dict): print("\nJSON structure:") for key, value in data.items(): if isinstance(value, list): print(f" {key}: list with {len(value)} items") if value: first_keys = list(value[0].keys()) if isinstance(value[0], dict) else type(value[0]) print(f" First item keys: {first_keys}") elif isinstance(value, dict): print(f" {key}: dict with keys {list(value.keys())[:5]}...") else: print(f" {key}: {type(value).__name__}") if __name__ == "__main__": main()