File size: 2,582 Bytes
781ed01 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
import json
import os
from datetime import datetime
import requests
LEADERBOARD_URL = "https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json"
def download_leaderboard(output_dir: str = "data") -> str:
response = requests.get(LEADERBOARD_URL)
response.raise_for_status()
data = response.json()
os.makedirs(output_dir, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"{output_dir}/swebench_leaderboard_{timestamp}.json"
with open(filename, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
print(f"Downloaded leaderboard to {filename}")
print(f"Available keys: {list(data.keys()) if isinstance(data, dict) else 'list'}")
return filename
def get_leaderboard(leaderboard_name: str = "bash-only") -> list:
response = requests.get(LEADERBOARD_URL)
response.raise_for_status()
data = response.json()
leaderboards = data.get("leaderboards", [])
for lb in leaderboards:
if lb.get("name") == leaderboard_name:
return lb.get("results", [])
available = [lb.get("name") for lb in leaderboards]
raise ValueError(f"Leaderboard '{leaderboard_name}' not found. Available: {available}")
def main():
import argparse
parser = argparse.ArgumentParser(description="Download SWE-bench leaderboard data")
parser.add_argument(
"--output-dir",
default="data",
help="Output directory for downloaded data",
)
parser.add_argument(
"--show-structure",
action="store_true",
help="Print the structure of the JSON data",
)
args = parser.parse_args()
filename = download_leaderboard(output_dir=args.output_dir)
if args.show_structure:
with open(filename, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
print("\nJSON structure:")
for key, value in data.items():
if isinstance(value, list):
print(f" {key}: list with {len(value)} items")
if value:
first_keys = list(value[0].keys()) if isinstance(value[0], dict) else type(value[0])
print(f" First item keys: {first_keys}")
elif isinstance(value, dict):
print(f" {key}: dict with keys {list(value.keys())[:5]}...")
else:
print(f" {key}: {type(value).__name__}")
if __name__ == "__main__":
main()
|