Add SWE-bench leaderboard viewer with S3 trajectory download

- Gradio app showing bash-only leaderboard
- Download trajectories from S3 bucket
- UV-based dependency management

.env.example

@@ -0,0 +1 @@
+HF_TOKEN=

.gitignore

@@ -0,0 +1,8 @@
+.env
+.venv/
+__pycache__/
+*.pyc
+data/
+.DS_Store
+
+

app.py

@@ -0,0 +1,172 @@
+import json
+import os
+import subprocess
+from pathlib import Path
+
+import gradio as gr
+import pandas as pd
+
+from src.download_swebench_leaderboard import download_leaderboard, get_leaderboard
+
+DATA_DIR = Path("data")
+TRAJS_DIR = DATA_DIR / "swebench_trajs"
+LEADERBOARD_CACHE = DATA_DIR / "swebench_leaderboard_latest.json"
+S3_BUCKET = "s3://swe-bench-experiments/bash-only"
+
+
+def load_or_download_leaderboard():
+    if LEADERBOARD_CACHE.exists():
+        with open(LEADERBOARD_CACHE) as f:
+            return json.load(f)
+
+    filename = download_leaderboard(output_dir=str(DATA_DIR))
+    os.rename(filename, LEADERBOARD_CACHE)
+    with open(LEADERBOARD_CACHE) as f:
+        return json.load(f)
+
+
+def get_bash_only_df():
+    data = load_or_download_leaderboard()
+    leaderboards = data.get("leaderboards", [])
+    bash_only = next((lb for lb in leaderboards if lb["name"] == "bash-only"), None)
+
+    if not bash_only:
+        return pd.DataFrame()
+
+    rows = []
+    for r in bash_only["results"]:
+        rows.append({
+            "name": r.get("name", ""),
+            "date": r.get("date", ""),
+            "cost": round(r.get("cost", 0), 2),
+            "instance_cost": round(r.get("instance_cost", 0), 4),
+            "instance_calls": r.get("instance_calls", 0),
+            "folder": r.get("folder", ""),
+            "os_model": "✅" if r.get("os_model") else "❌",
+            "os_system": "✅" if r.get("os_system") else "❌",
+        })
+
+    df = pd.DataFrame(rows)
+    return df
+
+
+def get_model_details(folder: str):
+    if not folder:
+        return None, "Select a model from the table"
+
+    data = load_or_download_leaderboard()
+    leaderboards = data.get("leaderboards", [])
+    bash_only = next((lb for lb in leaderboards if lb["name"] == "bash-only"), None)
+
+    if not bash_only:
+        return None, "Leaderboard not found"
+
+    model = next((r for r in bash_only["results"] if r.get("folder") == folder), None)
+    if not model:
+        return None, f"Model with folder '{folder}' not found"
+
+    return model, None
+
+
+def download_trajectories_from_s3(folder: str, progress=gr.Progress()):
+    if not folder:
+        return "❌ No model selected"
+
+    model, error = get_model_details(folder)
+    if error:
+        return f"❌ {error}"
+
+    output_dir = TRAJS_DIR / folder
+    if output_dir.exists() and any(output_dir.iterdir()):
+        file_count = len(list(output_dir.glob("*/*.traj.json")))
+        return f"✅ Already downloaded: {output_dir}\n\n{file_count} trajectory files"
+
+    s3_path = f"{S3_BUCKET}/{folder}/trajs/"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    progress(0, desc="Starting S3 download...")
+
+    try:
+        result = subprocess.run(
+            ["aws", "s3", "cp", "--recursive", s3_path, str(output_dir), "--no-sign-request"],
+            capture_output=True,
+            text=True,
+            timeout=600,
+        )
+
+        if result.returncode != 0:
+            return f"❌ S3 download failed:\n{result.stderr}"
+
+        file_count = len(list(output_dir.glob("*/*.traj.json")))
+        if file_count == 0:
+            file_count = len(list(output_dir.glob("*.json")))
+
+        per_instance = model.get("per_instance_details", {})
+        resolved_count = sum(1 for v in per_instance.values() if v.get("resolved"))
+        total_count = len(per_instance)
+
+        return f"✅ Downloaded to {output_dir}\n\n{file_count} trajectory files\nResolved: {resolved_count}/{total_count} ({100*resolved_count/total_count:.1f}%)"
+
+    except subprocess.TimeoutExpired:
+        return "❌ Download timed out (>10 min)"
+    except FileNotFoundError:
+        return "❌ AWS CLI not found. Install with: pip install awscli"
+    except Exception as e:
+        return f"❌ Error: {e}"
+
+
+def on_row_select(evt: gr.SelectData, df: pd.DataFrame):
+    if evt.index is None:
+        return "", "", gr.update()
+
+    row_idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index
+    row = df.iloc[row_idx]
+    folder = row["folder"]
+    name = row["name"]
+
+    return folder, name, gr.update(interactive=True)
+
+
+def build_app():
+    df = get_bash_only_df()
+
+    with gr.Blocks(title="SWE-bench Routing Cost Calculator") as app:
+        gr.Markdown("# 🧮 SWE-bench Bash-Only Leaderboard")
+        gr.Markdown("Select a model to use as base for cost analysis")
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                leaderboard_table = gr.Dataframe(
+                    value=df,
+                    label="Bash-Only Leaderboard",
+                    interactive=False,
+                    wrap=True,
+                )
+
+            with gr.Column(scale=1):
+                gr.Markdown("### Selected Model")
+                selected_name = gr.Textbox(label="Model Name", interactive=False)
+                selected_folder = gr.Textbox(label="Folder ID", interactive=False)
+
+                download_btn = gr.Button("📥 Download Trajectories", interactive=False)
+                download_status = gr.Textbox(label="Status", interactive=False, lines=3)
+
+        leaderboard_table.select(
+            fn=on_row_select,
+            inputs=[leaderboard_table],
+            outputs=[selected_folder, selected_name, download_btn],
+        )
+
+        download_btn.click(
+            fn=download_trajectories_from_s3,
+            inputs=[selected_folder],
+            outputs=[download_status],
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    app = build_app()
+    app.launch()
+

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "routing-money-calculation"
+version = "0.1.0"
+description = "Rough estimate of routing cost for AI agents"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = [
+    "gradio>=6.0.2",
+    "pandas>=2.0.0",
+    "requests>=2.31.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.8.0",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+

src/download_swebench_leaderboard.py

@@ -0,0 +1,84 @@
+import json
+import os
+from datetime import datetime
+
+import requests
+
+
+LEADERBOARD_URL = "https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json"
+
+
+def download_leaderboard(output_dir: str = "data") -> str:
+    response = requests.get(LEADERBOARD_URL)
+    response.raise_for_status()
+
+    data = response.json()
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"{output_dir}/swebench_leaderboard_{timestamp}.json"
+
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+    print(f"Downloaded leaderboard to {filename}")
+    print(f"Available keys: {list(data.keys()) if isinstance(data, dict) else 'list'}")
+
+    return filename
+
+
+def get_leaderboard(leaderboard_name: str = "bash-only") -> list:
+    response = requests.get(LEADERBOARD_URL)
+    response.raise_for_status()
+    data = response.json()
+
+    leaderboards = data.get("leaderboards", [])
+    for lb in leaderboards:
+        if lb.get("name") == leaderboard_name:
+            return lb.get("results", [])
+
+    available = [lb.get("name") for lb in leaderboards]
+    raise ValueError(f"Leaderboard '{leaderboard_name}' not found. Available: {available}")
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Download SWE-bench leaderboard data")
+    parser.add_argument(
+        "--output-dir",
+        default="data",
+        help="Output directory for downloaded data",
+    )
+    parser.add_argument(
+        "--show-structure",
+        action="store_true",
+        help="Print the structure of the JSON data",
+    )
+
+    args = parser.parse_args()
+
+    filename = download_leaderboard(output_dir=args.output_dir)
+
+    if args.show_structure:
+        with open(filename, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        if isinstance(data, dict):
+            print("\nJSON structure:")
+            for key, value in data.items():
+                if isinstance(value, list):
+                    print(f"  {key}: list with {len(value)} items")
+                    if value:
+                        first_keys = list(value[0].keys()) if isinstance(value[0], dict) else type(value[0])
+                        print(f"    First item keys: {first_keys}")
+                elif isinstance(value, dict):
+                    print(f"  {key}: dict with keys {list(value.keys())[:5]}...")
+                else:
+                    print(f"  {key}: {type(value).__name__}")
+
+
+if __name__ == "__main__":
+    main()
+