Address leaderboard review feedback

.gitignore

@@ -11,4 +11,5 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
-uv.lock.venv/
+uv.lock
+.venv/

app.py

@@ -1,11 +1,11 @@
 import os
-import re
 from pathlib import Path
 
 
 def patch_gradio_leaderboard():
     """Patch gradio_leaderboard JS to fix crash on tab switch with Gradio 5.x."""
     import gradio_leaderboard
+
     pkg_dir = Path(gradio_leaderboard.__file__).parent
     js_file = pkg_dir / "templates" / "component" / "Index-CzS_eGV6.js"
     if not js_file.exists():
@@ -57,7 +57,7 @@ from src.display.text_blocks import (
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
 )
-from src.leaderboard import get_benchmark_run_df, get_leaderboard_df, get_score_vs_cost_df
+from src.leaderboard import get_benchmark_names, get_benchmark_run_df, get_score_vs_cost_df
 
 REPO_ID = "taagarwa/coding-agent-leaderboard"
 TOKEN = os.environ.get("HF_TOKEN")
@@ -69,30 +69,12 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-LEADERBOARD_DF = get_leaderboard_df()
+BENCHMARK_NAMES = get_benchmark_names()
+DEFAULT_BENCHMARK = BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None
 BENCHMARK_RUN_DF = get_benchmark_run_df()
 SCORE_VS_COST_DF = get_score_vs_cost_df()
 
 
-def extract_body(s: str):
-    match = re.match(r"\[(.*?)\]", str(s))
-    return match.group(1) if match else str(s)
-
-
-def get_leaderboard_benchmark_names(dataframe):
-    meta_columns = {
-        " ",
-        "Harness",
-        "Model",
-        "Harness License",
-        "Model License",
-        "Model Num Params (B)",
-        "Precision",
-        "Avg Score",
-    }
-    return [col for col in dataframe.columns if col not in meta_columns]
-
-
 def render_leaderboard_benchmark_plot(benchmark_name, color_by):
     return create_leaderboard_benchmark_plot(
         BENCHMARK_RUN_DF,
@@ -101,9 +83,10 @@ def render_leaderboard_benchmark_plot(benchmark_name, color_by):
     )
 
 
-def render_score_vs_cost_plot(color_by):
+def render_score_vs_cost_plot(benchmark_name, color_by):
     return create_score_vs_cost_plot(
         SCORE_VS_COST_DF,
+        benchmark_name=benchmark_name,
         color_by=color_by,
     )
 
@@ -111,8 +94,8 @@ def render_score_vs_cost_plot(color_by):
 def build_header_html(df):
     n_results = len(df)
     n_models = df["Model"].nunique()
-    n_harnesses = df["Harness"].apply(lambda s: extract_body(s)).nunique()
-    n_benchmarks = df["Benchmark"].apply(lambda s: extract_body(s)).nunique()
+    n_harnesses = df["Harness"].apply(clean_markdown_link).nunique()
+    n_benchmarks = df["Benchmark"].apply(clean_markdown_link).nunique()
 
     return f"""
     <base target="_blank">
@@ -137,42 +120,14 @@ def build_header_html(df):
     """
 
 
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-
-    label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
-    meta_columns = [" ", "Harness", "Model", "Harness License", "Model License", "Model Num Params (B)", "Precision"]
-    benchmark_columns = [col for col in dataframe.columns if col not in meta_columns]
-    model_choices = sorted({(extract_body(v), v) for v in dataframe["Model"]})
-    harness_choices = sorted({(extract_body(v), v) for v in dataframe["Harness"]})
-
-    default_columns = [" ", "Harness", "Model"] + benchmark_columns
-    return Leaderboard(
-        value=dataframe,
-        select_columns=SelectColumns(
-            default_selection=default_columns,
-            label="Select Columns to Display:",
-        ),
-        datatype="markdown",
-        search_columns=["Harness", "Model"],
-        filter_columns=[
-            ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
-            ColumnFilter(label="Model", column="Model", type="checkboxgroup", choices=model_choices),
-            ColumnFilter(label="Harness", column="Harness", type="checkboxgroup", choices=harness_choices),
-            ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"),
-            ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"),
-        ],
-        interactive=False,
-    )
-
-
 def init_benchmark_runs(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
 
     label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
     benchmark_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Benchmark"]})
+    model_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Model"]})
+    harness_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Harness"]})
 
     return Leaderboard(
         value=dataframe,
@@ -196,6 +151,8 @@ def init_benchmark_runs(dataframe):
         filter_columns=[
             ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
             ColumnFilter(label="Benchmark", column="Benchmark", type="checkboxgroup", choices=benchmark_choices),
+            ColumnFilter(label="Model", column="Model", type="checkboxgroup", choices=model_choices),
+            ColumnFilter(label="Harness", column="Harness", type="checkboxgroup", choices=harness_choices),
             ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"),
             ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"),
         ],
@@ -203,15 +160,13 @@ def init_benchmark_runs(dataframe):
     )
 
 
-leaderboard_benchmark_names = get_leaderboard_benchmark_names(LEADERBOARD_DF)
-
 demo = gr.Blocks(theme="citrus")
 with demo:
     gr.HTML(build_header_html(BENCHMARK_RUN_DF))
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs():
-        with gr.Tab("Leaderboard"):
+        with gr.Tab("🏆 Leaderboard"):
             gr.Markdown("### Benchmark scores")
             leaderboard_color_by = gr.Radio(
                 choices=COLOR_BY_CHOICES,
@@ -220,7 +175,7 @@ with demo:
             )
 
             leaderboard_plots = []
-            for benchmark_name in leaderboard_benchmark_names:
+            for benchmark_name in BENCHMARK_NAMES:
                 gr.Markdown(f"#### {benchmark_name}")
                 plot = gr.Plot(
                     value=render_leaderboard_benchmark_plot(benchmark_name, "Model"),
@@ -235,29 +190,36 @@ with demo:
                     outputs=plot,
                 )
 
-            gr.Markdown("### Leaderboard table")
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.Tab("Cost vs Performance"):
+        with gr.Tab("💰 Cost vs Performance"):
+            cost_benchmark = gr.Dropdown(
+                choices=BENCHMARK_NAMES,
+                value=DEFAULT_BENCHMARK,
+                label="Benchmark",
+            )
             cost_color_by = gr.Radio(
                 choices=COLOR_BY_CHOICES,
                 value="Model",
                 label="Color by",
             )
             score_vs_cost_plot = gr.Plot(
-                value=render_score_vs_cost_plot("Model"),
+                value=render_score_vs_cost_plot(DEFAULT_BENCHMARK, "Model"),
                 show_label=False,
             )
+            cost_benchmark.change(
+                fn=render_score_vs_cost_plot,
+                inputs=[cost_benchmark, cost_color_by],
+                outputs=score_vs_cost_plot,
+            )
             cost_color_by.change(
                 fn=render_score_vs_cost_plot,
-                inputs=cost_color_by,
+                inputs=[cost_benchmark, cost_color_by],
                 outputs=score_vs_cost_plot,
             )
 
-        with gr.Tab("Benchmark Runs"):
+        with gr.Tab("🏃 Benchmark Runs"):
             benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF)
 
-        with gr.Tab("About"):
+        with gr.Tab("ℹ️ About"):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
             gr.Markdown(HOW_TO_USE_TEXT, elem_classes="markdown-text")
 

results/swe-bench-verified-claude-opus-4-8-claude-code.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-claude-opus-4-8-opencode.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-claude-sonnet-4-6-claude-code.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-gpt-5-5-codex.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-qwen3-6-35b-nvfp4-openclaw.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

results/swe-bench-verified-qwen3-6-36b-nvfp4-qwen-code.json

@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/SWE-Bench Verified",
+            "name": "swe-bench/swe-bench-verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,

src/charts.py

@@ -163,18 +163,23 @@ def create_leaderboard_benchmark_plot(
 
 def create_score_vs_cost_plot(
     dataframe: pd.DataFrame,
+    benchmark_name: str | None,
     color_by: ColorBy = "Model",
 ) -> Figure:
     if dataframe is None or dataframe.empty:
         return empty_figure("No cost data available.")
 
+    if not benchmark_name:
+        return empty_figure("Select a benchmark to view cost data.")
+
     plot_df = dataframe.copy()
-    plot_df["Avg Score"] = pd.to_numeric(plot_df["Avg Score"], errors="coerce")
-    plot_df["Avg Cost Per Task (USD)"] = pd.to_numeric(plot_df["Avg Cost Per Task (USD)"], errors="coerce")
-    plot_df = plot_df.dropna(subset=["Avg Score", "Avg Cost Per Task (USD)"])
+    plot_df = plot_df[plot_df["Benchmark"] == benchmark_name]
+    plot_df["Score"] = pd.to_numeric(plot_df["Score"], errors="coerce")
+    plot_df["Cost Per Task (USD)"] = pd.to_numeric(plot_df["Cost Per Task (USD)"], errors="coerce")
+    plot_df = plot_df.dropna(subset=["Score", "Cost Per Task (USD)"])
 
     if plot_df.empty:
-        return empty_figure("No cost data available.")
+        return empty_figure(f"No cost data available for {benchmark_name}.")
 
     colors = color_map_for(plot_df[color_by], color_by)
     fig = go.Figure()
@@ -182,8 +187,8 @@ def create_score_vs_cost_plot(
     for group, group_df in plot_df.groupby(color_by, sort=True):
         fig.add_trace(
             go.Scatter(
-                x=group_df["Avg Cost Per Task (USD)"],
-                y=group_df["Avg Score"],
+                x=group_df["Cost Per Task (USD)"],
+                y=group_df["Score"],
                 mode="markers+text",
                 name=str(group),
                 text=group_df["Label"],
@@ -193,21 +198,22 @@ def create_score_vs_cost_plot(
                     "color": colors[str(group)],
                     "line": {"width": 1, "color": "white"},
                 },
-                customdata=group_df[["Model", "Harness", "Avg Score", "Avg Cost Per Task (USD)"]],
+                customdata=group_df[["Model", "Harness", "Benchmark", "Score", "Cost Per Task (USD)"]],
                 hovertemplate=(
                     "<b>%{customdata[0]}</b><br>"
                     "Harness: %{customdata[1]}<br>"
-                    "Average score: %{customdata[2]:.1f}%<br>"
-                    "Average cost: $%{customdata[3]:.2f}/task"
+                    "Benchmark: %{customdata[2]}<br>"
+                    "Score: %{customdata[3]:.1f}%<br>"
+                    "Cost: $%{customdata[4]:.2f}/task"
                     "<extra></extra>"
                 ),
             )
         )
 
     fig.update_layout(
-        title={"text": "Cost vs Performance", "font": {"size": 18}},
-        xaxis={"title": "Average cost per task (USD)", "tickprefix": "$", "tickformat": ".2f"},
-        yaxis={"title": "Average score (%)", "range": [0, 105]},
+        title={"text": f"{benchmark_name}: Cost vs Performance", "font": {"size": 18}},
+        xaxis={"title": "Cost per task (USD)", "tickprefix": "$", "tickformat": ".2f"},
+        yaxis={"title": "Score (%)", "range": [0, 105]},
         legend_title_text=color_by,
     )
     return apply_plot_theme(fig, height=580)

src/display/text_blocks.py

@@ -49,5 +49,5 @@ Each benchmark measures the performance of the coding agent on different tasks:
 
 Higher scores indicate better performance on the benchmarks.
 If an agent scores better on a given benchmark than another, it can be generally considered to be better at those kinds of tasks.
-We take a weighted average of these scores so you can quickly compare the performance of different coding agents, but this is a relative score and the average itself is meaningless on its own.
+Compare agents within each benchmark rather than relying on a cross-benchmark average. Each benchmark emphasizes different task types, so per-benchmark scores are the clearest way to judge performance.
 """

src/leaderboard.py

@@ -3,15 +3,13 @@ import json
 
 import pandas as pd
 
-from src.models import Harness, Model, Result
+from src.models import Result
 
 RESULTS_DIR = Path(__file__).parent.parent / "results"
 
 BENCHMARK_SORT_ORDER = {
     "SWE-Bench Verified": 0,
-    "swe-bench-verified": 0,
     "SWE-Bench Pro -- Ansible": 1,
-    "swe-bench-pro--ansible": 1,
 }
 
 
@@ -19,7 +17,7 @@ def benchmark_sort_key(name: str) -> tuple[int, str]:
     return (BENCHMARK_SORT_ORDER.get(name, 99), name)
 
 
-def format_time(seconds: int):
+def format_time(seconds: int | None) -> str | None:
     if seconds is None:
         return None
     m, s = divmod(seconds, 60)
@@ -37,56 +35,11 @@ def get_results() -> list[Result]:
 
 
 def get_benchmark_names(results: list[Result] | None = None) -> list[str]:
-    results = results or get_results()
+    if results is None:
+        results = get_results()
     return sorted({r.benchmark.name for r in results}, key=benchmark_sort_key)
 
 
-def get_leaderboard_df():
-    results = get_results()
-
-    # Collect benchmark scores for each model-harness pair, and convert to percent out of 100.
-    benchmark_lookup: dict[tuple[str, str], dict[str, tuple[float, int]]] = {}
-    model_lookup: dict[str, Model] = {}
-    harness_lookup: dict[str, Harness] = {}
-    for result in results:
-        model_key = result.model.repo or result.model.name
-        pair = (model_key, result.harness.name)
-        harness_lookup[result.harness.name] = result.harness
-        model_lookup[model_key] = result.model
-        benchmark_lookup.setdefault(pair, {})[result.benchmark.name] = (
-            round(result.metrics.score * 100, 1),
-            result.benchmark.num_tasks,
-        )
-
-    rows = []
-    benchmark_names = get_benchmark_names(results=results)
-    for pair, benchmarks in benchmark_lookup.items():
-        model = model_lookup[pair[0]]
-        harness = harness_lookup[pair[1]]
-        avg_score = sum(score * size for score, size in benchmarks.values()) / sum(
-            size for _, size in benchmarks.values()
-        )
-        row = {
-            " ": "🟠" if model.is_oss and harness.is_oss else "🔶",
-            "Model": f"[{model.repo or model.name}]({model.url})",
-            "Harness": f"[{harness.name}]({harness.url})<sup>*</sup>"
-            if harness.name == "internal"
-            else f"[{harness.name}]({harness.url})",
-            "Precision": model.precision,
-            "Model License": "FOSS" if model.is_oss else "Proprietary",
-            "Harness License": "FOSS" if harness.is_oss else "Proprietary",
-            "Model Num Params (B)": model.num_params,
-            "Avg Score": round(avg_score, 1),
-        }
-        for benchmark_name in benchmark_names:
-            benchmark_score = benchmarks.get(benchmark_name)
-            row[benchmark_name] = benchmark_score[0] if benchmark_score else ""
-        rows.append(row)
-
-    leaderboard_df = pd.DataFrame(rows).sort_values("Avg Score", ascending=False).fillna("")
-    return leaderboard_df
-
-
 def get_benchmark_run_df():
     results = get_results()
 
@@ -114,44 +67,40 @@ def get_benchmark_run_df():
             }
         )
 
-    benchmark_run_df = pd.DataFrame(rows).sort_values(by=["Benchmark", "Score"], ascending=False).fillna("")
-    return benchmark_run_df
+    benchmark_run_df = pd.DataFrame(rows)
+    if benchmark_run_df.empty:
+        return benchmark_run_df
+
+    benchmark_run_df["_Benchmark Sort"] = benchmark_run_df["Benchmark"].str.extract(r"\[(.*?)\]", expand=False)
+    benchmark_run_df["_Benchmark Sort Key"] = benchmark_run_df["_Benchmark Sort"].map(benchmark_sort_key)
+    benchmark_run_df = benchmark_run_df.sort_values(
+        by=["_Benchmark Sort Key", "Score"],
+        ascending=[True, False],
+    ).drop(columns=["_Benchmark Sort", "_Benchmark Sort Key"])
+    return benchmark_run_df.fillna("")
 
 
 def get_score_vs_cost_df():
     results = get_results()
 
-    score_weighted_sum: dict[tuple[str, str], float] = {}
-    cost_weighted_sum: dict[tuple[str, str], float] = {}
-    weight_sum: dict[tuple[str, str], int] = {}
-    cost_weight_sum: dict[tuple[str, str], int] = {}
-    meta_lookup: dict[tuple[str, str], Result] = {}
-
+    rows = []
     for result in results:
-        pair = (result.model.name, result.harness.name)
-        weight = result.metrics.n_tasks or result.benchmark.num_tasks or 1
-        meta_lookup[pair] = result
-        score_weighted_sum[pair] = score_weighted_sum.get(pair, 0) + (result.metrics.score * 100 * weight)
-        weight_sum[pair] = weight_sum.get(pair, 0) + weight
-
         mean_cost = result.metrics.mean_cost_usd_per_task
-        if mean_cost is not None:
-            cost_weighted_sum[pair] = cost_weighted_sum.get(pair, 0) + (mean_cost * weight)
-            cost_weight_sum[pair] = cost_weight_sum.get(pair, 0) + weight
-
-    rows = []
-    for pair, weighted_score in score_weighted_sum.items():
-        if pair not in cost_weighted_sum:
+        if mean_cost is None:
             continue
-        result = meta_lookup[pair]
+
+        model_label = result.model.repo or result.model.name
+        harness_label = result.harness.name
+
         rows.append(
             {
-                "Label": f"{pair[0]} / {pair[1]}",
-                "Model": pair[0],
-                "Harness": pair[1],
+                "Label": f"{model_label} / {harness_label}",
+                "Model": model_label,
+                "Harness": harness_label,
+                "Benchmark": result.benchmark.name,
                 "Category": "FOSS" if result.model.is_oss and result.harness.is_oss else "Proprietary",
-                "Avg Score": round(weighted_score / weight_sum[pair], 1),
-                "Avg Cost Per Task (USD)": round(cost_weighted_sum[pair] / cost_weight_sum[pair], 2),
+                "Score": round(result.metrics.score * 100, 1),
+                "Cost Per Task (USD)": round(mean_cost, 2),
             }
         )
 
@@ -159,11 +108,16 @@ def get_score_vs_cost_df():
         "Label",
         "Model",
         "Harness",
+        "Benchmark",
         "Category",
-        "Avg Score",
-        "Avg Cost Per Task (USD)",
+        "Score",
+        "Cost Per Task (USD)",
     ]
     score_vs_cost_df = pd.DataFrame(rows, columns=columns)
     if score_vs_cost_df.empty:
         return score_vs_cost_df
-    return score_vs_cost_df.sort_values("Avg Score", ascending=False)
+
+    return score_vs_cost_df.sort_values(
+        ["Benchmark", "Score"],
+        ascending=[True, False],
+    )