Combine leaderboard and cost performance charts

app.py

@@ -43,20 +43,27 @@ def patch_gradio_leaderboard():
 patch_gradio_leaderboard()
 
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from apscheduler.schedulers.background import BackgroundScheduler
+from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 from huggingface_hub import HfApi
 
-from src.leaderboard import get_leaderboard_df, get_benchmark_run_df
+from src.charts import (
+    clean_markdown_link,
+    create_leaderboard_benchmark_plot,
+    create_score_vs_cost_plot,
+)
 from src.display.text_blocks import (
     HOW_TO_USE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
 )
+from src.leaderboard import get_benchmark_run_df, get_leaderboard_df, get_score_vs_cost_df
 
 REPO_ID = "taagarwa/coding-agent-leaderboard"
 TOKEN = os.environ.get("HF_TOKEN")
 API = HfApi(token=TOKEN)
+COLOR_BY_CHOICES = ["Model", "Harness"]
+
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
@@ -64,9 +71,41 @@ def restart_space():
 
 LEADERBOARD_DF = get_leaderboard_df()
 BENCHMARK_RUN_DF = get_benchmark_run_df()
+SCORE_VS_COST_DF = get_score_vs_cost_df()
+
 
 def extract_body(s: str):
-    return re.match(r'\[(.*?)\]', s).group(1)
+    match = re.match(r"\[(.*?)\]", str(s))
+    return match.group(1) if match else str(s)
+
+
+def get_leaderboard_benchmark_names(dataframe):
+    meta_columns = {
+        " ",
+        "Harness",
+        "Model",
+        "Harness License",
+        "Model License",
+        "Model Num Params (B)",
+        "Precision",
+        "Avg Score",
+    }
+    return [col for col in dataframe.columns if col not in meta_columns]
+
+
+def render_leaderboard_benchmark_plot(benchmark_name, color_by):
+    return create_leaderboard_benchmark_plot(
+        BENCHMARK_RUN_DF,
+        benchmark_name=benchmark_name,
+        color_by=color_by,
+    )
+
+
+def render_score_vs_cost_plot(color_by):
+    return create_score_vs_cost_plot(
+        SCORE_VS_COST_DF,
+        color_by=color_by,
+    )
 
 
 def build_header_html(df):
@@ -96,11 +135,12 @@ def build_header_html(df):
         </div>
     </div>
     """
-    
+
+
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
-    
+
     label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
     meta_columns = [" ", "Harness", "Model", "Harness License", "Model License", "Model Num Params (B)", "Precision"]
     benchmark_columns = [col for col in dataframe.columns if col not in meta_columns]
@@ -126,14 +166,14 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
+
 def init_benchmark_runs(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
-    
-    # Make ColumnFilter choices
+
     label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
-    benchmark_choices = sorted({(extract_body(v), v) for v in dataframe["Benchmark"]})
-    
+    benchmark_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Benchmark"]})
+
     return Leaderboard(
         value=dataframe,
         select_columns=SelectColumns(
@@ -162,22 +202,64 @@ def init_benchmark_runs(dataframe):
         interactive=False,
     )
 
+
+leaderboard_benchmark_names = get_leaderboard_benchmark_names(LEADERBOARD_DF)
+
 demo = gr.Blocks(theme="citrus")
 with demo:
     gr.HTML(build_header_html(BENCHMARK_RUN_DF))
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs():
-        with gr.Tab("🏆 Leaderboard"):
+        with gr.Tab("Leaderboard"):
+            gr.Markdown("### Benchmark scores")
+            leaderboard_color_by = gr.Radio(
+                choices=COLOR_BY_CHOICES,
+                value="Model",
+                label="Color by",
+            )
+
+            leaderboard_plots = []
+            for benchmark_name in leaderboard_benchmark_names:
+                gr.Markdown(f"#### {benchmark_name}")
+                plot = gr.Plot(
+                    value=render_leaderboard_benchmark_plot(benchmark_name, "Model"),
+                    show_label=False,
+                )
+                leaderboard_plots.append((benchmark_name, plot))
+
+            for benchmark_name, plot in leaderboard_plots:
+                leaderboard_color_by.change(
+                    fn=lambda color_by, name=benchmark_name: render_leaderboard_benchmark_plot(name, color_by),
+                    inputs=leaderboard_color_by,
+                    outputs=plot,
+                )
+
+            gr.Markdown("### Leaderboard table")
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.Tab("🏃 Benchmark Runs"):
+        with gr.Tab("Cost vs Performance"):
+            cost_color_by = gr.Radio(
+                choices=COLOR_BY_CHOICES,
+                value="Model",
+                label="Color by",
+            )
+            score_vs_cost_plot = gr.Plot(
+                value=render_score_vs_cost_plot("Model"),
+                show_label=False,
+            )
+            cost_color_by.change(
+                fn=render_score_vs_cost_plot,
+                inputs=cost_color_by,
+                outputs=score_vs_cost_plot,
+            )
+
+        with gr.Tab("Benchmark Runs"):
             benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF)
 
-        with gr.Tab("📝 About"):
+        with gr.Tab("About"):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-    
-    gr.Markdown(HOW_TO_USE_TEXT, elem_classes="markdown-text")
+            gr.Markdown(HOW_TO_USE_TEXT, elem_classes="markdown-text")
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)

requirements.txt

@@ -9,8 +9,9 @@ huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
+plotly
 python-dateutil
 tqdm
 transformers
 tokenizers>=0.15.0
-sentencepiece
+sentencepiece

results/swe-bench-pro--ansible-claude-opus-4-8-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 406,
         "mean_agent_time_seconds_per_task": 341
     }
-}
+}

results/swe-bench-pro--ansible-claude-opus-4-8-opencode.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 409,
         "mean_agent_time_seconds_per_task": 319
     }
-}
+}

results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 518,
         "mean_agent_time_seconds_per_task": 422
     }
-}
+}

results/swe-bench-pro--ansible-gpt-5-5-codex.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 411,
         "mean_agent_time_seconds_per_task": 342
     }
-}
+}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -12,7 +12,7 @@
         "url": "https://github.com/anthropics/claude-code"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 487,
         "mean_agent_time_seconds_per_task": 406
     }
-}
+}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -12,7 +12,7 @@
         "url": "https://github.com/anomalyco/opencode"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 596,
         "mean_agent_time_seconds_per_task": 515
     }
-}
+}

results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-openclaw.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -12,7 +12,7 @@
         "url": "https://github.com/openclaw/openclaw"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 528,
         "mean_agent_time_seconds_per_task": 396
     }
-}
+}

results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-pi.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -12,7 +12,7 @@
         "url": "https://github.com/earendil-works/pi/tree/main"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 650,
         "mean_agent_time_seconds_per_task": 568
     }
-}
+}

results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-qwen-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-pro--ansible",
+        "name": "SWE-Bench Pro -- Ansible",
         "repo": "ScaleAI/SWE-bench_Pro",
         "num_tasks": 96,
         "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
@@ -12,7 +12,7 @@
         "url": "https://github.com/QwenLM/qwen-code"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -57,4 +57,4 @@
         "mean_total_time_seconds_per_task": 398,
         "mean_agent_time_seconds_per_task": 350
     }
-}
+}

results/swe-bench-verified-claude-opus-4-8-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -55,4 +55,4 @@
         "mean_total_time_seconds_per_task": 319,
         "mean_agent_time_seconds_per_task": 208
     }
-}
+}

results/swe-bench-verified-claude-opus-4-8-opencode.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -55,4 +55,4 @@
         "mean_total_time_seconds_per_task": 343,
         "mean_agent_time_seconds_per_task": 229
     }
-}
+}

results/swe-bench-verified-claude-sonnet-4-6-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -39,4 +39,4 @@
     "metrics": {
         "score": 0.796
     }
-}
+}

results/swe-bench-verified-gpt-5-5-codex.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -55,4 +55,4 @@
         "mean_total_time_seconds_per_task": 283,
         "mean_agent_time_seconds_per_task": 185
     }
-}
+}

results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -12,7 +12,7 @@
         "url": "https://github.com/anthropics/claude-code"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -55,4 +55,4 @@
         "mean_total_time_seconds_per_task": 343,
         "mean_agent_time_seconds_per_task": 245
     }
-}
+}

results/swe-bench-verified-qwen3-6-35b-nvfp4-openclaw.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -12,7 +12,7 @@
         "url": "https://github.com/openclaw/openclaw"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -56,4 +56,4 @@
         "mean_total_time_seconds_per_task": 400,
         "mean_agent_time_seconds_per_task": 240
     }
-}
+}

results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -12,7 +12,7 @@
         "url": "https://github.com/anomalyco/opencode"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -56,4 +56,4 @@
         "mean_total_time_seconds_per_task": 370,
         "mean_agent_time_seconds_per_task": 240
     }
-}
+}

results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -12,7 +12,7 @@
         "url": "https://github.com/earendil-works/pi/tree/main"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -56,4 +56,4 @@
         "mean_total_time_seconds_per_task": 437,
         "mean_agent_time_seconds_per_task": 309
     }
-}
+}

results/swe-bench-verified-qwen3-6-36b-nvfp4-qwen-code.json

@@ -1,6 +1,6 @@
 {
     "benchmark": {
-        "name": "swe-bench-verified",
+        "name": "SWE-Bench Verified",
         "repo": "SWE-bench/SWE-bench_Verified",
         "num_tasks": 500,
         "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
@@ -12,7 +12,7 @@
         "url": "https://github.com/QwenLM/qwen-code"
     },
     "model": {
-        "name": "Qwen3.6-35B-A3B",
+        "name": "Qwen3.6-35B-A3B-NVFP4",
         "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
         "num_params": 35,
@@ -23,7 +23,7 @@
         "name": "harbor",
         "config": {
             "path": null,
-            "name": "swe-bench/swe-bench-verified",
+            "name": "swe-bench/SWE-Bench Verified",
             "version": null,
             "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
             "registry_url": null,
@@ -56,4 +56,4 @@
         "mean_total_time_seconds_per_task": 357,
         "mean_agent_time_seconds_per_task": 264
     }
-}
+}

src/charts.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from typing import Literal
+
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.graph_objs._figure import Figure
+
+ColorBy = Literal["Model", "Harness"]
+
+# One palette per grouping dimension. These are intentionally muted so they sit
+# comfortably next to Gradio's citrus theme instead of fighting the lime/orange
+# accents in the page header.
+MODEL_COLORS: dict[str, str] = {
+    "Opus 4.8": "#486C8F",
+    "Sonnet 4.6": "#7B5EA7",
+    "GPT 5.5 - high": "#357C76",
+    "Qwen3.6-35B-A3B": "#9A6B3F",
+    "Qwen3.6-35B-A3B-NVFP4": "#9A6B3F",
+    "RedHatAI/Qwen3.6-35B-A3B-NVFP4": "#9A6B3F",
+}
+
+HARNESS_COLORS: dict[str, str] = {
+    "Claude Code": "#486C8F",
+    "Codex": "#7B5EA7",
+    "OpenCode": "#357C76",
+    "OpenClaw": "#9B5B63",
+    "Pi": "#6A7C59",
+    "Qwen Code": "#8A6B45",
+    "internal": "#697386",
+}
+
+FALLBACK_PALETTE = [
+    "#486C8F",
+    "#357C76",
+    "#7B5EA7",
+    "#9B5B63",
+    "#6A7C59",
+    "#8A6B45",
+    "#697386",
+    "#8C6E8C",
+]
+
+
+def clean_markdown_link(value: object) -> str:
+    """Return human-readable text from Markdown links used in leaderboard tables."""
+    text = str(value).replace("<sup>*</sup>", "")
+    match = re.match(r"\[(.*?)\]\((.*?)\)", text)
+    if match:
+        return match.group(1)
+    return text
+
+
+def stable_color(name: str) -> str:
+    digest = hashlib.sha256(name.encode("utf-8")).hexdigest()
+    return FALLBACK_PALETTE[int(digest[:8], 16) % len(FALLBACK_PALETTE)]
+
+
+def get_color(name: str, color_by: ColorBy) -> str:
+    palette = MODEL_COLORS if color_by == "Model" else HARNESS_COLORS
+    return palette.get(name, stable_color(name))
+
+
+def color_map_for(values: pd.Series, color_by: ColorBy) -> dict[str, str]:
+    return {str(value): get_color(str(value), color_by) for value in sorted(values.dropna().unique())}
+
+
+def empty_figure(message: str) -> Figure:
+    fig = go.Figure()
+    fig.add_annotation(
+        text=message,
+        showarrow=False,
+        x=0.5,
+        y=0.5,
+        xref="paper",
+        yref="paper",
+        font={"size": 14, "color": "#5F6B75"},
+    )
+    return apply_plot_theme(fig, height=360)
+
+
+def apply_plot_theme(fig: Figure, height: int = 520) -> Figure:
+    fig.update_layout(
+        template="plotly_white",
+        height=height,
+        paper_bgcolor="rgba(0,0,0,0)",
+        plot_bgcolor="rgba(255,255,255,0.84)",
+        font={"color": "#2A2F33"},
+        margin={"t": 64, "b": 56, "l": 72, "r": 36},
+        legend={
+            "orientation": "h",
+            "yanchor": "bottom",
+            "y": 1.02,
+            "xanchor": "center",
+            "x": 0.5,
+        },
+    )
+    fig.update_xaxes(gridcolor="rgba(42,47,51,0.12)", zerolinecolor="rgba(42,47,51,0.16)")
+    fig.update_yaxes(gridcolor="rgba(42,47,51,0.12)", zerolinecolor="rgba(42,47,51,0.16)")
+    return fig
+
+
+def prepare_benchmark_run_plot_df(dataframe: pd.DataFrame) -> pd.DataFrame:
+    plot_df = dataframe.copy()
+    plot_df["Model Label"] = plot_df["Model"].map(clean_markdown_link)
+    plot_df["Harness Label"] = plot_df["Harness"].map(clean_markdown_link)
+    plot_df["Benchmark Label"] = plot_df["Benchmark"].map(clean_markdown_link)
+    plot_df["Run Label"] = plot_df["Model Label"] + "<br>" + plot_df["Harness Label"]
+    plot_df["Score"] = pd.to_numeric(plot_df["Score"], errors="coerce")
+    return plot_df
+
+
+def create_leaderboard_benchmark_plot(
+    dataframe: pd.DataFrame,
+    benchmark_name: str,
+    color_by: ColorBy = "Model",
+) -> Figure:
+    if dataframe is None or dataframe.empty:
+        return empty_figure("No benchmark data available.")
+
+    plot_df = prepare_benchmark_run_plot_df(dataframe)
+    plot_df = plot_df[plot_df["Benchmark Label"] == benchmark_name].dropna(subset=["Score"])
+    plot_df = plot_df.sort_values("Score", ascending=False)
+
+    if plot_df.empty:
+        return empty_figure(f"No results available for {benchmark_name}.")
+
+    color_source = "Model Label" if color_by == "Model" else "Harness Label"
+    colors = color_map_for(plot_df[color_source], color_by)
+    fig = go.Figure()
+
+    for group, group_df in plot_df.groupby(color_source, sort=True):
+        fig.add_trace(
+            go.Bar(
+                x=group_df["Run Label"],
+                y=group_df["Score"],
+                name=str(group),
+                marker={"color": colors[str(group)]},
+                text=group_df["Score"].map(lambda score: f"{score:.1f}"),
+                textposition="outside",
+                customdata=group_df[["Model Label", "Harness Label", "Score"]],
+                hovertemplate=(
+                    "<b>%{customdata[0]}</b><br>"
+                    "Harness: %{customdata[1]}<br>"
+                    "Score: %{customdata[2]:.1f}%"
+                    "<extra></extra>"
+                ),
+            )
+        )
+
+    fig.update_layout(
+        title={"text": benchmark_name, "font": {"size": 18}},
+        xaxis={"title": "Model / Harness", "categoryorder": "array", "categoryarray": plot_df["Run Label"].tolist()},
+        yaxis={"title": "Score (%)", "range": [0, 105]},
+        legend_title_text=color_by,
+        bargap=0.28,
+    )
+    fig.update_xaxes(tickangle=-28)
+    return apply_plot_theme(fig, height=560)
+
+
+def create_score_vs_cost_plot(
+    dataframe: pd.DataFrame,
+    color_by: ColorBy = "Model",
+) -> Figure:
+    if dataframe is None or dataframe.empty:
+        return empty_figure("No cost data available.")
+
+    plot_df = dataframe.copy()
+    plot_df["Avg Score"] = pd.to_numeric(plot_df["Avg Score"], errors="coerce")
+    plot_df["Avg Cost Per Task (USD)"] = pd.to_numeric(plot_df["Avg Cost Per Task (USD)"], errors="coerce")
+    plot_df = plot_df.dropna(subset=["Avg Score", "Avg Cost Per Task (USD)"])
+
+    if plot_df.empty:
+        return empty_figure("No cost data available.")
+
+    colors = color_map_for(plot_df[color_by], color_by)
+    fig = go.Figure()
+
+    for group, group_df in plot_df.groupby(color_by, sort=True):
+        fig.add_trace(
+            go.Scatter(
+                x=group_df["Avg Cost Per Task (USD)"],
+                y=group_df["Avg Score"],
+                mode="markers+text",
+                name=str(group),
+                text=group_df["Label"],
+                textposition="top center",
+                marker={
+                    "size": 15,
+                    "color": colors[str(group)],
+                    "line": {"width": 1, "color": "white"},
+                },
+                customdata=group_df[["Model", "Harness", "Avg Score", "Avg Cost Per Task (USD)"]],
+                hovertemplate=(
+                    "<b>%{customdata[0]}</b><br>"
+                    "Harness: %{customdata[1]}<br>"
+                    "Average score: %{customdata[2]:.1f}%<br>"
+                    "Average cost: $%{customdata[3]:.2f}/task"
+                    "<extra></extra>"
+                ),
+            )
+        )
+
+    fig.update_layout(
+        title={"text": "Cost vs Performance", "font": {"size": 18}},
+        xaxis={"title": "Average cost per task (USD)", "tickprefix": "$", "tickformat": ".2f"},
+        yaxis={"title": "Average score (%)", "range": [0, 105]},
+        legend_title_text=color_by,
+    )
+    return apply_plot_theme(fig, height=580)

src/display/text_blocks.py

@@ -43,8 +43,8 @@ All benchmarks are run using Harbor, a sandboxed environment for evaluating codi
 
 Each benchmark measures the performance of the coding agent on different tasks:
 
-* **swe-bench-verified**: Measures performance on solving GitHub issues in popular Python repositories.
-* **swe-bench-pro--ansible**: Measures performance on solving GitHub issues in the [ansible/ansible](https://github.com/ansible/ansible) repository. 
+* **SWE-Bench Verified**: Measures performance on solving GitHub issues in popular Python repositories.
+* **SWE-Bench Pro -- Ansible**: Measures performance on solving GitHub issues in the [ansible/ansible](https://github.com/ansible/ansible) repository.
     Demonstrates how benchmarking can be used to evaluate coding agents on enterprise-specific tasks.
 
 Higher scores indicate better performance on the benchmarks.

src/leaderboard.py

@@ -1,11 +1,24 @@
 from pathlib import Path
 import json
+
 import pandas as pd
 
-from src.models import Result, Model, Harness
+from src.models import Harness, Model, Result
 
 RESULTS_DIR = Path(__file__).parent.parent / "results"
 
+BENCHMARK_SORT_ORDER = {
+    "SWE-Bench Verified": 0,
+    "swe-bench-verified": 0,
+    "SWE-Bench Pro -- Ansible": 1,
+    "swe-bench-pro--ansible": 1,
+}
+
+
+def benchmark_sort_key(name: str) -> tuple[int, str]:
+    return (BENCHMARK_SORT_ORDER.get(name, 99), name)
+
+
 def format_time(seconds: int):
     if seconds is None:
         return None
@@ -14,70 +27,79 @@ def format_time(seconds: int):
     return f"{h}h{m}m{s}s"
 
 
-def get_benchmark_names(results: list[Result]):
-    return {r.benchmark.name for r in results}
-
-def get_leaderboard_df():
+def get_results() -> list[Result]:
     results: list[Result] = []
-    for file in RESULTS_DIR.glob("*.json"):
+    for file in sorted(RESULTS_DIR.glob("*.json")):
         with open(file, "r") as f:
             data = json.load(f)
-        result = Result(**data)
-        results.append(result)
-        
-    # Collect benchmark scores for each model-harness pair, and convert to percent out of 100
-    benchmark_lookup: dict[tuple[str, str], dict[str, float]] = {}
+        results.append(Result(**data))
+    return results
+
+
+def get_benchmark_names(results: list[Result] | None = None) -> list[str]:
+    results = results or get_results()
+    return sorted({r.benchmark.name for r in results}, key=benchmark_sort_key)
+
+
+def get_leaderboard_df():
+    results = get_results()
+
+    # Collect benchmark scores for each model-harness pair, and convert to percent out of 100.
+    benchmark_lookup: dict[tuple[str, str], dict[str, tuple[float, int]]] = {}
     model_lookup: dict[str, Model] = {}
     harness_lookup: dict[str, Harness] = {}
     for result in results:
-        pair = (result.model.repo, result.harness.name)
+        model_key = result.model.repo or result.model.name
+        pair = (model_key, result.harness.name)
         harness_lookup[result.harness.name] = result.harness
-        model_lookup[result.model.repo] = result.model
-        if pair not in benchmark_lookup:
-            benchmark_lookup[pair] = {}
-        benchmark_lookup[pair][result.benchmark.name] = (round(result.metrics.score * 100, 1), result.benchmark.num_tasks)
-    
-    # Collect results into df rows
+        model_lookup[model_key] = result.model
+        benchmark_lookup.setdefault(pair, {})[result.benchmark.name] = (
+            round(result.metrics.score * 100, 1),
+            result.benchmark.num_tasks,
+        )
+
     rows = []
     benchmark_names = get_benchmark_names(results=results)
     for pair, benchmarks in benchmark_lookup.items():
         model = model_lookup[pair[0]]
         harness = harness_lookup[pair[1]]
-        avg_score = sum([score * size for score, size in benchmarks.values()]) / sum([size for _, size in benchmarks.values()])
+        avg_score = sum(score * size for score, size in benchmarks.values()) / sum(
+            size for _, size in benchmarks.values()
+        )
         row = {
             " ": "🟠" if model.is_oss and harness.is_oss else "🔶",
-            "Model": f'[{model.repo}]({model.url})',
-            "Harness": f'[{harness.name}]({harness.url})<sup>*</sup>' if result.harness.name == "internal" else f'[{harness.name}]({harness.url})',
+            "Model": f"[{model.repo or model.name}]({model.url})",
+            "Harness": f"[{harness.name}]({harness.url})<sup>*</sup>"
+            if harness.name == "internal"
+            else f"[{harness.name}]({harness.url})",
             "Precision": model.precision,
             "Model License": "FOSS" if model.is_oss else "Proprietary",
             "Harness License": "FOSS" if harness.is_oss else "Proprietary",
             "Model Num Params (B)": model.num_params,
             "Avg Score": round(avg_score, 1),
         }
-        for benchmark_name in sorted(benchmark_names, key=lambda x: (0 if x == "swe-bench-verified" else 1)):
-            row[benchmark_name] = benchmarks.get(benchmark_name, "")[0]
+        for benchmark_name in benchmark_names:
+            benchmark_score = benchmarks.get(benchmark_name)
+            row[benchmark_name] = benchmark_score[0] if benchmark_score else ""
         rows.append(row)
-    
+
     leaderboard_df = pd.DataFrame(rows).sort_values("Avg Score", ascending=False).fillna("")
     return leaderboard_df
-    
-    
+
+
 def get_benchmark_run_df():
-    results: list[Result] = []
-    for file in RESULTS_DIR.glob("*.json"):
-        with open(file, "r") as f:
-            data = json.load(f)
-        result = Result(**data)
-        results.append(result)
+    results = get_results()
 
     rows = []
     for result in results:
         rows.append(
             {
                 " ": "🟠" if result.model.is_oss and result.harness.is_oss else "🔶",
-                "Model": f'[{result.model.repo}]({result.model.url})',
-                "Harness": f'[{result.harness.name}]({result.harness.url})<sup>*</sup>' if result.harness.name == "internal" else f'[{result.harness.name}]({result.harness.url})',
-                "Benchmark": f'[{result.benchmark.name}]({result.benchmark.url})',
+                "Model": f"[{result.model.repo or result.model.name}]({result.model.url})",
+                "Harness": f"[{result.harness.name}]({result.harness.url})<sup>*</sup>"
+                if result.harness.name == "internal"
+                else f"[{result.harness.name}]({result.harness.url})",
+                "Benchmark": f"[{result.benchmark.name}]({result.benchmark.url})",
                 "Base Model": result.model.name,
                 "Precision": result.model.precision,
                 "Skills": str(result.harness.skills) if result.harness.skills else "None",
@@ -94,3 +116,54 @@ def get_benchmark_run_df():
 
     benchmark_run_df = pd.DataFrame(rows).sort_values(by=["Benchmark", "Score"], ascending=False).fillna("")
     return benchmark_run_df
+
+
+def get_score_vs_cost_df():
+    results = get_results()
+
+    score_weighted_sum: dict[tuple[str, str], float] = {}
+    cost_weighted_sum: dict[tuple[str, str], float] = {}
+    weight_sum: dict[tuple[str, str], int] = {}
+    cost_weight_sum: dict[tuple[str, str], int] = {}
+    meta_lookup: dict[tuple[str, str], Result] = {}
+
+    for result in results:
+        pair = (result.model.name, result.harness.name)
+        weight = result.metrics.n_tasks or result.benchmark.num_tasks or 1
+        meta_lookup[pair] = result
+        score_weighted_sum[pair] = score_weighted_sum.get(pair, 0) + (result.metrics.score * 100 * weight)
+        weight_sum[pair] = weight_sum.get(pair, 0) + weight
+
+        mean_cost = result.metrics.mean_cost_usd_per_task
+        if mean_cost is not None:
+            cost_weighted_sum[pair] = cost_weighted_sum.get(pair, 0) + (mean_cost * weight)
+            cost_weight_sum[pair] = cost_weight_sum.get(pair, 0) + weight
+
+    rows = []
+    for pair, weighted_score in score_weighted_sum.items():
+        if pair not in cost_weighted_sum:
+            continue
+        result = meta_lookup[pair]
+        rows.append(
+            {
+                "Label": f"{pair[0]} / {pair[1]}",
+                "Model": pair[0],
+                "Harness": pair[1],
+                "Category": "FOSS" if result.model.is_oss and result.harness.is_oss else "Proprietary",
+                "Avg Score": round(weighted_score / weight_sum[pair], 1),
+                "Avg Cost Per Task (USD)": round(cost_weighted_sum[pair] / cost_weight_sum[pair], 2),
+            }
+        )
+
+    columns = [
+        "Label",
+        "Model",
+        "Harness",
+        "Category",
+        "Avg Score",
+        "Avg Cost Per Task (USD)",
+    ]
+    score_vs_cost_df = pd.DataFrame(rows, columns=columns)
+    if score_vs_cost_df.empty:
+        return score_vs_cost_df
+    return score_vs_cost_df.sort_values("Avg Score", ascending=False)

src/models.py

@@ -9,6 +9,9 @@ class Benchmark(BaseModel):
     num_tasks: int
     url: str
 
+    def __hash__(self):
+        return hash(self.name)
+
 
 class Harness(BaseModel):
     name: str