Add baseline qwen3.6

.gitattributes

@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/sync-to-hf-space-stage.yml

@@ -1,101 +0,0 @@
-name: Sync stage to HF Space (staging)
-
-# Mirrors every push to `stage` on GitHub into the HF Space git remote so
-# that the staging Space (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage)
-# always tracks the stage branch.
-#
-# Required repository secrets (Settings -> Secrets and variables -> Actions):
-#   HF_TOKEN     Hugging Face access token with write permission to the Space.
-#   HF_USERNAME  Optional fallback username if token introspection fails.
-
-on:
-  push:
-    branches: [stage]
-  workflow_dispatch:
-
-concurrency:
-  group: sync-to-hf-space-stage
-  cancel-in-progress: false
-
-jobs:
-  mirror:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout GitHub stage (full history + LFS)
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-
-      - name: Verify required secrets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          if [ -z "$HF_TOKEN" ]; then
-            echo "::error::HF_TOKEN repository secret must be set."
-            exit 1
-          fi
-
-      - name: Ensure HF Space exists
-        id: hf
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ secrets.HF_USERNAME }}
-        run: |
-          set -euo pipefail
-          python -m pip install --quiet 'huggingface_hub>=0.24,<2'
-          python - <<'PY'
-          import os
-
-          from huggingface_hub import HfApi
-
-          token = os.environ["HF_TOKEN"]
-          space_id = "taagarwa/coding-agent-leaderboard-stage"
-          fallback_username = os.environ.get("HF_USERNAME", "").strip()
-
-          api = HfApi(token=token)
-          username = fallback_username
-          try:
-              info = api.whoami(token=token)
-              username = str(info.get("name") or username).strip()
-          except Exception as exc:
-              if not username:
-                  raise RuntimeError("HF_USERNAME fallback is required when token introspection fails") from exc
-
-          api.create_repo(
-              repo_id=space_id,
-              repo_type="space",
-              space_sdk="docker",
-              token=token,
-              exist_ok=True,
-          )
-
-          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
-              output.write(f"username={username}\n")
-          print(f"HF Space ready: {space_id}")
-          PY
-
-      - name: Push to HF Space remote
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ steps.hf.outputs.username }}
-        run: |
-          set -euo pipefail
-          HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage"
-
-          git config user.name  "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          echo "Pushing $(git rev-parse --short HEAD) to taagarwa/coding-agent-leaderboard-stage..."
-          git push --force "${HF_REMOTE}" HEAD:main
-          echo "Sync complete."
-
-      - name: Summary
-        if: success()
-        run: |
-          echo "### HF Space mirror (staging)" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "Pushed \`$(git rev-parse --short HEAD)\` to \`taagarwa/coding-agent-leaderboard-stage\` Space." >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "View the Space: <https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage>" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/sync-to-hf-space.yml

@@ -1,124 +0,0 @@
-name: Sync main to HF Space
-
-# Mirrors every push to `main` on GitHub into the HF Space git remote so
-# that the public coding-agent-leaderboard Space (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard)
-# always tracks the source-of-truth repo.
-#
-# Required repository secrets (Settings -> Secrets and variables -> Actions):
-#   HF_TOKEN     Hugging Face access token with write permission to the Space.
-#                Create at https://huggingface.co/settings/tokens
-#                (token type "Write" is sufficient; no organization scope needed).
-#   HF_USERNAME  Optional fallback username if token introspection fails.
-#
-# Optional: set HF_SPACE_ID as a repo variable (not secret) to point the
-# workflow at a different Space; defaults to "taagarwa/coding-agent-leaderboard".
-
-on:
-  push:
-    branches: [main]
-  # Manual dispatch lets you re-mirror the latest main on demand from
-  # the Actions tab without pushing a new commit.
-  workflow_dispatch:
-
-# Only one mirror job at a time so we never race ourselves into
-# non-fast-forward pushes on the Space remote.
-concurrency:
-  group: sync-to-hf-space
-  cancel-in-progress: false
-
-jobs:
-  mirror:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout GitHub main (full history + LFS)
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-
-      - name: Verify required secrets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          if [ -z "$HF_TOKEN" ]; then
-            echo "::error::HF_TOKEN repository secret must be set."
-            echo "  Create HF_TOKEN at https://huggingface.co/settings/tokens (type: Write)"
-            exit 1
-          fi
-
-      - name: Ensure HF Space exists
-        id: hf
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ secrets.HF_USERNAME }}
-          HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}
-        run: |
-          set -euo pipefail
-          python -m pip install --quiet 'huggingface_hub>=0.24,<2'
-          python - <<'PY'
-          import os
-
-          from huggingface_hub import HfApi
-
-          token = os.environ["HF_TOKEN"]
-          space_id = os.environ["HF_SPACE_ID"]
-          fallback_username = os.environ.get("HF_USERNAME", "").strip()
-
-          api = HfApi(token=token)
-          username = fallback_username
-          try:
-              info = api.whoami(token=token)
-              username = str(info.get("name") or username).strip()
-          except Exception as exc:
-              if not username:
-                  raise RuntimeError("HF_USERNAME fallback is required when token introspection fails") from exc
-
-          api.create_repo(
-              repo_id=space_id,
-              repo_type="space",
-              space_sdk="docker",
-              token=token,
-              exist_ok=True,
-          )
-
-          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
-              output.write(f"username={username}\n")
-          print(f"HF Space ready: {space_id}")
-          PY
-
-      - name: Push to HF Space remote
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ steps.hf.outputs.username }}
-          HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}
-        run: |
-          set -euo pipefail
-          # Authenticate via token in the URL. HF Spaces accept the
-          # username + token basic-auth format over HTTPS git.
-          HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_SPACE_ID}"
-
-          # Configure identity for any metadata operations. The actual
-          # commits come from GitHub unchanged; we only push refs.
-          git config user.name  "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          echo "Pushing $(git rev-parse --short HEAD) to ${HF_SPACE_ID}..."
-
-          # --force is intentional: GitHub is the single source of truth
-          # for the Space's git history. Anything on the Space side that
-          # wasn't committed via GitHub is overwritten on the next sync.
-          # This prevents the drift situation where someone edits files
-          # in the HF Space UI and creates commits only visible there.
-          git push --force "${HF_REMOTE}" HEAD:main
-
-          echo "Sync complete."
-
-      - name: Summary
-        if: success()
-        run: |
-          echo "### HF Space mirror" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "Pushed \`$(git rev-parse --short HEAD)\` to \`${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}\` Space." >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "View the Space: <https://huggingface.co/spaces/${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}>" >> "$GITHUB_STEP_SUMMARY"

app.py

@@ -1,181 +1,73 @@
 import os
-import re
-from pathlib import Path
-
-
-def patch_gradio_leaderboard():
-    """Patch gradio_leaderboard JS to fix crash on tab switch with Gradio 5.x."""
-    import gradio_leaderboard
-    pkg_dir = Path(gradio_leaderboard.__file__).parent
-    js_file = pkg_dir / "templates" / "component" / "Index-CzS_eGV6.js"
-    if not js_file.exists():
-        return
-
-    src = js_file.read_text()
-
-    patches = [
-        # Fix 1 & 2: Guard r[39]/a[39] filter callback (undefined during Svelte outro)
-        (
-            'r[0].filter(\n        /*func*/\n        r[39]\n      ).map(qd)',
-            '(r[39] ? r[0].filter(r[39]) : r[0]).map(qd)',
-        ),
-        (
-            'a[0].filter(\n          /*func*/\n          a[39]\n        ).map(qd))',
-            '(a[39] ? a[0].filter(a[39]) : a[0]).map(qd))',
-        ),
-        # Fix 3: Lx (Boolean) extracted from Rx (globals) which is undefined in Gradio 5
-        (
-            '{ Boolean: Lx } = Rx,',
-            'Lx = (Rx && Rx.Boolean) || Boolean,',
-        ),
-    ]
-
-    patched = False
-    for old, new in patches:
-        if old in src:
-            src = src.replace(old, new)
-            patched = True
-
-    if patched:
-        js_file.write_text(src)
-
-
-patch_gradio_leaderboard()
 
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
 
-from src.leaderboard import get_leaderboard_df, get_benchmark_run_df
+from src.leaderboard import get_leaderboard_df, DISPLAY_BY_DEFAULT, SEARCH_COLUMNS
 from src.display.text_blocks import (
+    TITLE,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
 )
 
 REPO_ID = "taagarwa/coding-agent-leaderboard"
 TOKEN = os.environ.get("HF_TOKEN")
 API = HfApi(token=TOKEN)
 
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
 LEADERBOARD_DF = get_leaderboard_df()
-BENCHMARK_RUN_DF = get_benchmark_run_df()
 
-def extract_body(s: str):
-    return re.match(r'\[(.*?)\]', s).group(1)
 
-
-def build_header_html(df):
-    n_results = len(df)
-    n_models = df["Model"].nunique()
-    n_harnesses = df["Harness"].apply(lambda s: extract_body(s)).nunique()
-    n_benchmarks = df["Benchmark"].apply(lambda s: extract_body(s)).nunique()
-
-    return f"""
-    <base target="_blank">
-    <div style="padding: 1.5rem 0.5rem 1rem 0.5rem; text-align: left;">
-        <h1 style="margin: 0 0 0.5rem 0; font-size: 2rem;">
-            Coding Agent Leaderboard
-        </h1>
-        <div style="height: 4px; border-radius: 2px; background: linear-gradient(90deg, #84cc16, #f59e0b); margin-bottom: 0.75rem;"></div>
-        <p style="margin: 0 0 0.75rem 0; font-size: 1.1rem; opacity: 0.8;">
-            Compare coding agents across models and harnesses
-        </p>
-        <div style="display: flex; gap: 0.5rem; flex-wrap: wrap; font-size: 0.95rem; opacity: 0.7;">
-            <span style="font-weight: 600;">{n_results} Results</span>
-            <span>·</span>
-            <span style="font-weight: 600;">{n_models} Models</span>
-            <span>·</span>
-            <span style="font-weight: 600;">{n_harnesses} Harnesses</span>
-            <span>·</span>
-            <span style="font-weight: 600;">{n_benchmarks} Benchmarks</span>
-        </div>
-    </div>
-    """
-    
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
-    
-    label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
-    meta_columns = [" ", "Harness", "Model", "Harness License", "Model License", "Model Num Params (B)", "Precision"]
-    benchmark_columns = [col for col in dataframe.columns if col not in meta_columns]
-    model_choices = sorted({(extract_body(v), v) for v in dataframe["Model"]})
-    harness_choices = sorted({(extract_body(v), v) for v in dataframe["Harness"]})
-
-    default_columns = [" ", "Harness", "Model"] + benchmark_columns
     return Leaderboard(
         value=dataframe,
         select_columns=SelectColumns(
-            default_selection=default_columns,
+            default_selection=DISPLAY_BY_DEFAULT,
             label="Select Columns to Display:",
         ),
-        datatype="markdown",
-        search_columns=["Harness", "Model"],
+        search_columns=SEARCH_COLUMNS,
         filter_columns=[
-            ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
-            ColumnFilter(label="Model", column="Model", type="checkboxgroup", choices=model_choices),
-            ColumnFilter(label="Harness", column="Harness", type="checkboxgroup", choices=harness_choices),
-            ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"),
-            ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"),
+            ColumnFilter(label="Dataset", column="dataset", type="checkboxgroup"),
+            ColumnFilter(label="Number of Parameters (B)", column="model_num_params", type="slider", min=0.5, max=150),
+            ColumnFilter(label="Precision", column="precision", type="checkboxgroup"),
         ],
         interactive=False,
     )
 
-def init_benchmark_runs(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    
-    # Make ColumnFilter choices
-    label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
-    benchmark_choices = sorted({(extract_body(v), v) for v in dataframe["Benchmark"]})
-    
-    return Leaderboard(
-        value=dataframe,
-        select_columns=SelectColumns(
-            default_selection=[
-                " ",
-                "Model",
-                "Harness",
-                "Benchmark",
-                "Score",
-                "Avg Cost Per Task (USD)",
-            ],
-            label="Select Columns to Display:",
-        ),
-        datatype="markdown",
-        search_columns=[
-            "Benchmark",
-            "Harness",
-            "Model",
-        ],
-        filter_columns=[
-            ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
-            ColumnFilter(label="Benchmark", column="Benchmark", type="checkboxgroup", choices=benchmark_choices),
-            ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"),
-            ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"),
-        ],
-        interactive=False,
-    )
 
-demo = gr.Blocks(theme="citrus")
+demo = gr.Blocks()
 with demo:
-    gr.HTML(build_header_html(BENCHMARK_RUN_DF))
+    gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-    with gr.Tabs():
-        with gr.Tab("🏆 Leaderboard"):
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Coding Agent Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
+            gr.Markdown("\* `internal` refers to internal benchmarks performed by the model provider where the harness/environment were not made public")
 
-        with gr.Tab("🏃 Benchmark Runs"):
-            benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF)
-
-        with gr.Tab("📝 About"):
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()

requirements.txt

@@ -3,7 +3,7 @@ black
 datasets
 gradio
 gradio[oauth]
-gradio_leaderboard
+gradio_leaderboard==0.0.13
 gradio_client
 huggingface-hub>=0.18.0
 matplotlib

results/qwen3-6-35b-internal.json

@@ -0,0 +1,26 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500
+    },
+    "harness": {
+        "name": "internal",
+        "skills": []
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "Qwen/Qwen3.6-35B-A3B",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "bf16"
+    },
+    "environment": {
+        "name": "internal"
+    },
+    "metrics": {
+        "score": 0.734,
+        "time": null,
+        "costUSD": null
+    }
+}

results/{swe-bench-verified-claude-sonnet-4-6-claude-code.json → qwen3-6-35b-nvfp4-claude-code.json}

@@ -1,23 +1,19 @@
 {
-    "benchmark": {
+    "dataset": {
         "name": "swe-bench-verified",
         "repo": "SWE-bench/SWE-bench_Verified",
-        "num_tasks": 500,
-        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+        "num_tasks": 500
     },
     "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
+        "name": "claude-code",
+        "skills": []
     },
     "model": {
-        "name": "Sonnet 4.6",
-        "repo": "Sonnet 4.6",
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
         "is_oss": true,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
+        "num_params": 35,
+        "precision": "nvfp4"
     },
     "environment": {
         "name": "harbor",
@@ -33,10 +29,11 @@
             "task_names": null,
             "exclude_task_names": null,
             "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
+        }
     },
     "metrics": {
-        "score": 0.796
+        "score": 0.632,
+        "time": 21600,
+        "costUSD": 48.00
     }
 }

results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-pro--ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Sonnet 4.6",
-        "repo": "Sonnet 4.6",
-        "is_oss": true,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 1,
-        "score": 0.5,
-        "n_input_tokens": 190672390,
-        "n_cache_tokens": 184409111,
-        "n_output_tokens": 1593112,
-        "n_total_tokens": 376674613,
-        "agent_time_seconds": 40527,
-        "total_time_seconds": 49734,
-        "cost_usd": 184.42824125000004,
-        "mean_input_tokens_per_task": 1986170,
-        "mean_cache_tokens_per_task": 1920928,
-        "mean_output_tokens_per_task": 16594,
-        "mean_tokens_per_task": 3923693,
-        "mean_cost_usd_per_task": 1.92,
-        "mean_total_time_seconds_per_task": 518,
-        "mean_agent_time_seconds_per_task": 422
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-pro--ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 6,
-        "score": 0.458,
-        "n_input_tokens": 367897697,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1694885,
-        "n_total_tokens": 369592582,
-        "agent_time_seconds": 39024,
-        "total_time_seconds": 46758,
-        "cost_usd": 9.64,
-        "mean_input_tokens_per_task": 3832267,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 17655,
-        "mean_tokens_per_task": 3849922,
-        "mean_cost_usd_per_task": 0.1,
-        "mean_total_time_seconds_per_task": 487,
-        "mean_agent_time_seconds_per_task": 406
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-pro--ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 4,
-        "score": 0.375,
-        "n_input_tokens": 207164679,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1598703,
-        "n_total_tokens": 208763382,
-        "agent_time_seconds": 49450,
-        "total_time_seconds": 57287,
-        "cost_usd": 12.21,
-        "mean_input_tokens_per_task": 2157965,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 16653,
-        "mean_tokens_per_task": 2174618,
-        "mean_cost_usd_per_task": 0.13,
-        "mean_total_time_seconds_per_task": 596,
-        "mean_agent_time_seconds_per_task": 515
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-pi.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-pro--ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 1,
-        "score": 0.479,
-        "n_input_tokens": 742491363,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2387609,
-        "n_total_tokens": 744878972,
-        "agent_time_seconds": 54543,
-        "total_time_seconds": 62422,
-        "cost_usd": 13.47,
-        "mean_input_tokens_per_task": 7734285,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 24870,
-        "mean_tokens_per_task": 7759155,
-        "mean_cost_usd_per_task": 0.14,
-        "mean_total_time_seconds_per_task": 650,
-        "mean_agent_time_seconds_per_task": 568
-    }
-}

results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-verified",
-        "repo": "SWE-bench/SWE-bench_Verified",
-        "num_tasks": 500,
-        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "swe-bench/swe-bench-verified",
-            "version": null,
-            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 500,
-        "n_errors": 1,
-        "score": 0.632,
-        "n_input_tokens": 1106618897,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 5733245,
-        "n_total_tokens": 1112352142,
-        "agent_time_seconds": 122808,
-        "total_time_seconds": 171897,
-        "cost_usd": 34.11,
-        "mean_input_tokens_per_task": 2213237,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 11466,
-        "mean_tokens_per_task": 2224704,
-        "mean_cost_usd_per_task": 0.07,
-        "mean_total_time_seconds_per_task": 343,
-        "mean_agent_time_seconds_per_task": 245
-    }
-}

results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json

@@ -1,59 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-verified",
-        "repo": "SWE-bench/SWE-bench_Verified",
-        "num_tasks": 500,
-        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "swe-bench/swe-bench-verified",
-            "version": null,
-            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null,
-            "accelerated_images": true
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 500,
-        "n_errors": 4,
-        "score": 0.548,
-        "n_input_tokens": 469806650,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 4937761,
-        "n_total_tokens": 474744411,
-        "agent_time_seconds": 120473,
-        "total_time_seconds": 185168,
-        "cost_usd": 29.75,
-        "mean_input_tokens_per_task": 939613,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 9875,
-        "mean_tokens_per_task": 949488,
-        "mean_cost_usd_per_task": 0.06,
-        "mean_total_time_seconds_per_task": 370,
-        "mean_agent_time_seconds_per_task": 240
-    }
-}

results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json

@@ -1,59 +0,0 @@
-{
-    "benchmark": {
-        "name": "swe-bench-verified",
-        "repo": "SWE-bench/SWE-bench_Verified",
-        "num_tasks": 500,
-        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "swe-bench/swe-bench-verified",
-            "version": null,
-            "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null,
-            "accelerated_images": true
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 500,
-        "n_errors": 6,
-        "score": 0.65,
-        "n_input_tokens": 791183735,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 6333798,
-        "n_total_tokens": 797517533,
-        "agent_time_seconds": 154531,
-        "total_time_seconds": 218988,
-        "cost_usd": 38.16,
-        "mean_input_tokens_per_task": 1582367,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 12667,
-        "mean_tokens_per_task": 1595035,
-        "mean_cost_usd_per_task": 0.08,
-        "mean_total_time_seconds_per_task": 437,
-        "mean_agent_time_seconds_per_task": 309
-    }
-}

src/display/text_blocks.py

@@ -1,34 +1,19 @@
+TITLE = """<h1 align="center" id="space-title">Coding Agent Leaderboard</h1>"""
+
 INTRODUCTION_TEXT = """
-A **Coding Agent** is more than just a model - it's the combination of a **Model** and a **Harness** (the tool/framework driving the model).
-This leaderboard tracks how these components work together, because the same model can perform very differently depending on the harness it's paired with.
+Welcome to the Coding Agent Leaderboard!
 """
 
 LLM_BENCHMARKS_TEXT = """
-## What is a Coding Agent?
-
-A coding agent is a system that autonomously solves software engineering tasks - reading code, reasoning about bugs, and writing patches. Its performance depends on two components:
-
-- **Model** - The underlying language model (e.g. Claude Opus 4.7, Qwen3.6-35B)
-- **Harness** - The framework or tool that orchestrates the model's actions (e.g. Claude Code, OpenCode, Pi)
+## About
 
-## How to Read the Table
+Evaluate and compare Coding Agents.
 
-| Column | Description |
-|--------|-------------|
-| **Benchmark** | The benchmark used for evaluation (e.g. SWE-bench Verified - 500 real GitHub issues) |
-| **Harness** | The agent framework driving the model. |
-| **Model** | The language model being evaluated |
-| **Skills** | The set of instructions guiding the agent's behavior |
-| **Score** | Outcome of the benchmark, often the fraction of tasks solved correctly (higher is better) |
-| **Precision** | Model weight format (e.g. bf16, fp4) - affects speed, memory footprint, and quality |
+Coding Agent = Model + Harness + Skills.
 
-## Key Concepts
-
-- **FOSS vs Proprietary** - Filters let you compare fully open-source agents against proprietary ones. A FOSS model with a FOSS harness means anyone can reproduce the result
-- **Skills** - Some harnesses augment the model with extra capabilities (tools, retrieval, etc.). Listed in the "skills" column when present
-- **Internal results (`*`)** - Benchmarks run by the model provider where the harness and environment were not made public. These are useful reference points but are not independently reproducible
+Visit our [GitHub repo](https://github.com/redhat-et/coding_agent_bench) for more details about the project.
+"""
 
-## Learn More
+CITATION_BUTTON_TEXT = "TBD"
 
-Visit the [GitHub repo](https://github.com/redhat-et/coding_agent_bench) for details about the project, methodology, and how to submit your own results.
-"""
+CITATION_BUTTON_LABEL = "Citation"

src/leaderboard.py

@@ -2,10 +2,27 @@ from pathlib import Path
 import json
 import pandas as pd
 
-from src.models import Result, Model, Harness
+from src.models import Result
 
 RESULTS_DIR = Path(__file__).parent.parent / "results"
 
+DISPLAY_BY_DEFAULT = [
+    "dataset",
+    "model",
+    "precision",
+    "harness",
+    "skills",
+    "environment",
+    "score",
+]
+
+SEARCH_COLUMNS = [
+    "dataset",
+    "model",
+    "harness",
+]
+
+
 def format_time(seconds: int):
     if seconds is None:
         return None
@@ -14,9 +31,6 @@ def format_time(seconds: int):
     return f"{h}h{m}m{s}s"
 
 
-def get_benchmark_names(results: list[Result]):
-    return {r.benchmark.name for r in results}
-
 def get_leaderboard_df():
     results: list[Result] = []
     for file in RESULTS_DIR.glob("*.json"):
@@ -24,73 +38,25 @@ def get_leaderboard_df():
             data = json.load(f)
         result = Result(**data)
         results.append(result)
-        
-    # Collect benchmark scores for each model-harness pair, and convert to percent out of 100
-    benchmark_lookup: dict[tuple[str, str], dict[str, float]] = {}
-    model_lookup: dict[str, Model] = {}
-    harness_lookup: dict[str, Harness] = {}
-    for result in results:
-        pair = (result.model.repo, result.harness.name)
-        harness_lookup[result.harness.name] = result.harness
-        model_lookup[result.model.repo] = result.model
-        if pair not in benchmark_lookup:
-            benchmark_lookup[pair] = {}
-        benchmark_lookup[pair][result.benchmark.name] = round(result.metrics.score * 100, 1)
-    
-    # Collect results into df rows
-    rows = []
-    benchmark_names = get_benchmark_names(results=results)
-    for pair, benchmarks in benchmark_lookup.items():
-        model = model_lookup[pair[0]]
-        harness = harness_lookup[pair[1]]
-        avg_score = sum(benchmarks.values()) / len(benchmarks)
-        row = {
-            " ": "🟠" if model.is_oss and harness.is_oss else "🔶",
-            "Model": f'[{model.repo}]({model.url})',
-            "Harness": f'[{harness.name}]({harness.url})<sup>*</sup>' if result.harness.name == "internal" else f'[{harness.name}]({harness.url})',
-            "Precision": model.precision,
-            "Model License": "FOSS" if model.is_oss else "Proprietary",
-            "Harness License": "FOSS" if harness.is_oss else "Proprietary",
-            "Model Num Params (B)": model.num_params,
-            "Avg Score": round(avg_score, 1),
-        }
-        for benchmark_name in sorted(benchmark_names, key=lambda x: (0 if x == "swe-bench-verified" else 1)):
-            row[benchmark_name] = benchmarks.get(benchmark_name, "")
-        rows.append(row)
-    
-    leaderboard_df = pd.DataFrame(rows).sort_values("Avg Score", ascending=False).fillna("")
-    return leaderboard_df
-    
-    
-def get_benchmark_run_df():
-    results: list[Result] = []
-    for file in RESULTS_DIR.glob("*.json"):
-        with open(file, "r") as f:
-            data = json.load(f)
-        result = Result(**data)
-        results.append(result)
 
     rows = []
     for result in results:
         rows.append(
             {
-                " ": "🟠" if result.model.is_oss and result.harness.is_oss else "🔶",
-                "Model": f'[{result.model.repo}]({result.model.url})',
-                "Harness": f'[{result.harness.name}]({result.harness.url})<sup>*</sup>' if result.harness.name == "internal" else f'[{result.harness.name}]({result.harness.url})',
-                "Benchmark": f'[{result.benchmark.name}]({result.benchmark.url})',
-                "Base Model": result.model.name,
-                "Precision": result.model.precision,
-                "Skills": str(result.harness.skills) if result.harness.skills else "None",
-                "Score": round(result.metrics.score * 100, 1),
-                "Avg Cost Per Task (USD)": result.metrics.mean_cost_usd_per_task,
-                "Avg Seconds Per Task": result.metrics.mean_agent_time_seconds_per_task,
-                "Avg Input Tokens Per Task": result.metrics.mean_input_tokens_per_task,
-                "Avg Output Tokens Per Task": result.metrics.mean_output_tokens_per_task,
-                "Model License": "FOSS" if result.model.is_oss else "Proprietary",
-                "Harness License": "FOSS" if result.harness.is_oss else "Proprietary",
-                "Model Num Params (B)": result.model.num_params,
+                "dataset": result.dataset.name,
+                "model": result.model.name,
+                "model_id": result.model.repo,
+                "precision": result.model.precision,
+                "harness": result.harness.name,
+                "skills": str(result.harness.skills) if result.harness.skills else "None",
+                "environment": result.environment.name,
+                "score": result.metrics.score,
+                "costUSD": result.metrics.costUSD,
+                "time": format_time(result.metrics.time),
+                "model_is_oss": result.model.is_oss,
+                "model_num_params": result.model.num_params,
             }
         )
 
-    benchmark_run_df = pd.DataFrame(rows).sort_values(by=["Benchmark", "Score"], ascending=False).fillna("")
-    return benchmark_run_df
+    leaderboard_df = pd.DataFrame(rows)
+    return leaderboard_df

src/models.py

@@ -3,18 +3,15 @@ from typing import Any, Optional
 from pydantic import BaseModel
 
 
-class Benchmark(BaseModel):
+class Dataset(BaseModel):
     name: str
     repo: str
     num_tasks: int
-    url: str
 
 
 class Harness(BaseModel):
     name: str
     skills: list[str]
-    is_oss: bool
-    url: str
 
 
 class Model(BaseModel):
@@ -23,38 +20,21 @@ class Model(BaseModel):
     is_oss: bool
     num_params: int
     precision: str
-    url: str
 
 
 class Environment(BaseModel):
     name: str
     config: Optional[dict[str, Any]] = None
-    url: str
 
 
 class Metrics(BaseModel):
-    
     score: float
-    n_tasks: Optional[int] = None
-    n_errors: Optional[int] = None
-    n_input_tokens: Optional[int] = None
-    n_cache_tokens: Optional[int] = None
-    n_output_tokens: Optional[int] = None
-    n_total_tokens: Optional[int] = None
-    total_time_seconds: Optional[int] = None
-    agent_time_seconds: Optional[int] = None
-    cost_usd: Optional[float] = None
-    mean_input_tokens_per_task: Optional[int] = None
-    mean_cache_tokens_per_task: Optional[int] = None
-    mean_output_tokens_per_task: Optional[int] = None
-    mean_tokens_per_task: Optional[int] = None
-    mean_cost_usd_per_task: Optional[float] = None
-    mean_total_time_seconds_per_task: Optional[int] = None
-    mean_agent_time_seconds_per_task: Optional[int] = None
+    time: Optional[int] = None
+    costUSD: Optional[float] = None
 
 
 class Result(BaseModel):
-    benchmark: Benchmark
+    dataset: Dataset
     harness: Harness
     model: Model
     environment: Environment