Add baseline qwen3.6

.gitattributes

@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/sync-to-hf-space-stage.yml

@@ -1,101 +0,0 @@
-name: Sync stage to HF Space (staging)
-
-# Mirrors every push to `stage` on GitHub into the HF Space git remote so
-# that the staging Space (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage)
-# always tracks the stage branch.
-#
-# Required repository secrets (Settings -> Secrets and variables -> Actions):
-#   HF_TOKEN     Hugging Face access token with write permission to the Space.
-#   HF_USERNAME  Optional fallback username if token introspection fails.
-
-on:
-  push:
-    branches: [stage]
-  workflow_dispatch:
-
-concurrency:
-  group: sync-to-hf-space-stage
-  cancel-in-progress: false
-
-jobs:
-  mirror:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout GitHub stage (full history + LFS)
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-
-      - name: Verify required secrets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          if [ -z "$HF_TOKEN" ]; then
-            echo "::error::HF_TOKEN repository secret must be set."
-            exit 1
-          fi
-
-      - name: Ensure HF Space exists
-        id: hf
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ secrets.HF_USERNAME }}
-        run: |
-          set -euo pipefail
-          python -m pip install --quiet 'huggingface_hub>=0.24,<2'
-          python - <<'PY'
-          import os
-
-          from huggingface_hub import HfApi
-
-          token = os.environ["HF_TOKEN"]
-          space_id = "taagarwa/coding-agent-leaderboard-stage"
-          fallback_username = os.environ.get("HF_USERNAME", "").strip()
-
-          api = HfApi(token=token)
-          username = fallback_username
-          try:
-              info = api.whoami(token=token)
-              username = str(info.get("name") or username).strip()
-          except Exception as exc:
-              if not username:
-                  raise RuntimeError("HF_USERNAME fallback is required when token introspection fails") from exc
-
-          api.create_repo(
-              repo_id=space_id,
-              repo_type="space",
-              space_sdk="docker",
-              token=token,
-              exist_ok=True,
-          )
-
-          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
-              output.write(f"username={username}\n")
-          print(f"HF Space ready: {space_id}")
-          PY
-
-      - name: Push to HF Space remote
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ steps.hf.outputs.username }}
-        run: |
-          set -euo pipefail
-          HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage"
-
-          git config user.name  "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          echo "Pushing $(git rev-parse --short HEAD) to taagarwa/coding-agent-leaderboard-stage..."
-          git push --force "${HF_REMOTE}" HEAD:main
-          echo "Sync complete."
-
-      - name: Summary
-        if: success()
-        run: |
-          echo "### HF Space mirror (staging)" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "Pushed \`$(git rev-parse --short HEAD)\` to \`taagarwa/coding-agent-leaderboard-stage\` Space." >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "View the Space: <https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage>" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/sync-to-hf-space.yml

@@ -1,147 +0,0 @@
-name: Release to HF Space (production)
-
-# Releases the production HF Space
-# (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard) when a
-# version bump lands on `main`.
-#
-# Release flow:
-#   1. In your PR, bump the `VERSION` file (e.g. `make bump VERSION=1.2.3`).
-#   2. Merge the PR into `main`.
-#   3. This workflow runs, creates the `v<VERSION>` git tag on the merge
-#      commit, and pushes that commit to the HF Space.
-#
-# Pushes to `main` that do not change `VERSION` do NOT deploy. If the tag for
-# the current VERSION already exists (e.g. VERSION was edited without a bump),
-# the workflow fails instead of deploying, so an existing release tag is never
-# moved.
-#
-# Required repository secrets (Settings -> Secrets and variables -> Actions):
-#   HF_TOKEN     Hugging Face access token with write permission to the Space.
-#                Create at https://huggingface.co/settings/tokens
-#                (token type "Write" is sufficient; no organization scope needed).
-#   HF_USERNAME  Optional fallback username if token introspection fails.
-#
-# Optional: set HF_SPACE_ID as a repo variable (not secret) to point the
-# workflow at a different Space; defaults to "taagarwa/coding-agent-leaderboard".
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - VERSION
-  # Manual dispatch re-deploys an existing release tag on demand from the
-  # Actions tab (e.g. to recover the Space after a bad manual edit). It never
-  # creates tags.
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Existing release tag to redeploy (e.g. v1.2.3)"
-        required: true
-        type: string
-
-# Only one release job at a time so we never race ourselves into
-# non-fast-forward pushes on the Space remote.
-concurrency:
-  group: sync-to-hf-space
-  cancel-in-progress: false
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: write # needed to push the release tag
-    env:
-      HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }}
-    steps:
-      - name: Checkout (full history + LFS)
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
-          fetch-depth: 0
-          lfs: true
-
-      - name: Verify required secrets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          if [ -z "$HF_TOKEN" ]; then
-            echo "::error::HF_TOKEN repository secret must be set."
-            echo "  Create HF_TOKEN at https://huggingface.co/settings/tokens (type: Write)"
-            exit 1
-          fi
-
-      - name: Resolve release tag
-        id: tag
-        env:
-          EVENT_NAME: ${{ github.event_name }}
-          INPUT_TAG: ${{ inputs.tag }}
-        run: |
-          set -euo pipefail
-          FILE_VERSION="$(tr -d '[:space:]' < VERSION)"
-          if [[ ! "$FILE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "::error::VERSION file contains '$FILE_VERSION'; expected MAJOR.MINOR.PATCH."
-            exit 1
-          fi
-          TAG="v${FILE_VERSION}"
-
-          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
-            if [ "$INPUT_TAG" != "$TAG" ]; then
-              echo "::error::Requested tag '$INPUT_TAG' does not match VERSION file at that ref ('$FILE_VERSION')."
-              exit 1
-            fi
-            echo "Redeploying existing release ${TAG}."
-          else
-            if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
-              echo "::error::Tag ${TAG} already exists. Bump VERSION to a new number to release; existing release tags are never moved."
-              exit 1
-            fi
-          fi
-
-          echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
-          echo "Release ${TAG} at $(git rev-parse --short HEAD)."
-
-      - name: Create release tag
-        if: github.event_name == 'push'
-        env:
-          RELEASE_TAG: ${{ steps.tag.outputs.tag }}
-        run: |
-          set -euo pipefail
-          git config user.name  "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git tag -a "$RELEASE_TAG" -m "Release $RELEASE_TAG"
-          git push origin "refs/tags/$RELEASE_TAG"
-          echo "Created and pushed ${RELEASE_TAG}."
-
-      - name: Push to HF Space remote
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          HF_USERNAME: ${{ secrets.HF_USERNAME || github.repository_owner }}
-          RELEASE_TAG: ${{ steps.tag.outputs.tag }}
-        run: |
-          set -euo pipefail
-          # Authenticate via token in the URL. HF Spaces accept the
-          # username + token basic-auth format over HTTPS git.
-          HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_SPACE_ID}"
-
-          echo "Pushing ${RELEASE_TAG} ($(git rev-parse --short HEAD)) to ${HF_SPACE_ID}..."
-
-          # --force is intentional: GitHub is the single source of truth
-          # for the Space's git history. Anything on the Space side that
-          # wasn't committed via GitHub is overwritten on the next sync.
-          # This prevents the drift situation where someone edits files
-          # in the HF Space UI and creates commits only visible there.
-          git push --force "${HF_REMOTE}" HEAD:main
-
-          echo "Sync complete."
-
-      - name: Summary
-        if: success()
-        env:
-          RELEASE_TAG: ${{ steps.tag.outputs.tag }}
-        run: |
-          echo "### HF Space release" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "Deployed \`${RELEASE_TAG}\` (\`$(git rev-parse --short HEAD)\`) to \`${HF_SPACE_ID}\` Space." >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "View the Space: <https://huggingface.co/spaces/${HF_SPACE_ID}>" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/update-csv.yml

@@ -1,35 +0,0 @@
-name: Update results CSV
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "results/*.json"
-
-jobs:
-  update-csv:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.PAT_TOKEN }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-
-      - name: Install dependencies
-        run: pip install pandas==2.2.3
-
-      - name: Generate CSV
-        run: python src/results_to_csv.py
-
-      - name: Commit if changed
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add results.csv
-          git diff --staged --quiet || (git commit -m "Auto-update results.csv" && git push)

.gitignore

@@ -6,11 +6,9 @@ __pycache__/
 *ipynb
 .vscode/
 
-Backup/
 eval-queue/
 eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
-uv.lock
-.venv/
+uv.lock

Makefile

@@ -1,17 +1,4 @@
-.PHONY: style format quality csv bump
-
-# Bump the VERSION file and commit it. Do this in your PR; when the PR merges
-# into main, CI creates the v<VERSION> tag and deploys to the production Space.
-# Usage: make bump VERSION=1.2.3
-bump:
-	@test -n "$(VERSION)" || { echo "usage: make bump VERSION=X.Y.Z"; exit 1; }
-	@echo "$(VERSION)" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$$' || { echo "VERSION must be MAJOR.MINOR.PATCH"; exit 1; }
-	@git fetch --tags --quiet origin 2>/dev/null || true
-	@! git rev-parse -q --verify "refs/tags/v$(VERSION)" >/dev/null || { echo "tag v$(VERSION) already exists"; exit 1; }
-	@echo "$(VERSION)" > VERSION
-	git add VERSION
-	git commit -m "Bump version to $(VERSION)"
-	@echo "VERSION is now $(VERSION). Merge to main to release v$(VERSION)."
+.PHONY: style format
 
 
 style:
@@ -24,7 +11,3 @@ quality:
 	python -m black --check --line-length 119 .
 	python -m isort --check-only .
 	ruff check .
-
-
-csv:
-	python src/results_to_csv.py

README.md

@@ -15,127 +15,26 @@ tags:
 
 # Coding Agent Leaderboard
 
-Compare coding-agent models and harnesses across benchmark performance, cost, latency, and token usage.
-
-## Leaderboard views
-
-### Efficiency
-
-The **Efficiency** tab compares benchmark score with resource use for one selected benchmark at a time. Keeping the view benchmark-specific avoids misleading comparisons when model and harness combinations have incomplete benchmark coverage.
-The generalized performance-vs-resource chart covers tokens, cost, and agent time in one benchmark-specific view.
-
-The scatter plot supports three resource metrics on the x-axis:
-
-- **Total tokens**: mean total token usage per task.
-- **Cost per task**: the repository's reported `mean_cost_usd_per_task` value, in USD.
-- **Agent time per task**: the repository's reported `mean_agent_time_seconds_per_task` value, in seconds.
-
-Only positive, available values for the selected resource are plotted. Missing, zero, and negative values are treated as unavailable rather than as perfect efficiency. The UI reports how many runs for the selected benchmark were excluded for the chosen resource metric.
-
-Scores are stored internally as fractions from 0 to 1 and displayed as percentages. Point labels are optional and coloring can be grouped by **Model** or **Harness**. Linear and logarithmic resource axes are available.
-
-#### Pareto frontier
-
-For all three resource metrics, lower resource use and higher score are better. A displayed run is Pareto-efficient when no other valid run on the selected benchmark:
-
-- uses an equal or lower amount of the selected resource, and
-- achieves an equal or higher score,
-
-with at least one strict improvement. Tied nondominated points are preserved. The dashed line connects the Pareto-efficient observations.
-
-#### Ranking table
-
-The ranking table remains benchmark-specific and begins with **Model**, **Harness**, and **Benchmark**. Displayed scores are rounded to one decimal place without reducing the precision used for calculations.
-
-**Tokens Per Solved Task** remains available in the table as a reference metric even though it is no longer an Efficiency scatter x-axis option. It is defined as `Total Tokens Per Task / Score`, where Score is the fractional value from 0 to 1. It is unavailable when total token data is missing/non-positive or when Score is zero or negative.
-
-#### Color palettes and themes
-
-The Efficiency chart uses the shared palette registry and supports Citrus, Okabe-Ito, High contrast, Rainbow, Grayscale, Viridis, Plasma, and Cividis palettes with light and dark chart backgrounds.
-
 ## Adding a New Leaderboard Entry
 
-Create a pull request adding a new entry to the `results/` folder. See [`results/qwen3-6-35b-nvfp4-claude-code.json`](./results/qwen3-6-35b-nvfp4-claude-code.json) for an example result.
-
-Do not change the result JSON schema for Efficiency analysis. The feature uses resource and performance metrics already present in the existing result model.
+Create a PR adding a new entry into the `results/` folder. 
+Check out `results/qwen3-6-35b-nvfp4-claude-code.json`(./results/qwen3-6-35b-nvfp4-claude-code.json) for an example result.
 
 ## Development
 
-1. Install dependencies:
-
-   ```sh
-   pip install -r requirements.txt
-
-   # or
-
-   uv venv
-   uv pip install -r requirements.txt
-   ```
-
-2. Run the app:
-
-   ```sh
-   python app.py
-   ```
-
-3. Run tests:
-
-   ```sh
-   pytest
-   ```
-
-## Releasing
-
-Versions are managed with git tags. The `VERSION` file is the single source of truth for the version number (shown in the app header). Each release is tagged `v<VERSION>` on `main` by CI.
-
-Deployment targets:
-
-- **Staging** (`taagarwa/coding-agent-leaderboard-stage`): every push to the `stage` branch.
-- **Production** (`taagarwa/coding-agent-leaderboard`): only when a commit that changes `VERSION` lands on `main`. Other pushes to `main` do not deploy.
-
-To cut a release, bump the version in your PR:
-
-```sh
-make bump VERSION=1.2.3             # writes VERSION and commits "Bump version to 1.2.3"
-git push                            # open / update your PR as usual
-```
-
-When the PR is merged into `main`, the **Release to HF Space (production)** workflow creates the `v1.2.3` tag on the merge commit and pushes it to the Space. If a tag for the current `VERSION` already exists, the workflow fails rather than moving the tag, so bump to a new number for every release.
-
-To redeploy an existing release (e.g. after a manual edit on the Space), run the workflow from the Actions tab and supply the tag.
-
-## Manual validation
-
-After automated checks pass, launch the app from a clean process with `python app.py` and verify the Efficiency view in a real browser. Unit tests and figure-level smoke tests do not replace this browser validation.
-
-### Efficiency behavior
-
-- Confirm there is no **All benchmarks** option and that a valid benchmark is selected by default.
-- Confirm **Color By** offers only **Model** and **Harness**.
-- Confirm the resource selector offers **Total tokens**, **Cost per task**, and **Agent time per task**.
-- Switch through all three resource metrics and verify axis labels and hover formatting.
-- Verify the Pareto frontier for all three metrics.
-- Verify linear and logarithmic scales where valid.
-- Verify point labels off and on.
-- Confirm the ranking table starts with **Model**, **Harness**, **Benchmark**, shows scores to one decimal place, and retains **Tokens Per Solved Task**.
-- Confirm the existing Leaderboard and Benchmark Runs tabs still work without new terminal tracebacks.
-
-### Responsive layout verification
+1. Install dependencies
 
-- Restart the app from a clean launch.
-- Open **Efficiency** as the first non-default tab and confirm the plot is not squished.
-- Switch away from Efficiency and back several times.
-- Resize the browser narrower and wider.
-- Confirm the plot resizes correctly without requiring a control change.
-- Verify there is no legend overlap or clipping.
-- Verify the ranking table below does not force the plot into a narrow column.
-- Repeat the checks with point labels off and on, Pareto off and on, each resource metric, and light and dark plot backgrounds.
-- Also switch **Leaderboard → Efficiency** and **Benchmark Runs → Efficiency** to confirm hidden-tab initialization does not collapse the chart.
+    ```sh
+    pip install -r requirements.txt
 
-### PR completion report
+    # or
 
-The final PR report should state the files changed, metric definitions, tests and results, data-quality limitations, and whether the app was manually verified. For the responsive-layout fix, it must additionally record:
+    uv venv
+    uv pip install -r requirements.txt
+    ```
 
-- the root cause of the initial squished-chart issue;
-- the exact responsive-layout fix used; and
-- whether the behavior was manually verified from a fresh app launch.
+2. Run the app
+   
+    ```sh
+    python app.py
+    ```

VERSION

@@ -1 +0,0 @@
-0.1.0

app.py

@@ -1,1068 +1,72 @@
 import os
-from pathlib import Path
-
-
-def patch_gradio_leaderboard():
-    """Patch gradio_leaderboard JS to fix crash on tab switch with Gradio 5.x."""
-    import gradio_leaderboard
-
-    pkg_dir = Path(gradio_leaderboard.__file__).parent
-    js_file = pkg_dir / "templates" / "component" / "Index-CzS_eGV6.js"
-    if not js_file.exists():
-        return
-
-    src = js_file.read_text()
-
-    patches = [
-        # Fix 1 & 2: Guard r[39]/a[39] filter callback (undefined during Svelte outro)
-        (
-            'r[0].filter(\n        /*func*/\n        r[39]\n      ).map(qd)',
-            '(r[39] ? r[0].filter(r[39]) : r[0]).map(qd)',
-        ),
-        (
-            'a[0].filter(\n          /*func*/\n          a[39]\n        ).map(qd))',
-            '(a[39] ? a[0].filter(a[39]) : a[0]).map(qd))',
-        ),
-        # Fix 3: Lx (Boolean) extracted from Rx (globals) which is undefined in Gradio 5
-        (
-            '{ Boolean: Lx } = Rx,',
-            'Lx = (Rx && Rx.Boolean) || Boolean,',
-        ),
-    ]
-
-    patched = False
-    for old, new in patches:
-        if old in src:
-            src = src.replace(old, new)
-            patched = True
-
-    if patched:
-        js_file.write_text(src)
-
-
-patch_gradio_leaderboard()
 
 import gradio as gr
-import pandas as pd
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from apscheduler.schedulers.background import BackgroundScheduler
-from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 from huggingface_hub import HfApi
 
-from src.analytics import (
-    MATRIX_METRICS,
-    RANKING_METRICS,
-    TRADEOFF_METRICS,
-    benchmarks_for_category,
-    coverage_summary,
-    cross_benchmark_ranking_df,
-    enrich_analysis_df,
-    filter_category,
-    matrix_df,
-    ranking_df,
-)
-from src.charts import (
-    clean_markdown_link,
-    create_coverage_matrix_plot,
-    create_leaderboard_benchmark_plot,
-    create_matrix_plot,
-    create_performance_vs_resource_plot,
-    create_ranking_plot,
-    create_tradeoff_plot,
-)
+from src.leaderboard import get_leaderboard_df, DISPLAY_BY_DEFAULT, SEARCH_COLUMNS
 from src.display.text_blocks import (
-    HOW_TO_USE_TEXT,
+    TITLE,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
 )
-from src.leaderboard import (
-    EFFICIENCY_RESOURCE_METRICS,
-    get_analysis_df,
-    get_benchmark_names,
-    get_benchmark_run_df,
-    get_efficiency_df,
-)
-from src.rankings import EXCLUDED_BENCHMARKS, RANK_BY_OPTIONS, load_and_rank
-from src.version import __version__
 
 REPO_ID = "taagarwa/coding-agent-leaderboard"
 TOKEN = os.environ.get("HF_TOKEN")
 API = HfApi(token=TOKEN)
-COLOR_BY_CHOICES = ["Model", "Harness"]
-EFFICIENCY_COLOR_BY_CHOICES = ["Model", "Harness"]
-COLOR_PALETTE_CHOICES = ["Citrus", "Okabe-Ito", "High contrast", "Rainbow"]
-DEFAULT_COLOR_PALETTE = "Citrus"
-PLOT_BACKGROUND_CHOICES = ["Dark", "White"]
-DEFAULT_PLOT_BACKGROUND = "Dark"
-RESPONSIVE_PLOT_MIN_HEIGHT_PX = 420
-TABLE_MAX_HEIGHT_PX = 720
-RESPONSIVE_PLOT_CSS = f"""
-<style>
-.responsive-plot {{
-  overflow-x: hidden;
-  width: 100%;
-  min-width: 0;
-  min-height: {RESPONSIVE_PLOT_MIN_HEIGHT_PX}px;
-}}
-.responsive-plot .plot-container,
-.responsive-plot .js-plotly-plot,
-.responsive-plot .plotly-graph-div,
-.responsive-plot .svg-container {{
-  width: 100% !important;
-  max-width: 100% !important;
-  min-width: 0 !important;
-  min-height: {RESPONSIVE_PLOT_MIN_HEIGHT_PX}px;
-}}
-.summary-cards {{
-  display: grid;
-  grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
-  gap: 0.6rem;
-}}
-.summary-card {{
-  border: 1px solid rgba(203,213,225,.18);
-  border-radius: 10px;
-  padding: .8rem;
-  background: rgba(39,33,30,.7);
-}}
-.summary-card strong {{ font-size: 1.35rem; display:block; }}
-</style>
-"""
-FORCE_DARK_MODE_HEAD = (
-    """
-<script>
-(() => {
-  const url = new URL(window.location.href);
-  if (url.searchParams.get("__theme") !== "dark") {
-    url.searchParams.set("__theme", "dark");
-    window.location.replace(url.toString());
-    return;
-  }
-
-  // Plotly often measures hidden Gradio tabs before they are visible. Resize every
-  // visible chart when a tab opens, its container changes size, or the window resizes.
-  const resizeVisiblePlots = () => {
-    if (!window.Plotly) return;
-    document.querySelectorAll('.responsive-plot .js-plotly-plot').forEach((plot) => {
-      if (plot.offsetParent !== null) window.Plotly.Plots.resize(plot);
-    });
-  };
-  const scheduleResize = () => {
-    requestAnimationFrame(() => {
-      resizeVisiblePlots();
-      setTimeout(resizeVisiblePlots, 80);
-      setTimeout(resizeVisiblePlots, 250);
-    });
-  };
-  window.addEventListener('resize', scheduleResize);
-  document.addEventListener('click', scheduleResize, true);
-  document.addEventListener('DOMContentLoaded', () => {
-    const observer = new ResizeObserver(scheduleResize);
-    document.querySelectorAll('.responsive-plot').forEach((el) => observer.observe(el));
-    const mutation = new MutationObserver(() => {
-      document.querySelectorAll('.responsive-plot').forEach((el) => observer.observe(el));
-      scheduleResize();
-    });
-    mutation.observe(document.body, {childList: true, subtree: true});
-    scheduleResize();
-  });
-})();
-</script>
-"""
-    + RESPONSIVE_PLOT_CSS
-)
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-BENCHMARK_NAMES = get_benchmark_names()
-DEFAULT_BENCHMARK = BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None
-BENCHMARK_RUN_DF = get_benchmark_run_df()
-ANALYSIS_DF = get_analysis_df()
-PR2_DF = enrich_analysis_df(ANALYSIS_DF)
-CODING_BENCHMARKS = benchmarks_for_category(PR2_DF, "Coding")
-GENERALIST_BENCHMARKS = benchmarks_for_category(PR2_DF, "Generalist")
-
-
-def render_leaderboard_benchmark_plot(
-    benchmark_name,
-    color_by,
-    color_palette=DEFAULT_COLOR_PALETTE,
-    plot_background=DEFAULT_PLOT_BACKGROUND,
-):
-    return create_leaderboard_benchmark_plot(
-        BENCHMARK_RUN_DF,
-        benchmark_name=benchmark_name,
-        color_by=color_by,
-        palette_name=color_palette,
-        background_name=plot_background,
-    )
-
-
-def render_efficiency(
-    benchmark_name,
-    token_metric,
-    color_by,
-    x_scale,
-    show_pareto_frontier,
-    show_labels,
-    color_palette=DEFAULT_COLOR_PALETTE,
-    plot_background=DEFAULT_PLOT_BACKGROUND,
-):
-    plot_df = get_efficiency_df(
-        benchmark_name=benchmark_name,
-        resource_metric=token_metric,
-        analysis_df=ANALYSIS_DF,
-    )
-    exclusion_count = plot_df.attrs.get("exclusion_count", 0)
-    note = (
-        f"{exclusion_count} runs excluded for this benchmark because "
-        f"{token_metric.lower()} was missing or non-positive."
-    )
-    figure = create_performance_vs_resource_plot(
-        plot_df,
-        resource_metric=token_metric,
-        color_by=color_by,
-        x_scale=x_scale,
-        show_pareto_frontier=show_pareto_frontier,
-        show_labels=show_labels,
-        palette_name=color_palette,
-        background_name=plot_background,
-    )
-    return figure, note
-
-
-PAGE_TABLE_COLUMNS = [
-    "Model",
-    "Harness",
-    "Benchmark",
-    "Score (%)",
-    "Within-Benchmark Rank",
-    "Within-Benchmark Percentile",
-    "Total Tokens Per Task",
-    "Cost Per Task",
-    "Total Time Per Task",
-    "Agent Time Per Task",
-    "Execution Error Rate (%)",
-    "Tokens Per Successful Task",
-    "Cost Per Successful Task",
-    "Time Per Successful Task",
-]
-
-PAGE_TABLE_SORT_COLUMNS = {
-    "Score": "Score (%)",
-    "Rank": "Within-Benchmark Rank",
-    "Percentile": "Within-Benchmark Percentile",
-    "Total tokens": "Total Tokens Per Task",
-    "Cost": "Cost Per Task",
-    "Response time": "Total Time Per Task",
-    "Agent time": "Agent Time Per Task",
-    "Execution error rate": "Execution Error Rate (%)",
-}
-
-PAGE_TABLE_HIGHER_IS_BETTER = {
-    "Score": True,
-    "Rank": False,
-    "Percentile": True,
-    "Total tokens": False,
-    "Cost": False,
-    "Response time": False,
-    "Agent time": False,
-    "Execution error rate": False,
-}
-
-
-def render_page_table(benchmark, sort_metric="Score", sort_order="Best first"):
-    """One compact table per page with all metrics relevant to ranking/trade-off views."""
-    data = PR2_DF.copy()
-    if benchmark and benchmark != "All benchmarks":
-        data = data[data["Benchmark"] == benchmark].copy()
-    columns = [column for column in PAGE_TABLE_COLUMNS if column in data.columns]
-    data = data[columns].copy()
-    if data.empty:
-        return data
-
-    data["_agent"] = data["Model"].astype(str) + " / " + data["Harness"].astype(str)
-    if sort_order == "Alphabetical (A–Z)":
-        data = data.sort_values(["_agent", "Benchmark"], ascending=[True, True], kind="mergesort")
-    elif sort_order == "Alphabetical (Z–A)":
-        data = data.sort_values(["_agent", "Benchmark"], ascending=[False, True], kind="mergesort")
-    else:
-        sort_column = PAGE_TABLE_SORT_COLUMNS.get(sort_metric, "Score (%)")
-        values = pd.to_numeric(data.get(sort_column), errors="coerce")
-        data["_sort_value"] = values
-        if sort_order == "Best first":
-            ascending = not PAGE_TABLE_HIGHER_IS_BETTER.get(sort_metric, True)
-        elif sort_order == "Best last":
-            ascending = PAGE_TABLE_HIGHER_IS_BETTER.get(sort_metric, True)
-        else:
-            ascending = sort_order == "Lowest value first"
-        data = data.sort_values(
-            ["_sort_value", "_agent", "Benchmark"],
-            ascending=[ascending, True, True],
-            na_position="last",
-            kind="mergesort",
-        ).drop(columns="_sort_value")
-    data = data.drop(columns="_agent").reset_index(drop=True)
-
-    # Metrics displayed to 2 decimal places
-    decimal_columns = [
-        "Score (%)",
-        "Within-Benchmark Percentile",
-        "Execution Error Rate (%)",
-        "Cost Per Successful Task",
-    ]
-
-    for column in decimal_columns:
-        if column in data.columns:
-            data[column] = pd.to_numeric(data[column], errors="coerce").round(2)
-
-    # Token and time metrics displayed as whole numbers
-    whole_number_columns = [
-        "Total Tokens Per Task",
-        "Total Time Per Task",
-        "Agent Time Per Task",
-        "Tokens Per Successful Task",
-        "Time Per Successful Task",
-    ]
-
-    for column in whole_number_columns:
-        if column in data.columns:
-            data[column] = pd.to_numeric(data[column], errors="coerce").round(0)
-
-    return data
-
-
-def render_ranking(
-    metric,
-    benchmark,
-    color_by="Model",
-    color_palette=DEFAULT_COLOR_PALETTE,
-    plot_background=DEFAULT_PLOT_BACKGROUND,
-    sort_order="Best first",
-):
-    def sort_table(table, metric_column, higher_is_better):
-        if table is None or table.empty:
-            return table
-        work = table.copy()
-        work["_agent"] = work["Model"].astype(str) + " / " + work["Harness"].astype(str)
-        if sort_order == "Alphabetical (A–Z)":
-            work = work.sort_values("_agent", ascending=True, kind="mergesort")
-        elif sort_order == "Alphabetical (Z–A)":
-            work = work.sort_values("_agent", ascending=False, kind="mergesort")
-        elif sort_order in {"Best first", "Best last"}:
-            ascending = not higher_is_better
-            if sort_order == "Best last":
-                ascending = not ascending
-            work = work.sort_values(
-                [metric_column, "_agent"],
-                ascending=[ascending, True],
-                kind="mergesort",
-            )
-        elif sort_order == "Lowest value first":
-            work = work.sort_values(
-                [metric_column, "_agent"],
-                ascending=[True, True],
-                kind="mergesort",
-            )
-        else:
-            work = work.sort_values(
-                [metric_column, "_agent"],
-                ascending=[False, True],
-                kind="mergesort",
-            )
-        return work.drop(columns="_agent").reset_index(drop=True)
-
-    table = ranking_df(PR2_DF, metric, benchmark=benchmark)
-    spec = RANKING_METRICS[metric]
-    table = sort_table(table, spec.column, spec.higher_is_better)
-    figure = create_ranking_plot(
-        table,
-        spec.column,
-        spec.label,
-        spec.higher_is_better,
-        color_by=color_by,
-        palette_name=color_palette,
-        background_name=plot_background,
-        sort_order=sort_order,
-    )
-    return figure, table
-
-
-TRADEOFF_PAIRS = {
-    "Score vs cost": ("Cost Per Task", "Score (%)", "Cost per task (USD)", "Score (%)", True, True),
-    "Score vs tokens": ("Total Tokens Per Task", "Score (%)", "Total tokens per task", "Score (%)", True, True),
-    "Score vs total time": ("Total Time Per Task", "Score (%)", "Total time per task (seconds)", "Score (%)", True, True),
-    "Score vs agent time": ("Agent Time Per Task", "Score (%)", "Agent time per task (seconds)", "Score (%)", True, True),
-    "Score vs execution error rate": (
-        "Execution Error Rate (%)", "Score (%)", "Execution error rate (%)", "Score (%)", True, True
-    ),
-    "Tokens vs cost": ("Total Tokens Per Task", "Cost Per Task", "Total tokens per task", "Cost per task (USD)", True, False),
-    "Cost vs total time": ("Cost Per Task", "Total Time Per Task", "Cost per task (USD)", "Total time per task (seconds)", True, False),
-}
-
-
-def render_tradeoff(
-    pair,
-    benchmark,
-    color_by,
-    show_labels,
-    x_scale,
-    show_pareto,
-    color_palette,
-    plot_background,
-):
-    x_column, y_column, x_label, y_label, lower_x, higher_y = TRADEOFF_PAIRS[pair]
-    data = PR2_DF[PR2_DF["Benchmark"] == benchmark].copy()
-    figure = create_tradeoff_plot(
-        data,
-        x_column=x_column,
-        y_column=y_column,
-        x_label=x_label,
-        y_label=y_label,
-        color_by=color_by,
-        show_labels=show_labels,
-        x_scale=x_scale,
-        show_pareto_frontier=show_pareto,
-        lower_x_is_better=lower_x,
-        higher_y_is_better=higher_y,
-        palette_name=color_palette,
-        background_name=plot_background,
-    )
-    valid = data[[x_column, y_column]].apply(pd.to_numeric, errors="coerce").dropna()
-    note = f"{len(valid)} comparable runs shown for {benchmark}; missing metrics are omitted, not treated as zero."
-    return figure, note
-
-
-def render_matrix(
-    metric,
-    category,
-    include_incomplete,
-    sort_by,
-    show_values,
-    reverse_scale,
-    plot_background,
-):
-    category_filter = None if category == "All" else category
-    matrix = matrix_df(
-        PR2_DF,
-        metric,
-        category=category_filter,
-        include_incomplete=include_incomplete,
-        sort_by=sort_by,
-    )
-    if metric == "Coverage":
-        return create_coverage_matrix_plot(matrix, plot_background)
-    spec = MATRIX_METRICS[metric]
-    display_matrix = None
-    display_metric_label = None
-    if metric == "Within-benchmark percentile":
-        display_matrix = matrix_df(
-            PR2_DF,
-            "Score",
-            category=category_filter,
-            include_incomplete=include_incomplete,
-            sort_by=sort_by,
-        ).reindex(index=matrix.index, columns=matrix.columns)
-        display_metric_label = "Benchmark score (%)"
-    return create_matrix_plot(
-        matrix,
-        f"{metric} matrix",
-        spec.label,
-        higher_is_better=spec.higher_is_better,
-        show_values=show_values,
-        reverse_scale=reverse_scale,
-        background_name=plot_background,
-        display_matrix=display_matrix,
-        display_metric_label=display_metric_label,
-    )
-
-
-def category_leaderboard(category):
-    data = filter_category(PR2_DF, category)
-    if data.empty:
-        return pd.DataFrame()
-    normalized = cross_benchmark_ranking_df(data, minimum_coverage=0)
-    return normalized
-
-
-def render_category_tradeoff(category, benchmark, metric, color_by, show_labels, plot_background):
-    data = filter_category(PR2_DF, category)
-    data = data[data["Benchmark"] == benchmark]
-    spec = TRADEOFF_METRICS[metric]
-    return create_tradeoff_plot(
-        data,
-        x_column=spec.column,
-        y_column="Score (%)",
-        x_label=spec.label,
-        y_label="Score (%)",
-        color_by=color_by,
-        show_labels=show_labels,
-        show_pareto_frontier=metric != "Execution error rate",
-        lower_x_is_better=True,
-        higher_y_is_better=True,
-        background_name=plot_background,
-    )
-
-
-def render_category_matrix(category, metric, show_values, plot_background):
-    matrix = matrix_df(PR2_DF, metric, category=category, include_incomplete=True)
-    if metric == "Coverage":
-        return create_coverage_matrix_plot(matrix, plot_background)
-    spec = MATRIX_METRICS[metric]
-    display_matrix = None
-    display_metric_label = None
-    if metric == "Within-benchmark percentile":
-        display_matrix = matrix_df(
-            PR2_DF, "Score", category=category, include_incomplete=True
-        ).reindex(index=matrix.index, columns=matrix.columns)
-        display_metric_label = "Benchmark score (%)"
-    return create_matrix_plot(
-        matrix,
-        f"{category} — {metric}",
-        spec.label,
-        higher_is_better=spec.higher_is_better,
-        show_values=show_values,
-        background_name=plot_background,
-        display_matrix=display_matrix,
-        display_metric_label=display_metric_label,
-    )
-
+LEADERBOARD_DF = get_leaderboard_df()
 
-def build_header_html(df):
-    summary = coverage_summary(PR2_DF)
-    return f"""
-    <base target="_blank">
-    <div style="padding: 1.5rem 0.5rem 1rem 0.5rem; text-align: left; color: #F8FAFC;">
-        <h1 style="margin: 0 0 0.5rem 0; font-size: 2rem;">Coding Agent Leaderboard
-            <span style="font-size: 0.9rem; font-weight: 500; color: #94A3B8; margin-left: 0.5rem;">v{__version__}</span>
-        </h1>
-        <div style="height: 4px; border-radius: 2px; background: linear-gradient(90deg, #84cc16, #f59e0b); margin-bottom: 0.75rem;"></div>
-        <p style="margin: 0 0 0.9rem 0; font-size: 1.1rem; color: #E5E7EB;">
-            Performance, efficiency, coverage, and reliability across coding-agent benchmarks.
-            Each result is one model + harness run on one benchmark.
-        </p>
-        <div class="summary-cards">
-          <div class="summary-card"><strong>{summary['results']}</strong>benchmark results</div>
-          <div class="summary-card"><strong>{summary['models']}</strong>models</div>
-          <div class="summary-card"><strong>{summary['harnesses']}</strong>harnesses</div>
-          <div class="summary-card"><strong>{summary['benchmarks']}</strong>benchmarks</div>
-          <div class="summary-card"><strong>{summary['token_coverage_pct']:.0f}%</strong>token coverage</div>
-          <div class="summary-card"><strong>{summary['cost_coverage_pct']:.0f}%</strong>cost coverage</div>
-          <div class="summary-card"><strong>{summary['time_coverage_pct']:.0f}%</strong>timing coverage</div>
-        </div>
-        <p style="margin: 0.9rem 0 0 0; color: #CBD5E1; font-size: 0.95rem;">
-            Cross-benchmark ordering uses within-benchmark percentiles rather than averaging incompatible raw score scales.
-            Missing metrics remain missing and reduce coverage; they are never converted to zero.
-        </p>
-    </div>
-    """
 
-
-def build_overview_html():
-    summary = coverage_summary(PR2_DF)
-    return f"""
-    <div class="summary-cards">
-      <div class="summary-card"><strong>{summary['results']}</strong>benchmark results</div>
-      <div class="summary-card"><strong>{summary['models']}</strong>models</div>
-      <div class="summary-card"><strong>{summary['harnesses']}</strong>harnesses</div>
-      <div class="summary-card"><strong>{summary['benchmarks']}</strong>benchmarks</div>
-      <div class="summary-card"><strong>{summary['token_coverage_pct']:.0f}%</strong>token coverage</div>
-      <div class="summary-card"><strong>{summary['cost_coverage_pct']:.0f}%</strong>cost coverage</div>
-      <div class="summary-card"><strong>{summary['time_coverage_pct']:.0f}%</strong>timing coverage</div>
-    </div>
-    """
-
-
-def init_benchmark_runs(dataframe):
+def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
-
-    label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")]
-    benchmark_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Benchmark"]})
-    model_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Model"]})
-    harness_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Harness"]})
-
     return Leaderboard(
         value=dataframe,
         select_columns=SelectColumns(
-            default_selection=[
-                " ",
-                "Model",
-                "Harness",
-                "Benchmark",
-                "Score",
-                "Avg Cost Per Task (USD)",
-            ],
+            default_selection=DISPLAY_BY_DEFAULT,
             label="Select Columns to Display:",
         ),
-        datatype="markdown",
-        search_columns=[
-            "Benchmark",
-            "Harness",
-            "Model",
-        ],
+        search_columns=SEARCH_COLUMNS,
         filter_columns=[
-            ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices),
-            ColumnFilter(label="Benchmark", column="Benchmark", type="checkboxgroup", choices=benchmark_choices),
-            ColumnFilter(label="Model", column="Model", type="checkboxgroup", choices=model_choices),
-            ColumnFilter(label="Harness", column="Harness", type="checkboxgroup", choices=harness_choices),
-            ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"),
-            ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"),
+            ColumnFilter(label="Dataset", column="dataset", type="checkboxgroup"),
+            ColumnFilter(label="Number of Parameters (B)", column="model_num_params", type="slider", min=0.5, max=150),
+            ColumnFilter(label="Precision", column="precision", type="checkboxgroup"),
         ],
         interactive=False,
     )
 
 
-def add_category_section(category, benchmarks):
-    if not benchmarks:
-        gr.Markdown(f"No active benchmarks are currently classified as **{category}**.")
-        return
-    gr.Markdown(
-        f"Results classified as **{category}**. Cross-benchmark ordering uses within-benchmark "
-        "percentiles and reports coverage; raw benchmark scores are not averaged together."
-    )
-    gr.Markdown("#### Trade-offs")
-    with gr.Row():
-        benchmark = gr.Dropdown(choices=benchmarks, value=benchmarks[0], label="Benchmark")
-        metric = gr.Dropdown(
-            choices=["Cost per task", "Total tokens per task", "Total time per task", "Execution error rate"],
-            value="Cost per task",
-            label="X metric",
-        )
-        color_by = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by")
-        labels = gr.Checkbox(value=False, label="Show point labels")
-        background = gr.Dropdown(
-            choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-        )
-    plot = gr.Plot(
-        value=render_category_tradeoff(category, benchmarks[0], "Cost per task", "Model", False, "Dark"),
-        show_label=False,
-        elem_classes="responsive-plot",
-    )
-    controls = [benchmark, metric, color_by, labels, background]
-    for control in controls:
-        control.change(
-            fn=lambda b, m, c, l, bg, cat=category: render_category_tradeoff(cat, b, m, c, l, bg),
-            inputs=controls,
-            outputs=plot,
-        )
-
-    gr.Markdown("#### Matrix")
-    with gr.Row():
-        matrix_metric = gr.Dropdown(
-            choices=[
-                "Score", "Within-benchmark percentile", "Within-benchmark rank",
-                "Total tokens", "Cost", "Total time", "Agent time", "Execution error rate", "Coverage"
-            ],
-            value="Within-benchmark percentile",
-            label="Metric",
-        )
-        matrix_values = gr.Checkbox(value=True, label="Show cell values")
-        matrix_background = gr.Dropdown(
-            choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-        )
-    matrix_plot = gr.Plot(
-        value=render_category_matrix(category, "Within-benchmark percentile", True, "Dark"),
-        show_label=False,
-        elem_classes="responsive-plot",
-    )
-    matrix_controls = [matrix_metric, matrix_values, matrix_background]
-    for control in matrix_controls:
-        control.change(
-            fn=lambda m, v, bg, cat=category: render_category_matrix(cat, m, v, bg),
-            inputs=matrix_controls,
-            outputs=matrix_plot,
-        )
-
-    gr.Markdown("#### Category ranking data")
-    gr.Dataframe(
-        value=category_leaderboard(category), interactive=False, show_label=False, max_height=TABLE_MAX_HEIGHT_PX
-    )
-
-
-demo = gr.Blocks(theme="citrus", head=FORCE_DARK_MODE_HEAD)
+demo = gr.Blocks()
 with demo:
-    # Overview is deliberately always visible above the page navigation.
-    gr.HTML(build_header_html(BENCHMARK_RUN_DF))
-
-    with gr.Tabs():
-        with gr.Tab("Rankings"):
-            gr.Markdown(
-                "### Rankings\n"
-                "Paired comparisons are shown first, followed by benchmark-specific metric rankings."
-            )
-            gr.Markdown("### Paired Comparisons")
-            gr.Markdown(
-                "Rankings computed from head-to-head benchmark results using "
-                "a Bradley–Terry paired-comparison model. "
-                "Handles missing data and inconsistent orderings."
-            )
-
-            rank_csv = pd.read_csv("results.csv")
-            rank_csv = rank_csv.dropna(subset=["metrics.score"])
-            rank_csv = rank_csv.loc[~rank_csv["benchmark.name"].isin(EXCLUDED_BENCHMARKS)]
-            rank_model_choices = sorted(rank_csv["model.name"].unique().tolist())
-            rank_harness_choices = sorted(rank_csv["harness.name"].unique().tolist())
-            rank_benchmark_choices = sorted(rank_csv["benchmark.name"].unique().tolist())
-
-            with gr.Row():
-                rank_by = gr.Dropdown(
-                    choices=list(RANK_BY_OPTIONS.keys()),
-                    value="Benchmark Score",
-                    label="Rank by",
-                )
-                rank_oss_models = gr.Checkbox(value=False, label="Open models only")
-                rank_oss_harnesses = gr.Checkbox(value=False, label="Open harnesses only")
-
-            with gr.Row():
-                rank_benchmark_filter = gr.CheckboxGroup(
-                    choices=rank_benchmark_choices,
-                    value=rank_benchmark_choices,
-                    label="Benchmarks",
-                )
-
-            with gr.Row():
-                rank_model_filter = gr.CheckboxGroup(
-                    choices=rank_model_choices,
-                    value=rank_model_choices,
-                    label="Models",
-                )
-
-            with gr.Row():
-                rank_harness_filter = gr.CheckboxGroup(
-                    choices=rank_harness_choices,
-                    value=rank_harness_choices,
-                    label="Harnesses",
-                )
-
-            harness_df_init, model_df_init, pair_df_init = load_and_rank("results.csv")
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-            gr.Markdown("#### Harness ranking")
-            harness_table = gr.Dataframe(value=harness_df_init, interactive=False)
-            gr.Markdown("#### Model ranking")
-            model_table = gr.Dataframe(value=model_df_init, interactive=False)
-            gr.Markdown("#### Model + harness ranking")
-            pair_table = gr.Dataframe(value=pair_df_init, interactive=False)
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Coding Agent Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            gr.Markdown("\* `internal` refers to internal benchmarks performed by the model provider where the harness/environment were not made public")
 
-            def update_rankings(rank_by_val, oss_models, oss_harnesses, benchmarks, models, harnesses):
-                return load_and_rank(
-                    "results.csv",
-                    open_models_only=oss_models,
-                    open_harnesses_only=oss_harnesses,
-                    benchmarks=benchmarks,
-                    models=models,
-                    harnesses=harnesses,
-                    rank_by=rank_by_val,
-                )
-
-            ranking_inputs = [
-                rank_by,
-                rank_oss_models,
-                rank_oss_harnesses,
-                rank_benchmark_filter,
-                rank_model_filter,
-                rank_harness_filter,
-            ]
-            for control in ranking_inputs:
-                control.change(
-                    fn=update_rankings,
-                    inputs=ranking_inputs,
-                    outputs=[harness_table, model_table, pair_table],
-                )
-
-            gr.Markdown(
-                "### Metric rankings\n"
-                "Use the shared display controls below, then choose a benchmark for each metric. "
-                "Tables are capped to a scrollable height so the visualizations stay primary."
-            )
-            with gr.Row():
-                ranking_color = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by")
-                ranking_sort = gr.Dropdown(
-                    choices=[
-                        "Best first",
-                        "Best last",
-                        "Alphabetical (A–Z)",
-                        "Alphabetical (Z–A)",
-                    ],
-                    value="Best first",
-                    label="Chart order",
-                )
-                ranking_palette = gr.Dropdown(
-                    choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette"
-                )
-                ranking_background = gr.Dropdown(
-                    choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-                )
-
-            ranking_sections = [("Score", "Score", BENCHMARK_NAMES, DEFAULT_BENCHMARK)]
-            ranking_sections += [
-                ("Token usage", "Total tokens", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Cost", "Cost", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Response time", "Response time", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Reliability", "Reliability", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Tokens per successful task", "Tokens per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Cost per successful task", "Cost per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-                ("Time per successful task", "Time per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK),
-            ]
-            shared_ranking_controls = [ranking_color, ranking_palette, ranking_background, ranking_sort]
-            for section_label, metric_name, benchmark_choices, default_benchmark in ranking_sections:
-                spec = RANKING_METRICS[metric_name]
-                gr.Markdown(f"#### {section_label}\n{spec.label}. {'Higher' if spec.higher_is_better else 'Lower'} is better.")
-                benchmark = gr.Dropdown(
-                    choices=benchmark_choices, value=default_benchmark, label="Benchmark"
-                )
-                initial = render_ranking(
-                    metric_name,
-                    default_benchmark,
-                    "Model",
-                    DEFAULT_COLOR_PALETTE,
-                    DEFAULT_PLOT_BACKGROUND,
-                    "Best first",
-                )
-                plot = gr.Plot(value=initial[0], show_label=False, elem_classes="responsive-plot")
-                controls = [benchmark, *shared_ranking_controls]
-                for control in controls:
-                    control.change(
-                        fn=lambda b, c, p, bg, so, m=metric_name: render_ranking(m, b, c, p, bg, so)[0],
-                        inputs=controls,
-                        outputs=plot,
-                    )
-
-            gr.Markdown(
-                "### Ranking data\n"
-                "One table for the page, placed after all charts. It includes the score, resource, timing, "
-                "reliability, and per-success values for the selected benchmark."
-            )
-            with gr.Row():
-                ranking_table_benchmark = gr.Dropdown(
-                    choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Table benchmark"
-                )
-                ranking_table_metric = gr.Dropdown(
-                    choices=list(PAGE_TABLE_SORT_COLUMNS), value="Score", label="Sort table by"
-                )
-                ranking_table_order = gr.Dropdown(
-                    choices=[
-                        "Best first",
-                        "Best last",
-                        "Alphabetical (A–Z)",
-                        "Alphabetical (Z–A)",
-                    ],
-                    value="Best first",
-                    label="Table order",
-                )
-            ranking_page_table = gr.Dataframe(
-                value=render_page_table(DEFAULT_BENCHMARK, "Score", "Best first"),
-                interactive=False,
-                show_label=False,
-                max_height=TABLE_MAX_HEIGHT_PX,
-            )
-            ranking_table_controls = [ranking_table_benchmark, ranking_table_metric, ranking_table_order]
-            for control in ranking_table_controls:
-                control.change(
-                    fn=render_page_table,
-                    inputs=ranking_table_controls,
-                    outputs=ranking_page_table,
-                )
-
-        with gr.Tab("Trade-offs"):
-            gr.Markdown(
-                "### Trade-offs\n"
-                "Efficiency and metric-pair views are displayed together. Pareto frontiers support both maximize and "
-                "minimize directions, so a checked frontier is shown whenever valid comparable points exist."
-            )
-            gr.Markdown("#### Efficiency")
-            with gr.Row():
-                efficiency_benchmark = gr.Dropdown(
-                    choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Benchmark"
-                )
-                efficiency_metric = gr.Dropdown(
-                    choices=list(EFFICIENCY_RESOURCE_METRICS), value="Total tokens", label="Resource metric"
-                )
-                efficiency_color_by = gr.Radio(
-                    choices=EFFICIENCY_COLOR_BY_CHOICES, value="Model", label="Color by"
-                )
-                efficiency_scale = gr.Radio(choices=["Log", "Linear"], value="Log", label="X-axis scale")
-            with gr.Row():
-                efficiency_pareto = gr.Checkbox(value=True, label="Show Pareto frontier")
-                efficiency_labels = gr.Checkbox(value=False, label="Show point labels")
-                efficiency_palette = gr.Dropdown(
-                    choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette"
-                )
-                efficiency_background = gr.Dropdown(
-                    choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-                )
-            initial_efficiency = render_efficiency(
-                DEFAULT_BENCHMARK, "Total tokens", "Model", "Log", True, False,
-                DEFAULT_COLOR_PALETTE, DEFAULT_PLOT_BACKGROUND,
-            )
-            efficiency_note = gr.Markdown(initial_efficiency[1])
-            efficiency_plot = gr.Plot(
-                value=initial_efficiency[0], show_label=False, elem_classes="responsive-plot"
-            )
-            efficiency_controls = [
-                efficiency_benchmark, efficiency_metric, efficiency_color_by, efficiency_scale,
-                efficiency_pareto, efficiency_labels, efficiency_palette, efficiency_background,
-            ]
-            for control in efficiency_controls:
-                control.change(
-                    fn=render_efficiency,
-                    inputs=efficiency_controls,
-                    outputs=[efficiency_plot, efficiency_note],
-                )
-
-            gr.Markdown("#### Metric pairs")
-            with gr.Row():
-                tradeoff_pair = gr.Dropdown(
-                    choices=list(TRADEOFF_PAIRS), value="Score vs cost", label="Trade-off"
-                )
-                tradeoff_benchmark = gr.Dropdown(
-                    choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Benchmark"
-                )
-                tradeoff_color = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by")
-                tradeoff_scale = gr.Radio(choices=["Linear", "Log"], value="Linear", label="X-axis scale")
-            with gr.Row():
-                tradeoff_labels = gr.Checkbox(value=False, label="Show point labels")
-                tradeoff_pareto = gr.Checkbox(value=True, label="Show Pareto frontier")
-                tradeoff_palette = gr.Dropdown(
-                    choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette"
-                )
-                tradeoff_background = gr.Dropdown(
-                    choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-                )
-            initial_tradeoff = render_tradeoff(
-                "Score vs cost", DEFAULT_BENCHMARK, "Model", False, "Linear", True,
-                DEFAULT_COLOR_PALETTE, DEFAULT_PLOT_BACKGROUND,
-            )
-            tradeoff_note = gr.Markdown(initial_tradeoff[1])
-            tradeoff_plot = gr.Plot(
-                value=initial_tradeoff[0], show_label=False, elem_classes="responsive-plot"
-            )
-            tradeoff_controls = [
-                tradeoff_pair, tradeoff_benchmark, tradeoff_color, tradeoff_labels,
-                tradeoff_scale, tradeoff_pareto, tradeoff_palette, tradeoff_background,
-            ]
-            for control in tradeoff_controls:
-                control.change(
-                    fn=render_tradeoff,
-                    inputs=tradeoff_controls,
-                    outputs=[tradeoff_plot, tradeoff_note],
-                )
-
-            gr.Markdown(
-                "### Trade-off data\n"
-                "A single table for this page appears after both charts and includes every metric used by the trade-off views."
-            )
-            with gr.Row():
-                tradeoff_table_benchmark = gr.Dropdown(
-                    choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Table benchmark"
-                )
-                tradeoff_table_metric = gr.Dropdown(
-                    choices=list(PAGE_TABLE_SORT_COLUMNS), value="Score", label="Sort table by"
-                )
-                tradeoff_table_order = gr.Dropdown(
-                    choices=[
-                        "Largest value first",
-                        "Lowest value first",
-                        "Alphabetical (A–Z)",
-                        "Alphabetical (Z–A)",
-                    ],
-                    value="Largest value first",
-                    label="Table order",
-                )
-            tradeoff_page_table = gr.Dataframe(
-                value=render_page_table(DEFAULT_BENCHMARK, "Score", "Largest value first"),
-                interactive=False,
-                show_label=False,
-                max_height=TABLE_MAX_HEIGHT_PX,
-            )
-            tradeoff_table_controls = [tradeoff_table_benchmark, tradeoff_table_metric, tradeoff_table_order]
-            for control in tradeoff_table_controls:
-                control.change(
-                    fn=render_page_table,
-                    inputs=tradeoff_table_controls,
-                    outputs=tradeoff_page_table,
-                )
-
-        with gr.Tab("Matrices"):
-            gr.Markdown(
-                "### Model × benchmark matrices\n"
-                "The within-benchmark percentile controls the color scale, while cell labels show the actual benchmark "
-                "score. Missing cells stay missing."
-            )
-            with gr.Row():
-                matrix_metric = gr.Dropdown(
-                    choices=[*MATRIX_METRICS.keys(), "Coverage"],
-                    value="Within-benchmark percentile",
-                    label="Metric",
-                )
-                matrix_category = gr.Dropdown(
-                    choices=["All", "Coding", "Generalist"], value="All", label="Benchmark category"
-                )
-                matrix_sort = gr.Dropdown(
-                    choices=[
-                        "Normalized performance (high to low)",
-                        "Normalized performance (low to high)",
-                        "Coverage (high to low)",
-                        "Coverage (low to high)",
-                        "Alphabetical",
-                        "Alphabetical (Z–A)",
-                    ],
-                    value="Normalized performance (high to low)",
-                    label="Sort rows",
-                )
-                matrix_background = gr.Dropdown(
-                    choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background"
-                )
-            with gr.Row():
-                matrix_incomplete = gr.Checkbox(value=True, label="Include incomplete rows")
-                matrix_values = gr.Checkbox(value=True, label="Show cell values")
-                matrix_reverse = gr.Checkbox(value=False, label="Reverse color scale")
-            initial_matrix = render_matrix(
-                "Within-benchmark percentile", "All", True,
-                "Normalized performance (high to low)", True, False, "Dark"
-            )
-            matrix_plot = gr.Plot(value=initial_matrix, show_label=False, elem_classes="responsive-plot")
-            matrix_controls = [
-                matrix_metric, matrix_category, matrix_incomplete, matrix_sort,
-                matrix_values, matrix_reverse, matrix_background,
-            ]
-            for control in matrix_controls:
-                control.change(fn=render_matrix, inputs=matrix_controls, outputs=matrix_plot)
-
-
-        with gr.Tab("Coding"):
-            gr.Markdown(
-                "### Coding\n"
-                "Coding benchmarks are defined centrally in the benchmark catalog. The aggregate leaderboard uses "
-                "within-benchmark percentiles and displays benchmark coverage."
-            )
-            add_category_section("Coding", CODING_BENCHMARKS)
-
-        with gr.Tab("Generalist"):
-            gr.Markdown(
-                "### Terminal & Generalist\n"
-                "This category reflects active terminal/generalist benchmarks present in the repository."
-            )
-            add_category_section("Generalist", GENERALIST_BENCHMARKS)
-
-        with gr.Tab("Results Explorer"):
-            gr.Markdown("### Benchmark runs")
-            benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF)
-            gr.Markdown("### Methodology")
-            gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-            gr.Markdown(
-                "### Analytics methodology notes\n"
-                "- **Normalized ordering:** rank/percentile is calculated independently inside each benchmark, then "
-                "aggregated by model + harness with coverage shown beside it.\n"
-                "- **Execution error rate:** recorded errors divided by recorded task count; unresolved tasks are not "
-                "relabeled as errors.\n"
-                "- **Missing metrics:** omitted from metric-specific comparisons and preserved as missing matrix cells.\n"
-                "- **Pareto frontier:** benchmark-specific and direction-aware for maximize/minimize metric pairs."
-            )
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-            gr.Markdown(HOW_TO_USE_TEXT, elem_classes="markdown-text")
 
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)

requirements.txt

@@ -3,17 +3,14 @@ black
 datasets
 gradio
 gradio[oauth]
-gradio_leaderboard
+gradio_leaderboard==0.0.13
 gradio_client
 huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
-plotly
 python-dateutil
 tqdm
 transformers
 tokenizers>=0.15.0
-sentencepiece
-choix
-scipy
+sentencepiece

results.csv

@@ -1,67 +0,0 @@
-benchmark.name,benchmark.repo,benchmark.num_tasks,benchmark.url,harness.name,harness.skills,harness.is_oss,harness.url,model.name,model.repo,model.is_oss,model.num_params,model.precision,model.url,environment.name,environment.url,metrics.n_tasks,metrics.n_errors,metrics.score,metrics.n_input_tokens,metrics.n_cache_tokens,metrics.n_output_tokens,metrics.n_total_tokens,metrics.agent_time_seconds,metrics.total_time_seconds,metrics.cost_usd,metrics.mean_input_tokens_per_task,metrics.mean_cache_tokens_per_task,metrics.mean_output_tokens_per_task,metrics.mean_tokens_per_task,metrics.mean_cost_usd_per_task,metrics.mean_total_time_seconds_per_task,metrics.mean_agent_time_seconds_per_task
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.6,Opus 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.633,438501505.0,419608736.0,2823496.0,860933737.0,126529.0,243310.0,434.54,1228295.0,1175374.0,7908.0,2411579.0,1.22,681.0,354.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.557,500304168.0,479035103.0,4937782.0,984277053.0,150652.0,269008.0,349.77,1401412.0,1341835.0,13831.0,2757078.0,0.98,753.0,421.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,31.0,0.317,1088634865.0,0.0,6200414.0,1094835279.0,398444.0,575536.0,15.05,3049397.0,0.0,17368.0,3066765.0,0.04,1612.0,1116.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.294,419671770.0,0.0,2277963.0,421949733.0,98909.0,169021.0,3.74,1175551.0,0.0,6380.0,1181932.0,0.01,473.0,277.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,20.0,0.23,419282595.0,0.0,1954817.0,421237412.0,100632.0,168697.0,3.8,1174461.0,0.0,5475.0,1179936.0,0.01,472.0,281.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,14.0,0.224,783670817.0,0.0,5510291.0,789181108.0,144352.0,355122.0,38.03,2195156.0,0.0,15434.0,2210591.0,0.11,994.0,404.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.308,839586046.0,0.0,5706139.0,845292185.0,94762.0,251102.0,37.44,2351781.0,0.0,15983.0,2367765.0,0.1,703.0,265.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,13.0,0.216,770574986.0,0.0,6519128.0,777094114.0,116348.0,360934.0,30.65,2158473.0,0.0,18260.0,2176734.0,0.09,1011.0,325.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,9.0,0.493,885869433.0,0.0,4763916.0,890633349.0,201374.0,401657.0,73.84,2481426.0,0.0,13344.0,2494771.0,0.21,1125.0,564.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,10.0,0.44,410001939.0,0.0,2851571.0,412853510.0,115135.0,315451.0,42.22,1148464.0,0.0,7987.0,1156452.0,0.12,883.0,322.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,1.0,0.468,560672462.0,0.0,3618673.0,564291135.0,114791.0,333385.0,42.09,1570511.0,0.0,10136.0,1580647.0,0.12,933.0,321.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,115.0,6.0,0.0,0.0,0.0,0.0,0.0,29895.0,36503.0,8.3,0.0,0.0,0.0,0.0,0.07,317.0,259.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.0,0.0,0.0,0.0,0.0,8170.0,13296.0,2.27,0.0,0.0,0.0,0.0,0.02,115.0,71.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,fp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,2.0,0.009,0.0,0.0,0.0,0.0,4846.0,9862.0,1.35,0.0,0.0,0.0,0.0,0.01,85.0,42.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,fp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.014,0.0,0.0,0.0,0.0,21575.0,26392.0,5.99,0.0,0.0,0.0,0.0,0.05,229.0,187.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.128,0.0,0.0,0.0,0.0,10812.0,16949.0,6.01,0.0,0.0,0.0,0.0,0.05,147.0,94.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.698,192346997.0,186506482.0,2179945.0,381033424.0,32745.0,39030.0,185.66155285,2003614.0,1942775.0,22707.0,3969098.0,1.93,406.0,341.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.781,187217712.0,187209844.0,1280944.0,375708500.0,30686.0,39352.0,151.4104807499999,1950184.0,1950102.0,13343.0,3913630.0,1.58,409.0,319.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.5,190672390.0,184409111.0,1593112.0,376674613.0,40527.0,49734.0,184.42824125000004,1986170.0,1920928.0,16594.0,3923693.0,1.92,518.0,422.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,16.0,0.417,122366824.0,0.0,1307027.0,123673851.0,138148.0,158812.0,54.82,1274654.0,0.0,13614.0,1288269.0,0.57,1654.0,1439.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,12.0,0.417,100068358.0,0.0,1261237.0,101329595.0,60747.0,77623.0,19.28,1042378.0,0.0,13137.0,1055516.0,0.2,808.0,632.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.469,78678324.0,0.0,1370430.0,80048754.0,51767.0,66845.0,16.43,819565.0,0.0,14275.0,833841.0,0.17,696.0,539.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.604,198924339.0,189578624.0,1560836.0,390063799.0,32914.0,39468.0,188.34296700000002,2072128.0,1974777.0,16258.0,4063164.0,1.96,411.0,342.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.375,347744043.0,0.0,1903023.0,349647066.0,59653.0,67586.0,2.25,3622333.0,0.0,19823.0,3642156.0,0.02,704.0,621.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.333,112041538.0,0.0,781173.0,112822711.0,25086.0,31180.0,0.95,1167099.0,0.0,8137.0,1175236.0,0.01,324.0,261.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,15.0,0.292,125247664.0,0.0,597435.0,125845099.0,75715.0,79741.0,2.86,1304663.0,0.0,6223.0,1310886.0,0.03,830.0,788.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.26,147452235.0,0.0,953241.0,148405476.0,33613.0,55241.0,5.84,1535960.0,0.0,9929.0,1545890.0,0.06,575.0,350.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.385,140362798.0,0.0,1230697.0,141593495.0,23543.0,49289.0,6.54,1462112.0,0.0,12819.0,1474932.0,0.07,513.0,245.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.333,104994429.0,0.0,1095179.0,106089608.0,15004.0,55176.0,4.17,1093691.0,0.0,11408.0,1105100.0,0.04,574.0,156.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.432,406026150.0,0.0,2839236.0,408865386.0,100355.0,115220.0,26.44,4229439.0,0.0,29575.0,4259014.0,0.28,1200.0,1045.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.323,686392098.0,0.0,2915876.0,689307974.0,86130.0,99705.0,19.45,7149917.0,0.0,30373.0,7180291.0,0.2,1038.0,897.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,14.0,0.375,1051387065.0,0.0,3651367.0,1055038432.0,113768.0,127326.0,25.69,10951948.0,0.0,38035.0,10989983.0,0.27,1326.0,1185.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,8.0,0.521,386929017.0,0.0,1847087.0,388776104.0,118693.0,137736.0,43.52,4030510.0,0.0,19240.0,4049751.0,0.45,1434.0,1236.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.573,344853232.0,0.0,1661197.0,346514429.0,118356.0,196235.0,43.4,3592221.0,0.0,17304.0,3609525.0,0.45,2044.0,1232.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.49,395189021.0,0.0,2206097.0,397395118.0,80305.0,91995.0,29.45,4116552.0,0.0,22980.0,4139532.0,0.31,958.0,836.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,6.0,0.458,367897697.0,0.0,1694885.0,369592582.0,39024.0,46758.0,9.64,3832267.0,0.0,17655.0,3849922.0,0.1,487.0,406.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.375,207164679.0,0.0,1598703.0,208763382.0,49450.0,57287.0,12.21,2157965.0,0.0,16653.0,2174618.0,0.13,596.0,515.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,5.0,0.406,0.0,0.0,0.0,0.0,38085.0,50779.0,9.4,0.0,0.0,0.0,0.0,0.1,528.0,396.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.479,742491363.0,0.0,2387609.0,744878972.0,54543.0,62422.0,13.47,7734285.0,0.0,24870.0,7759155.0,0.14,650.0,568.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.438,159198517.0,0.0,972133.0,160170650.0,33610.0,38272.0,9.34,1658317.0,0.0,10126.0,1668444.0,0.1,398.0,350.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.868,402225158.0,388781382.0,4588859.0,795595399.0,104467.0,159566.0,394.7823929,804450.0,777562.0,9177.0,1591190.0,0.79,319.0,208.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.834,347751942.0,347725246.0,3640204.0,699117392.0,114633.0,171540.0,319.53304049999974,695503.0,695450.0,7280.0,1398234.0,0.64,343.0,229.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,,,0.796,,,,,,,,,,,,,,
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,35.0,0.612,651306166.0,0.0,4591859.0,655898025.0,524857.0,669355.0,104.14,1302612.0,0.0,9183.0,1311796.0,0.21,1338.0,1049.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.606,525563275.0,0.0,4183675.0,529746950.0,166092.0,444527.0,52.73,1051126.0,0.0,8367.0,1059493.0,0.11,889.0,332.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.574,377479979.0,0.0,4177117.0,381657096.0,122274.0,385336.0,38.82,754959.0,0.0,8354.0,763314.0,0.08,770.0,244.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.798,415795756.0,386352384.0,3431224.0,805579364.0,92588.0,141637.0,443.32977200000016,831591.0,772704.0,6862.0,1611158.0,0.89,283.0,185.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,13.0,0.376,833651681.0,0.0,5656173.0,839307854.0,150981.0,200547.0,5.7,1667303.0,0.0,11312.0,1678615.0,0.01,401.0,301.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,33.0,0.326,202856940.0,0.0,1444371.0,204301311.0,35190.0,322482.0,3.32,405713.0,0.0,2888.0,408602.0,0.01,644.0,70.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,47.0,0.234,90107247.0,0.0,905031.0,91012278.0,146747.0,176174.0,13.86,180214.0,0.0,1810.0,182024.0,0.03,352.0,293.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,53.0,0.118,309857096.0,0.0,2416878.0,312273974.0,54338.0,555599.0,9.43,619714.0,0.0,4833.0,624547.0,0.02,1111.0,108.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.318,373713878.0,0.0,4051687.0,377765565.0,49103.0,543025.0,13.64,747427.0,0.0,8103.0,755531.0,0.03,1086.0,98.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.232,195246618.0,0.0,2726988.0,197973606.0,29236.0,458591.0,8.12,390493.0,0.0,5453.0,395947.0,0.02,917.0,58.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.472,882330271.0,0.0,6689285.0,889019556.0,174288.0,410828.0,45.91,1764660.0,0.0,13378.0,1778039.0,0.09,821.0,348.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.41,724984042.0,0.0,5941303.0,730925345.0,93088.0,518565.0,21.02,1449968.0,0.0,11882.0,1461850.0,0.04,1037.0,186.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.498,996839122.0,0.0,7196671.0,1004035793.0,144084.0,446845.0,32.53,1993678.0,0.0,14393.0,2008071.0,0.07,893.0,288.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.694,1005844088.0,0.0,6255800.0,1012099888.0,228858.0,411348.0,83.91,2011688.0,0.0,12511.0,2024199.0,0.17,822.0,457.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,8.0,0.642,395291929.0,0.0,3296668.0,398588597.0,92700.0,374913.0,33.99,790583.0,0.0,6593.0,797177.0,0.07,749.0,185.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,7.0,0.694,544070591.0,0.0,4965652.0,549036243.0,130591.0,408706.0,47.88,1088141.0,0.0,9931.0,1098072.0,0.1,817.0,261.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.632,1106618897.0,0.0,5733245.0,1112352142.0,122808.0,171897.0,34.11,2213237.0,0.0,11466.0,2224704.0,0.07,343.0,245.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.588,0.0,0.0,0.0,0.0,120399.0,200354.0,33.44,0.0,0.0,0.0,0.0,0.07,400.0,240.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.548,469806650.0,0.0,4937761.0,474744411.0,120473.0,185168.0,29.75,939613.0,0.0,9875.0,949488.0,0.06,370.0,240.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.65,791183735.0,0.0,6333798.0,797517533.0,154531.0,218988.0,38.16,1582367.0,0.0,12667.0,1595035.0,0.08,437.0,309.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.638,609589099.0,0.0,3964252.0,613553351.0,132273.0,178984.0,36.74,1219178.0,0.0,7928.0,1227106.0,0.07,357.0,264.0
-Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.303,47607780.0,0.0,1657188.0,49264968.0,46467.0,58507.0,11.47,534918.0,0.0,18620.0,553538.0,0.13,657.0,522.0
-Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.36,82108716.0,0.0,2056390.0,84165106.0,44991.0,64760.0,11.11,922569.0,0.0,23105.0,945675.0,0.12,727.0,505.0

results/qwen3-6-35b-internal.json

@@ -0,0 +1,26 @@
+{
+    "dataset": {
+        "name": "swe-bench-verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500
+    },
+    "harness": {
+        "name": "internal",
+        "skills": []
+    },
+    "model": {
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "Qwen/Qwen3.6-35B-A3B",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "bf16"
+    },
+    "environment": {
+        "name": "internal"
+    },
+    "metrics": {
+        "score": 0.734,
+        "time": null,
+        "costUSD": null
+    }
+}

results/{swe-bench-verified-claude-sonnet-4-6-claude-code.json → qwen3-6-35b-nvfp4-claude-code.json}

@@ -1,23 +1,19 @@
 {
-    "benchmark": {
-        "name": "SWE-Bench Verified",
+    "dataset": {
+        "name": "swe-bench-verified",
         "repo": "SWE-bench/SWE-bench_Verified",
-        "num_tasks": 500,
-        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+        "num_tasks": 500
     },
     "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
+        "name": "claude-code",
+        "skills": []
     },
     "model": {
-        "name": "Sonnet 4.6",
-        "repo": "Sonnet 4.6",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
+        "name": "Qwen3.6-35B-A3B",
+        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
+        "is_oss": true,
+        "num_params": 35,
+        "precision": "nvfp4"
     },
     "environment": {
         "name": "harbor",
@@ -33,10 +29,11 @@
             "task_names": null,
             "exclude_task_names": null,
             "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
+        }
     },
     "metrics": {
-        "score": 0.796
+        "score": 0.632,
+        "time": 21600,
+        "costUSD": 48.00
     }
-}
+}

results/rh-swe-bench-claude-opus-4-6-claude-code.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Opus 4.6",
-        "repo": "Opus 4.6",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-opus-4-6"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "rh-swe-bench",
-            "version": null,
-            "ref": null,
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 0,
-        "score": 0.633,
-        "n_input_tokens": 438501505,
-        "n_cache_tokens": 419608736,
-        "n_output_tokens": 2823496,
-        "n_total_tokens": 860933737,
-        "agent_time_seconds": 126529,
-        "total_time_seconds": 243310,
-        "cost_usd": 434.54,
-        "mean_input_tokens_per_task": 1228295,
-        "mean_cache_tokens_per_task": 1175374,
-        "mean_output_tokens_per_task": 7908,
-        "mean_tokens_per_task": 2411579,
-        "mean_cost_usd_per_task": 1.22,
-        "mean_total_time_seconds_per_task": 681,
-        "mean_agent_time_seconds_per_task": 354
-    }
-}

results/rh-swe-bench-claude-sonnet-4-6-claude-code.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Sonnet 4.6",
-        "repo": "Sonnet 4.6",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "rh-swe-bench",
-            "version": null,
-            "ref": null,
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 0,
-        "score": 0.557,
-        "n_input_tokens": 500304168,
-        "n_cache_tokens": 479035103,
-        "n_output_tokens": 4937782,
-        "n_total_tokens": 984277053,
-        "agent_time_seconds": 150652,
-        "total_time_seconds": 269008,
-        "cost_usd": 349.77,
-        "mean_input_tokens_per_task": 1401412,
-        "mean_cache_tokens_per_task": 1341835,
-        "mean_output_tokens_per_task": 13831,
-        "mean_tokens_per_task": 2757078,
-        "mean_cost_usd_per_task": 0.98,
-        "mean_total_time_seconds_per_task": 753,
-        "mean_agent_time_seconds_per_task": 421
-    }
-}

results/rh-swe-bench-gpt-oss-120b-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://docs.anthropic.com/en/docs/claude-code"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 31,
-        "score": 0.317,
-        "n_input_tokens": 1088634865,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 6200414,
-        "n_total_tokens": 1094835279,
-        "agent_time_seconds": 398444,
-        "total_time_seconds": 575536,
-        "cost_usd": 15.05,
-        "mean_input_tokens_per_task": 3049397,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 17368,
-        "mean_tokens_per_task": 3066765,
-        "mean_cost_usd_per_task": 0.04,
-        "mean_total_time_seconds_per_task": 1612,
-        "mean_agent_time_seconds_per_task": 1116
-    }
-}

results/rh-swe-bench-gpt-oss-120b-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/opencode-ai/opencode"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 7,
-        "score": 0.294,
-        "n_input_tokens": 419671770,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2277963,
-        "n_total_tokens": 421949733,
-        "agent_time_seconds": 98909,
-        "total_time_seconds": 169021,
-        "cost_usd": 3.74,
-        "mean_input_tokens_per_task": 1175551,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 6380,
-        "mean_tokens_per_task": 1181932,
-        "mean_cost_usd_per_task": 0.01,
-        "mean_total_time_seconds_per_task": 473,
-        "mean_agent_time_seconds_per_task": 277
-    }
-}

results/rh-swe-bench-gpt-oss-120b-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 20,
-        "score": 0.23,
-        "n_input_tokens": 419282595,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1954817,
-        "n_total_tokens": 421237412,
-        "agent_time_seconds": 100632,
-        "total_time_seconds": 168697,
-        "cost_usd": 3.8,
-        "mean_input_tokens_per_task": 1174461,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 5475,
-        "mean_tokens_per_task": 1179936,
-        "mean_cost_usd_per_task": 0.01,
-        "mean_total_time_seconds_per_task": 472,
-        "mean_agent_time_seconds_per_task": 281
-    }
-}

results/rh-swe-bench-nemotron-120b-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 14,
-        "score": 0.224,
-        "n_input_tokens": 783670817,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 5510291,
-        "n_total_tokens": 789181108,
-        "agent_time_seconds": 144352,
-        "total_time_seconds": 355122,
-        "cost_usd": 38.03,
-        "mean_input_tokens_per_task": 2195156,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 15434,
-        "mean_tokens_per_task": 2210591,
-        "mean_cost_usd_per_task": 0.11,
-        "mean_total_time_seconds_per_task": 994,
-        "mean_agent_time_seconds_per_task": 404
-    }
-}

results/rh-swe-bench-nemotron-120b-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 7,
-        "score": 0.308,
-        "n_input_tokens": 839586046,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 5706139,
-        "n_total_tokens": 845292185,
-        "agent_time_seconds": 94762,
-        "total_time_seconds": 251102,
-        "cost_usd": 37.44,
-        "mean_input_tokens_per_task": 2351781,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 15983,
-        "mean_tokens_per_task": 2367765,
-        "mean_cost_usd_per_task": 0.1,
-        "mean_total_time_seconds_per_task": 703,
-        "mean_agent_time_seconds_per_task": 265
-    }
-}

results/rh-swe-bench-nemotron-120b-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende10/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 13,
-        "score": 0.216,
-        "n_input_tokens": 770574986,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 6519128,
-        "n_total_tokens": 777094114,
-        "agent_time_seconds": 116348,
-        "total_time_seconds": 360934,
-        "cost_usd": 30.65,
-        "mean_input_tokens_per_task": 2158473,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 18260,
-        "mean_tokens_per_task": 2176734,
-        "mean_cost_usd_per_task": 0.09,
-        "mean_total_time_seconds_per_task": 1011,
-        "mean_agent_time_seconds_per_task": 325
-    }
-}

results/rh-swe-bench-qwen3-6-27b-fp8-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 9,
-        "score": 0.493,
-        "n_input_tokens": 885869433,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 4763916,
-        "n_total_tokens": 890633349,
-        "agent_time_seconds": 201374,
-        "total_time_seconds": 401657,
-        "cost_usd": 73.84,
-        "mean_input_tokens_per_task": 2481426,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 13344,
-        "mean_tokens_per_task": 2494771,
-        "mean_cost_usd_per_task": 0.21,
-        "mean_total_time_seconds_per_task": 1125,
-        "mean_agent_time_seconds_per_task": 564
-    }
-}

results/rh-swe-bench-qwen3-6-27b-fp8-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/opencode-ai/opencode"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 10,
-        "score": 0.44,
-        "n_input_tokens": 410001939,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2851571,
-        "n_total_tokens": 412853510,
-        "agent_time_seconds": 115135,
-        "total_time_seconds": 315451,
-        "cost_usd": 42.22,
-        "mean_input_tokens_per_task": 1148464,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 7987,
-        "mean_tokens_per_task": 1156452,
-        "mean_cost_usd_per_task": 0.12,
-        "mean_total_time_seconds_per_task": 883,
-        "mean_agent_time_seconds_per_task": 322
-    }
-}

results/rh-swe-bench-qwen3-6-27b-fp8-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "RH SWE-Bench",
-        "repo": "rounakbende/rh-swe-bench",
-        "num_tasks": 357,
-        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/plandex-ai/plandex"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "rh-swe-bench"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 357,
-        "n_errors": 1,
-        "score": 0.468,
-        "n_input_tokens": 560672462,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 3618673,
-        "n_total_tokens": 564291135,
-        "agent_time_seconds": 114791,
-        "total_time_seconds": 333385,
-        "cost_usd": 42.09,
-        "mean_input_tokens_per_task": 1570511,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 10136,
-        "mean_tokens_per_task": 1580647,
-        "mean_cost_usd_per_task": 0.12,
-        "mean_total_time_seconds_per_task": 933,
-        "mean_agent_time_seconds_per_task": 321
-    }
-}

results/shellbench-gemma4-31b-fp8-openclaw.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "Shellbench",
-        "repo": "ShellBench/public-tasks",
-        "num_tasks": 115,
-        "url": "https://github.com/ShellBench/public-tasks"
-    },
-    "harness": {
-        "name": "OpenClaw",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/OpenClaw/OpenClaw"
-    },
-    "model": {
-        "name": "Gemma4-31B-FP8",
-        "repo": "RedHatAI/gemma-4-31B-it-FP8-block",
-        "is_oss": true,
-        "num_params": 31,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 115,
-        "n_errors": 6,
-        "score": 0.0,
-        "n_input_tokens": 0,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 0,
-        "n_total_tokens": 0,
-        "agent_time_seconds": 29895,
-        "total_time_seconds": 36503,
-        "cost_usd": 8.3,
-        "mean_input_tokens_per_task": 0,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 0,
-        "mean_tokens_per_task": 0,
-        "mean_cost_usd_per_task": 0.07,
-        "mean_total_time_seconds_per_task": 317,
-        "mean_agent_time_seconds_per_task": 259
-    }
-}

results/shellbench-gpt-oss-120b-openclaw.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "Shellbench",
-        "repo": "ShellBench/public-tasks",
-        "num_tasks": 115,
-        "url": "https://github.com/ShellBench/public-tasks"
-    },
-    "harness": {
-        "name": "OpenClaw",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/OpenClaw/OpenClaw"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 115,
-        "n_errors": 1,
-        "score": 0.0,
-        "n_input_tokens": 0,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 0,
-        "n_total_tokens": 0,
-        "agent_time_seconds": 8170,
-        "total_time_seconds": 13296,
-        "cost_usd": 2.27,
-        "mean_input_tokens_per_task": 0,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 0,
-        "mean_tokens_per_task": 0,
-        "mean_cost_usd_per_task": 0.02,
-        "mean_total_time_seconds_per_task": 115,
-        "mean_agent_time_seconds_per_task": 71
-    }
-}

results/shellbench-mistral4-119b-fp4-openclaw.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "Shellbench",
-        "repo": "ShellBench/public-tasks",
-        "num_tasks": 115,
-        "url": "https://github.com/ShellBench/public-tasks"
-    },
-    "harness": {
-        "name": "OpenClaw",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/OpenClaw/OpenClaw"
-    },
-    "model": {
-        "name": "Mistral-Small-4-119B-2603-NVFP4",
-        "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4",
-        "is_oss": true,
-        "num_params": 119,
-        "precision": "fp4",
-        "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 115,
-        "n_errors": 2,
-        "score": 0.009,
-        "n_input_tokens": 0,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 0,
-        "n_total_tokens": 0,
-        "agent_time_seconds": 4846,
-        "total_time_seconds": 9862,
-        "cost_usd": 1.35,
-        "mean_input_tokens_per_task": 0,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 0,
-        "mean_tokens_per_task": 0,
-        "mean_cost_usd_per_task": 0.01,
-        "mean_total_time_seconds_per_task": 85,
-        "mean_agent_time_seconds_per_task": 42
-    }
-}

results/shellbench-nemotron-120b-openclaw.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "Shellbench",
-        "repo": "ShellBench/public-tasks",
-        "num_tasks": 115,
-        "url": "https://github.com/ShellBench/public-tasks"
-    },
-    "harness": {
-        "name": "OpenClaw",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/OpenClaw/OpenClaw"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "fp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 115,
-        "n_errors": 1,
-        "score": 0.014,
-        "n_input_tokens": 0,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 0,
-        "n_total_tokens": 0,
-        "agent_time_seconds": 21575,
-        "total_time_seconds": 26392,
-        "cost_usd": 5.99,
-        "mean_input_tokens_per_task": 0,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 0,
-        "mean_tokens_per_task": 0,
-        "mean_cost_usd_per_task": 0.05,
-        "mean_total_time_seconds_per_task": 229,
-        "mean_agent_time_seconds_per_task": 187
-    }
-}

results/shellbench-qwen3-6-27b-fp8-openclaw.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "Shellbench",
-        "repo": "ShellBench/public-tasks",
-        "num_tasks": 115,
-        "url": "https://github.com/ShellBench/public-tasks"
-    },
-    "harness": {
-        "name": "OpenClaw",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/OpenClaw/OpenClaw"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 115,
-        "n_errors": 1,
-        "score": 0.128,
-        "n_input_tokens": 0,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 0,
-        "n_total_tokens": 0,
-        "agent_time_seconds": 10812,
-        "total_time_seconds": 16949,
-        "cost_usd": 6.01,
-        "mean_input_tokens_per_task": 0,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 0,
-        "mean_tokens_per_task": 0,
-        "mean_cost_usd_per_task": 0.05,
-        "mean_total_time_seconds_per_task": 147,
-        "mean_agent_time_seconds_per_task": 94
-    }
-}

results/swe-bench-pro--ansible-claude-opus-4-8-claude-code.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Opus 4.8",
-        "repo": "Opus 4.8",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-opus-4-8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.698,
-        "n_input_tokens": 192346997,
-        "n_cache_tokens": 186506482,
-        "n_output_tokens": 2179945,
-        "n_total_tokens": 381033424,
-        "agent_time_seconds": 32745,
-        "total_time_seconds": 39030,
-        "cost_usd": 185.66155285,
-        "mean_input_tokens_per_task": 2003614,
-        "mean_cache_tokens_per_task": 1942775,
-        "mean_output_tokens_per_task": 22707,
-        "mean_tokens_per_task": 3969098,
-        "mean_cost_usd_per_task": 1.93,
-        "mean_total_time_seconds_per_task": 406,
-        "mean_agent_time_seconds_per_task": 341
-    }
-}

results/swe-bench-pro--ansible-claude-opus-4-8-opencode.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Opus 4.8",
-        "repo": "Opus 4.8",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-opus-4-8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.781,
-        "n_input_tokens": 187217712,
-        "n_cache_tokens": 187209844,
-        "n_output_tokens": 1280944,
-        "n_total_tokens": 375708500,
-        "agent_time_seconds": 30686,
-        "total_time_seconds": 39352,
-        "cost_usd": 151.4104807499999,
-        "mean_input_tokens_per_task": 1950184,
-        "mean_cache_tokens_per_task": 1950102,
-        "mean_output_tokens_per_task": 13343,
-        "mean_tokens_per_task": 3913630,
-        "mean_cost_usd_per_task": 1.58,
-        "mean_total_time_seconds_per_task": 409,
-        "mean_agent_time_seconds_per_task": 319
-    }
-}

results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Sonnet 4.6",
-        "repo": "Sonnet 4.6",
-        "is_oss": false,
-        "num_params": 1000,
-        "precision": "bf16",
-        "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 1,
-        "score": 0.5,
-        "n_input_tokens": 190672390,
-        "n_cache_tokens": 184409111,
-        "n_output_tokens": 1593112,
-        "n_total_tokens": 376674613,
-        "agent_time_seconds": 40527,
-        "total_time_seconds": 49734,
-        "cost_usd": 184.42824125000004,
-        "mean_input_tokens_per_task": 1986170,
-        "mean_cache_tokens_per_task": 1920928,
-        "mean_output_tokens_per_task": 16594,
-        "mean_tokens_per_task": 3923693,
-        "mean_cost_usd_per_task": 1.92,
-        "mean_total_time_seconds_per_task": 518,
-        "mean_agent_time_seconds_per_task": 422
-    }
-}

results/swe-bench-pro--ansible-gemma4-31b-fp8-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Gemma4-31B-FP8",
-        "repo": "RedHatAI/gemma-4-31B-it-FP8-block",
-        "is_oss": true,
-        "num_params": 31,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "SWE-bench_Pro"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 16,
-        "score": 0.417,
-        "n_input_tokens": 122366824,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1307027,
-        "n_total_tokens": 123673851,
-        "agent_time_seconds": 138148,
-        "total_time_seconds": 158812,
-        "cost_usd": 54.82,
-        "mean_input_tokens_per_task": 1274654,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 13614,
-        "mean_tokens_per_task": 1288269,
-        "mean_cost_usd_per_task": 0.57,
-        "mean_total_time_seconds_per_task": 1654,
-        "mean_agent_time_seconds_per_task": 1439
-    }
-}

results/swe-bench-pro--ansible-gemma4-31b-fp8-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Gemma4-31B-FP8",
-        "repo": "RedHatAI/gemma-4-31B-it-FP8-block",
-        "is_oss": true,
-        "num_params": 31,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "SWE-bench_Pro"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 12,
-        "score": 0.417,
-        "n_input_tokens": 100068358,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1261237,
-        "n_total_tokens": 101329595,
-        "agent_time_seconds": 60747,
-        "total_time_seconds": 77623,
-        "cost_usd": 19.28,
-        "mean_input_tokens_per_task": 1042378,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 13137,
-        "mean_tokens_per_task": 1055516,
-        "mean_cost_usd_per_task": 0.2,
-        "mean_total_time_seconds_per_task": 808,
-        "mean_agent_time_seconds_per_task": 632
-    }
-}

results/swe-bench-pro--ansible-gemma4-31b-fp8-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Gemma4-31B-FP8",
-        "repo": "RedHatAI/gemma-4-31B-it-FP8-block",
-        "is_oss": true,
-        "num_params": 31,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "SWE-bench_Pro"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 2,
-        "score": 0.469,
-        "n_input_tokens": 78678324,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1370430,
-        "n_total_tokens": 80048754,
-        "agent_time_seconds": 51767,
-        "total_time_seconds": 66845,
-        "cost_usd": 16.43,
-        "mean_input_tokens_per_task": 819565,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 14275,
-        "mean_tokens_per_task": 833841,
-        "mean_cost_usd_per_task": 0.17,
-        "mean_total_time_seconds_per_task": 696,
-        "mean_agent_time_seconds_per_task": 539
-    }
-}

results/swe-bench-pro--ansible-gpt-5-5-codex.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Codex",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/openai/codex"
-    },
-    "model": {
-        "name": "GPT 5.5 - high",
-        "repo": "GPT 5.5 - high",
-        "is_oss": false,
-        "num_params": 9700,
-        "precision": "bf16",
-        "url": "https://openai.com/index/introducing-gpt-5-5"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.604,
-        "n_input_tokens": 198924339,
-        "n_cache_tokens": 189578624,
-        "n_output_tokens": 1560836,
-        "n_total_tokens": 390063799,
-        "agent_time_seconds": 32914,
-        "total_time_seconds": 39468,
-        "cost_usd": 188.34296700000002,
-        "mean_input_tokens_per_task": 2072128,
-        "mean_cache_tokens_per_task": 1974777,
-        "mean_output_tokens_per_task": 16258,
-        "mean_tokens_per_task": 4063164,
-        "mean_cost_usd_per_task": 1.96,
-        "mean_total_time_seconds_per_task": 411,
-        "mean_agent_time_seconds_per_task": 342
-    }
-}

results/swe-bench-pro--ansible-gpt-oss-120b-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://docs.anthropic.com/en/docs/claude-code"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swebench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 2,
-        "score": 0.375,
-        "n_input_tokens": 347744043,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1903023,
-        "n_total_tokens": 349647066,
-        "agent_time_seconds": 59653,
-        "total_time_seconds": 67586,
-        "cost_usd": 2.25,
-        "mean_input_tokens_per_task": 3622333,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 19823,
-        "mean_tokens_per_task": 3642156,
-        "mean_cost_usd_per_task": 0.02,
-        "mean_total_time_seconds_per_task": 704,
-        "mean_agent_time_seconds_per_task": 621
-    }
-}

results/swe-bench-pro--ansible-gpt-oss-120b-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/opencode-ai/opencode"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swebench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 1,
-        "score": 0.333,
-        "n_input_tokens": 112041538,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 781173,
-        "n_total_tokens": 112822711,
-        "agent_time_seconds": 25086,
-        "total_time_seconds": 31180,
-        "cost_usd": 0.95,
-        "mean_input_tokens_per_task": 1167099,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 8137,
-        "mean_tokens_per_task": 1175236,
-        "mean_cost_usd_per_task": 0.01,
-        "mean_total_time_seconds_per_task": 324,
-        "mean_agent_time_seconds_per_task": 261
-    }
-}

results/swe-bench-pro--ansible-gpt-oss-120b-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "GPT-OSS-120B",
-        "repo": "RedHatAI/gpt-oss-120b",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "mxfp4",
-        "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swebench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 15,
-        "score": 0.292,
-        "n_input_tokens": 125247664,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 597435,
-        "n_total_tokens": 125845099,
-        "agent_time_seconds": 75715,
-        "total_time_seconds": 79741,
-        "cost_usd": 2.86,
-        "mean_input_tokens_per_task": 1304663,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 6223,
-        "mean_tokens_per_task": 1310886,
-        "mean_cost_usd_per_task": 0.03,
-        "mean_total_time_seconds_per_task": 830,
-        "mean_agent_time_seconds_per_task": 788
-    }
-}

results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-claude-code.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Mistral-Small-4-119B-2603-NVFP4",
-        "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4",
-        "is_oss": true,
-        "num_params": 119,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 1,
-        "score": 0.26,
-        "n_input_tokens": 147452235,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 953241,
-        "n_total_tokens": 148405476,
-        "agent_time_seconds": 33613,
-        "total_time_seconds": 55241,
-        "cost_usd": 5.84,
-        "mean_input_tokens_per_task": 1535960,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 9929,
-        "mean_tokens_per_task": 1545890,
-        "mean_cost_usd_per_task": 0.06,
-        "mean_total_time_seconds_per_task": 575,
-        "mean_agent_time_seconds_per_task": 350
-    }
-}

results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-opencode.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Mistral-Small-4-119B-2603-NVFP4",
-        "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4",
-        "is_oss": true,
-        "num_params": 119,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.385,
-        "n_input_tokens": 140362798,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1230697,
-        "n_total_tokens": 141593495,
-        "agent_time_seconds": 23543,
-        "total_time_seconds": 49289,
-        "cost_usd": 6.54,
-        "mean_input_tokens_per_task": 1462112,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 12819,
-        "mean_tokens_per_task": 1474932,
-        "mean_cost_usd_per_task": 0.07,
-        "mean_total_time_seconds_per_task": 513,
-        "mean_agent_time_seconds_per_task": 245
-    }
-}

results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-pi.json

@@ -1,58 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Mistral-Small-4-119B-2603-NVFP4",
-        "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4",
-        "is_oss": true,
-        "num_params": 119,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": null,
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.333,
-        "n_input_tokens": 104994429,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1095179,
-        "n_total_tokens": 106089608,
-        "agent_time_seconds": 15004,
-        "total_time_seconds": 55176,
-        "cost_usd": 4.17,
-        "mean_input_tokens_per_task": 1093691,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 11408,
-        "mean_tokens_per_task": 1105100,
-        "mean_cost_usd_per_task": 0.04,
-        "mean_total_time_seconds_per_task": 574,
-        "mean_agent_time_seconds_per_task": 156
-    }
-}

results/swe-bench-pro--ansible-nemotron-120b-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.432,
-        "n_input_tokens": 406026150,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2839236,
-        "n_total_tokens": 408865386,
-        "agent_time_seconds": 100355,
-        "total_time_seconds": 115220,
-        "cost_usd": 26.44,
-        "mean_input_tokens_per_task": 4229439,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 29575,
-        "mean_tokens_per_task": 4259014,
-        "mean_cost_usd_per_task": 0.28,
-        "mean_total_time_seconds_per_task": 1200,
-        "mean_agent_time_seconds_per_task": 1045
-    }
-}

results/swe-bench-pro--ansible-nemotron-120b-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 4,
-        "score": 0.323,
-        "n_input_tokens": 686392098,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2915876,
-        "n_total_tokens": 689307974,
-        "agent_time_seconds": 86130,
-        "total_time_seconds": 99705,
-        "cost_usd": 19.45,
-        "mean_input_tokens_per_task": 7149917,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 30373,
-        "mean_tokens_per_task": 7180291,
-        "mean_cost_usd_per_task": 0.2,
-        "mean_total_time_seconds_per_task": 1038,
-        "mean_agent_time_seconds_per_task": 897
-    }
-}

results/swe-bench-pro--ansible-nemotron-120b-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/earendil-works/pi/tree/main"
-    },
-    "model": {
-        "name": "Nemotron-3-Super-120B-NVFP4",
-        "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
-        "is_oss": true,
-        "num_params": 120,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 14,
-        "score": 0.375,
-        "n_input_tokens": 1051387065,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 3651367,
-        "n_total_tokens": 1055038432,
-        "agent_time_seconds": 113768,
-        "total_time_seconds": 127326,
-        "cost_usd": 25.69,
-        "mean_input_tokens_per_task": 10951948,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 38035,
-        "mean_tokens_per_task": 10989983,
-        "mean_cost_usd_per_task": 0.27,
-        "mean_total_time_seconds_per_task": 1326,
-        "mean_agent_time_seconds_per_task": 1185
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-27b-fp8-claude-code.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 8,
-        "score": 0.521,
-        "n_input_tokens": 386929017,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1847087,
-        "n_total_tokens": 388776104,
-        "agent_time_seconds": 118693,
-        "total_time_seconds": 137736,
-        "cost_usd": 43.52,
-        "mean_input_tokens_per_task": 4030510,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 19240,
-        "mean_tokens_per_task": 4049751,
-        "mean_cost_usd_per_task": 0.45,
-        "mean_total_time_seconds_per_task": 1434,
-        "mean_agent_time_seconds_per_task": 1236
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-27b-fp8-opencode.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/opencode-ai/opencode"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.573,
-        "n_input_tokens": 344853232,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1661197,
-        "n_total_tokens": 346514429,
-        "agent_time_seconds": 118356,
-        "total_time_seconds": 196235,
-        "cost_usd": 43.4,
-        "mean_input_tokens_per_task": 3592221,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 17304,
-        "mean_tokens_per_task": 3609525,
-        "mean_cost_usd_per_task": 0.45,
-        "mean_total_time_seconds_per_task": 2044,
-        "mean_agent_time_seconds_per_task": 1232
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-27b-fp8-pi.json

@@ -1,48 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "scale-ai/swe-bench-pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro"
-    },
-    "harness": {
-        "name": "Pi",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/plandex-ai/plandex"
-    },
-    "model": {
-        "name": "Qwen3.6-27B-FP8",
-        "repo": "RedHatAI/Qwen3.6-27B-FP8",
-        "is_oss": true,
-        "num_params": 27,
-        "precision": "fp8",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "name": "swe-bench-pro-ansible"
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 0,
-        "score": 0.49,
-        "n_input_tokens": 395189021,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 2206097,
-        "n_total_tokens": 397395118,
-        "agent_time_seconds": 80305,
-        "total_time_seconds": 91995,
-        "cost_usd": 29.45,
-        "mean_input_tokens_per_task": 4116552,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 22980,
-        "mean_tokens_per_task": 4139532,
-        "mean_cost_usd_per_task": 0.31,
-        "mean_total_time_seconds_per_task": 958,
-        "mean_agent_time_seconds_per_task": 836
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "Claude Code",
-        "skills": [],
-        "is_oss": false,
-        "url": "https://github.com/anthropics/claude-code"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B-NVFP4",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 6,
-        "score": 0.458,
-        "n_input_tokens": 367897697,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1694885,
-        "n_total_tokens": 369592582,
-        "agent_time_seconds": 39024,
-        "total_time_seconds": 46758,
-        "cost_usd": 9.64,
-        "mean_input_tokens_per_task": 3832267,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 17655,
-        "mean_tokens_per_task": 3849922,
-        "mean_cost_usd_per_task": 0.1,
-        "mean_total_time_seconds_per_task": 487,
-        "mean_agent_time_seconds_per_task": 406
-    }
-}

results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json

@@ -1,60 +0,0 @@
-{
-    "benchmark": {
-        "name": "SWE-Bench Pro -- Ansible",
-        "repo": "ScaleAI/SWE-bench_Pro",
-        "num_tasks": 96,
-        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
-    },
-    "harness": {
-        "name": "OpenCode",
-        "skills": [],
-        "is_oss": true,
-        "url": "https://github.com/anomalyco/opencode"
-    },
-    "model": {
-        "name": "Qwen3.6-35B-A3B-NVFP4",
-        "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
-        "is_oss": true,
-        "num_params": 35,
-        "precision": "nvfp4",
-        "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4"
-    },
-    "environment": {
-        "name": "harbor",
-        "config": {
-            "path": null,
-            "name": "scale-ai/swe-bench-pro",
-            "version": null,
-            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
-            "registry_url": null,
-            "registry_path": null,
-            "overwrite": false,
-            "download_dir": null,
-            "task_names": [
-                "*ansible*"
-            ],
-            "exclude_task_names": null,
-            "n_tasks": null
-        },
-        "url": "https://github.com/harbor-framework/harbor"
-    },
-    "metrics": {
-        "n_tasks": 96,
-        "n_errors": 4,
-        "score": 0.375,
-        "n_input_tokens": 207164679,
-        "n_cache_tokens": 0,
-        "n_output_tokens": 1598703,
-        "n_total_tokens": 208763382,
-        "agent_time_seconds": 49450,
-        "total_time_seconds": 57287,
-        "cost_usd": 12.21,
-        "mean_input_tokens_per_task": 2157965,
-        "mean_cache_tokens_per_task": 0,
-        "mean_output_tokens_per_task": 16653,
-        "mean_tokens_per_task": 2174618,
-        "mean_cost_usd_per_task": 0.13,
-        "mean_total_time_seconds_per_task": 596,
-        "mean_agent_time_seconds_per_task": 515
-    }
-}