diff --git a/.gitattributes b/.gitattributes index 98672030fa021426e67558e20d2a79e439527aba..e2fdd3c78bfef7d609c3b755310e6439737447f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text -*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/sync-to-hf-space-stage.yml b/.github/workflows/sync-to-hf-space-stage.yml deleted file mode 100644 index a08aaa31cd2e6f07e210addacef2e3b77cbef8f3..0000000000000000000000000000000000000000 --- a/.github/workflows/sync-to-hf-space-stage.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Sync stage to HF Space (staging) - -# Mirrors every push to `stage` on GitHub into the HF Space git remote so -# that the staging Space (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage) -# always tracks the stage branch. -# -# Required repository secrets (Settings -> Secrets and variables -> Actions): -# HF_TOKEN Hugging Face access token with write permission to the Space. -# HF_USERNAME Optional fallback username if token introspection fails. - -on: - push: - branches: [stage] - workflow_dispatch: - -concurrency: - group: sync-to-hf-space-stage - cancel-in-progress: false - -jobs: - mirror: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout GitHub stage (full history + LFS) - uses: actions/checkout@v4 - with: - fetch-depth: 0 - lfs: true - - - name: Verify required secrets - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - if [ -z "$HF_TOKEN" ]; then - echo "::error::HF_TOKEN repository secret must be set." - exit 1 - fi - - - name: Ensure HF Space exists - id: hf - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_USERNAME: ${{ secrets.HF_USERNAME }} - run: | - set -euo pipefail - python -m pip install --quiet 'huggingface_hub>=0.24,<2' - python - <<'PY' - import os - - from huggingface_hub import HfApi - - token = os.environ["HF_TOKEN"] - space_id = "taagarwa/coding-agent-leaderboard-stage" - fallback_username = os.environ.get("HF_USERNAME", "").strip() - - api = HfApi(token=token) - username = fallback_username - try: - info = api.whoami(token=token) - username = str(info.get("name") or username).strip() - except Exception as exc: - if not username: - raise RuntimeError("HF_USERNAME fallback is required when token introspection fails") from exc - - api.create_repo( - repo_id=space_id, - repo_type="space", - space_sdk="docker", - token=token, - exist_ok=True, - ) - - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write(f"username={username}\n") - print(f"HF Space ready: {space_id}") - PY - - - name: Push to HF Space remote - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_USERNAME: ${{ steps.hf.outputs.username }} - run: | - set -euo pipefail - HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/taagarwa/coding-agent-leaderboard-stage" - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - echo "Pushing $(git rev-parse --short HEAD) to taagarwa/coding-agent-leaderboard-stage..." - git push --force "${HF_REMOTE}" HEAD:main - echo "Sync complete." - - - name: Summary - if: success() - run: | - echo "### HF Space mirror (staging)" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Pushed \`$(git rev-parse --short HEAD)\` to \`taagarwa/coding-agent-leaderboard-stage\` Space." >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "View the Space: " >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/sync-to-hf-space.yml b/.github/workflows/sync-to-hf-space.yml deleted file mode 100644 index f25c095a21aeae32f80d8c906f8cb7fbb4a35481..0000000000000000000000000000000000000000 --- a/.github/workflows/sync-to-hf-space.yml +++ /dev/null @@ -1,147 +0,0 @@ -name: Release to HF Space (production) - -# Releases the production HF Space -# (https://huggingface.co/spaces/taagarwa/coding-agent-leaderboard) when a -# version bump lands on `main`. -# -# Release flow: -# 1. In your PR, bump the `VERSION` file (e.g. `make bump VERSION=1.2.3`). -# 2. Merge the PR into `main`. -# 3. This workflow runs, creates the `v` git tag on the merge -# commit, and pushes that commit to the HF Space. -# -# Pushes to `main` that do not change `VERSION` do NOT deploy. If the tag for -# the current VERSION already exists (e.g. VERSION was edited without a bump), -# the workflow fails instead of deploying, so an existing release tag is never -# moved. -# -# Required repository secrets (Settings -> Secrets and variables -> Actions): -# HF_TOKEN Hugging Face access token with write permission to the Space. -# Create at https://huggingface.co/settings/tokens -# (token type "Write" is sufficient; no organization scope needed). -# HF_USERNAME Optional fallback username if token introspection fails. -# -# Optional: set HF_SPACE_ID as a repo variable (not secret) to point the -# workflow at a different Space; defaults to "taagarwa/coding-agent-leaderboard". - -on: - push: - branches: [main] - paths: - - VERSION - # Manual dispatch re-deploys an existing release tag on demand from the - # Actions tab (e.g. to recover the Space after a bad manual edit). It never - # creates tags. - workflow_dispatch: - inputs: - tag: - description: "Existing release tag to redeploy (e.g. v1.2.3)" - required: true - type: string - -# Only one release job at a time so we never race ourselves into -# non-fast-forward pushes on the Space remote. -concurrency: - group: sync-to-hf-space - cancel-in-progress: false - -jobs: - release: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write # needed to push the release tag - env: - HF_SPACE_ID: ${{ vars.HF_SPACE_ID || 'taagarwa/coding-agent-leaderboard' }} - steps: - - name: Checkout (full history + LFS) - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }} - fetch-depth: 0 - lfs: true - - - name: Verify required secrets - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - if [ -z "$HF_TOKEN" ]; then - echo "::error::HF_TOKEN repository secret must be set." - echo " Create HF_TOKEN at https://huggingface.co/settings/tokens (type: Write)" - exit 1 - fi - - - name: Resolve release tag - id: tag - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - FILE_VERSION="$(tr -d '[:space:]' < VERSION)" - if [[ ! "$FILE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::VERSION file contains '$FILE_VERSION'; expected MAJOR.MINOR.PATCH." - exit 1 - fi - TAG="v${FILE_VERSION}" - - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - if [ "$INPUT_TAG" != "$TAG" ]; then - echo "::error::Requested tag '$INPUT_TAG' does not match VERSION file at that ref ('$FILE_VERSION')." - exit 1 - fi - echo "Redeploying existing release ${TAG}." - else - if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then - echo "::error::Tag ${TAG} already exists. Bump VERSION to a new number to release; existing release tags are never moved." - exit 1 - fi - fi - - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "Release ${TAG} at $(git rev-parse --short HEAD)." - - - name: Create release tag - if: github.event_name == 'push' - env: - RELEASE_TAG: ${{ steps.tag.outputs.tag }} - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git tag -a "$RELEASE_TAG" -m "Release $RELEASE_TAG" - git push origin "refs/tags/$RELEASE_TAG" - echo "Created and pushed ${RELEASE_TAG}." - - - name: Push to HF Space remote - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_USERNAME: ${{ secrets.HF_USERNAME || github.repository_owner }} - RELEASE_TAG: ${{ steps.tag.outputs.tag }} - run: | - set -euo pipefail - # Authenticate via token in the URL. HF Spaces accept the - # username + token basic-auth format over HTTPS git. - HF_REMOTE="https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_SPACE_ID}" - - echo "Pushing ${RELEASE_TAG} ($(git rev-parse --short HEAD)) to ${HF_SPACE_ID}..." - - # --force is intentional: GitHub is the single source of truth - # for the Space's git history. Anything on the Space side that - # wasn't committed via GitHub is overwritten on the next sync. - # This prevents the drift situation where someone edits files - # in the HF Space UI and creates commits only visible there. - git push --force "${HF_REMOTE}" HEAD:main - - echo "Sync complete." - - - name: Summary - if: success() - env: - RELEASE_TAG: ${{ steps.tag.outputs.tag }} - run: | - echo "### HF Space release" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Deployed \`${RELEASE_TAG}\` (\`$(git rev-parse --short HEAD)\`) to \`${HF_SPACE_ID}\` Space." >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "View the Space: " >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/update-csv.yml b/.github/workflows/update-csv.yml deleted file mode 100644 index 020995296815c73ebbc2ffe7fc39aeb2dc581a78..0000000000000000000000000000000000000000 --- a/.github/workflows/update-csv.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Update results CSV - -on: - push: - branches: [main] - paths: - - "results/*.json" - -jobs: - update-csv: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.PAT_TOKEN }} - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install dependencies - run: pip install pandas==2.2.3 - - - name: Generate CSV - run: python src/results_to_csv.py - - - name: Commit if changed - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add results.csv - git diff --staged --quiet || (git commit -m "Auto-update results.csv" && git push) diff --git a/.gitignore b/.gitignore index 09b25b652d942d44002e4e7fec53510c1325d9bd..e1c647ae54425f512445dd04a6d8cd40585e7b2e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,9 @@ __pycache__/ *ipynb .vscode/ -Backup/ eval-queue/ eval-results/ eval-queue-bk/ eval-results-bk/ logs/ -uv.lock -.venv/ +uv.lock \ No newline at end of file diff --git a/Makefile b/Makefile index 95e69942f9142d6711534045692889d281dc4aa1..b5685772804c8af4235a8504dc6752bfc9ae5d1d 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,4 @@ -.PHONY: style format quality csv bump - -# Bump the VERSION file and commit it. Do this in your PR; when the PR merges -# into main, CI creates the v tag and deploys to the production Space. -# Usage: make bump VERSION=1.2.3 -bump: - @test -n "$(VERSION)" || { echo "usage: make bump VERSION=X.Y.Z"; exit 1; } - @echo "$(VERSION)" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$$' || { echo "VERSION must be MAJOR.MINOR.PATCH"; exit 1; } - @git fetch --tags --quiet origin 2>/dev/null || true - @! git rev-parse -q --verify "refs/tags/v$(VERSION)" >/dev/null || { echo "tag v$(VERSION) already exists"; exit 1; } - @echo "$(VERSION)" > VERSION - git add VERSION - git commit -m "Bump version to $(VERSION)" - @echo "VERSION is now $(VERSION). Merge to main to release v$(VERSION)." +.PHONY: style format style: @@ -24,7 +11,3 @@ quality: python -m black --check --line-length 119 . python -m isort --check-only . ruff check . - - -csv: - python src/results_to_csv.py diff --git a/README.md b/README.md index 196f8f8c928ac00cd82cff2fbfec4fb6c8207229..6020218eb70d45e3b4b5afc3911dc9050c1e4eae 100644 --- a/README.md +++ b/README.md @@ -15,127 +15,26 @@ tags: # Coding Agent Leaderboard -Compare coding-agent models and harnesses across benchmark performance, cost, latency, and token usage. - -## Leaderboard views - -### Efficiency - -The **Efficiency** tab compares benchmark score with resource use for one selected benchmark at a time. Keeping the view benchmark-specific avoids misleading comparisons when model and harness combinations have incomplete benchmark coverage. -The generalized performance-vs-resource chart covers tokens, cost, and agent time in one benchmark-specific view. - -The scatter plot supports three resource metrics on the x-axis: - -- **Total tokens**: mean total token usage per task. -- **Cost per task**: the repository's reported `mean_cost_usd_per_task` value, in USD. -- **Agent time per task**: the repository's reported `mean_agent_time_seconds_per_task` value, in seconds. - -Only positive, available values for the selected resource are plotted. Missing, zero, and negative values are treated as unavailable rather than as perfect efficiency. The UI reports how many runs for the selected benchmark were excluded for the chosen resource metric. - -Scores are stored internally as fractions from 0 to 1 and displayed as percentages. Point labels are optional and coloring can be grouped by **Model** or **Harness**. Linear and logarithmic resource axes are available. - -#### Pareto frontier - -For all three resource metrics, lower resource use and higher score are better. A displayed run is Pareto-efficient when no other valid run on the selected benchmark: - -- uses an equal or lower amount of the selected resource, and -- achieves an equal or higher score, - -with at least one strict improvement. Tied nondominated points are preserved. The dashed line connects the Pareto-efficient observations. - -#### Ranking table - -The ranking table remains benchmark-specific and begins with **Model**, **Harness**, and **Benchmark**. Displayed scores are rounded to one decimal place without reducing the precision used for calculations. - -**Tokens Per Solved Task** remains available in the table as a reference metric even though it is no longer an Efficiency scatter x-axis option. It is defined as `Total Tokens Per Task / Score`, where Score is the fractional value from 0 to 1. It is unavailable when total token data is missing/non-positive or when Score is zero or negative. - -#### Color palettes and themes - -The Efficiency chart uses the shared palette registry and supports Citrus, Okabe-Ito, High contrast, Rainbow, Grayscale, Viridis, Plasma, and Cividis palettes with light and dark chart backgrounds. - ## Adding a New Leaderboard Entry -Create a pull request adding a new entry to the `results/` folder. See [`results/qwen3-6-35b-nvfp4-claude-code.json`](./results/qwen3-6-35b-nvfp4-claude-code.json) for an example result. - -Do not change the result JSON schema for Efficiency analysis. The feature uses resource and performance metrics already present in the existing result model. +Create a PR adding a new entry into the `results/` folder. +Check out `results/qwen3-6-35b-nvfp4-claude-code.json`(./results/qwen3-6-35b-nvfp4-claude-code.json) for an example result. ## Development -1. Install dependencies: - - ```sh - pip install -r requirements.txt - - # or - - uv venv - uv pip install -r requirements.txt - ``` - -2. Run the app: - - ```sh - python app.py - ``` - -3. Run tests: - - ```sh - pytest - ``` - -## Releasing - -Versions are managed with git tags. The `VERSION` file is the single source of truth for the version number (shown in the app header). Each release is tagged `v` on `main` by CI. - -Deployment targets: - -- **Staging** (`taagarwa/coding-agent-leaderboard-stage`): every push to the `stage` branch. -- **Production** (`taagarwa/coding-agent-leaderboard`): only when a commit that changes `VERSION` lands on `main`. Other pushes to `main` do not deploy. - -To cut a release, bump the version in your PR: - -```sh -make bump VERSION=1.2.3 # writes VERSION and commits "Bump version to 1.2.3" -git push # open / update your PR as usual -``` - -When the PR is merged into `main`, the **Release to HF Space (production)** workflow creates the `v1.2.3` tag on the merge commit and pushes it to the Space. If a tag for the current `VERSION` already exists, the workflow fails rather than moving the tag, so bump to a new number for every release. - -To redeploy an existing release (e.g. after a manual edit on the Space), run the workflow from the Actions tab and supply the tag. - -## Manual validation - -After automated checks pass, launch the app from a clean process with `python app.py` and verify the Efficiency view in a real browser. Unit tests and figure-level smoke tests do not replace this browser validation. - -### Efficiency behavior - -- Confirm there is no **All benchmarks** option and that a valid benchmark is selected by default. -- Confirm **Color By** offers only **Model** and **Harness**. -- Confirm the resource selector offers **Total tokens**, **Cost per task**, and **Agent time per task**. -- Switch through all three resource metrics and verify axis labels and hover formatting. -- Verify the Pareto frontier for all three metrics. -- Verify linear and logarithmic scales where valid. -- Verify point labels off and on. -- Confirm the ranking table starts with **Model**, **Harness**, **Benchmark**, shows scores to one decimal place, and retains **Tokens Per Solved Task**. -- Confirm the existing Leaderboard and Benchmark Runs tabs still work without new terminal tracebacks. - -### Responsive layout verification +1. Install dependencies -- Restart the app from a clean launch. -- Open **Efficiency** as the first non-default tab and confirm the plot is not squished. -- Switch away from Efficiency and back several times. -- Resize the browser narrower and wider. -- Confirm the plot resizes correctly without requiring a control change. -- Verify there is no legend overlap or clipping. -- Verify the ranking table below does not force the plot into a narrow column. -- Repeat the checks with point labels off and on, Pareto off and on, each resource metric, and light and dark plot backgrounds. -- Also switch **Leaderboard → Efficiency** and **Benchmark Runs → Efficiency** to confirm hidden-tab initialization does not collapse the chart. + ```sh + pip install -r requirements.txt -### PR completion report + # or -The final PR report should state the files changed, metric definitions, tests and results, data-quality limitations, and whether the app was manually verified. For the responsive-layout fix, it must additionally record: + uv venv + uv pip install -r requirements.txt + ``` -- the root cause of the initial squished-chart issue; -- the exact responsive-layout fix used; and -- whether the behavior was manually verified from a fresh app launch. +2. Run the app + + ```sh + python app.py + ``` diff --git a/VERSION b/VERSION deleted file mode 100644 index 6e8bf73aa550d4c57f6f35830f1bcdc7a4a62f38..0000000000000000000000000000000000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.0 diff --git a/app.py b/app.py index 5b21f329ab48438078efe3dee74e9286393748bc..5d096b4c14a363959d702dd3367199fe60e6555e 100644 --- a/app.py +++ b/app.py @@ -1,1068 +1,70 @@ import os -from pathlib import Path - - -def patch_gradio_leaderboard(): - """Patch gradio_leaderboard JS to fix crash on tab switch with Gradio 5.x.""" - import gradio_leaderboard - - pkg_dir = Path(gradio_leaderboard.__file__).parent - js_file = pkg_dir / "templates" / "component" / "Index-CzS_eGV6.js" - if not js_file.exists(): - return - - src = js_file.read_text() - - patches = [ - # Fix 1 & 2: Guard r[39]/a[39] filter callback (undefined during Svelte outro) - ( - 'r[0].filter(\n /*func*/\n r[39]\n ).map(qd)', - '(r[39] ? r[0].filter(r[39]) : r[0]).map(qd)', - ), - ( - 'a[0].filter(\n /*func*/\n a[39]\n ).map(qd))', - '(a[39] ? a[0].filter(a[39]) : a[0]).map(qd))', - ), - # Fix 3: Lx (Boolean) extracted from Rx (globals) which is undefined in Gradio 5 - ( - '{ Boolean: Lx } = Rx,', - 'Lx = (Rx && Rx.Boolean) || Boolean,', - ), - ] - - patched = False - for old, new in patches: - if old in src: - src = src.replace(old, new) - patched = True - - if patched: - js_file.write_text(src) - - -patch_gradio_leaderboard() import gradio as gr -import pandas as pd +from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns from apscheduler.schedulers.background import BackgroundScheduler -from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from huggingface_hub import HfApi -from src.analytics import ( - MATRIX_METRICS, - RANKING_METRICS, - TRADEOFF_METRICS, - benchmarks_for_category, - coverage_summary, - cross_benchmark_ranking_df, - enrich_analysis_df, - filter_category, - matrix_df, - ranking_df, -) -from src.charts import ( - clean_markdown_link, - create_coverage_matrix_plot, - create_leaderboard_benchmark_plot, - create_matrix_plot, - create_performance_vs_resource_plot, - create_ranking_plot, - create_tradeoff_plot, -) +from src.leaderboard import get_leaderboard_df, DISPLAY_BY_DEFAULT, SEARCH_COLUMNS from src.display.text_blocks import ( - HOW_TO_USE_TEXT, + TITLE, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, + CITATION_BUTTON_LABEL, + CITATION_BUTTON_TEXT, ) -from src.leaderboard import ( - EFFICIENCY_RESOURCE_METRICS, - get_analysis_df, - get_benchmark_names, - get_benchmark_run_df, - get_efficiency_df, -) -from src.rankings import EXCLUDED_BENCHMARKS, RANK_BY_OPTIONS, load_and_rank -from src.version import __version__ REPO_ID = "taagarwa/coding-agent-leaderboard" TOKEN = os.environ.get("HF_TOKEN") API = HfApi(token=TOKEN) -COLOR_BY_CHOICES = ["Model", "Harness"] -EFFICIENCY_COLOR_BY_CHOICES = ["Model", "Harness"] -COLOR_PALETTE_CHOICES = ["Citrus", "Okabe-Ito", "High contrast", "Rainbow"] -DEFAULT_COLOR_PALETTE = "Citrus" -PLOT_BACKGROUND_CHOICES = ["Dark", "White"] -DEFAULT_PLOT_BACKGROUND = "Dark" -RESPONSIVE_PLOT_MIN_HEIGHT_PX = 420 -TABLE_MAX_HEIGHT_PX = 720 -RESPONSIVE_PLOT_CSS = f""" - -""" -FORCE_DARK_MODE_HEAD = ( - """ - -""" - + RESPONSIVE_PLOT_CSS -) def restart_space(): API.restart_space(repo_id=REPO_ID) -BENCHMARK_NAMES = get_benchmark_names() -DEFAULT_BENCHMARK = BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None -BENCHMARK_RUN_DF = get_benchmark_run_df() -ANALYSIS_DF = get_analysis_df() -PR2_DF = enrich_analysis_df(ANALYSIS_DF) -CODING_BENCHMARKS = benchmarks_for_category(PR2_DF, "Coding") -GENERALIST_BENCHMARKS = benchmarks_for_category(PR2_DF, "Generalist") - - -def render_leaderboard_benchmark_plot( - benchmark_name, - color_by, - color_palette=DEFAULT_COLOR_PALETTE, - plot_background=DEFAULT_PLOT_BACKGROUND, -): - return create_leaderboard_benchmark_plot( - BENCHMARK_RUN_DF, - benchmark_name=benchmark_name, - color_by=color_by, - palette_name=color_palette, - background_name=plot_background, - ) - - -def render_efficiency( - benchmark_name, - token_metric, - color_by, - x_scale, - show_pareto_frontier, - show_labels, - color_palette=DEFAULT_COLOR_PALETTE, - plot_background=DEFAULT_PLOT_BACKGROUND, -): - plot_df = get_efficiency_df( - benchmark_name=benchmark_name, - resource_metric=token_metric, - analysis_df=ANALYSIS_DF, - ) - exclusion_count = plot_df.attrs.get("exclusion_count", 0) - note = ( - f"{exclusion_count} runs excluded for this benchmark because " - f"{token_metric.lower()} was missing or non-positive." - ) - figure = create_performance_vs_resource_plot( - plot_df, - resource_metric=token_metric, - color_by=color_by, - x_scale=x_scale, - show_pareto_frontier=show_pareto_frontier, - show_labels=show_labels, - palette_name=color_palette, - background_name=plot_background, - ) - return figure, note - - -PAGE_TABLE_COLUMNS = [ - "Model", - "Harness", - "Benchmark", - "Score (%)", - "Within-Benchmark Rank", - "Within-Benchmark Percentile", - "Total Tokens Per Task", - "Cost Per Task", - "Total Time Per Task", - "Agent Time Per Task", - "Execution Error Rate (%)", - "Tokens Per Successful Task", - "Cost Per Successful Task", - "Time Per Successful Task", -] - -PAGE_TABLE_SORT_COLUMNS = { - "Score": "Score (%)", - "Rank": "Within-Benchmark Rank", - "Percentile": "Within-Benchmark Percentile", - "Total tokens": "Total Tokens Per Task", - "Cost": "Cost Per Task", - "Response time": "Total Time Per Task", - "Agent time": "Agent Time Per Task", - "Execution error rate": "Execution Error Rate (%)", -} - -PAGE_TABLE_HIGHER_IS_BETTER = { - "Score": True, - "Rank": False, - "Percentile": True, - "Total tokens": False, - "Cost": False, - "Response time": False, - "Agent time": False, - "Execution error rate": False, -} - - -def render_page_table(benchmark, sort_metric="Score", sort_order="Best first"): - """One compact table per page with all metrics relevant to ranking/trade-off views.""" - data = PR2_DF.copy() - if benchmark and benchmark != "All benchmarks": - data = data[data["Benchmark"] == benchmark].copy() - columns = [column for column in PAGE_TABLE_COLUMNS if column in data.columns] - data = data[columns].copy() - if data.empty: - return data - - data["_agent"] = data["Model"].astype(str) + " / " + data["Harness"].astype(str) - if sort_order == "Alphabetical (A–Z)": - data = data.sort_values(["_agent", "Benchmark"], ascending=[True, True], kind="mergesort") - elif sort_order == "Alphabetical (Z–A)": - data = data.sort_values(["_agent", "Benchmark"], ascending=[False, True], kind="mergesort") - else: - sort_column = PAGE_TABLE_SORT_COLUMNS.get(sort_metric, "Score (%)") - values = pd.to_numeric(data.get(sort_column), errors="coerce") - data["_sort_value"] = values - if sort_order == "Best first": - ascending = not PAGE_TABLE_HIGHER_IS_BETTER.get(sort_metric, True) - elif sort_order == "Best last": - ascending = PAGE_TABLE_HIGHER_IS_BETTER.get(sort_metric, True) - else: - ascending = sort_order == "Lowest value first" - data = data.sort_values( - ["_sort_value", "_agent", "Benchmark"], - ascending=[ascending, True, True], - na_position="last", - kind="mergesort", - ).drop(columns="_sort_value") - data = data.drop(columns="_agent").reset_index(drop=True) - - # Metrics displayed to 2 decimal places - decimal_columns = [ - "Score (%)", - "Within-Benchmark Percentile", - "Execution Error Rate (%)", - "Cost Per Successful Task", - ] - - for column in decimal_columns: - if column in data.columns: - data[column] = pd.to_numeric(data[column], errors="coerce").round(2) - - # Token and time metrics displayed as whole numbers - whole_number_columns = [ - "Total Tokens Per Task", - "Total Time Per Task", - "Agent Time Per Task", - "Tokens Per Successful Task", - "Time Per Successful Task", - ] - - for column in whole_number_columns: - if column in data.columns: - data[column] = pd.to_numeric(data[column], errors="coerce").round(0) - - return data - - -def render_ranking( - metric, - benchmark, - color_by="Model", - color_palette=DEFAULT_COLOR_PALETTE, - plot_background=DEFAULT_PLOT_BACKGROUND, - sort_order="Best first", -): - def sort_table(table, metric_column, higher_is_better): - if table is None or table.empty: - return table - work = table.copy() - work["_agent"] = work["Model"].astype(str) + " / " + work["Harness"].astype(str) - if sort_order == "Alphabetical (A–Z)": - work = work.sort_values("_agent", ascending=True, kind="mergesort") - elif sort_order == "Alphabetical (Z–A)": - work = work.sort_values("_agent", ascending=False, kind="mergesort") - elif sort_order in {"Best first", "Best last"}: - ascending = not higher_is_better - if sort_order == "Best last": - ascending = not ascending - work = work.sort_values( - [metric_column, "_agent"], - ascending=[ascending, True], - kind="mergesort", - ) - elif sort_order == "Lowest value first": - work = work.sort_values( - [metric_column, "_agent"], - ascending=[True, True], - kind="mergesort", - ) - else: - work = work.sort_values( - [metric_column, "_agent"], - ascending=[False, True], - kind="mergesort", - ) - return work.drop(columns="_agent").reset_index(drop=True) - - table = ranking_df(PR2_DF, metric, benchmark=benchmark) - spec = RANKING_METRICS[metric] - table = sort_table(table, spec.column, spec.higher_is_better) - figure = create_ranking_plot( - table, - spec.column, - spec.label, - spec.higher_is_better, - color_by=color_by, - palette_name=color_palette, - background_name=plot_background, - sort_order=sort_order, - ) - return figure, table - - -TRADEOFF_PAIRS = { - "Score vs cost": ("Cost Per Task", "Score (%)", "Cost per task (USD)", "Score (%)", True, True), - "Score vs tokens": ("Total Tokens Per Task", "Score (%)", "Total tokens per task", "Score (%)", True, True), - "Score vs total time": ("Total Time Per Task", "Score (%)", "Total time per task (seconds)", "Score (%)", True, True), - "Score vs agent time": ("Agent Time Per Task", "Score (%)", "Agent time per task (seconds)", "Score (%)", True, True), - "Score vs execution error rate": ( - "Execution Error Rate (%)", "Score (%)", "Execution error rate (%)", "Score (%)", True, True - ), - "Tokens vs cost": ("Total Tokens Per Task", "Cost Per Task", "Total tokens per task", "Cost per task (USD)", True, False), - "Cost vs total time": ("Cost Per Task", "Total Time Per Task", "Cost per task (USD)", "Total time per task (seconds)", True, False), -} - - -def render_tradeoff( - pair, - benchmark, - color_by, - show_labels, - x_scale, - show_pareto, - color_palette, - plot_background, -): - x_column, y_column, x_label, y_label, lower_x, higher_y = TRADEOFF_PAIRS[pair] - data = PR2_DF[PR2_DF["Benchmark"] == benchmark].copy() - figure = create_tradeoff_plot( - data, - x_column=x_column, - y_column=y_column, - x_label=x_label, - y_label=y_label, - color_by=color_by, - show_labels=show_labels, - x_scale=x_scale, - show_pareto_frontier=show_pareto, - lower_x_is_better=lower_x, - higher_y_is_better=higher_y, - palette_name=color_palette, - background_name=plot_background, - ) - valid = data[[x_column, y_column]].apply(pd.to_numeric, errors="coerce").dropna() - note = f"{len(valid)} comparable runs shown for {benchmark}; missing metrics are omitted, not treated as zero." - return figure, note - - -def render_matrix( - metric, - category, - include_incomplete, - sort_by, - show_values, - reverse_scale, - plot_background, -): - category_filter = None if category == "All" else category - matrix = matrix_df( - PR2_DF, - metric, - category=category_filter, - include_incomplete=include_incomplete, - sort_by=sort_by, - ) - if metric == "Coverage": - return create_coverage_matrix_plot(matrix, plot_background) - spec = MATRIX_METRICS[metric] - display_matrix = None - display_metric_label = None - if metric == "Within-benchmark percentile": - display_matrix = matrix_df( - PR2_DF, - "Score", - category=category_filter, - include_incomplete=include_incomplete, - sort_by=sort_by, - ).reindex(index=matrix.index, columns=matrix.columns) - display_metric_label = "Benchmark score (%)" - return create_matrix_plot( - matrix, - f"{metric} matrix", - spec.label, - higher_is_better=spec.higher_is_better, - show_values=show_values, - reverse_scale=reverse_scale, - background_name=plot_background, - display_matrix=display_matrix, - display_metric_label=display_metric_label, - ) - - -def category_leaderboard(category): - data = filter_category(PR2_DF, category) - if data.empty: - return pd.DataFrame() - normalized = cross_benchmark_ranking_df(data, minimum_coverage=0) - return normalized - - -def render_category_tradeoff(category, benchmark, metric, color_by, show_labels, plot_background): - data = filter_category(PR2_DF, category) - data = data[data["Benchmark"] == benchmark] - spec = TRADEOFF_METRICS[metric] - return create_tradeoff_plot( - data, - x_column=spec.column, - y_column="Score (%)", - x_label=spec.label, - y_label="Score (%)", - color_by=color_by, - show_labels=show_labels, - show_pareto_frontier=metric != "Execution error rate", - lower_x_is_better=True, - higher_y_is_better=True, - background_name=plot_background, - ) - - -def render_category_matrix(category, metric, show_values, plot_background): - matrix = matrix_df(PR2_DF, metric, category=category, include_incomplete=True) - if metric == "Coverage": - return create_coverage_matrix_plot(matrix, plot_background) - spec = MATRIX_METRICS[metric] - display_matrix = None - display_metric_label = None - if metric == "Within-benchmark percentile": - display_matrix = matrix_df( - PR2_DF, "Score", category=category, include_incomplete=True - ).reindex(index=matrix.index, columns=matrix.columns) - display_metric_label = "Benchmark score (%)" - return create_matrix_plot( - matrix, - f"{category} — {metric}", - spec.label, - higher_is_better=spec.higher_is_better, - show_values=show_values, - background_name=plot_background, - display_matrix=display_matrix, - display_metric_label=display_metric_label, - ) - +LEADERBOARD_DF = get_leaderboard_df() -def build_header_html(df): - summary = coverage_summary(PR2_DF) - return f""" - -
-

Coding Agent Leaderboard - v{__version__} -

-
-

- Performance, efficiency, coverage, and reliability across coding-agent benchmarks. - Each result is one model + harness run on one benchmark. -

-
-
{summary['results']}benchmark results
-
{summary['models']}models
-
{summary['harnesses']}harnesses
-
{summary['benchmarks']}benchmarks
-
{summary['token_coverage_pct']:.0f}%token coverage
-
{summary['cost_coverage_pct']:.0f}%cost coverage
-
{summary['time_coverage_pct']:.0f}%timing coverage
-
-

- Cross-benchmark ordering uses within-benchmark percentiles rather than averaging incompatible raw score scales. - Missing metrics remain missing and reduce coverage; they are never converted to zero. -

-
- """ - -def build_overview_html(): - summary = coverage_summary(PR2_DF) - return f""" -
-
{summary['results']}benchmark results
-
{summary['models']}models
-
{summary['harnesses']}harnesses
-
{summary['benchmarks']}benchmarks
-
{summary['token_coverage_pct']:.0f}%token coverage
-
{summary['cost_coverage_pct']:.0f}%cost coverage
-
{summary['time_coverage_pct']:.0f}%timing coverage
-
- """ - - -def init_benchmark_runs(dataframe): +def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") - - label_choices = [("🟠 Fully FOSS", "🟠"), ("🔶 Proprietary", "🔶")] - benchmark_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Benchmark"]}) - model_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Model"]}) - harness_choices = sorted({(clean_markdown_link(v), v) for v in dataframe["Harness"]}) - return Leaderboard( value=dataframe, select_columns=SelectColumns( - default_selection=[ - " ", - "Model", - "Harness", - "Benchmark", - "Score", - "Avg Cost Per Task (USD)", - ], + default_selection=DISPLAY_BY_DEFAULT, label="Select Columns to Display:", ), - datatype="markdown", - search_columns=[ - "Benchmark", - "Harness", - "Model", - ], + search_columns=SEARCH_COLUMNS, filter_columns=[ - ColumnFilter(label="Category", column=" ", type="checkboxgroup", choices=label_choices), - ColumnFilter(label="Benchmark", column="Benchmark", type="checkboxgroup", choices=benchmark_choices), - ColumnFilter(label="Model", column="Model", type="checkboxgroup", choices=model_choices), - ColumnFilter(label="Harness", column="Harness", type="checkboxgroup", choices=harness_choices), - ColumnFilter(label="Number of Parameters (B)", column="Model Num Params (B)", type="slider"), - ColumnFilter(label="Precision", column="Precision", type="checkboxgroup"), + ColumnFilter(label="Dataset", column="dataset", type="checkboxgroup"), + ColumnFilter(label="Number of Parameters (B)", column="model_num_params", type="slider", min=0.5, max=150), ], interactive=False, ) -def add_category_section(category, benchmarks): - if not benchmarks: - gr.Markdown(f"No active benchmarks are currently classified as **{category}**.") - return - gr.Markdown( - f"Results classified as **{category}**. Cross-benchmark ordering uses within-benchmark " - "percentiles and reports coverage; raw benchmark scores are not averaged together." - ) - gr.Markdown("#### Trade-offs") - with gr.Row(): - benchmark = gr.Dropdown(choices=benchmarks, value=benchmarks[0], label="Benchmark") - metric = gr.Dropdown( - choices=["Cost per task", "Total tokens per task", "Total time per task", "Execution error rate"], - value="Cost per task", - label="X metric", - ) - color_by = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by") - labels = gr.Checkbox(value=False, label="Show point labels") - background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - plot = gr.Plot( - value=render_category_tradeoff(category, benchmarks[0], "Cost per task", "Model", False, "Dark"), - show_label=False, - elem_classes="responsive-plot", - ) - controls = [benchmark, metric, color_by, labels, background] - for control in controls: - control.change( - fn=lambda b, m, c, l, bg, cat=category: render_category_tradeoff(cat, b, m, c, l, bg), - inputs=controls, - outputs=plot, - ) - - gr.Markdown("#### Matrix") - with gr.Row(): - matrix_metric = gr.Dropdown( - choices=[ - "Score", "Within-benchmark percentile", "Within-benchmark rank", - "Total tokens", "Cost", "Total time", "Agent time", "Execution error rate", "Coverage" - ], - value="Within-benchmark percentile", - label="Metric", - ) - matrix_values = gr.Checkbox(value=True, label="Show cell values") - matrix_background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - matrix_plot = gr.Plot( - value=render_category_matrix(category, "Within-benchmark percentile", True, "Dark"), - show_label=False, - elem_classes="responsive-plot", - ) - matrix_controls = [matrix_metric, matrix_values, matrix_background] - for control in matrix_controls: - control.change( - fn=lambda m, v, bg, cat=category: render_category_matrix(cat, m, v, bg), - inputs=matrix_controls, - outputs=matrix_plot, - ) - - gr.Markdown("#### Category ranking data") - gr.Dataframe( - value=category_leaderboard(category), interactive=False, show_label=False, max_height=TABLE_MAX_HEIGHT_PX - ) - - -demo = gr.Blocks(theme="citrus", head=FORCE_DARK_MODE_HEAD) +demo = gr.Blocks() with demo: - # Overview is deliberately always visible above the page navigation. - gr.HTML(build_header_html(BENCHMARK_RUN_DF)) - - with gr.Tabs(): - with gr.Tab("Rankings"): - gr.Markdown( - "### Rankings\n" - "Paired comparisons are shown first, followed by benchmark-specific metric rankings." - ) - gr.Markdown("### Paired Comparisons") - gr.Markdown( - "Rankings computed from head-to-head benchmark results using " - "a Bradley–Terry paired-comparison model. " - "Handles missing data and inconsistent orderings." - ) - - rank_csv = pd.read_csv("results.csv") - rank_csv = rank_csv.dropna(subset=["metrics.score"]) - rank_csv = rank_csv.loc[~rank_csv["benchmark.name"].isin(EXCLUDED_BENCHMARKS)] - rank_model_choices = sorted(rank_csv["model.name"].unique().tolist()) - rank_harness_choices = sorted(rank_csv["harness.name"].unique().tolist()) - rank_benchmark_choices = sorted(rank_csv["benchmark.name"].unique().tolist()) - - with gr.Row(): - rank_by = gr.Dropdown( - choices=list(RANK_BY_OPTIONS.keys()), - value="Benchmark Score", - label="Rank by", - ) - rank_oss_models = gr.Checkbox(value=False, label="Open models only") - rank_oss_harnesses = gr.Checkbox(value=False, label="Open harnesses only") - - with gr.Row(): - rank_benchmark_filter = gr.CheckboxGroup( - choices=rank_benchmark_choices, - value=rank_benchmark_choices, - label="Benchmarks", - ) - - with gr.Row(): - rank_model_filter = gr.CheckboxGroup( - choices=rank_model_choices, - value=rank_model_choices, - label="Models", - ) - - with gr.Row(): - rank_harness_filter = gr.CheckboxGroup( - choices=rank_harness_choices, - value=rank_harness_choices, - label="Harnesses", - ) - - harness_df_init, model_df_init, pair_df_init = load_and_rank("results.csv") + gr.HTML(TITLE) + gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") - gr.Markdown("#### Harness ranking") - harness_table = gr.Dataframe(value=harness_df_init, interactive=False) - gr.Markdown("#### Model ranking") - model_table = gr.Dataframe(value=model_df_init, interactive=False) - gr.Markdown("#### Model + harness ranking") - pair_table = gr.Dataframe(value=pair_df_init, interactive=False) + with gr.Tabs(elem_classes="tab-buttons") as tabs: + with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0): + leaderboard = init_leaderboard(LEADERBOARD_DF) - def update_rankings(rank_by_val, oss_models, oss_harnesses, benchmarks, models, harnesses): - return load_and_rank( - "results.csv", - open_models_only=oss_models, - open_harnesses_only=oss_harnesses, - benchmarks=benchmarks, - models=models, - harnesses=harnesses, - rank_by=rank_by_val, - ) - - ranking_inputs = [ - rank_by, - rank_oss_models, - rank_oss_harnesses, - rank_benchmark_filter, - rank_model_filter, - rank_harness_filter, - ] - for control in ranking_inputs: - control.change( - fn=update_rankings, - inputs=ranking_inputs, - outputs=[harness_table, model_table, pair_table], - ) - - gr.Markdown( - "### Metric rankings\n" - "Use the shared display controls below, then choose a benchmark for each metric. " - "Tables are capped to a scrollable height so the visualizations stay primary." - ) - with gr.Row(): - ranking_color = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by") - ranking_sort = gr.Dropdown( - choices=[ - "Best first", - "Best last", - "Alphabetical (A–Z)", - "Alphabetical (Z–A)", - ], - value="Best first", - label="Chart order", - ) - ranking_palette = gr.Dropdown( - choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette" - ) - ranking_background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - - ranking_sections = [("Score", "Score", BENCHMARK_NAMES, DEFAULT_BENCHMARK)] - ranking_sections += [ - ("Token usage", "Total tokens", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Cost", "Cost", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Response time", "Response time", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Reliability", "Reliability", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Tokens per successful task", "Tokens per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Cost per successful task", "Cost per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ("Time per successful task", "Time per successful task", BENCHMARK_NAMES, DEFAULT_BENCHMARK), - ] - shared_ranking_controls = [ranking_color, ranking_palette, ranking_background, ranking_sort] - for section_label, metric_name, benchmark_choices, default_benchmark in ranking_sections: - spec = RANKING_METRICS[metric_name] - gr.Markdown(f"#### {section_label}\n{spec.label}. {'Higher' if spec.higher_is_better else 'Lower'} is better.") - benchmark = gr.Dropdown( - choices=benchmark_choices, value=default_benchmark, label="Benchmark" - ) - initial = render_ranking( - metric_name, - default_benchmark, - "Model", - DEFAULT_COLOR_PALETTE, - DEFAULT_PLOT_BACKGROUND, - "Best first", - ) - plot = gr.Plot(value=initial[0], show_label=False, elem_classes="responsive-plot") - controls = [benchmark, *shared_ranking_controls] - for control in controls: - control.change( - fn=lambda b, c, p, bg, so, m=metric_name: render_ranking(m, b, c, p, bg, so)[0], - inputs=controls, - outputs=plot, - ) - - gr.Markdown( - "### Ranking data\n" - "One table for the page, placed after all charts. It includes the score, resource, timing, " - "reliability, and per-success values for the selected benchmark." - ) - with gr.Row(): - ranking_table_benchmark = gr.Dropdown( - choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Table benchmark" - ) - ranking_table_metric = gr.Dropdown( - choices=list(PAGE_TABLE_SORT_COLUMNS), value="Score", label="Sort table by" - ) - ranking_table_order = gr.Dropdown( - choices=[ - "Best first", - "Best last", - "Alphabetical (A–Z)", - "Alphabetical (Z–A)", - ], - value="Best first", - label="Table order", - ) - ranking_page_table = gr.Dataframe( - value=render_page_table(DEFAULT_BENCHMARK, "Score", "Best first"), - interactive=False, - show_label=False, - max_height=TABLE_MAX_HEIGHT_PX, - ) - ranking_table_controls = [ranking_table_benchmark, ranking_table_metric, ranking_table_order] - for control in ranking_table_controls: - control.change( - fn=render_page_table, - inputs=ranking_table_controls, - outputs=ranking_page_table, - ) - - with gr.Tab("Trade-offs"): - gr.Markdown( - "### Trade-offs\n" - "Efficiency and metric-pair views are displayed together. Pareto frontiers support both maximize and " - "minimize directions, so a checked frontier is shown whenever valid comparable points exist." - ) - gr.Markdown("#### Efficiency") - with gr.Row(): - efficiency_benchmark = gr.Dropdown( - choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Benchmark" - ) - efficiency_metric = gr.Dropdown( - choices=list(EFFICIENCY_RESOURCE_METRICS), value="Total tokens", label="Resource metric" - ) - efficiency_color_by = gr.Radio( - choices=EFFICIENCY_COLOR_BY_CHOICES, value="Model", label="Color by" - ) - efficiency_scale = gr.Radio(choices=["Log", "Linear"], value="Log", label="X-axis scale") - with gr.Row(): - efficiency_pareto = gr.Checkbox(value=True, label="Show Pareto frontier") - efficiency_labels = gr.Checkbox(value=False, label="Show point labels") - efficiency_palette = gr.Dropdown( - choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette" - ) - efficiency_background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - initial_efficiency = render_efficiency( - DEFAULT_BENCHMARK, "Total tokens", "Model", "Log", True, False, - DEFAULT_COLOR_PALETTE, DEFAULT_PLOT_BACKGROUND, - ) - efficiency_note = gr.Markdown(initial_efficiency[1]) - efficiency_plot = gr.Plot( - value=initial_efficiency[0], show_label=False, elem_classes="responsive-plot" - ) - efficiency_controls = [ - efficiency_benchmark, efficiency_metric, efficiency_color_by, efficiency_scale, - efficiency_pareto, efficiency_labels, efficiency_palette, efficiency_background, - ] - for control in efficiency_controls: - control.change( - fn=render_efficiency, - inputs=efficiency_controls, - outputs=[efficiency_plot, efficiency_note], - ) - - gr.Markdown("#### Metric pairs") - with gr.Row(): - tradeoff_pair = gr.Dropdown( - choices=list(TRADEOFF_PAIRS), value="Score vs cost", label="Trade-off" - ) - tradeoff_benchmark = gr.Dropdown( - choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Benchmark" - ) - tradeoff_color = gr.Radio(choices=COLOR_BY_CHOICES, value="Model", label="Color by") - tradeoff_scale = gr.Radio(choices=["Linear", "Log"], value="Linear", label="X-axis scale") - with gr.Row(): - tradeoff_labels = gr.Checkbox(value=False, label="Show point labels") - tradeoff_pareto = gr.Checkbox(value=True, label="Show Pareto frontier") - tradeoff_palette = gr.Dropdown( - choices=COLOR_PALETTE_CHOICES, value=DEFAULT_COLOR_PALETTE, label="Color palette" - ) - tradeoff_background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - initial_tradeoff = render_tradeoff( - "Score vs cost", DEFAULT_BENCHMARK, "Model", False, "Linear", True, - DEFAULT_COLOR_PALETTE, DEFAULT_PLOT_BACKGROUND, - ) - tradeoff_note = gr.Markdown(initial_tradeoff[1]) - tradeoff_plot = gr.Plot( - value=initial_tradeoff[0], show_label=False, elem_classes="responsive-plot" - ) - tradeoff_controls = [ - tradeoff_pair, tradeoff_benchmark, tradeoff_color, tradeoff_labels, - tradeoff_scale, tradeoff_pareto, tradeoff_palette, tradeoff_background, - ] - for control in tradeoff_controls: - control.change( - fn=render_tradeoff, - inputs=tradeoff_controls, - outputs=[tradeoff_plot, tradeoff_note], - ) - - gr.Markdown( - "### Trade-off data\n" - "A single table for this page appears after both charts and includes every metric used by the trade-off views." - ) - with gr.Row(): - tradeoff_table_benchmark = gr.Dropdown( - choices=BENCHMARK_NAMES, value=DEFAULT_BENCHMARK, label="Table benchmark" - ) - tradeoff_table_metric = gr.Dropdown( - choices=list(PAGE_TABLE_SORT_COLUMNS), value="Score", label="Sort table by" - ) - tradeoff_table_order = gr.Dropdown( - choices=[ - "Largest value first", - "Lowest value first", - "Alphabetical (A–Z)", - "Alphabetical (Z–A)", - ], - value="Largest value first", - label="Table order", - ) - tradeoff_page_table = gr.Dataframe( - value=render_page_table(DEFAULT_BENCHMARK, "Score", "Largest value first"), - interactive=False, - show_label=False, - max_height=TABLE_MAX_HEIGHT_PX, - ) - tradeoff_table_controls = [tradeoff_table_benchmark, tradeoff_table_metric, tradeoff_table_order] - for control in tradeoff_table_controls: - control.change( - fn=render_page_table, - inputs=tradeoff_table_controls, - outputs=tradeoff_page_table, - ) - - with gr.Tab("Matrices"): - gr.Markdown( - "### Model × benchmark matrices\n" - "The within-benchmark percentile controls the color scale, while cell labels show the actual benchmark " - "score. Missing cells stay missing." - ) - with gr.Row(): - matrix_metric = gr.Dropdown( - choices=[*MATRIX_METRICS.keys(), "Coverage"], - value="Within-benchmark percentile", - label="Metric", - ) - matrix_category = gr.Dropdown( - choices=["All", "Coding", "Generalist"], value="All", label="Benchmark category" - ) - matrix_sort = gr.Dropdown( - choices=[ - "Normalized performance (high to low)", - "Normalized performance (low to high)", - "Coverage (high to low)", - "Coverage (low to high)", - "Alphabetical", - "Alphabetical (Z–A)", - ], - value="Normalized performance (high to low)", - label="Sort rows", - ) - matrix_background = gr.Dropdown( - choices=PLOT_BACKGROUND_CHOICES, value=DEFAULT_PLOT_BACKGROUND, label="Image background" - ) - with gr.Row(): - matrix_incomplete = gr.Checkbox(value=True, label="Include incomplete rows") - matrix_values = gr.Checkbox(value=True, label="Show cell values") - matrix_reverse = gr.Checkbox(value=False, label="Reverse color scale") - initial_matrix = render_matrix( - "Within-benchmark percentile", "All", True, - "Normalized performance (high to low)", True, False, "Dark" - ) - matrix_plot = gr.Plot(value=initial_matrix, show_label=False, elem_classes="responsive-plot") - matrix_controls = [ - matrix_metric, matrix_category, matrix_incomplete, matrix_sort, - matrix_values, matrix_reverse, matrix_background, - ] - for control in matrix_controls: - control.change(fn=render_matrix, inputs=matrix_controls, outputs=matrix_plot) - - - with gr.Tab("Coding"): - gr.Markdown( - "### Coding\n" - "Coding benchmarks are defined centrally in the benchmark catalog. The aggregate leaderboard uses " - "within-benchmark percentiles and displays benchmark coverage." - ) - add_category_section("Coding", CODING_BENCHMARKS) - - with gr.Tab("Generalist"): - gr.Markdown( - "### Terminal & Generalist\n" - "This category reflects active terminal/generalist benchmarks present in the repository." - ) - add_category_section("Generalist", GENERALIST_BENCHMARKS) - - with gr.Tab("Results Explorer"): - gr.Markdown("### Benchmark runs") - benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF) - gr.Markdown("### Methodology") - gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") - gr.Markdown( - "### Analytics methodology notes\n" - "- **Normalized ordering:** rank/percentile is calculated independently inside each benchmark, then " - "aggregated by model + harness with coverage shown beside it.\n" - "- **Execution error rate:** recorded errors divided by recorded task count; unresolved tasks are not " - "relabeled as errors.\n" - "- **Missing metrics:** omitted from metric-specific comparisons and preserved as missing matrix cells.\n" - "- **Pareto frontier:** benchmark-specific and direction-aware for maximize/minimize metric pairs." - ) + with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") - gr.Markdown(HOW_TO_USE_TEXT, elem_classes="markdown-text") + with gr.Row(): + with gr.Accordion("📙 Citation", open=False): + citation_button = gr.Textbox( + value=CITATION_BUTTON_TEXT, + label=CITATION_BUTTON_LABEL, + lines=20, + elem_id="citation-button", + show_copy_button=True, + ) scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) diff --git a/requirements.txt b/requirements.txt index 65037834c70c97752bc502c2b5492204f3da1412..3cacab3e9afab55f2ce3493ac25d7a0ea5c96255 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,17 +3,14 @@ black datasets gradio gradio[oauth] -gradio_leaderboard +gradio_leaderboard==0.0.13 gradio_client huggingface-hub>=0.18.0 matplotlib numpy pandas -plotly python-dateutil tqdm transformers tokenizers>=0.15.0 -sentencepiece -choix -scipy +sentencepiece \ No newline at end of file diff --git a/results.csv b/results.csv deleted file mode 100644 index 23541d9ec6fd5a686673a0607976b1f0b9670b2f..0000000000000000000000000000000000000000 --- a/results.csv +++ /dev/null @@ -1,67 +0,0 @@ -benchmark.name,benchmark.repo,benchmark.num_tasks,benchmark.url,harness.name,harness.skills,harness.is_oss,harness.url,model.name,model.repo,model.is_oss,model.num_params,model.precision,model.url,environment.name,environment.url,metrics.n_tasks,metrics.n_errors,metrics.score,metrics.n_input_tokens,metrics.n_cache_tokens,metrics.n_output_tokens,metrics.n_total_tokens,metrics.agent_time_seconds,metrics.total_time_seconds,metrics.cost_usd,metrics.mean_input_tokens_per_task,metrics.mean_cache_tokens_per_task,metrics.mean_output_tokens_per_task,metrics.mean_tokens_per_task,metrics.mean_cost_usd_per_task,metrics.mean_total_time_seconds_per_task,metrics.mean_agent_time_seconds_per_task -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.6,Opus 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.633,438501505.0,419608736.0,2823496.0,860933737.0,126529.0,243310.0,434.54,1228295.0,1175374.0,7908.0,2411579.0,1.22,681.0,354.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.557,500304168.0,479035103.0,4937782.0,984277053.0,150652.0,269008.0,349.77,1401412.0,1341835.0,13831.0,2757078.0,0.98,753.0,421.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,31.0,0.317,1088634865.0,0.0,6200414.0,1094835279.0,398444.0,575536.0,15.05,3049397.0,0.0,17368.0,3066765.0,0.04,1612.0,1116.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.294,419671770.0,0.0,2277963.0,421949733.0,98909.0,169021.0,3.74,1175551.0,0.0,6380.0,1181932.0,0.01,473.0,277.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,20.0,0.23,419282595.0,0.0,1954817.0,421237412.0,100632.0,168697.0,3.8,1174461.0,0.0,5475.0,1179936.0,0.01,472.0,281.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,14.0,0.224,783670817.0,0.0,5510291.0,789181108.0,144352.0,355122.0,38.03,2195156.0,0.0,15434.0,2210591.0,0.11,994.0,404.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.308,839586046.0,0.0,5706139.0,845292185.0,94762.0,251102.0,37.44,2351781.0,0.0,15983.0,2367765.0,0.1,703.0,265.0 -RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,13.0,0.216,770574986.0,0.0,6519128.0,777094114.0,116348.0,360934.0,30.65,2158473.0,0.0,18260.0,2176734.0,0.09,1011.0,325.0 -RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,9.0,0.493,885869433.0,0.0,4763916.0,890633349.0,201374.0,401657.0,73.84,2481426.0,0.0,13344.0,2494771.0,0.21,1125.0,564.0 -RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,10.0,0.44,410001939.0,0.0,2851571.0,412853510.0,115135.0,315451.0,42.22,1148464.0,0.0,7987.0,1156452.0,0.12,883.0,322.0 -RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,1.0,0.468,560672462.0,0.0,3618673.0,564291135.0,114791.0,333385.0,42.09,1570511.0,0.0,10136.0,1580647.0,0.12,933.0,321.0 -Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,115.0,6.0,0.0,0.0,0.0,0.0,0.0,29895.0,36503.0,8.3,0.0,0.0,0.0,0.0,0.07,317.0,259.0 -Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.0,0.0,0.0,0.0,0.0,8170.0,13296.0,2.27,0.0,0.0,0.0,0.0,0.02,115.0,71.0 -Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,fp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,2.0,0.009,0.0,0.0,0.0,0.0,4846.0,9862.0,1.35,0.0,0.0,0.0,0.0,0.01,85.0,42.0 -Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,fp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.014,0.0,0.0,0.0,0.0,21575.0,26392.0,5.99,0.0,0.0,0.0,0.0,0.05,229.0,187.0 -Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.128,0.0,0.0,0.0,0.0,10812.0,16949.0,6.01,0.0,0.0,0.0,0.0,0.05,147.0,94.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.698,192346997.0,186506482.0,2179945.0,381033424.0,32745.0,39030.0,185.66155285,2003614.0,1942775.0,22707.0,3969098.0,1.93,406.0,341.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.781,187217712.0,187209844.0,1280944.0,375708500.0,30686.0,39352.0,151.4104807499999,1950184.0,1950102.0,13343.0,3913630.0,1.58,409.0,319.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.5,190672390.0,184409111.0,1593112.0,376674613.0,40527.0,49734.0,184.42824125000004,1986170.0,1920928.0,16594.0,3923693.0,1.92,518.0,422.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,16.0,0.417,122366824.0,0.0,1307027.0,123673851.0,138148.0,158812.0,54.82,1274654.0,0.0,13614.0,1288269.0,0.57,1654.0,1439.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,12.0,0.417,100068358.0,0.0,1261237.0,101329595.0,60747.0,77623.0,19.28,1042378.0,0.0,13137.0,1055516.0,0.2,808.0,632.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.469,78678324.0,0.0,1370430.0,80048754.0,51767.0,66845.0,16.43,819565.0,0.0,14275.0,833841.0,0.17,696.0,539.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.604,198924339.0,189578624.0,1560836.0,390063799.0,32914.0,39468.0,188.34296700000002,2072128.0,1974777.0,16258.0,4063164.0,1.96,411.0,342.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.375,347744043.0,0.0,1903023.0,349647066.0,59653.0,67586.0,2.25,3622333.0,0.0,19823.0,3642156.0,0.02,704.0,621.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.333,112041538.0,0.0,781173.0,112822711.0,25086.0,31180.0,0.95,1167099.0,0.0,8137.0,1175236.0,0.01,324.0,261.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,15.0,0.292,125247664.0,0.0,597435.0,125845099.0,75715.0,79741.0,2.86,1304663.0,0.0,6223.0,1310886.0,0.03,830.0,788.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.26,147452235.0,0.0,953241.0,148405476.0,33613.0,55241.0,5.84,1535960.0,0.0,9929.0,1545890.0,0.06,575.0,350.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.385,140362798.0,0.0,1230697.0,141593495.0,23543.0,49289.0,6.54,1462112.0,0.0,12819.0,1474932.0,0.07,513.0,245.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.333,104994429.0,0.0,1095179.0,106089608.0,15004.0,55176.0,4.17,1093691.0,0.0,11408.0,1105100.0,0.04,574.0,156.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.432,406026150.0,0.0,2839236.0,408865386.0,100355.0,115220.0,26.44,4229439.0,0.0,29575.0,4259014.0,0.28,1200.0,1045.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.323,686392098.0,0.0,2915876.0,689307974.0,86130.0,99705.0,19.45,7149917.0,0.0,30373.0,7180291.0,0.2,1038.0,897.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,14.0,0.375,1051387065.0,0.0,3651367.0,1055038432.0,113768.0,127326.0,25.69,10951948.0,0.0,38035.0,10989983.0,0.27,1326.0,1185.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,8.0,0.521,386929017.0,0.0,1847087.0,388776104.0,118693.0,137736.0,43.52,4030510.0,0.0,19240.0,4049751.0,0.45,1434.0,1236.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.573,344853232.0,0.0,1661197.0,346514429.0,118356.0,196235.0,43.4,3592221.0,0.0,17304.0,3609525.0,0.45,2044.0,1232.0 -SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.49,395189021.0,0.0,2206097.0,397395118.0,80305.0,91995.0,29.45,4116552.0,0.0,22980.0,4139532.0,0.31,958.0,836.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,6.0,0.458,367897697.0,0.0,1694885.0,369592582.0,39024.0,46758.0,9.64,3832267.0,0.0,17655.0,3849922.0,0.1,487.0,406.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.375,207164679.0,0.0,1598703.0,208763382.0,49450.0,57287.0,12.21,2157965.0,0.0,16653.0,2174618.0,0.13,596.0,515.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,5.0,0.406,0.0,0.0,0.0,0.0,38085.0,50779.0,9.4,0.0,0.0,0.0,0.0,0.1,528.0,396.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.479,742491363.0,0.0,2387609.0,744878972.0,54543.0,62422.0,13.47,7734285.0,0.0,24870.0,7759155.0,0.14,650.0,568.0 -SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.438,159198517.0,0.0,972133.0,160170650.0,33610.0,38272.0,9.34,1658317.0,0.0,10126.0,1668444.0,0.1,398.0,350.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.868,402225158.0,388781382.0,4588859.0,795595399.0,104467.0,159566.0,394.7823929,804450.0,777562.0,9177.0,1591190.0,0.79,319.0,208.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.834,347751942.0,347725246.0,3640204.0,699117392.0,114633.0,171540.0,319.53304049999974,695503.0,695450.0,7280.0,1398234.0,0.64,343.0,229.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,,,0.796,,,,,,,,,,,,,, -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,35.0,0.612,651306166.0,0.0,4591859.0,655898025.0,524857.0,669355.0,104.14,1302612.0,0.0,9183.0,1311796.0,0.21,1338.0,1049.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.606,525563275.0,0.0,4183675.0,529746950.0,166092.0,444527.0,52.73,1051126.0,0.0,8367.0,1059493.0,0.11,889.0,332.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.574,377479979.0,0.0,4177117.0,381657096.0,122274.0,385336.0,38.82,754959.0,0.0,8354.0,763314.0,0.08,770.0,244.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.798,415795756.0,386352384.0,3431224.0,805579364.0,92588.0,141637.0,443.32977200000016,831591.0,772704.0,6862.0,1611158.0,0.89,283.0,185.0 -SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,13.0,0.376,833651681.0,0.0,5656173.0,839307854.0,150981.0,200547.0,5.7,1667303.0,0.0,11312.0,1678615.0,0.01,401.0,301.0 -SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,33.0,0.326,202856940.0,0.0,1444371.0,204301311.0,35190.0,322482.0,3.32,405713.0,0.0,2888.0,408602.0,0.01,644.0,70.0 -SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,47.0,0.234,90107247.0,0.0,905031.0,91012278.0,146747.0,176174.0,13.86,180214.0,0.0,1810.0,182024.0,0.03,352.0,293.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,53.0,0.118,309857096.0,0.0,2416878.0,312273974.0,54338.0,555599.0,9.43,619714.0,0.0,4833.0,624547.0,0.02,1111.0,108.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.318,373713878.0,0.0,4051687.0,377765565.0,49103.0,543025.0,13.64,747427.0,0.0,8103.0,755531.0,0.03,1086.0,98.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.232,195246618.0,0.0,2726988.0,197973606.0,29236.0,458591.0,8.12,390493.0,0.0,5453.0,395947.0,0.02,917.0,58.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.472,882330271.0,0.0,6689285.0,889019556.0,174288.0,410828.0,45.91,1764660.0,0.0,13378.0,1778039.0,0.09,821.0,348.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.41,724984042.0,0.0,5941303.0,730925345.0,93088.0,518565.0,21.02,1449968.0,0.0,11882.0,1461850.0,0.04,1037.0,186.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.498,996839122.0,0.0,7196671.0,1004035793.0,144084.0,446845.0,32.53,1993678.0,0.0,14393.0,2008071.0,0.07,893.0,288.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.694,1005844088.0,0.0,6255800.0,1012099888.0,228858.0,411348.0,83.91,2011688.0,0.0,12511.0,2024199.0,0.17,822.0,457.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,8.0,0.642,395291929.0,0.0,3296668.0,398588597.0,92700.0,374913.0,33.99,790583.0,0.0,6593.0,797177.0,0.07,749.0,185.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,7.0,0.694,544070591.0,0.0,4965652.0,549036243.0,130591.0,408706.0,47.88,1088141.0,0.0,9931.0,1098072.0,0.1,817.0,261.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.632,1106618897.0,0.0,5733245.0,1112352142.0,122808.0,171897.0,34.11,2213237.0,0.0,11466.0,2224704.0,0.07,343.0,245.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.588,0.0,0.0,0.0,0.0,120399.0,200354.0,33.44,0.0,0.0,0.0,0.0,0.07,400.0,240.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.548,469806650.0,0.0,4937761.0,474744411.0,120473.0,185168.0,29.75,939613.0,0.0,9875.0,949488.0,0.06,370.0,240.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.65,791183735.0,0.0,6333798.0,797517533.0,154531.0,218988.0,38.16,1582367.0,0.0,12667.0,1595035.0,0.08,437.0,309.0 -SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.638,609589099.0,0.0,3964252.0,613553351.0,132273.0,178984.0,36.74,1219178.0,0.0,7928.0,1227106.0,0.07,357.0,264.0 -Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.303,47607780.0,0.0,1657188.0,49264968.0,46467.0,58507.0,11.47,534918.0,0.0,18620.0,553538.0,0.13,657.0,522.0 -Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.36,82108716.0,0.0,2056390.0,84165106.0,44991.0,64760.0,11.11,922569.0,0.0,23105.0,945675.0,0.12,727.0,505.0 diff --git a/results/swe-bench-verified-claude-sonnet-4-6-claude-code.json b/results/qwen3-6-35b-nvfp4-claude-code.json similarity index 51% rename from results/swe-bench-verified-claude-sonnet-4-6-claude-code.json rename to results/qwen3-6-35b-nvfp4-claude-code.json index 3a7e545a0f252f65c21b40cb6ed310aea586588d..de91b29f0aa1fdf34fb2677fe5f5b956baf67a47 100644 --- a/results/swe-bench-verified-claude-sonnet-4-6-claude-code.json +++ b/results/qwen3-6-35b-nvfp4-claude-code.json @@ -1,23 +1,19 @@ { - "benchmark": { - "name": "SWE-Bench Verified", + "dataset": { + "name": "swe-bench-verified", "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" + "num_tasks": 500 }, "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" + "name": "claude-code", + "skills": [] }, "model": { - "name": "Sonnet 4.6", - "repo": "Sonnet 4.6", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-sonnet-4-6" + "name": "Qwen3.6-35B-A3B-NVFP4", + "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", + "is_oss": true, + "num_params": 35, + "precision": "nvfp4" }, "environment": { "name": "harbor", @@ -33,10 +29,11 @@ "task_names": null, "exclude_task_names": null, "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" + } }, "metrics": { - "score": 0.796 + "score": 0.632, + "time": 21600, + "costUSD": 48.00 } -} +} \ No newline at end of file diff --git a/results/rh-swe-bench-claude-opus-4-6-claude-code.json b/results/rh-swe-bench-claude-opus-4-6-claude-code.json deleted file mode 100644 index 133663e7c36154af0a21ba1004b0a6d63cc02c11..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-claude-opus-4-6-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Opus 4.6", - "repo": "Opus 4.6", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-opus-4-6" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "rh-swe-bench", - "version": null, - "ref": null, - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 0, - "score": 0.633, - "n_input_tokens": 438501505, - "n_cache_tokens": 419608736, - "n_output_tokens": 2823496, - "n_total_tokens": 860933737, - "agent_time_seconds": 126529, - "total_time_seconds": 243310, - "cost_usd": 434.54, - "mean_input_tokens_per_task": 1228295, - "mean_cache_tokens_per_task": 1175374, - "mean_output_tokens_per_task": 7908, - "mean_tokens_per_task": 2411579, - "mean_cost_usd_per_task": 1.22, - "mean_total_time_seconds_per_task": 681, - "mean_agent_time_seconds_per_task": 354 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-claude-sonnet-4-6-claude-code.json b/results/rh-swe-bench-claude-sonnet-4-6-claude-code.json deleted file mode 100644 index 11e22f947ffec4509823361a84e8bd359716f0b9..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-claude-sonnet-4-6-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Sonnet 4.6", - "repo": "Sonnet 4.6", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-sonnet-4-6" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "rh-swe-bench", - "version": null, - "ref": null, - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 0, - "score": 0.557, - "n_input_tokens": 500304168, - "n_cache_tokens": 479035103, - "n_output_tokens": 4937782, - "n_total_tokens": 984277053, - "agent_time_seconds": 150652, - "total_time_seconds": 269008, - "cost_usd": 349.77, - "mean_input_tokens_per_task": 1401412, - "mean_cache_tokens_per_task": 1341835, - "mean_output_tokens_per_task": 13831, - "mean_tokens_per_task": 2757078, - "mean_cost_usd_per_task": 0.98, - "mean_total_time_seconds_per_task": 753, - "mean_agent_time_seconds_per_task": 421 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-gpt-oss-120b-claude-code.json b/results/rh-swe-bench-gpt-oss-120b-claude-code.json deleted file mode 100644 index fe0013f2638e82e5ff0f5350b143420bba126241..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-gpt-oss-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://docs.anthropic.com/en/docs/claude-code" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 31, - "score": 0.317, - "n_input_tokens": 1088634865, - "n_cache_tokens": 0, - "n_output_tokens": 6200414, - "n_total_tokens": 1094835279, - "agent_time_seconds": 398444, - "total_time_seconds": 575536, - "cost_usd": 15.05, - "mean_input_tokens_per_task": 3049397, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 17368, - "mean_tokens_per_task": 3066765, - "mean_cost_usd_per_task": 0.04, - "mean_total_time_seconds_per_task": 1612, - "mean_agent_time_seconds_per_task": 1116 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-gpt-oss-120b-opencode.json b/results/rh-swe-bench-gpt-oss-120b-opencode.json deleted file mode 100644 index 18cca5b445f565c7c82c2b3d44c23ac12d3b246d..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-gpt-oss-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 7, - "score": 0.294, - "n_input_tokens": 419671770, - "n_cache_tokens": 0, - "n_output_tokens": 2277963, - "n_total_tokens": 421949733, - "agent_time_seconds": 98909, - "total_time_seconds": 169021, - "cost_usd": 3.74, - "mean_input_tokens_per_task": 1175551, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 6380, - "mean_tokens_per_task": 1181932, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 473, - "mean_agent_time_seconds_per_task": 277 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-gpt-oss-120b-pi.json b/results/rh-swe-bench-gpt-oss-120b-pi.json deleted file mode 100644 index 344351b4dcc16be06edadb5c026c2f506e761603..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-gpt-oss-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 20, - "score": 0.23, - "n_input_tokens": 419282595, - "n_cache_tokens": 0, - "n_output_tokens": 1954817, - "n_total_tokens": 421237412, - "agent_time_seconds": 100632, - "total_time_seconds": 168697, - "cost_usd": 3.8, - "mean_input_tokens_per_task": 1174461, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 5475, - "mean_tokens_per_task": 1179936, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 472, - "mean_agent_time_seconds_per_task": 281 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-nemotron-120b-claude-code.json b/results/rh-swe-bench-nemotron-120b-claude-code.json deleted file mode 100644 index 6bd26fd0b4925b325efb912baee235d4a6c8ea54..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-nemotron-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 14, - "score": 0.224, - "n_input_tokens": 783670817, - "n_cache_tokens": 0, - "n_output_tokens": 5510291, - "n_total_tokens": 789181108, - "agent_time_seconds": 144352, - "total_time_seconds": 355122, - "cost_usd": 38.03, - "mean_input_tokens_per_task": 2195156, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 15434, - "mean_tokens_per_task": 2210591, - "mean_cost_usd_per_task": 0.11, - "mean_total_time_seconds_per_task": 994, - "mean_agent_time_seconds_per_task": 404 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-nemotron-120b-opencode.json b/results/rh-swe-bench-nemotron-120b-opencode.json deleted file mode 100644 index ed00d50edf483ac197ff5264dd4496ee44192f01..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-nemotron-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 7, - "score": 0.308, - "n_input_tokens": 839586046, - "n_cache_tokens": 0, - "n_output_tokens": 5706139, - "n_total_tokens": 845292185, - "agent_time_seconds": 94762, - "total_time_seconds": 251102, - "cost_usd": 37.44, - "mean_input_tokens_per_task": 2351781, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 15983, - "mean_tokens_per_task": 2367765, - "mean_cost_usd_per_task": 0.1, - "mean_total_time_seconds_per_task": 703, - "mean_agent_time_seconds_per_task": 265 - } -} diff --git a/results/rh-swe-bench-nemotron-120b-pi.json b/results/rh-swe-bench-nemotron-120b-pi.json deleted file mode 100644 index 68e070b57b25fa3bc16d7ebe2b34f703346b6ed5..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-nemotron-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende10/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 13, - "score": 0.216, - "n_input_tokens": 770574986, - "n_cache_tokens": 0, - "n_output_tokens": 6519128, - "n_total_tokens": 777094114, - "agent_time_seconds": 116348, - "total_time_seconds": 360934, - "cost_usd": 30.65, - "mean_input_tokens_per_task": 2158473, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 18260, - "mean_tokens_per_task": 2176734, - "mean_cost_usd_per_task": 0.09, - "mean_total_time_seconds_per_task": 1011, - "mean_agent_time_seconds_per_task": 325 - } -} diff --git a/results/rh-swe-bench-qwen3-6-27b-fp8-claude-code.json b/results/rh-swe-bench-qwen3-6-27b-fp8-claude-code.json deleted file mode 100644 index 666676e9328a2b127ddf5ca7b969898037da2f2e..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-qwen3-6-27b-fp8-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 9, - "score": 0.493, - "n_input_tokens": 885869433, - "n_cache_tokens": 0, - "n_output_tokens": 4763916, - "n_total_tokens": 890633349, - "agent_time_seconds": 201374, - "total_time_seconds": 401657, - "cost_usd": 73.84, - "mean_input_tokens_per_task": 2481426, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 13344, - "mean_tokens_per_task": 2494771, - "mean_cost_usd_per_task": 0.21, - "mean_total_time_seconds_per_task": 1125, - "mean_agent_time_seconds_per_task": 564 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-qwen3-6-27b-fp8-opencode.json b/results/rh-swe-bench-qwen3-6-27b-fp8-opencode.json deleted file mode 100644 index 3ffc2a4c4b8c1970798bf4ecbc0983c1ce71307f..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-qwen3-6-27b-fp8-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 10, - "score": 0.44, - "n_input_tokens": 410001939, - "n_cache_tokens": 0, - "n_output_tokens": 2851571, - "n_total_tokens": 412853510, - "agent_time_seconds": 115135, - "total_time_seconds": 315451, - "cost_usd": 42.22, - "mean_input_tokens_per_task": 1148464, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 7987, - "mean_tokens_per_task": 1156452, - "mean_cost_usd_per_task": 0.12, - "mean_total_time_seconds_per_task": 883, - "mean_agent_time_seconds_per_task": 322 - } -} \ No newline at end of file diff --git a/results/rh-swe-bench-qwen3-6-27b-fp8-pi.json b/results/rh-swe-bench-qwen3-6-27b-fp8-pi.json deleted file mode 100644 index a9a262fa7b170c3c0b8234a33358fe6aaea57031..0000000000000000000000000000000000000000 --- a/results/rh-swe-bench-qwen3-6-27b-fp8-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "RH SWE-Bench", - "repo": "rounakbende/rh-swe-bench", - "num_tasks": 357, - "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/plandex-ai/plandex" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "rh-swe-bench" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 357, - "n_errors": 1, - "score": 0.468, - "n_input_tokens": 560672462, - "n_cache_tokens": 0, - "n_output_tokens": 3618673, - "n_total_tokens": 564291135, - "agent_time_seconds": 114791, - "total_time_seconds": 333385, - "cost_usd": 42.09, - "mean_input_tokens_per_task": 1570511, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 10136, - "mean_tokens_per_task": 1580647, - "mean_cost_usd_per_task": 0.12, - "mean_total_time_seconds_per_task": 933, - "mean_agent_time_seconds_per_task": 321 - } -} \ No newline at end of file diff --git a/results/shellbench-gemma4-31b-fp8-openclaw.json b/results/shellbench-gemma4-31b-fp8-openclaw.json deleted file mode 100644 index 95f6271b9b59ed11af03dce19fffd4713ee7997a..0000000000000000000000000000000000000000 --- a/results/shellbench-gemma4-31b-fp8-openclaw.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "Shellbench", - "repo": "ShellBench/public-tasks", - "num_tasks": 115, - "url": "https://github.com/ShellBench/public-tasks" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/OpenClaw/OpenClaw" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "path": "/Users/hveeradh/public-tasks/tasks/115-tasks" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 115, - "n_errors": 6, - "score": 0.0, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 29895, - "total_time_seconds": 36503, - "cost_usd": 8.3, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 317, - "mean_agent_time_seconds_per_task": 259 - } -} \ No newline at end of file diff --git a/results/shellbench-gpt-oss-120b-openclaw.json b/results/shellbench-gpt-oss-120b-openclaw.json deleted file mode 100644 index 32e0658f8c94014f278a2e5d1806da32310c8597..0000000000000000000000000000000000000000 --- a/results/shellbench-gpt-oss-120b-openclaw.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "Shellbench", - "repo": "ShellBench/public-tasks", - "num_tasks": 115, - "url": "https://github.com/ShellBench/public-tasks" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/OpenClaw/OpenClaw" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "path": "/Users/hveeradh/public-tasks/tasks/115-tasks" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 115, - "n_errors": 1, - "score": 0.0, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 8170, - "total_time_seconds": 13296, - "cost_usd": 2.27, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.02, - "mean_total_time_seconds_per_task": 115, - "mean_agent_time_seconds_per_task": 71 - } -} \ No newline at end of file diff --git a/results/shellbench-mistral4-119b-fp4-openclaw.json b/results/shellbench-mistral4-119b-fp4-openclaw.json deleted file mode 100644 index dc7f401eefc7cd8a2f49d513cc6e25867e83406d..0000000000000000000000000000000000000000 --- a/results/shellbench-mistral4-119b-fp4-openclaw.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "Shellbench", - "repo": "ShellBench/public-tasks", - "num_tasks": 115, - "url": "https://github.com/ShellBench/public-tasks" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/OpenClaw/OpenClaw" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "fp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": "/Users/hveeradh/public-tasks/tasks/115-tasks" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 115, - "n_errors": 2, - "score": 0.009, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 4846, - "total_time_seconds": 9862, - "cost_usd": 1.35, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 85, - "mean_agent_time_seconds_per_task": 42 - } -} \ No newline at end of file diff --git a/results/shellbench-nemotron-120b-openclaw.json b/results/shellbench-nemotron-120b-openclaw.json deleted file mode 100644 index 9495f0eb2d270ab948e1c38c1abbef22c185c419..0000000000000000000000000000000000000000 --- a/results/shellbench-nemotron-120b-openclaw.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "Shellbench", - "repo": "ShellBench/public-tasks", - "num_tasks": 115, - "url": "https://github.com/ShellBench/public-tasks" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/OpenClaw/OpenClaw" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "fp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": "/Users/hveeradh/public-tasks/tasks/115-tasks" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 115, - "n_errors": 1, - "score": 0.014, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 21575, - "total_time_seconds": 26392, - "cost_usd": 5.99, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.05, - "mean_total_time_seconds_per_task": 229, - "mean_agent_time_seconds_per_task": 187 - } -} \ No newline at end of file diff --git a/results/shellbench-qwen3-6-27b-fp8-openclaw.json b/results/shellbench-qwen3-6-27b-fp8-openclaw.json deleted file mode 100644 index c7a14b5335c9769ef45b935ba4c47bc0ca8633d0..0000000000000000000000000000000000000000 --- a/results/shellbench-qwen3-6-27b-fp8-openclaw.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "Shellbench", - "repo": "ShellBench/public-tasks", - "num_tasks": 115, - "url": "https://github.com/ShellBench/public-tasks" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/OpenClaw/OpenClaw" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "path": "/Users/hveeradh/public-tasks/tasks/115-tasks" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 115, - "n_errors": 1, - "score": 0.128, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 10812, - "total_time_seconds": 16949, - "cost_usd": 6.01, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.05, - "mean_total_time_seconds_per_task": 147, - "mean_agent_time_seconds_per_task": 94 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-claude-opus-4-8-claude-code.json b/results/swe-bench-pro--ansible-claude-opus-4-8-claude-code.json deleted file mode 100644 index 6b587d9c4ae5c06acd7adf93468890ebda174125..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-claude-opus-4-8-claude-code.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Opus 4.8", - "repo": "Opus 4.8", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-opus-4-8" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.698, - "n_input_tokens": 192346997, - "n_cache_tokens": 186506482, - "n_output_tokens": 2179945, - "n_total_tokens": 381033424, - "agent_time_seconds": 32745, - "total_time_seconds": 39030, - "cost_usd": 185.66155285, - "mean_input_tokens_per_task": 2003614, - "mean_cache_tokens_per_task": 1942775, - "mean_output_tokens_per_task": 22707, - "mean_tokens_per_task": 3969098, - "mean_cost_usd_per_task": 1.93, - "mean_total_time_seconds_per_task": 406, - "mean_agent_time_seconds_per_task": 341 - } -} diff --git a/results/swe-bench-pro--ansible-claude-opus-4-8-opencode.json b/results/swe-bench-pro--ansible-claude-opus-4-8-opencode.json deleted file mode 100644 index 8326879f6399e2b48ba8adfe151663ebb599d87b..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-claude-opus-4-8-opencode.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Opus 4.8", - "repo": "Opus 4.8", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-opus-4-8" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.781, - "n_input_tokens": 187217712, - "n_cache_tokens": 187209844, - "n_output_tokens": 1280944, - "n_total_tokens": 375708500, - "agent_time_seconds": 30686, - "total_time_seconds": 39352, - "cost_usd": 151.4104807499999, - "mean_input_tokens_per_task": 1950184, - "mean_cache_tokens_per_task": 1950102, - "mean_output_tokens_per_task": 13343, - "mean_tokens_per_task": 3913630, - "mean_cost_usd_per_task": 1.58, - "mean_total_time_seconds_per_task": 409, - "mean_agent_time_seconds_per_task": 319 - } -} diff --git a/results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json b/results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json deleted file mode 100644 index 308bd0651e53511e4fc2616161c699f8db93ff2a..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-claude-sonnet-4-6-claude-code.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Sonnet 4.6", - "repo": "Sonnet 4.6", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-sonnet-4-6" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 1, - "score": 0.5, - "n_input_tokens": 190672390, - "n_cache_tokens": 184409111, - "n_output_tokens": 1593112, - "n_total_tokens": 376674613, - "agent_time_seconds": 40527, - "total_time_seconds": 49734, - "cost_usd": 184.42824125000004, - "mean_input_tokens_per_task": 1986170, - "mean_cache_tokens_per_task": 1920928, - "mean_output_tokens_per_task": 16594, - "mean_tokens_per_task": 3923693, - "mean_cost_usd_per_task": 1.92, - "mean_total_time_seconds_per_task": 518, - "mean_agent_time_seconds_per_task": 422 - } -} diff --git a/results/swe-bench-pro--ansible-gemma4-31b-fp8-claude-code.json b/results/swe-bench-pro--ansible-gemma4-31b-fp8-claude-code.json deleted file mode 100644 index 6594b3002848b13f7c090d8ec717a9027ea687af..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gemma4-31b-fp8-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Pro" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 16, - "score": 0.417, - "n_input_tokens": 122366824, - "n_cache_tokens": 0, - "n_output_tokens": 1307027, - "n_total_tokens": 123673851, - "agent_time_seconds": 138148, - "total_time_seconds": 158812, - "cost_usd": 54.82, - "mean_input_tokens_per_task": 1274654, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 13614, - "mean_tokens_per_task": 1288269, - "mean_cost_usd_per_task": 0.57, - "mean_total_time_seconds_per_task": 1654, - "mean_agent_time_seconds_per_task": 1439 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-gemma4-31b-fp8-opencode.json b/results/swe-bench-pro--ansible-gemma4-31b-fp8-opencode.json deleted file mode 100644 index 2fd4bd2ad8cbde1815094a7cc2298776c5714d60..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gemma4-31b-fp8-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Pro" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 12, - "score": 0.417, - "n_input_tokens": 100068358, - "n_cache_tokens": 0, - "n_output_tokens": 1261237, - "n_total_tokens": 101329595, - "agent_time_seconds": 60747, - "total_time_seconds": 77623, - "cost_usd": 19.28, - "mean_input_tokens_per_task": 1042378, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 13137, - "mean_tokens_per_task": 1055516, - "mean_cost_usd_per_task": 0.2, - "mean_total_time_seconds_per_task": 808, - "mean_agent_time_seconds_per_task": 632 - } -} diff --git a/results/swe-bench-pro--ansible-gemma4-31b-fp8-pi.json b/results/swe-bench-pro--ansible-gemma4-31b-fp8-pi.json deleted file mode 100644 index f00cf700d556c0b0b18f7f1919f9554f4d280af4..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gemma4-31b-fp8-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Pro" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 2, - "score": 0.469, - "n_input_tokens": 78678324, - "n_cache_tokens": 0, - "n_output_tokens": 1370430, - "n_total_tokens": 80048754, - "agent_time_seconds": 51767, - "total_time_seconds": 66845, - "cost_usd": 16.43, - "mean_input_tokens_per_task": 819565, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 14275, - "mean_tokens_per_task": 833841, - "mean_cost_usd_per_task": 0.17, - "mean_total_time_seconds_per_task": 696, - "mean_agent_time_seconds_per_task": 539 - } -} diff --git a/results/swe-bench-pro--ansible-gpt-5-5-codex.json b/results/swe-bench-pro--ansible-gpt-5-5-codex.json deleted file mode 100644 index 8c7485c476f540076722c8ce3db2362d38ca09f2..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gpt-5-5-codex.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Codex", - "skills": [], - "is_oss": true, - "url": "https://github.com/openai/codex" - }, - "model": { - "name": "GPT 5.5 - high", - "repo": "GPT 5.5 - high", - "is_oss": false, - "num_params": 9700, - "precision": "bf16", - "url": "https://openai.com/index/introducing-gpt-5-5" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.604, - "n_input_tokens": 198924339, - "n_cache_tokens": 189578624, - "n_output_tokens": 1560836, - "n_total_tokens": 390063799, - "agent_time_seconds": 32914, - "total_time_seconds": 39468, - "cost_usd": 188.34296700000002, - "mean_input_tokens_per_task": 2072128, - "mean_cache_tokens_per_task": 1974777, - "mean_output_tokens_per_task": 16258, - "mean_tokens_per_task": 4063164, - "mean_cost_usd_per_task": 1.96, - "mean_total_time_seconds_per_task": 411, - "mean_agent_time_seconds_per_task": 342 - } -} diff --git a/results/swe-bench-pro--ansible-gpt-oss-120b-claude-code.json b/results/swe-bench-pro--ansible-gpt-oss-120b-claude-code.json deleted file mode 100644 index 84587f81ebca4b2a412440bb9400de2a349b8997..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gpt-oss-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://docs.anthropic.com/en/docs/claude-code" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 2, - "score": 0.375, - "n_input_tokens": 347744043, - "n_cache_tokens": 0, - "n_output_tokens": 1903023, - "n_total_tokens": 349647066, - "agent_time_seconds": 59653, - "total_time_seconds": 67586, - "cost_usd": 2.25, - "mean_input_tokens_per_task": 3622333, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 19823, - "mean_tokens_per_task": 3642156, - "mean_cost_usd_per_task": 0.02, - "mean_total_time_seconds_per_task": 704, - "mean_agent_time_seconds_per_task": 621 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-gpt-oss-120b-opencode.json b/results/swe-bench-pro--ansible-gpt-oss-120b-opencode.json deleted file mode 100644 index d278255d356114a65574566fb939c13a54c58ff3..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gpt-oss-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 1, - "score": 0.333, - "n_input_tokens": 112041538, - "n_cache_tokens": 0, - "n_output_tokens": 781173, - "n_total_tokens": 112822711, - "agent_time_seconds": 25086, - "total_time_seconds": 31180, - "cost_usd": 0.95, - "mean_input_tokens_per_task": 1167099, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 8137, - "mean_tokens_per_task": 1175236, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 324, - "mean_agent_time_seconds_per_task": 261 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-gpt-oss-120b-pi.json b/results/swe-bench-pro--ansible-gpt-oss-120b-pi.json deleted file mode 100644 index 621d798192745d82af0cc1e4c29177af3e65737b..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-gpt-oss-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 15, - "score": 0.292, - "n_input_tokens": 125247664, - "n_cache_tokens": 0, - "n_output_tokens": 597435, - "n_total_tokens": 125845099, - "agent_time_seconds": 75715, - "total_time_seconds": 79741, - "cost_usd": 2.86, - "mean_input_tokens_per_task": 1304663, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 6223, - "mean_tokens_per_task": 1310886, - "mean_cost_usd_per_task": 0.03, - "mean_total_time_seconds_per_task": 830, - "mean_agent_time_seconds_per_task": 788 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-claude-code.json b/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-claude-code.json deleted file mode 100644 index 66d3db013dd3e1800050193c26b350af60d7dd44..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 1, - "score": 0.26, - "n_input_tokens": 147452235, - "n_cache_tokens": 0, - "n_output_tokens": 953241, - "n_total_tokens": 148405476, - "agent_time_seconds": 33613, - "total_time_seconds": 55241, - "cost_usd": 5.84, - "mean_input_tokens_per_task": 1535960, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 9929, - "mean_tokens_per_task": 1545890, - "mean_cost_usd_per_task": 0.06, - "mean_total_time_seconds_per_task": 575, - "mean_agent_time_seconds_per_task": 350 - } -} diff --git a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-opencode.json b/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-opencode.json deleted file mode 100644 index e4545e1f258b4195176211fca0fa3a6836d9981e..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-opencode.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.385, - "n_input_tokens": 140362798, - "n_cache_tokens": 0, - "n_output_tokens": 1230697, - "n_total_tokens": 141593495, - "agent_time_seconds": 23543, - "total_time_seconds": 49289, - "cost_usd": 6.54, - "mean_input_tokens_per_task": 1462112, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 12819, - "mean_tokens_per_task": 1474932, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 513, - "mean_agent_time_seconds_per_task": 245 - } -} diff --git a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-pi.json b/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-pi.json deleted file mode 100644 index b458d3214b893909c24aa25970a795b035e89e3c..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-mistral-small-4-119b-nvfp4-pi.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.333, - "n_input_tokens": 104994429, - "n_cache_tokens": 0, - "n_output_tokens": 1095179, - "n_total_tokens": 106089608, - "agent_time_seconds": 15004, - "total_time_seconds": 55176, - "cost_usd": 4.17, - "mean_input_tokens_per_task": 1093691, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 11408, - "mean_tokens_per_task": 1105100, - "mean_cost_usd_per_task": 0.04, - "mean_total_time_seconds_per_task": 574, - "mean_agent_time_seconds_per_task": 156 - } -} diff --git a/results/swe-bench-pro--ansible-nemotron-120b-claude-code.json b/results/swe-bench-pro--ansible-nemotron-120b-claude-code.json deleted file mode 100644 index 0b64849954049135c8925f46ac514e066f102aa4..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-nemotron-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.432, - "n_input_tokens": 406026150, - "n_cache_tokens": 0, - "n_output_tokens": 2839236, - "n_total_tokens": 408865386, - "agent_time_seconds": 100355, - "total_time_seconds": 115220, - "cost_usd": 26.44, - "mean_input_tokens_per_task": 4229439, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 29575, - "mean_tokens_per_task": 4259014, - "mean_cost_usd_per_task": 0.28, - "mean_total_time_seconds_per_task": 1200, - "mean_agent_time_seconds_per_task": 1045 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-nemotron-120b-opencode.json b/results/swe-bench-pro--ansible-nemotron-120b-opencode.json deleted file mode 100644 index 30df7d1f6e8316239a7331fc4513b1393a534fe9..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-nemotron-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 4, - "score": 0.323, - "n_input_tokens": 686392098, - "n_cache_tokens": 0, - "n_output_tokens": 2915876, - "n_total_tokens": 689307974, - "agent_time_seconds": 86130, - "total_time_seconds": 99705, - "cost_usd": 19.45, - "mean_input_tokens_per_task": 7149917, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 30373, - "mean_tokens_per_task": 7180291, - "mean_cost_usd_per_task": 0.2, - "mean_total_time_seconds_per_task": 1038, - "mean_agent_time_seconds_per_task": 897 - } -} diff --git a/results/swe-bench-pro--ansible-nemotron-120b-pi.json b/results/swe-bench-pro--ansible-nemotron-120b-pi.json deleted file mode 100644 index 39ebbd7f4bb0c9f2199aee038f980e12c2e63c2e..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-nemotron-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 14, - "score": 0.375, - "n_input_tokens": 1051387065, - "n_cache_tokens": 0, - "n_output_tokens": 3651367, - "n_total_tokens": 1055038432, - "agent_time_seconds": 113768, - "total_time_seconds": 127326, - "cost_usd": 25.69, - "mean_input_tokens_per_task": 10951948, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 38035, - "mean_tokens_per_task": 10989983, - "mean_cost_usd_per_task": 0.27, - "mean_total_time_seconds_per_task": 1326, - "mean_agent_time_seconds_per_task": 1185 - } -} diff --git a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-claude-code.json b/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-claude-code.json deleted file mode 100644 index fa3bec1b778cba41dd21f589a41d6929a06a1018..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 8, - "score": 0.521, - "n_input_tokens": 386929017, - "n_cache_tokens": 0, - "n_output_tokens": 1847087, - "n_total_tokens": 388776104, - "agent_time_seconds": 118693, - "total_time_seconds": 137736, - "cost_usd": 43.52, - "mean_input_tokens_per_task": 4030510, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 19240, - "mean_tokens_per_task": 4049751, - "mean_cost_usd_per_task": 0.45, - "mean_total_time_seconds_per_task": 1434, - "mean_agent_time_seconds_per_task": 1236 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-opencode.json b/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-opencode.json deleted file mode 100644 index 83d80203710eaf537c596755b8818c0aa9fcd349..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.573, - "n_input_tokens": 344853232, - "n_cache_tokens": 0, - "n_output_tokens": 1661197, - "n_total_tokens": 346514429, - "agent_time_seconds": 118356, - "total_time_seconds": 196235, - "cost_usd": 43.4, - "mean_input_tokens_per_task": 3592221, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 17304, - "mean_tokens_per_task": 3609525, - "mean_cost_usd_per_task": 0.45, - "mean_total_time_seconds_per_task": 2044, - "mean_agent_time_seconds_per_task": 1232 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-pi.json b/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-pi.json deleted file mode 100644 index 9aa033d1b611ccd8ab51d171c823e14a3e507d21..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-27b-fp8-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "scale-ai/swe-bench-pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/scale-ai/swe-bench-pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/plandex-ai/plandex" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-pro-ansible" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 0, - "score": 0.49, - "n_input_tokens": 395189021, - "n_cache_tokens": 0, - "n_output_tokens": 2206097, - "n_total_tokens": 397395118, - "agent_time_seconds": 80305, - "total_time_seconds": 91995, - "cost_usd": 29.45, - "mean_input_tokens_per_task": 4116552, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 22980, - "mean_tokens_per_task": 4139532, - "mean_cost_usd_per_task": 0.31, - "mean_total_time_seconds_per_task": 958, - "mean_agent_time_seconds_per_task": 836 - } -} \ No newline at end of file diff --git a/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json b/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json deleted file mode 100644 index 35d3122b2fb5a8ef7a507233fda34b7cf7b24c67..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-claude-code.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 6, - "score": 0.458, - "n_input_tokens": 367897697, - "n_cache_tokens": 0, - "n_output_tokens": 1694885, - "n_total_tokens": 369592582, - "agent_time_seconds": 39024, - "total_time_seconds": 46758, - "cost_usd": 9.64, - "mean_input_tokens_per_task": 3832267, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 17655, - "mean_tokens_per_task": 3849922, - "mean_cost_usd_per_task": 0.1, - "mean_total_time_seconds_per_task": 487, - "mean_agent_time_seconds_per_task": 406 - } -} diff --git a/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json b/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json deleted file mode 100644 index 62a464d2a58dffcea876e66b3a4b3d705a05f9a2..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-35b-nvfp4-opencode.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 4, - "score": 0.375, - "n_input_tokens": 207164679, - "n_cache_tokens": 0, - "n_output_tokens": 1598703, - "n_total_tokens": 208763382, - "agent_time_seconds": 49450, - "total_time_seconds": 57287, - "cost_usd": 12.21, - "mean_input_tokens_per_task": 2157965, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 16653, - "mean_tokens_per_task": 2174618, - "mean_cost_usd_per_task": 0.13, - "mean_total_time_seconds_per_task": 596, - "mean_agent_time_seconds_per_task": 515 - } -} diff --git a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-openclaw.json b/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-openclaw.json deleted file mode 100644 index 53bb5af578ae3eb4f49e552e7574ddb60f8f8aeb..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-openclaw.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/openclaw/openclaw" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 5, - "score": 0.406, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 38085, - "total_time_seconds": 50779, - "cost_usd": 9.4, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.1, - "mean_total_time_seconds_per_task": 528, - "mean_agent_time_seconds_per_task": 396 - } -} diff --git a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-pi.json b/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-pi.json deleted file mode 100644 index 79d0b3be80151adf63165ec4d8e14ac3c289a60c..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-pi.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 1, - "score": 0.479, - "n_input_tokens": 742491363, - "n_cache_tokens": 0, - "n_output_tokens": 2387609, - "n_total_tokens": 744878972, - "agent_time_seconds": 54543, - "total_time_seconds": 62422, - "cost_usd": 13.47, - "mean_input_tokens_per_task": 7734285, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 24870, - "mean_tokens_per_task": 7759155, - "mean_cost_usd_per_task": 0.14, - "mean_total_time_seconds_per_task": 650, - "mean_agent_time_seconds_per_task": 568 - } -} diff --git a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-qwen-code.json b/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-qwen-code.json deleted file mode 100644 index 386a6797f0ade5f6088a76ceb8d8924c3aeae493..0000000000000000000000000000000000000000 --- a/results/swe-bench-pro--ansible-qwen3-6-36b-nvfp4-qwen-code.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Pro -- Ansible", - "repo": "ScaleAI/SWE-bench_Pro", - "num_tasks": 96, - "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" - }, - "harness": { - "name": "Qwen Code", - "skills": [], - "is_oss": true, - "url": "https://github.com/QwenLM/qwen-code" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "scale-ai/swe-bench-pro", - "version": null, - "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": [ - "*ansible*" - ], - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 96, - "n_errors": 9, - "score": 0.438, - "n_input_tokens": 159198517, - "n_cache_tokens": 0, - "n_output_tokens": 972133, - "n_total_tokens": 160170650, - "agent_time_seconds": 33610, - "total_time_seconds": 38272, - "cost_usd": 9.34, - "mean_input_tokens_per_task": 1658317, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 10126, - "mean_tokens_per_task": 1668444, - "mean_cost_usd_per_task": 0.1, - "mean_total_time_seconds_per_task": 398, - "mean_agent_time_seconds_per_task": 350 - } -} diff --git a/results/swe-bench-verified-claude-opus-4-8-claude-code.json b/results/swe-bench-verified-claude-opus-4-8-claude-code.json deleted file mode 100644 index 95f94a839b938ab216852a24649c05a80fbb4a28..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-claude-opus-4-8-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Opus 4.8", - "repo": "Opus 4.8", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-opus-4-8" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 0, - "score": 0.868, - "n_input_tokens": 402225158, - "n_cache_tokens": 388781382, - "n_output_tokens": 4588859, - "n_total_tokens": 795595399, - "agent_time_seconds": 104467, - "total_time_seconds": 159566, - "cost_usd": 394.7823929, - "mean_input_tokens_per_task": 804450, - "mean_cache_tokens_per_task": 777562, - "mean_output_tokens_per_task": 9177, - "mean_tokens_per_task": 1591190, - "mean_cost_usd_per_task": 0.79, - "mean_total_time_seconds_per_task": 319, - "mean_agent_time_seconds_per_task": 208 - } -} diff --git a/results/swe-bench-verified-claude-opus-4-8-opencode.json b/results/swe-bench-verified-claude-opus-4-8-opencode.json deleted file mode 100644 index 3f7491a487d969ccb9f78b9efe1d5c0ce0d94f1d..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-claude-opus-4-8-opencode.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Opus 4.8", - "repo": "Opus 4.8", - "is_oss": false, - "num_params": 1000, - "precision": "bf16", - "url": "https://www.anthropic.com/news/claude-opus-4-8" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 0, - "score": 0.834, - "n_input_tokens": 347751942, - "n_cache_tokens": 347725246, - "n_output_tokens": 3640204, - "n_total_tokens": 699117392, - "agent_time_seconds": 114633, - "total_time_seconds": 171540, - "cost_usd": 319.53304049999974, - "mean_input_tokens_per_task": 695503, - "mean_cache_tokens_per_task": 695450, - "mean_output_tokens_per_task": 7280, - "mean_tokens_per_task": 1398234, - "mean_cost_usd_per_task": 0.64, - "mean_total_time_seconds_per_task": 343, - "mean_agent_time_seconds_per_task": 229 - } -} diff --git a/results/swe-bench-verified-gemma4-31b-fp8-claude-code.json b/results/swe-bench-verified-gemma4-31b-fp8-claude-code.json deleted file mode 100644 index 0465977973201478765a262cf2cebeceae85eb4f..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gemma4-31b-fp8-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 35, - "score": 0.612, - "n_input_tokens": 651306166, - "n_cache_tokens": 0, - "n_output_tokens": 4591859, - "n_total_tokens": 655898025, - "agent_time_seconds": 524857, - "total_time_seconds": 669355, - "cost_usd": 104.14, - "mean_input_tokens_per_task": 1302612, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 9183, - "mean_tokens_per_task": 1311796, - "mean_cost_usd_per_task": 0.21, - "mean_total_time_seconds_per_task": 1338, - "mean_agent_time_seconds_per_task": 1049 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-gemma4-31b-fp8-opencode.json b/results/swe-bench-verified-gemma4-31b-fp8-opencode.json deleted file mode 100644 index 011dd2ad2f07d78b362299897e06a28f4712ed54..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gemma4-31b-fp8-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 12, - "score": 0.606, - "n_input_tokens": 525563275, - "n_cache_tokens": 0, - "n_output_tokens": 4183675, - "n_total_tokens": 529746950, - "agent_time_seconds": 166092, - "total_time_seconds": 444527, - "cost_usd": 52.73, - "mean_input_tokens_per_task": 1051126, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 8367, - "mean_tokens_per_task": 1059493, - "mean_cost_usd_per_task": 0.11, - "mean_total_time_seconds_per_task": 889, - "mean_agent_time_seconds_per_task": 332 - } -} diff --git a/results/swe-bench-verified-gemma4-31b-fp8-pi.json b/results/swe-bench-verified-gemma4-31b-fp8-pi.json deleted file mode 100644 index d6e6bc2a31dd1b6c9008bcf558441cab0dffa7c1..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gemma4-31b-fp8-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Gemma4-31B-FP8", - "repo": "RedHatAI/gemma-4-31B-it-FP8-block", - "is_oss": true, - "num_params": 31, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block" - }, - "environment": { - "name": "harbor", - "config": { - "name": "SWE-bench_Verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 0, - "score": 0.574, - "n_input_tokens": 377479979, - "n_cache_tokens": 0, - "n_output_tokens": 4177117, - "n_total_tokens": 381657096, - "agent_time_seconds": 122274, - "total_time_seconds": 385336, - "cost_usd": 38.82, - "mean_input_tokens_per_task": 754959, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 8354, - "mean_tokens_per_task": 763314, - "mean_cost_usd_per_task": 0.08, - "mean_total_time_seconds_per_task": 770, - "mean_agent_time_seconds_per_task": 244 - } -} diff --git a/results/swe-bench-verified-gpt-5-5-codex.json b/results/swe-bench-verified-gpt-5-5-codex.json deleted file mode 100644 index 9a1e122e5da6b2b17b95b17bdbe35b484ed24b64..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gpt-5-5-codex.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Codex", - "skills": [], - "is_oss": true, - "url": "https://github.com/openai/codex" - }, - "model": { - "name": "GPT 5.5 - high", - "repo": "GPT 5.5 - high", - "is_oss": false, - "num_params": 9700, - "precision": "bf16", - "url": "https://openai.com/index/introducing-gpt-5-5" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 0, - "score": 0.798, - "n_input_tokens": 415795756, - "n_cache_tokens": 386352384, - "n_output_tokens": 3431224, - "n_total_tokens": 805579364, - "agent_time_seconds": 92588, - "total_time_seconds": 141637, - "cost_usd": 443.32977200000016, - "mean_input_tokens_per_task": 831591, - "mean_cache_tokens_per_task": 772704, - "mean_output_tokens_per_task": 6862, - "mean_tokens_per_task": 1611158, - "mean_cost_usd_per_task": 0.89, - "mean_total_time_seconds_per_task": 283, - "mean_agent_time_seconds_per_task": 185 - } -} diff --git a/results/swe-bench-verified-gpt-oss-120b-claude-code.json b/results/swe-bench-verified-gpt-oss-120b-claude-code.json deleted file mode 100644 index 21fd356713443d79682e98eda6b3b764d75db655..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gpt-oss-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "swe-bench/swe-bench-verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://docs.anthropic.com/en/docs/claude-code" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 13, - "score": 0.376, - "n_input_tokens": 833651681, - "n_cache_tokens": 0, - "n_output_tokens": 5656173, - "n_total_tokens": 839307854, - "agent_time_seconds": 150981, - "total_time_seconds": 200547, - "cost_usd": 5.7, - "mean_input_tokens_per_task": 1667303, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 11312, - "mean_tokens_per_task": 1678615, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 401, - "mean_agent_time_seconds_per_task": 301 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-gpt-oss-120b-opencode.json b/results/swe-bench-verified-gpt-oss-120b-opencode.json deleted file mode 100644 index 59d52c2f263fa36a5493777d550946e5f551e27f..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gpt-oss-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "swe-bench/swe-bench-verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 33, - "score": 0.326, - "n_input_tokens": 202856940, - "n_cache_tokens": 0, - "n_output_tokens": 1444371, - "n_total_tokens": 204301311, - "agent_time_seconds": 35190, - "total_time_seconds": 322482, - "cost_usd": 3.32, - "mean_input_tokens_per_task": 405713, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 2888, - "mean_tokens_per_task": 408602, - "mean_cost_usd_per_task": 0.01, - "mean_total_time_seconds_per_task": 644, - "mean_agent_time_seconds_per_task": 70 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-gpt-oss-120b-pi.json b/results/swe-bench-verified-gpt-oss-120b-pi.json deleted file mode 100644 index 50fa2fe33d755e25b49d82959390f81ae18c86f5..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-gpt-oss-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "swe-bench/swe-bench-verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "GPT-OSS-120B", - "repo": "RedHatAI/gpt-oss-120b", - "is_oss": true, - "num_params": 120, - "precision": "mxfp4", - "url": "https://huggingface.co/RedHatAI/gpt-oss-120b" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swebench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 47, - "score": 0.234, - "n_input_tokens": 90107247, - "n_cache_tokens": 0, - "n_output_tokens": 905031, - "n_total_tokens": 91012278, - "agent_time_seconds": 146747, - "total_time_seconds": 176174, - "cost_usd": 13.86, - "mean_input_tokens_per_task": 180214, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 1810, - "mean_tokens_per_task": 182024, - "mean_cost_usd_per_task": 0.03, - "mean_total_time_seconds_per_task": 352, - "mean_agent_time_seconds_per_task": 293 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-claude-code.json b/results/swe-bench-verified-mistral-small-4-119b-nvfp4-claude-code.json deleted file mode 100644 index 39d9d8ffe3ab0dd10adec72f1b94d2afa570dd8c..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:b934b0cc3dc800fe945eaf9f1623329db97ee3133c706d20644524c7759fb341", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 53, - "score": 0.118, - "n_input_tokens": 309857096, - "n_cache_tokens": 0, - "n_output_tokens": 2416878, - "n_total_tokens": 312273974, - "agent_time_seconds": 54338, - "total_time_seconds": 555599, - "cost_usd": 9.43, - "mean_input_tokens_per_task": 619714, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 4833, - "mean_tokens_per_task": 624547, - "mean_cost_usd_per_task": 0.02, - "mean_total_time_seconds_per_task": 1111, - "mean_agent_time_seconds_per_task": 108 - } -} diff --git a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-opencode.json b/results/swe-bench-verified-mistral-small-4-119b-nvfp4-opencode.json deleted file mode 100644 index 2c9d097ceda311225a839be5243d997effc10aa9..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-opencode.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:b934b0cc3dc800fe945eaf9f1623329db97ee3133c706d20644524c7759fb341", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 6, - "score": 0.318, - "n_input_tokens": 373713878, - "n_cache_tokens": 0, - "n_output_tokens": 4051687, - "n_total_tokens": 377765565, - "agent_time_seconds": 49103, - "total_time_seconds": 543025, - "cost_usd": 13.64, - "mean_input_tokens_per_task": 747427, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 8103, - "mean_tokens_per_task": 755531, - "mean_cost_usd_per_task": 0.03, - "mean_total_time_seconds_per_task": 1086, - "mean_agent_time_seconds_per_task": 98 - } -} diff --git a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-pi.json b/results/swe-bench-verified-mistral-small-4-119b-nvfp4-pi.json deleted file mode 100644 index 541bdc7c6adb7dd4c8673b0d5a796ee295efc6c7..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-mistral-small-4-119b-nvfp4-pi.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Mistral-Small-4-119B-2603-NVFP4", - "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4", - "is_oss": true, - "num_params": 119, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:b934b0cc3dc800fe945eaf9f1623329db97ee3133c706d20644524c7759fb341", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 12, - "score": 0.232, - "n_input_tokens": 195246618, - "n_cache_tokens": 0, - "n_output_tokens": 2726988, - "n_total_tokens": 197973606, - "agent_time_seconds": 29236, - "total_time_seconds": 458591, - "cost_usd": 8.12, - "mean_input_tokens_per_task": 390493, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 5453, - "mean_tokens_per_task": 395947, - "mean_cost_usd_per_task": 0.02, - "mean_total_time_seconds_per_task": 917, - "mean_agent_time_seconds_per_task": 58 - } -} diff --git a/results/swe-bench-verified-nemotron-120b-claude-code.json b/results/swe-bench-verified-nemotron-120b-claude-code.json deleted file mode 100644 index abee82d422ea0f4a78fce9f00cbe30d8abc02002..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-nemotron-120b-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 0, - "score": 0.472, - "n_input_tokens": 882330271, - "n_cache_tokens": 0, - "n_output_tokens": 6689285, - "n_total_tokens": 889019556, - "agent_time_seconds": 174288, - "total_time_seconds": 410828, - "cost_usd": 45.91, - "mean_input_tokens_per_task": 1764660, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 13378, - "mean_tokens_per_task": 1778039, - "mean_cost_usd_per_task": 0.09, - "mean_total_time_seconds_per_task": 821, - "mean_agent_time_seconds_per_task": 348 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-nemotron-120b-opencode.json b/results/swe-bench-verified-nemotron-120b-opencode.json deleted file mode 100644 index ca6fecc59820a6494c59a15ff2f58e4554605bc7..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-nemotron-120b-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 1, - "score": 0.41, - "n_input_tokens": 724984042, - "n_cache_tokens": 0, - "n_output_tokens": 5941303, - "n_total_tokens": 730925345, - "agent_time_seconds": 93088, - "total_time_seconds": 518565, - "cost_usd": 21.02, - "mean_input_tokens_per_task": 1449968, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 11882, - "mean_tokens_per_task": 1461850, - "mean_cost_usd_per_task": 0.04, - "mean_total_time_seconds_per_task": 1037, - "mean_agent_time_seconds_per_task": 186 - } -} diff --git a/results/swe-bench-verified-nemotron-120b-pi.json b/results/swe-bench-verified-nemotron-120b-pi.json deleted file mode 100644 index 8fbd40ab44226773435e6d5cf7b5831bd107b8b4..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-nemotron-120b-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Nemotron-3-Super-120B-NVFP4", - "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - "is_oss": true, - "num_params": 120, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 6, - "score": 0.498, - "n_input_tokens": 996839122, - "n_cache_tokens": 0, - "n_output_tokens": 7196671, - "n_total_tokens": 1004035793, - "agent_time_seconds": 144084, - "total_time_seconds": 446845, - "cost_usd": 32.53, - "mean_input_tokens_per_task": 1993678, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 14393, - "mean_tokens_per_task": 2008071, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 893, - "mean_agent_time_seconds_per_task": 288 - } -} diff --git a/results/swe-bench-verified-qwen3-6-27b-fp8-claude-code.json b/results/swe-bench-verified-qwen3-6-27b-fp8-claude-code.json deleted file mode 100644 index abd30b5a4db546ec7704f100fac7538df59e67f5..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-27b-fp8-claude-code.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 3, - "score": 0.694, - "n_input_tokens": 1005844088, - "n_cache_tokens": 0, - "n_output_tokens": 6255800, - "n_total_tokens": 1012099888, - "agent_time_seconds": 228858, - "total_time_seconds": 411348, - "cost_usd": 83.91, - "mean_input_tokens_per_task": 2011688, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 12511, - "mean_tokens_per_task": 2024199, - "mean_cost_usd_per_task": 0.17, - "mean_total_time_seconds_per_task": 822, - "mean_agent_time_seconds_per_task": 457 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-qwen3-6-27b-fp8-opencode.json b/results/swe-bench-verified-qwen3-6-27b-fp8-opencode.json deleted file mode 100644 index 82cbdcc31b9d912dc60b2636e1108096119d8261..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-27b-fp8-opencode.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/opencode-ai/opencode" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 8, - "score": 0.642, - "n_input_tokens": 395291929, - "n_cache_tokens": 0, - "n_output_tokens": 3296668, - "n_total_tokens": 398588597, - "agent_time_seconds": 92700, - "total_time_seconds": 374913, - "cost_usd": 33.99, - "mean_input_tokens_per_task": 790583, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 6593, - "mean_tokens_per_task": 797177, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 749, - "mean_agent_time_seconds_per_task": 185 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-qwen3-6-27b-fp8-pi.json b/results/swe-bench-verified-qwen3-6-27b-fp8-pi.json deleted file mode 100644 index a64912b22aa813de5270409791b6851f707906d2..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-27b-fp8-pi.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/plandex-ai/plandex" - }, - "model": { - "name": "Qwen3.6-27B-FP8", - "repo": "RedHatAI/Qwen3.6-27B-FP8", - "is_oss": true, - "num_params": 27, - "precision": "fp8", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8" - }, - "environment": { - "name": "harbor", - "config": { - "name": "swe-bench-verified" - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 7, - "score": 0.694, - "n_input_tokens": 544070591, - "n_cache_tokens": 0, - "n_output_tokens": 4965652, - "n_total_tokens": 549036243, - "agent_time_seconds": 130591, - "total_time_seconds": 408706, - "cost_usd": 47.88, - "mean_input_tokens_per_task": 1088141, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 9931, - "mean_tokens_per_task": 1098072, - "mean_cost_usd_per_task": 0.1, - "mean_total_time_seconds_per_task": 817, - "mean_agent_time_seconds_per_task": 261 - } -} \ No newline at end of file diff --git a/results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json b/results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json deleted file mode 100644 index 69970631d62e42907630bd7078725e8d75d0373f..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-35b-nvfp4-claude-code.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Claude Code", - "skills": [], - "is_oss": false, - "url": "https://github.com/anthropics/claude-code" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 1, - "score": 0.632, - "n_input_tokens": 1106618897, - "n_cache_tokens": 0, - "n_output_tokens": 5733245, - "n_total_tokens": 1112352142, - "agent_time_seconds": 122808, - "total_time_seconds": 171897, - "cost_usd": 34.11, - "mean_input_tokens_per_task": 2213237, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 11466, - "mean_tokens_per_task": 2224704, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 343, - "mean_agent_time_seconds_per_task": 245 - } -} diff --git a/results/swe-bench-verified-qwen3-6-35b-nvfp4-openclaw.json b/results/swe-bench-verified-qwen3-6-35b-nvfp4-openclaw.json deleted file mode 100644 index 06892776680b937d11ef1f97f0d41e8b8a872643..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-35b-nvfp4-openclaw.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenClaw", - "skills": [], - "is_oss": true, - "url": "https://github.com/openclaw/openclaw" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null, - "accelerated_images": true - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 3, - "score": 0.588, - "n_input_tokens": 0, - "n_cache_tokens": 0, - "n_output_tokens": 0, - "n_total_tokens": 0, - "agent_time_seconds": 120399, - "total_time_seconds": 200354, - "cost_usd": 33.44, - "mean_input_tokens_per_task": 0, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 0, - "mean_tokens_per_task": 0, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 400, - "mean_agent_time_seconds_per_task": 240 - } -} diff --git a/results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json b/results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json deleted file mode 100644 index 4a9ef8c494c2528aca8f54e80450898d4fb67cfe..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-35b-nvfp4-opencode.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null, - "accelerated_images": true - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 4, - "score": 0.548, - "n_input_tokens": 469806650, - "n_cache_tokens": 0, - "n_output_tokens": 4937761, - "n_total_tokens": 474744411, - "agent_time_seconds": 120473, - "total_time_seconds": 185168, - "cost_usd": 29.75, - "mean_input_tokens_per_task": 939613, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 9875, - "mean_tokens_per_task": 949488, - "mean_cost_usd_per_task": 0.06, - "mean_total_time_seconds_per_task": 370, - "mean_agent_time_seconds_per_task": 240 - } -} diff --git a/results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json b/results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json deleted file mode 100644 index 652d374637c1873d1b693c6d07964636a924ee1b..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-36b-nvfp4-pi.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null, - "accelerated_images": true - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 6, - "score": 0.65, - "n_input_tokens": 791183735, - "n_cache_tokens": 0, - "n_output_tokens": 6333798, - "n_total_tokens": 797517533, - "agent_time_seconds": 154531, - "total_time_seconds": 218988, - "cost_usd": 38.16, - "mean_input_tokens_per_task": 1582367, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 12667, - "mean_tokens_per_task": 1595035, - "mean_cost_usd_per_task": 0.08, - "mean_total_time_seconds_per_task": 437, - "mean_agent_time_seconds_per_task": 309 - } -} diff --git a/results/swe-bench-verified-qwen3-6-36b-nvfp4-qwen-code.json b/results/swe-bench-verified-qwen3-6-36b-nvfp4-qwen-code.json deleted file mode 100644 index 852439488fc663a76ef4aad80bfecdeab4d76f40..0000000000000000000000000000000000000000 --- a/results/swe-bench-verified-qwen3-6-36b-nvfp4-qwen-code.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "benchmark": { - "name": "SWE-Bench Verified", - "repo": "SWE-bench/SWE-bench_Verified", - "num_tasks": 500, - "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" - }, - "harness": { - "name": "Qwen Code", - "skills": [], - "is_oss": true, - "url": "https://github.com/QwenLM/qwen-code" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "swe-bench/swe-bench-verified", - "version": null, - "ref": "sha256:235d6032d549851a936db3b5fe08807c4d385c12ee10e7be9c9786a1ff60563c", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null, - "accelerated_images": true - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 500, - "n_errors": 3, - "score": 0.638, - "n_input_tokens": 609589099, - "n_cache_tokens": 0, - "n_output_tokens": 3964252, - "n_total_tokens": 613553351, - "agent_time_seconds": 132273, - "total_time_seconds": 178984, - "cost_usd": 36.74, - "mean_input_tokens_per_task": 1219178, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 7928, - "mean_tokens_per_task": 1227106, - "mean_cost_usd_per_task": 0.07, - "mean_total_time_seconds_per_task": 357, - "mean_agent_time_seconds_per_task": 264 - } -} diff --git a/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-opencode.json b/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-opencode.json deleted file mode 100644 index 78ee639a9709c365e76e9e54db65de3c56c12c00..0000000000000000000000000000000000000000 --- a/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-opencode.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "Terminal Bench 2.0", - "repo": "terminal-bench/terminal-bench-2", - "num_tasks": 89, - "url": "https://www.tbench.ai/benchmarks/terminal-bench-2" - }, - "harness": { - "name": "OpenCode", - "skills": [], - "is_oss": true, - "url": "https://github.com/anomalyco/opencode" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "terminal-bench/terminal-bench-2", - "version": null, - "ref": "sha256:c6fc2e2382c1dbae99b2d5ecd2f4f4a60c3c01e0d84642d69b4afd92e99d078b", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 89, - "n_errors": 5, - "score": 0.303, - "n_input_tokens": 47607780, - "n_cache_tokens": 0, - "n_output_tokens": 1657188, - "n_total_tokens": 49264968, - "agent_time_seconds": 46467, - "total_time_seconds": 58507, - "cost_usd": 11.47, - "mean_input_tokens_per_task": 534918, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 18620, - "mean_tokens_per_task": 553538, - "mean_cost_usd_per_task": 0.13, - "mean_total_time_seconds_per_task": 657, - "mean_agent_time_seconds_per_task": 522 - } -} diff --git a/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-pi.json b/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-pi.json deleted file mode 100644 index dabd91c40bd2e9bada52a4673bc473db5faeb94f..0000000000000000000000000000000000000000 --- a/results/terminal-bench-2-0-qwen-3-6-36b-nvfp4-pi.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "benchmark": { - "name": "Terminal Bench 2.0", - "repo": "terminal-bench/terminal-bench-2", - "num_tasks": 89, - "url": "https://www.tbench.ai/benchmarks/terminal-bench-2" - }, - "harness": { - "name": "Pi", - "skills": [], - "is_oss": true, - "url": "https://github.com/earendil-works/pi/tree/main" - }, - "model": { - "name": "Qwen3.6-35B-A3B-NVFP4", - "repo": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", - "is_oss": true, - "num_params": 35, - "precision": "nvfp4", - "url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4" - }, - "environment": { - "name": "harbor", - "config": { - "path": null, - "name": "terminal-bench/terminal-bench-2", - "version": null, - "ref": "sha256:c6fc2e2382c1dbae99b2d5ecd2f4f4a60c3c01e0d84642d69b4afd92e99d078b", - "registry_url": null, - "registry_path": null, - "overwrite": false, - "download_dir": null, - "task_names": null, - "exclude_task_names": null, - "n_tasks": null - }, - "url": "https://github.com/harbor-framework/harbor" - }, - "metrics": { - "n_tasks": 89, - "n_errors": 5, - "score": 0.36, - "n_input_tokens": 82108716, - "n_cache_tokens": 0, - "n_output_tokens": 2056390, - "n_total_tokens": 84165106, - "agent_time_seconds": 44991, - "total_time_seconds": 64760, - "cost_usd": 11.11, - "mean_input_tokens_per_task": 922569, - "mean_cache_tokens_per_task": 0, - "mean_output_tokens_per_task": 23105, - "mean_tokens_per_task": 945675, - "mean_cost_usd_per_task": 0.12, - "mean_total_time_seconds_per_task": 727, - "mean_agent_time_seconds_per_task": 505 - } -} diff --git a/src/analytics.py b/src/analytics.py deleted file mode 100644 index d222d3a9feb3879ec06f6c2a158afbec6e9c8ce2..0000000000000000000000000000000000000000 --- a/src/analytics.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -import pandas as pd - - -BENCHMARK_CATALOG: dict[str, dict[str, object]] = { - "SWE-Bench Verified": { - "category": "Coding", - "capabilities": ["repository-repair", "software-engineering"], - }, - "SWE-Bench Pro -- Ansible": { - "category": "Coding", - "capabilities": ["repository-repair", "software-engineering", "ansible"], - }, - "RH SWE-Bench": { - "category": "Coding", - "capabilities": ["repository-repair", "software-engineering"], - }, - "Terminal Bench 2.0": { - "category": "Generalist", - "capabilities": ["shell", "tool-use"], - }, - "Shellbench": { - "category": "Generalist", - "capabilities": ["shell", "tool-use"], - }, -} - -DEFAULT_BENCHMARK_CATEGORY = "Other" - - -@dataclass(frozen=True) -class MetricSpec: - column: str - label: str - higher_is_better: bool - positive_only: bool = False - - -RANKING_METRICS: dict[str, MetricSpec] = { - "Score": MetricSpec("Score (%)", "Score (%)", True), - "Total tokens": MetricSpec("Total Tokens Per Task", "Total tokens per task", False, True), - "Input tokens": MetricSpec("Input Tokens Per Task", "Input tokens per task", False, True), - "Output tokens": MetricSpec("Output Tokens Per Task", "Output tokens per task", False, True), - "Cache tokens": MetricSpec("Cache Tokens Per Task", "Cache tokens per task", False, True), - "Cost": MetricSpec("Cost Per Task", "Cost per task (USD)", False, True), - "Response time": MetricSpec("Total Time Per Task", "Total time per task (seconds)", False, True), - "Agent time": MetricSpec("Agent Time Per Task", "Agent time per task (seconds)", False, True), - "Reliability": MetricSpec("Execution Error Rate (%)", "Execution error rate (%)", False), - "Tokens per successful task": MetricSpec("Tokens Per Successful Task", "Tokens per successful task", False, True), - "Cost per successful task": MetricSpec("Cost Per Successful Task", "Cost per successful task (USD)", False, True), - "Time per successful task": MetricSpec("Time Per Successful Task", "Time per successful task (seconds)", False, True), -} - - -TRADEOFF_METRICS: dict[str, MetricSpec] = { - "Score": RANKING_METRICS["Score"], - "Cost per task": RANKING_METRICS["Cost"], - "Total tokens per task": RANKING_METRICS["Total tokens"], - "Total time per task": RANKING_METRICS["Response time"], - "Agent time per task": RANKING_METRICS["Agent time"], - "Execution error rate": RANKING_METRICS["Reliability"], -} - - -MATRIX_METRICS: dict[str, MetricSpec] = { - "Score": RANKING_METRICS["Score"], - "Within-benchmark percentile": MetricSpec("Within-Benchmark Percentile", "Within-benchmark percentile", True), - "Within-benchmark rank": MetricSpec("Within-Benchmark Rank", "Within-benchmark rank", False), - "Total tokens": RANKING_METRICS["Total tokens"], - "Cost": RANKING_METRICS["Cost"], - "Total time": RANKING_METRICS["Response time"], - "Agent time": RANKING_METRICS["Agent time"], - "Execution error rate": RANKING_METRICS["Reliability"], -} - - -def benchmark_metadata(name: str) -> dict[str, object]: - metadata = BENCHMARK_CATALOG.get(name) - if metadata is not None: - return metadata - return {"category": DEFAULT_BENCHMARK_CATEGORY, "capabilities": []} - - -def benchmark_category(name: str) -> str: - return str(benchmark_metadata(name)["category"]) - - -def benchmarks_for_category(dataframe: pd.DataFrame, category: str) -> list[str]: - if dataframe is None or dataframe.empty or "Benchmark" not in dataframe: - return [] - names = sorted(str(value) for value in dataframe["Benchmark"].dropna().unique()) - return [name for name in names if benchmark_category(name) == category] - - -def enrich_analysis_df(dataframe: pd.DataFrame) -> pd.DataFrame: - """Add PR2 taxonomy, reliability, per-success and normalized performance fields.""" - if dataframe is None: - return pd.DataFrame() - df = dataframe.copy() - if df.empty: - for column in ( - "Benchmark Category", - "Execution Error Rate (%)", - "Tokens Per Successful Task", - "Cost Per Successful Task", - "Time Per Successful Task", - "Total Benchmark Cost", - "Within-Benchmark Rank", - "Within-Benchmark Percentile", - ): - if column not in df: - df[column] = pd.Series(dtype="float64" if column != "Benchmark Category" else "object") - return df - - df["Benchmark Category"] = df["Benchmark"].map(lambda value: benchmark_category(str(value))) - - tasks = pd.to_numeric(df.get("Tasks"), errors="coerce") - errors = pd.to_numeric(df.get("Errors"), errors="coerce") - valid_tasks = tasks.notna() & (tasks > 0) - df["Execution Error Rate (%)"] = (errors / tasks * 100).where(valid_tasks & errors.notna()) - - score_fraction = pd.to_numeric(df.get("Score"), errors="coerce") - successful = score_fraction.notna() & (score_fraction > 0) - for source, target in ( - ("Total Tokens Per Task", "Tokens Per Successful Task"), - ("Cost Per Task", "Cost Per Successful Task"), - ("Total Time Per Task", "Time Per Successful Task"), - ): - values = pd.to_numeric(df.get(source), errors="coerce") - df[target] = (values / score_fraction).where(successful & values.notna() & (values > 0)) - - cost = pd.to_numeric(df.get("Cost Per Task"), errors="coerce") - df["Total Benchmark Cost"] = (cost * tasks).where(cost.notna() & (cost > 0) & valid_tasks) - - scores = pd.to_numeric(df.get("Score (%)"), errors="coerce") - df["Within-Benchmark Rank"] = scores.groupby(df["Benchmark"]).rank(method="min", ascending=False) - - def percentile(group: pd.Series) -> pd.Series: - valid = group.dropna() - out = pd.Series(index=group.index, dtype=float) - if valid.empty: - return out - ranks = valid.rank(method="average", ascending=False) - if len(valid) == 1: - out.loc[valid.index] = 100.0 - else: - out.loc[valid.index] = 100.0 * (len(valid) - ranks) / (len(valid) - 1) - return out - - df["Within-Benchmark Percentile"] = scores.groupby(df["Benchmark"], group_keys=False).apply(percentile) - return df - - -def filter_category(dataframe: pd.DataFrame, category: str) -> pd.DataFrame: - df = enrich_analysis_df(dataframe) - return df[df["Benchmark Category"] == category].copy() - - -def ranking_df( - dataframe: pd.DataFrame, - metric: str, - benchmark: str | None = None, - category: str | None = None, -) -> pd.DataFrame: - df = enrich_analysis_df(dataframe) - spec = RANKING_METRICS[metric] - if category: - df = df[df["Benchmark Category"] == category] - if benchmark and benchmark != "All benchmarks": - df = df[df["Benchmark"] == benchmark] - elif benchmark == "All benchmarks": - if metric != "Score": - # Resource units can be compared across benchmarks, but rows remain per benchmark; - # do not silently aggregate them. - pass - else: - return cross_benchmark_ranking_df(df) - - values = pd.to_numeric(df[spec.column], errors="coerce") - valid = values.notna() - if spec.positive_only: - valid &= values > 0 - df = df.loc[valid].copy() - df[spec.column] = values.loc[valid] - columns = [ - "Model", "Harness", "Benchmark", "Benchmark Category", spec.column, - "Score (%)", "Execution Error Rate (%)" - ] - columns = list(dict.fromkeys(column for column in columns if column in df)) - return df.sort_values( - [spec.column, "Model", "Harness"], - ascending=[not spec.higher_is_better, True, True], - kind="mergesort", - )[columns].reset_index(drop=True) - - -def cross_benchmark_ranking_df( - dataframe: pd.DataFrame, - minimum_coverage: float | int = 0.5, -) -> pd.DataFrame: - """Aggregate within-benchmark percentiles without averaging incompatible raw scores.""" - df = enrich_analysis_df(dataframe) - if df.empty: - return pd.DataFrame(columns=[ - "Model", "Harness", "Normalized Performance", "Benchmarks Covered", - "Eligible Benchmarks", "Coverage (%)", - ]) - eligible = int(df["Benchmark"].nunique()) - grouped = ( - df.dropna(subset=["Within-Benchmark Percentile"]) - .groupby(["Model", "Harness"], as_index=False) - .agg( - **{ - "Normalized Performance": ("Within-Benchmark Percentile", "mean"), - "Benchmarks Covered": ("Benchmark", "nunique"), - } - ) - ) - grouped["Eligible Benchmarks"] = eligible - grouped["Coverage (%)"] = grouped["Benchmarks Covered"] / eligible * 100 if eligible else 0.0 - - if isinstance(minimum_coverage, float) and minimum_coverage <= 1: - threshold_pct = minimum_coverage * 100 - grouped = grouped[grouped["Coverage (%)"] >= threshold_pct] - else: - grouped = grouped[grouped["Benchmarks Covered"] >= int(minimum_coverage)] - return grouped.sort_values( - ["Normalized Performance", "Benchmarks Covered", "Model", "Harness"], - ascending=[False, False, True, True], - kind="mergesort", - ).reset_index(drop=True) - - -def matrix_df( - dataframe: pd.DataFrame, - metric: str, - category: str | None = None, - include_incomplete: bool = True, - sort_by: str = "Normalized performance", -) -> pd.DataFrame: - df = enrich_analysis_df(dataframe) - if category: - df = df[df["Benchmark Category"] == category] - if df.empty: - return pd.DataFrame() - - df["Agent"] = df["Model"].astype(str) + " / " + df["Harness"].astype(str) - benchmarks = sorted(df["Benchmark"].dropna().unique()) - agents = sorted(df["Agent"].dropna().unique()) - - if metric == "Coverage": - available = df.assign(_coverage=1).pivot_table( - index="Agent", columns="Benchmark", values="_coverage", aggfunc="max" - ) - matrix = available.reindex(index=agents, columns=benchmarks) - else: - spec = MATRIX_METRICS[metric] - values = pd.to_numeric(df[spec.column], errors="coerce") - work = df.assign(_value=values) - matrix = work.pivot_table(index="Agent", columns="Benchmark", values="_value", aggfunc="mean") - matrix = matrix.reindex(index=agents, columns=benchmarks) - - if not include_incomplete: - matrix = matrix.dropna(axis=0, how="any") - - if matrix.empty: - return matrix - - if sort_by in {"Coverage", "Coverage (high to low)", "Coverage (low to high)"}: - ascending = sort_by == "Coverage (low to high)" - coverage = matrix.notna().sum(axis=1) - order = coverage.sort_values(ascending=ascending, kind="mergesort").index - matrix = matrix.loc[order] - elif sort_by in {"Normalized performance", "Normalized performance (high to low)", "Normalized performance (low to high)"}: - perf = cross_benchmark_ranking_df(df, minimum_coverage=0) - perf["Agent"] = perf["Model"] + " / " + perf["Harness"] - perf = perf.sort_values( - ["Normalized Performance", "Agent"], - ascending=[sort_by == "Normalized performance (low to high)", True], - kind="mergesort", - ) - order = [agent for agent in perf["Agent"] if agent in matrix.index] - order += [agent for agent in matrix.index if agent not in order] - matrix = matrix.loc[order] - elif sort_by == "Alphabetical (Z–A)": - matrix = matrix.loc[sorted(matrix.index, reverse=True)] - elif sort_by in {"Alphabetical", "Stable"}: - matrix = matrix.loc[sorted(matrix.index)] - return matrix - - -def coverage_summary(dataframe: pd.DataFrame) -> dict[str, float | int]: - df = enrich_analysis_df(dataframe) - total = len(df) - def pct(column: str, positive: bool = False) -> float: - if total == 0: - return 0.0 - values = pd.to_numeric(df[column], errors="coerce") - mask = values.notna() - if positive: - mask &= values > 0 - return float(mask.mean() * 100) - return { - "results": total, - "models": int(df["Model"].nunique()) if total else 0, - "harnesses": int(df["Harness"].nunique()) if total else 0, - "benchmarks": int(df["Benchmark"].nunique()) if total else 0, - "token_coverage_pct": pct("Total Tokens Per Task", True), - "cost_coverage_pct": pct("Cost Per Task", True), - "time_coverage_pct": pct("Total Time Per Task", True), - } diff --git a/src/charts.py b/src/charts.py deleted file mode 100644 index 2b33ffc43f006aadc3efa4718101770a79c2121c..0000000000000000000000000000000000000000 --- a/src/charts.py +++ /dev/null @@ -1,971 +0,0 @@ -from __future__ import annotations - -import hashlib -import re -from typing import Literal - -import pandas as pd -import plotly.colors as pc -import plotly.graph_objects as go -from plotly.graph_objs._figure import Figure - -ColorBy = Literal["Model", "Harness"] -PaletteName = Literal[ - "Citrus", - "Okabe-Ito", - "High contrast", - "Rainbow", - "Grayscale", - "Viridis", - "Plasma", - "Cividis", -] -PlotBackground = Literal["Dark", "White"] -DEFAULT_PALETTE: PaletteName = "Citrus" -DEFAULT_BACKGROUND: PlotBackground = "Dark" - -RANKING_MIN_HEIGHT_PX = 340 -RANKING_ROW_HEIGHT_PX = 22 -RANKING_VERTICAL_PADDING_PX = 280 -MATRIX_MIN_HEIGHT_PX = 720 -MATRIX_ROW_HEIGHT_PX = 36 -MATRIX_VERTICAL_PADDING_PX = 260 - -# Separate categorical palettes for each grouping dimension. -# Model and harness colors intentionally start from different hue families so -# switching "Color by" remains visually obvious. -MODEL_COLORS: dict[str, str] = { - "GPT 5.5 - high": "#F8FAFC", # white - "Opus 4.8": "#FEF3C7", # cream - "RedHatAI/Qwen3.6-35B-A3B-NVFP4": "#F97316", # orange - "Sonnet 4.6": "#DC2626", # red -} - -HARNESS_COLORS: dict[str, str] = { - "Claude Code": "#06B6D4", # cyan - "Codex": "#3B82F6", # blue - "OpenCode": "#8B5CF6", # violet - "OpenClaw": "#EC4899", # pink - "Pi": "#14B8A6", # teal - "Qwen Code": "#F43F5E", # rose - "internal": "#94A3B8", -} - -MODEL_FALLBACK_PALETTE = [ - "#F8FAFC", # white - "#FEF3C7", # cream - "#FACC15", # yellow - "#FB923C", # orange - "#DC2626", # red - "#93C5FD", # blue fallback - "#22C55E", # green fallback - "#C084FC", # violet fallback - "#F472B6", # pink fallback - "#14B8A6", # teal fallback -] - -HARNESS_FALLBACK_PALETTE = [ - "#06B6D4", # cyan - "#3B82F6", # blue - "#8B5CF6", # violet - "#EC4899", # pink - "#14B8A6", # teal - "#F43F5E", # rose - "#6366F1", # indigo - "#10B981", # emerald - "#A855F7", # purple - "#94A3B8", # slate -] - -DARK_PAPER = "#15110F" -DARK_PLOT = "#1F1A17" -DARK_CARD = "#27211E" -TEXT_PRIMARY = "#F8FAFC" -TEXT_MUTED = "#CBD5E1" -GRID_COLOR = "rgba(248,250,252,0.14)" -ZERO_LINE_COLOR = "rgba(248,250,252,0.24)" - -PLOT_BACKGROUNDS: dict[PlotBackground, dict[str, str]] = { - "Dark": { - "template": "plotly_dark", - "paper_bgcolor": DARK_CARD, - "plot_bgcolor": DARK_PLOT, - "text_primary": TEXT_PRIMARY, - "text_muted": TEXT_MUTED, - "grid_color": GRID_COLOR, - "zero_line_color": ZERO_LINE_COLOR, - "marker_line_color": DARK_PAPER, - }, - "White": { - "template": "plotly_white", - "paper_bgcolor": "#FFFFFF", - "plot_bgcolor": "#FFFFFF", - "text_primary": "#0F172A", - "text_muted": "#475569", - "grid_color": "rgba(15,23,42,0.12)", - "zero_line_color": "rgba(15,23,42,0.25)", - "marker_line_color": "#334155", - }, -} - - - -def clean_markdown_link(value: object) -> str: - """Return human-readable text from Markdown links used in leaderboard tables.""" - text = str(value).replace("*", "") - match = re.match(r"\[(.*?)\]\((.*?)\)", text) - if match: - return match.group(1) - return text - - -COLOR_PALETTES: dict[PaletteName, list[str]] = { - "Citrus": MODEL_FALLBACK_PALETTE, - "Okabe-Ito": [ - "#E69F00", - "#56B4E9", - "#009E73", - "#F0E442", - "#0072B2", - "#D55E00", - "#CC79A7", - "#999999", - ], - "High contrast": ["#FFD166", "#06D6A0", "#118AB2", "#EF476F", "#A78BFA", "#F97316", "#22D3EE", "#E5E7EB"], - "Rainbow": ["#E6194B", "#F58231", "#FFE119", "#3CB44B", "#42D4F4", "#4363D8", "#911EB4", "#F032E6", "#469990", "#9A6324"], - # Near-white and near-black endpoints remain visible against both supported backgrounds. - "Grayscale": ["#E2E8F0", "#CBD5E1", "#94A3B8", "#64748B", "#475569", "#334155", "#1E293B", "#111827"], - "Viridis": list(pc.sequential.Viridis), - "Plasma": list(pc.sequential.Plasma), - "Cividis": list(pc.sequential.Cividis), -} - -# Harness categories use the same central registry, with the default palette retaining -# its established cyan/blue/violet identity. -HARNESS_PALETTES: dict[PaletteName, list[str]] = { - **COLOR_PALETTES, - "Citrus": HARNESS_FALLBACK_PALETTE, -} -MODEL_PALETTES = COLOR_PALETTES - - -def get_color_palette(name: str | None) -> list[str]: - """Return a copy of the requested palette, falling back to Citrus.""" - palette_name = normalize_palette_name(name) - return list(COLOR_PALETTES[palette_name]) - - -def normalize_palette_name(palette_name: str | None) -> PaletteName: - if palette_name in MODEL_PALETTES: - return palette_name # type: ignore[return-value] - return DEFAULT_PALETTE - - -def normalize_background_name(background_name: str | None) -> PlotBackground: - if background_name == "Current": - return "Dark" - if background_name in PLOT_BACKGROUNDS: - return background_name # type: ignore[return-value] - return DEFAULT_BACKGROUND - - -def get_plot_background(background_name: str | None = DEFAULT_BACKGROUND) -> dict[str, str]: - return PLOT_BACKGROUNDS[normalize_background_name(background_name)] - - -def stable_color(name: str, color_by: ColorBy, palette_name: str | None = DEFAULT_PALETTE) -> str: - palette_key = normalize_palette_name(palette_name) - palettes = MODEL_PALETTES if color_by == "Model" else HARNESS_PALETTES - palette = palettes[palette_key] - digest = hashlib.sha256(f"{palette_key}:{color_by}:{name}".encode("utf-8")).hexdigest() - return palette[int(digest[:8], 16) % len(palette)] - - -def get_color(name: str, color_by: ColorBy, palette_name: str | None = DEFAULT_PALETTE) -> str: - palette_key = normalize_palette_name(palette_name) - if palette_key == "Citrus": - palette = MODEL_COLORS if color_by == "Model" else HARNESS_COLORS - if name in palette: - return palette[name] - return stable_color(name, color_by, palette_key) - - -def palette_colors_for(color_by: ColorBy, palette_name: str | None = DEFAULT_PALETTE) -> list[str]: - palette_key = normalize_palette_name(palette_name) - palettes = MODEL_PALETTES if color_by == "Model" else HARNESS_PALETTES - return list(palettes[palette_key]) - - -def color_map_for( - values: pd.Series, - color_by: ColorBy, - palette_name: str | None = DEFAULT_PALETTE, -) -> dict[str, str]: - unique_values = [str(value) for value in sorted(values.dropna().unique())] - palette_key = normalize_palette_name(palette_name) - - # For the default Citrus palette, preserve hand-picked colors for known labels. - # Unknown labels still get sequential fallback colors to avoid hash collisions. - if palette_key == "Citrus": - named_colors = MODEL_COLORS if color_by == "Model" else HARNESS_COLORS - fallback_colors = palette_colors_for(color_by, palette_key) - color_map: dict[str, str] = {} - fallback_index = 0 - for value in unique_values: - if value in named_colors: - color_map[value] = named_colors[value] - else: - color_map[value] = fallback_colors[fallback_index % len(fallback_colors)] - fallback_index += 1 - return color_map - - # Non-default palettes are assigned sequentially rather than by hash. Hashing can - # map multiple visible categories to the same color, which made the high-contrast - # harness palette look like only gray/blue/purple buckets. - palette = palette_colors_for(color_by, palette_key) - return { - value: palette[index % len(palette)] - for index, value in enumerate(unique_values) - } - - -def empty_figure(message: str, background_name: str | None = DEFAULT_BACKGROUND) -> Figure: - theme = get_plot_background(background_name) - fig = go.Figure() - fig.add_annotation( - text=message, - showarrow=False, - x=0.5, - y=0.5, - xref="paper", - yref="paper", - font={"size": 14, "color": theme["text_muted"]}, - ) - return apply_plot_theme(fig, background_name) - - -def apply_plot_theme(fig: Figure, background_name: str | None = DEFAULT_BACKGROUND) -> Figure: - theme = get_plot_background(background_name) - fig.update_layout( - template=theme["template"], - autosize=True, - paper_bgcolor=theme["paper_bgcolor"], - plot_bgcolor=theme["plot_bgcolor"], - font={"color": theme["text_primary"]}, - title={"font": {"color": theme["text_primary"]}}, - showlegend=True, - margin={"t": 60, "b": 0, "l": 0, "r": 0}, - legend={ - "orientation": "h", - "yanchor": "top", - "y": 1, - "yref": "container", - "xanchor": "center", - "x": 0.5, - "font": {"color": theme["text_muted"]}, - "itemclick": False, - "itemdoubleclick": False, - }, - ) - # Width remains responsive. Preserve any explicit height set by dense - # categorical charts so Plotly has enough vertical room for every label. - fig.update_layout(width=None) - fig.update_xaxes( - automargin=True, - color=theme["text_muted"], - gridcolor=theme["grid_color"], - zerolinecolor=theme["zero_line_color"], - linecolor=theme["grid_color"], - title_font={"color": theme["text_muted"]}, - tickfont={"color": theme["text_muted"]}, - ) - fig.update_yaxes( - automargin=True, - color=theme["text_muted"], - gridcolor=theme["grid_color"], - zerolinecolor=theme["zero_line_color"], - linecolor=theme["grid_color"], - title_font={"color": theme["text_muted"]}, - tickfont={"color": theme["text_muted"]}, - ) - return fig - -def prepare_benchmark_run_plot_df(dataframe: pd.DataFrame) -> pd.DataFrame: - plot_df = dataframe.copy() - plot_df["Model Label"] = plot_df["Model"].map(clean_markdown_link) - plot_df["Harness Label"] = plot_df["Harness"].map(clean_markdown_link) - plot_df["Benchmark Label"] = plot_df["Benchmark"].map(clean_markdown_link) - plot_df["Run Label"] = plot_df["Model Label"] + "
" + plot_df["Harness Label"] - plot_df["Score"] = pd.to_numeric(plot_df["Score"], errors="coerce") - return plot_df - - -def create_leaderboard_benchmark_plot( - dataframe: pd.DataFrame, - benchmark_name: str, - color_by: ColorBy = "Model", - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - if dataframe is None or dataframe.empty: - return empty_figure("No benchmark data available.", background_name) - - plot_df = prepare_benchmark_run_plot_df(dataframe) - plot_df = plot_df[plot_df["Benchmark Label"] == benchmark_name].dropna(subset=["Score"]) - plot_df = plot_df.sort_values("Score", ascending=False) - - if plot_df.empty: - return empty_figure(f"No results available for {benchmark_name}.", background_name) - - color_source = "Model Label" if color_by == "Model" else "Harness Label" - colors = color_map_for(plot_df[color_source], color_by, palette_name) - theme = get_plot_background(background_name) - fig = go.Figure() - - for group, group_df in plot_df.groupby(color_source, sort=True): - fig.add_trace( - go.Bar( - x=group_df["Run Label"], - y=group_df["Score"], - name=str(group), - marker={ - "color": colors[str(group)], - "line": {"width": 1, "color": theme["marker_line_color"]}, - }, - text=group_df["Score"].map(lambda score: f"{score:.1f}"), - textposition="outside", - customdata=group_df[["Model Label", "Harness Label", "Score"]], - hovertemplate=( - "%{customdata[0]}
" - "Harness: %{customdata[1]}
" - "Score: %{customdata[2]:.1f}%" - "" - ), - ) - ) - - fig.update_layout( - title=None, - xaxis={"title": "Model / Harness", "categoryorder": "total descending"}, - yaxis={"title": "Score (%)", "range": [0, plot_df["Score"].max() * 1.12]}, - legend_title_text=color_by, - bargap=0.28, - ) - fig.update_xaxes(tickangle=-28) - fig = apply_plot_theme(fig, background_name) - return fig - - -def scatter_label_kwargs( - dataframe: pd.DataFrame, - show_labels: bool, - preferred_columns: tuple[str, ...] = ("Run Label", "Label"), -) -> dict[str, object]: - """Return consistent Plotly scatter label arguments without affecting hover data.""" - if not show_labels: - return {"mode": "markers", "text": None, "textposition": "top center"} - label_column = next((column for column in preferred_columns if column in dataframe.columns), None) - labels = dataframe[label_column] if label_column else None - return {"mode": "markers+text", "text": labels, "textposition": "top center"} - - -def create_score_vs_cost_plot( - dataframe: pd.DataFrame, - benchmark_name: str | None, - color_by: ColorBy = "Model", - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - if dataframe is None or dataframe.empty: - return empty_figure("No cost data available.", background_name) - - if not benchmark_name: - return empty_figure("Select a benchmark to view cost data.", background_name) - - plot_df = dataframe.copy() - plot_df = plot_df[plot_df["Benchmark"] == benchmark_name] - plot_df["Score"] = pd.to_numeric(plot_df["Score"], errors="coerce") - plot_df["Cost Per Task (USD)"] = pd.to_numeric(plot_df["Cost Per Task (USD)"], errors="coerce") - plot_df = plot_df.dropna(subset=["Score", "Cost Per Task (USD)"]) - - if plot_df.empty: - return empty_figure(f"No cost data available for {benchmark_name}.", background_name) - - colors = color_map_for(plot_df[color_by], color_by, palette_name) - theme = get_plot_background(background_name) - fig = go.Figure() - - for group, group_df in plot_df.groupby(color_by, sort=True): - label_kwargs = scatter_label_kwargs(group_df, show_labels) - fig.add_trace( - go.Scatter( - x=group_df["Cost Per Task (USD)"], - y=group_df["Score"], - name=str(group), - **label_kwargs, - marker={ - "size": 15, - "color": colors[str(group)], - "line": {"width": 1, "color": theme["marker_line_color"]}, - }, - customdata=group_df[["Model", "Harness", "Benchmark", "Score", "Cost Per Task (USD)"]], - hovertemplate=( - "%{customdata[0]}
" - "Harness: %{customdata[1]}
" - "Benchmark: %{customdata[2]}
" - "Score: %{customdata[3]:.1f}%
" - "Cost: $%{customdata[4]:.2f}/task" - "" - ), - ) - ) - - fig.update_layout( - title=None, - xaxis={"title": "Cost per task (USD)", "tickprefix": "$", "tickformat": ".2f"}, - yaxis={"title": "Score (%)", "range": [0, 105]}, - legend_title_text=color_by, - ) - return apply_plot_theme(fig, background_name) - - -RESOURCE_AXIS_CONFIG = { - "Total tokens": { - "column": "Total Tokens Per Task", - "axis_title": "Total tokens per task", - "hover_label": "Total tokens/task", - "hover_format": ",.0f", - }, - "Cost per task": { - "column": "Cost Per Task", - "axis_title": "Cost per task (USD)", - "hover_label": "Cost/task", - "hover_format": ".4f", - "tickprefix": "$", - }, - "Agent time per task": { - "column": "Agent Time Per Task", - "axis_title": "Agent time per task (seconds)", - "hover_label": "Agent time/task", - "hover_format": ",.1f", - "ticksuffix": "s", - }, -} - - -def _resource_axis_config(resource_metric: str) -> dict[str, str]: - """Return display metadata for a supported Efficiency resource metric.""" - if resource_metric in RESOURCE_AXIS_CONFIG: - return RESOURCE_AXIS_CONFIG[resource_metric] - for config in RESOURCE_AXIS_CONFIG.values(): - if resource_metric == config["column"]: - return config - raise ValueError(f"Unsupported efficiency resource metric: {resource_metric}") - - -def create_performance_vs_resource_plot( - dataframe: pd.DataFrame, - resource_metric: str = "Total tokens", - color_by: ColorBy = "Model", - x_scale: Literal["Linear", "Log"] = "Log", - show_pareto_frontier: bool = True, - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - """Plot benchmark score against one positive resource metric. - - Lower resource use and higher score define the optional Pareto frontier. - The caller is expected to provide rows for one benchmark only. - """ - from src.leaderboard import get_resource_pareto_frontier_df - - try: - resource_config = _resource_axis_config(resource_metric) - except ValueError: - return empty_figure(f"Resource metric not available: {resource_metric}.", background_name) - resource_column = resource_config["column"] - - if dataframe is None or dataframe.empty: - return empty_figure("No valid resource data available for this benchmark.", background_name) - if resource_column not in dataframe.columns: - return empty_figure(f"Resource metric not available: {resource_column}.", background_name) - if color_by not in ("Model", "Harness") or color_by not in dataframe.columns: - return empty_figure(f"Color dimension not available: {color_by}.", background_name) - - plot_df = dataframe.copy() - if "Benchmark" in plot_df.columns and plot_df["Benchmark"].dropna().nunique() > 1: - return empty_figure("Select one benchmark for the Efficiency view.", background_name) - - plot_df[resource_column] = pd.to_numeric(plot_df[resource_column], errors="coerce") - plot_df["Score (%)"] = pd.to_numeric(plot_df["Score (%)"], errors="coerce") - plot_df = plot_df.dropna(subset=[resource_column, "Score (%)"]) - plot_df = plot_df[plot_df[resource_column] > 0] - if plot_df.empty: - return empty_figure("No valid resource data available for this benchmark.", background_name) - - colors = color_map_for(plot_df[color_by], color_by, palette_name) - theme = get_plot_background(background_name) - fig = go.Figure() - hover_columns = [ - "Model", - "Harness", - "Benchmark", - "Score (%)", - "Input Tokens Per Task", - "Output Tokens Per Task", - "Cache Tokens Per Task", - "Total Tokens Per Task", - "Cost Per Task", - "Total Time Per Task", - "Agent Time Per Task", - ] - for column in hover_columns: - if column not in plot_df: - plot_df[column] = None - - resource_hover = f"{resource_config['hover_label']}: %{{x:{resource_config['hover_format']}}}" - if resource_metric == "Cost per task" or resource_column == "Cost Per Task": - resource_hover = f"{resource_config['hover_label']}: $%{{x:{resource_config['hover_format']}}}" - elif resource_metric == "Agent time per task" or resource_column == "Agent Time Per Task": - resource_hover += "s" - - for group, group_df in plot_df.groupby(color_by, sort=True): - label_kwargs = scatter_label_kwargs(group_df, show_labels) - fig.add_trace( - go.Scatter( - x=group_df[resource_column], - y=group_df["Score (%)"], - name=str(group), - **label_kwargs, - marker={ - "size": 13, - "color": colors[str(group)], - "line": {"width": 1, "color": theme["marker_line_color"]}, - }, - customdata=group_df[hover_columns], - hovertemplate=( - "%{customdata[0]}
" - "Harness: %{customdata[1]}
" - "Benchmark: %{customdata[2]}
" - "Score: %{customdata[3]:.1f}%
" - f"{resource_hover}
" - "Input tokens/task: %{customdata[4]:,.0f}
" - "Output tokens/task: %{customdata[5]:,.0f}
" - "Cache tokens/task: %{customdata[6]:,.0f}
" - "Total tokens/task: %{customdata[7]:,.0f}
" - "Cost/task: $%{customdata[8]:.4f}
" - "Total time/task: %{customdata[9]:,.1f}s
" - "Agent time/task: %{customdata[10]:,.1f}s" - "" - ), - ) - ) - - if show_pareto_frontier: - frontier_df = get_resource_pareto_frontier_df(plot_df, resource_column) - if not frontier_df.empty: - frontier_hover = f"{resource_config['hover_label']}: %{{x:{resource_config['hover_format']}}}" - if resource_column == "Cost Per Task": - frontier_hover = f"{resource_config['hover_label']}: $%{{x:{resource_config['hover_format']}}}" - elif resource_column == "Agent Time Per Task": - frontier_hover += "s" - fig.add_trace( - go.Scatter( - x=frontier_df[resource_column], - y=frontier_df["Score (%)"], - mode="lines+markers", - name="Pareto frontier", - line={"width": 3, "dash": "dash", "color": theme["text_primary"]}, - marker={ - "size": 10, - "symbol": "diamond-open", - "color": theme["text_primary"], - "line": {"width": 2, "color": theme["text_primary"]}, - }, - customdata=frontier_df[["Run Label"]], - hovertemplate=( - "Pareto frontier
" - "%{customdata[0]}
" - f"{frontier_hover}
" - "Score: %{y:.1f}%" - ), - ) - ) - - - xaxis = { - "title": resource_config["axis_title"], - "type": "log" if x_scale == "Log" else "linear", - } - if "tickprefix" in resource_config: - xaxis["tickprefix"] = resource_config["tickprefix"] - if "ticksuffix" in resource_config: - xaxis["ticksuffix"] = resource_config["ticksuffix"] - - fig.update_layout( - title=None, - xaxis=xaxis, - yaxis={"title": "Score (%)", "range": [0, 105]}, - legend_title_text=color_by, - ) - return apply_plot_theme(fig, background_name) - - -def create_score_vs_tokens_plot( - dataframe: pd.DataFrame, - token_metric: str = "Total tokens", - color_by: ColorBy = "Model", - x_scale: Literal["Linear", "Log"] = "Log", - show_pareto_frontier: bool = True, - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - """Backward-compatible wrapper around the performance-vs-resource chart.""" - return create_performance_vs_resource_plot( - dataframe=dataframe, - resource_metric=token_metric, - color_by=color_by, - x_scale=x_scale, - show_pareto_frontier=show_pareto_frontier, - show_labels=show_labels, - palette_name=palette_name, - background_name=background_name, - ) - - -def create_token_pareto_frontier_plot( - dataframe: pd.DataFrame, - token_metric: str = "Total tokens", - color_by: ColorBy = "Model", - x_scale: Literal["Linear", "Log"] = "Log", - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - """Backward-compatible convenience wrapper with the Pareto frontier enabled.""" - return create_performance_vs_resource_plot( - dataframe=dataframe, - resource_metric=token_metric, - color_by=color_by, - x_scale=x_scale, - show_pareto_frontier=True, - show_labels=show_labels, - palette_name=palette_name, - background_name=background_name, - ) - - -def _categorical_chart_height( - n_rows: int, - *, - min_height: int, - row_height: int, - vertical_padding: int, -) -> int: - """Scale dense categorical charts vertically so labels remain readable.""" - return max(min_height, row_height * max(n_rows, 0) + vertical_padding) - - -def create_ranking_plot( - dataframe: pd.DataFrame, - metric_column: str, - metric_label: str, - higher_is_better: bool, - color_by: ColorBy = "Model", - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, - sort_order: str = "Best first", -) -> Figure: - """Generic horizontal ranking chart for any numeric metric.""" - if dataframe is None or dataframe.empty or metric_column not in dataframe: - return empty_figure(f"No data available for {metric_label}.", background_name) - plot_df = dataframe.copy() - plot_df[metric_column] = pd.to_numeric(plot_df[metric_column], errors="coerce") - plot_df = plot_df.dropna(subset=[metric_column]) - if plot_df.empty: - return empty_figure(f"No data available for {metric_label}.", background_name) - plot_df["Agent"] = plot_df["Model"].astype(str) + " / " + plot_df["Harness"].astype(str) - if sort_order == "Alphabetical (A–Z)": - plot_df = plot_df.sort_values(["Agent", metric_column], ascending=[True, False], kind="mergesort") - elif sort_order == "Alphabetical (Z–A)": - plot_df = plot_df.sort_values(["Agent", metric_column], ascending=[False, False], kind="mergesort") - elif sort_order in {"Best first", "Best last"}: - ascending = not higher_is_better - if sort_order == "Best last": - ascending = not ascending - plot_df = plot_df.sort_values( - [metric_column, "Agent"], - ascending=[ascending, True], - kind="mergesort", - ) - elif sort_order == "Lowest value first": - plot_df = plot_df.sort_values( - [metric_column, "Agent"], - ascending=[True, True], - kind="mergesort", - ) - else: - plot_df = plot_df.sort_values( - [metric_column, "Agent"], - ascending=[False, True], - kind="mergesort", - ) - colors = color_map_for(plot_df[color_by], color_by, palette_name) - theme = get_plot_background(background_name) - fig = go.Figure() - for group, group_df in plot_df.groupby(color_by, sort=True): - fig.add_trace( - go.Bar( - x=group_df[metric_column], - y=group_df["Agent"], - orientation="h", - name=str(group), - marker={ - "color": colors[str(group)], - "line": {"width": 1, "color": theme["marker_line_color"]}, - }, - customdata=group_df[["Benchmark", "Model", "Harness"]], - hovertemplate=( - "%{customdata[1]}
" - "Harness: %{customdata[2]}
" - "Benchmark: %{customdata[0]}
" - f"{metric_label}: %{{x:.4g}}" - ), - ) - ) - agent_order = plot_df["Agent"].drop_duplicates().tolist() - fig.update_layout( - xaxis={"title": metric_label}, - yaxis={ - "title": None, - "autorange": "reversed", - "categoryorder": "array", - "categoryarray": agent_order, - "tickmode": "array", - "tickvals": agent_order, - "ticktext": agent_order, - }, - legend_title_text=color_by, - barmode="group", - height=_categorical_chart_height( - len(agent_order), - min_height=RANKING_MIN_HEIGHT_PX, - row_height=RANKING_ROW_HEIGHT_PX, - vertical_padding=RANKING_VERTICAL_PADDING_PX, - ), - ) - return apply_plot_theme(fig, background_name) - - -def create_tradeoff_plot( - dataframe: pd.DataFrame, - x_column: str, - y_column: str, - x_label: str, - y_label: str, - color_by: ColorBy = "Model", - show_labels: bool = False, - palette_name: str | None = DEFAULT_PALETTE, - background_name: str | None = DEFAULT_BACKGROUND, - x_scale: Literal["Linear", "Log"] = "Linear", - y_scale: Literal["Linear", "Log"] = "Linear", - show_pareto_frontier: bool = False, - lower_x_is_better: bool = True, - higher_y_is_better: bool = True, -) -> Figure: - """Generic two-metric scatter used by PR2 trade-off views.""" - if dataframe is None or dataframe.empty: - return empty_figure("No trade-off data available.", background_name) - if x_column not in dataframe or y_column not in dataframe: - return empty_figure("Selected trade-off metric is unavailable.", background_name) - if color_by not in ("Model", "Harness") or color_by not in dataframe: - return empty_figure(f"Color dimension not available: {color_by}.", background_name) - - plot_df = dataframe.copy() - plot_df[x_column] = pd.to_numeric(plot_df[x_column], errors="coerce") - plot_df[y_column] = pd.to_numeric(plot_df[y_column], errors="coerce") - plot_df = plot_df.dropna(subset=[x_column, y_column]) - if x_scale == "Log": - plot_df = plot_df[plot_df[x_column] > 0] - if y_scale == "Log": - plot_df = plot_df[plot_df[y_column] > 0] - if plot_df.empty: - return empty_figure("No valid points for the selected trade-off.", background_name) - - colors = color_map_for(plot_df[color_by], color_by, palette_name) - theme = get_plot_background(background_name) - fig = go.Figure() - for group, group_df in plot_df.groupby(color_by, sort=True): - label_kwargs = scatter_label_kwargs(group_df, show_labels) - fig.add_trace( - go.Scatter( - x=group_df[x_column], - y=group_df[y_column], - name=str(group), - **label_kwargs, - marker={ - "size": 13, - "color": colors[str(group)], - "line": {"width": 1, "color": theme["marker_line_color"]}, - }, - customdata=group_df[["Model", "Harness", "Benchmark"]], - hovertemplate=( - "%{customdata[0]}
" - "Harness: %{customdata[1]}
" - "Benchmark: %{customdata[2]}
" - f"{x_label}: %{{x:.4g}}
" - f"{y_label}: %{{y:.4g}}" - ), - ) - ) - - if show_pareto_frontier: - from src.leaderboard import get_pareto_frontier_df - - frontier = get_pareto_frontier_df( - plot_df, - x_column, - y_column, - lower_x_is_better=lower_x_is_better, - higher_y_is_better=higher_y_is_better, - ) - if not frontier.empty: - fig.add_trace( - go.Scatter( - x=frontier[x_column], - y=frontier[y_column], - mode="lines+markers", - name="Pareto frontier", - line={"width": 3, "dash": "dash", "color": theme["text_primary"]}, - marker={"size": 9, "symbol": "diamond-open"}, - hovertemplate=f"{x_label}: %{{x:.4g}}
{y_label}: %{{y:.4g}}", - ) - ) - - fig.update_layout( - xaxis={"title": x_label, "type": "log" if x_scale == "Log" else "linear"}, - yaxis={"title": y_label, "type": "log" if y_scale == "Log" else "linear"}, - legend_title_text=color_by, - ) - return apply_plot_theme(fig, background_name) - - -def create_matrix_plot( - matrix: pd.DataFrame, - title: str, - metric_label: str, - higher_is_better: bool = True, - show_values: bool = True, - reverse_scale: bool | None = None, - background_name: str | None = DEFAULT_BACKGROUND, - display_matrix: pd.DataFrame | None = None, - display_metric_label: str | None = None, -) -> Figure: - """Render a reusable model/harness × benchmark matrix.""" - if matrix is None or matrix.empty: - return empty_figure(f"No data available for {title}.", background_name) - reverse = (not higher_is_better) if reverse_scale is None else reverse_scale - colorscale = "Viridis_r" if reverse else "Viridis" - z = matrix.to_numpy(dtype=float) - text = None - texttemplate = None - display_values = matrix if display_matrix is None else display_matrix.reindex( - index=matrix.index, columns=matrix.columns - ) - display_z = display_values.to_numpy(dtype=float) - if show_values: - text = [[("" if pd.isna(value) else f"{value:.4g}") for value in row] for row in display_z] - texttemplate = "%{text}" - hover_label = display_metric_label or metric_label - customdata = display_z if display_matrix is not None else None - hovertemplate = ( - "Agent: %{y}
" - "Benchmark: %{x}
" - + ( - f"{hover_label}: %{{customdata:.4g}}
{metric_label}: %{{z:.4g}}" - if display_matrix is not None - else f"{metric_label}: %{{z:.4g}}" - ) - ) - fig = go.Figure( - go.Heatmap( - z=z, - x=[str(value) for value in matrix.columns], - y=[str(value) for value in matrix.index], - colorscale=colorscale, - colorbar={"title": metric_label}, - text=text, - texttemplate=texttemplate, - customdata=customdata, - hovertemplate=hovertemplate, - hoverongaps=False, - ) - ) - row_labels = [str(value) for value in matrix.index] - fig.update_layout( - title=title, - xaxis={"title": "Benchmark"}, - yaxis={ - "title": "Model / Harness", - "autorange": "reversed", - "tickmode": "array", - "tickvals": row_labels, - "ticktext": row_labels, - }, - height=_categorical_chart_height( - len(row_labels), - min_height=MATRIX_MIN_HEIGHT_PX, - row_height=MATRIX_ROW_HEIGHT_PX, - vertical_padding=MATRIX_VERTICAL_PADDING_PX, - ), - ) - return apply_plot_theme(fig, background_name) - - -def create_coverage_matrix_plot( - matrix: pd.DataFrame, - background_name: str | None = DEFAULT_BACKGROUND, -) -> Figure: - """Render coverage as available/missing without converting missing data to score zero.""" - if matrix is None or matrix.empty: - return empty_figure("No benchmark coverage data available.", background_name) - display = matrix.copy() - z = display.notna().astype(int).to_numpy() - text = [["Available" if value else "Missing" for value in row] for row in z] - fig = go.Figure( - go.Heatmap( - z=z, - x=[str(value) for value in display.columns], - y=[str(value) for value in display.index], - zmin=0, - zmax=1, - colorscale=[[0, "#475569"], [1, "#84cc16"]], - showscale=False, - text=text, - texttemplate="%{text}", - hovertemplate="Agent: %{y}
Benchmark: %{x}
Status: %{text}", - ) - ) - row_labels = [str(value) for value in display.index] - fig.update_layout( - title="Benchmark coverage", - xaxis={"title": "Benchmark"}, - yaxis={ - "title": "Model / Harness", - "autorange": "reversed", - "tickmode": "array", - "tickvals": row_labels, - "ticktext": row_labels, - }, - height=_categorical_chart_height( - len(row_labels), - min_height=MATRIX_MIN_HEIGHT_PX, - row_height=MATRIX_ROW_HEIGHT_PX, - vertical_padding=MATRIX_VERTICAL_PADDING_PX, - ), - ) - return apply_plot_theme(fig, background_name) diff --git a/src/configuration.py b/src/configuration.py deleted file mode 100644 index d38d70ec0c66268a0c5bedb07f38c7b4d8f3c18b..0000000000000000000000000000000000000000 --- a/src/configuration.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -from typing import Any - -from src.models import Configuration - -DISPLAY_ONLY_FIELDS = {"configuration_label"} -SET_LIKE_FIELDS = {"skills", "tools"} - - -def _normalize_value(value: Any, key: str | None = None) -> Any: - if isinstance(value, dict): - normalized: dict[str, Any] = {} - for child_key in sorted(value): - if child_key in DISPLAY_ONLY_FIELDS: - continue - child = _normalize_value(value[child_key], child_key) - if child is not None: - normalized[child_key] = child - return normalized - if isinstance(value, list): - items = [_normalize_value(item) for item in value] - if key in SET_LIKE_FIELDS: - # Skills/tools describe membership, not declaration order. - return sorted(dict.fromkeys(items)) - return items - return value - - -def normalize_configuration(configuration: Configuration | dict[str, Any] | None) -> dict[str, Any]: - """Return a canonical structured representation suitable for comparison. - - Missing values stay missing and display-only labels are excluded. Skills and tools - are treated as set-like fields so equivalent ordering produces the same identity. - """ - if configuration is None: - return {} - if isinstance(configuration, Configuration): - raw = configuration.model_dump(exclude_none=True) - else: - raw = dict(configuration) - return _normalize_value(raw) - - -def configuration_fingerprint(configuration: Configuration | dict[str, Any] | None) -> str | None: - """Return a deterministic SHA-256 identity for a non-empty configuration.""" - normalized = normalize_configuration(configuration) - if not normalized: - return None - payload = json.dumps(normalized, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def configuration_label(configuration: Configuration | None) -> str: - if configuration is None: - return "Configuration metadata unavailable" - if configuration.configuration_label: - return configuration.configuration_label - - parts: list[str] = [] - if configuration.shell_access is True: - parts.append("shell") - if configuration.planning_enabled is True: - parts.append("planning") - if configuration.context_compression and configuration.context_compression.enabled is True: - method = configuration.context_compression.method - parts.append(f"compression:{method}" if method else "compression") - if configuration.memory and configuration.memory.enabled is True: - method = configuration.memory.method - parts.append(f"memory:{method}" if method else "memory") - if configuration.reasoning_effort: - parts.append(f"reasoning:{configuration.reasoning_effort}") - if configuration.max_context_tokens is not None: - parts.append(f"context:{configuration.max_context_tokens}") - if configuration.skills: - parts.append("skills:" + ",".join(sorted(set(configuration.skills)))) - if configuration.tools: - parts.append("tools:" + ",".join(sorted(set(configuration.tools)))) - return " + ".join(parts) if parts else "Structured configuration" - - -def _flatten(value: Any, prefix: str = "") -> dict[str, Any]: - if not isinstance(value, dict): - return {prefix: value} - flattened: dict[str, Any] = {} - for key, child in value.items(): - path = f"{prefix}.{key}" if prefix else key - if isinstance(child, dict): - flattened.update(_flatten(child, path)) - else: - flattened[path] = child - return flattened - - -def changed_configuration_fields( - baseline: Configuration | dict[str, Any] | None, - treatment: Configuration | dict[str, Any] | None, -) -> list[str]: - """Return stable dotted paths whose known structured values differ.""" - left = _flatten(normalize_configuration(baseline)) - right = _flatten(normalize_configuration(treatment)) - keys = sorted(set(left) | set(right)) - return [key for key in keys if left.get(key) != right.get(key)] diff --git a/src/configuration_analysis.py b/src/configuration_analysis.py deleted file mode 100644 index e647e154fc2ae87a6062f79731c8cd5c043528b1..0000000000000000000000000000000000000000 --- a/src/configuration_analysis.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from src.configuration import changed_configuration_fields -from src.models import Result - - -@dataclass(frozen=True) -class MetricDelta: - baseline: float | None - treatment: float | None - absolute: float | None - percent: float | None - - -@dataclass(frozen=True) -class Comparison: - baseline: Result - treatment: Result - changed_fields: tuple[str, ...] - matched_dimensions: tuple[str, ...] - unmatched_dimensions: tuple[str, ...] - status: str - metric_deltas: dict[str, MetricDelta] - - -def safe_percent_change(baseline: float | int | None, treatment: float | int | None) -> float | None: - if baseline is None or treatment is None or baseline == 0: - return None - return (float(treatment) - float(baseline)) / abs(float(baseline)) * 100.0 - - -def metric_delta(baseline: float | int | None, treatment: float | int | None) -> MetricDelta: - absolute = None if baseline is None or treatment is None else float(treatment) - float(baseline) - return MetricDelta( - baseline=None if baseline is None else float(baseline), - treatment=None if treatment is None else float(treatment), - absolute=absolute, - percent=safe_percent_change(baseline, treatment), - ) - - -def _error_rate(result: Result) -> float | None: - tasks = result.metrics.n_tasks - errors = result.metrics.n_errors - if tasks is None or errors is None or tasks <= 0: - return None - return errors / tasks * 100.0 - - -def _per_success(value: float | int | None, result: Result) -> float | None: - if value is None or result.metrics.score <= 0: - return None - return float(value) / result.metrics.score - - -def comparison_metric_deltas(baseline: Result, treatment: Result) -> dict[str, MetricDelta]: - bm = baseline.metrics - tm = treatment.metrics - return { - # Score absolute delta is percentage points; relative delta remains percent. - "score_percentage_points": metric_delta(bm.score * 100.0, tm.score * 100.0), - "total_tokens": metric_delta(bm.n_total_tokens, tm.n_total_tokens), - "cost_usd": metric_delta(bm.cost_usd, tm.cost_usd), - "total_time_seconds": metric_delta(bm.total_time_seconds, tm.total_time_seconds), - "agent_time_seconds": metric_delta(bm.agent_time_seconds, tm.agent_time_seconds), - "execution_error_rate_percentage_points": metric_delta(_error_rate(baseline), _error_rate(treatment)), - "tokens_per_successful_task": metric_delta( - _per_success(bm.mean_tokens_per_task, baseline), - _per_success(tm.mean_tokens_per_task, treatment), - ), - "cost_per_successful_task": metric_delta( - _per_success(bm.mean_cost_usd_per_task, baseline), - _per_success(tm.mean_cost_usd_per_task, treatment), - ), - "time_per_successful_task": metric_delta( - _per_success(bm.mean_total_time_seconds_per_task, baseline), - _per_success(tm.mean_total_time_seconds_per_task, treatment), - ), - } - - -def _dimension_values(result: Result) -> dict[str, Any]: - # Use only metadata represented by the current schema. Future schema additions - # (benchmark revision, budgets, seed, protocol, harness version) can extend this - # mapping without changing the comparison API. - return { - "benchmark": result.benchmark.name, - "model": result.model.repo or result.model.name, - "harness": result.harness.name, - "environment": result.environment.name, - "environment_config": result.environment.config, - } - - -def compare_runs(baseline: Result, treatment: Result) -> Comparison: - changed = tuple(changed_configuration_fields(baseline.configuration, treatment.configuration)) - left = _dimension_values(baseline) - right = _dimension_values(treatment) - matched = tuple(key for key in left if left[key] is not None and left[key] == right[key]) - unmatched = tuple(key for key in left if left[key] != right[key]) - - if baseline.configuration is None or treatment.configuration is None: - status = "observational" - elif not changed: - status = "ambiguous" - elif not unmatched: - status = "exact" - elif {"benchmark", "model"}.issubset(matched): - status = "partial" - else: - status = "unmatched" - - return Comparison( - baseline=baseline, - treatment=treatment, - changed_fields=changed, - matched_dimensions=matched, - unmatched_dimensions=unmatched, - status=status, - metric_deltas=comparison_metric_deltas(baseline, treatment), - ) diff --git a/src/display/text_blocks.py b/src/display/text_blocks.py index 97b8b0423af897123df3759ed3f6adb8e164d971..7818daa427a6e4b1a1f48a6dd8d123abf89a031d 100644 --- a/src/display/text_blocks.py +++ b/src/display/text_blocks.py @@ -1,53 +1,19 @@ +TITLE = """

Coding Agent Leaderboard

""" + INTRODUCTION_TEXT = """ -A **Coding Agent** is more than just a model - it's the combination of a **Model** and a **Harness** (the tool/framework driving the model). -This leaderboard tracks how these components work together, because the same model can perform very differently depending on the harness it's paired with. +Welcome to the Coding Agent Leaderboard! """ LLM_BENCHMARKS_TEXT = """ -## What is a Coding Agent? - -A coding agent is a system that autonomously solves software engineering tasks - reading code, reasoning about bugs, and writing patches. Its performance depends on two components: - -- **Model** - The underlying language model (e.g. Claude Opus 4.7, Qwen3.6-35B) -- **Harness** - The framework or tool that orchestrates the model's actions (e.g. Claude Code, OpenCode, Pi) - -## How to Read the Table - -| Column | Description | -|--------|-------------| -| **Benchmark** | The benchmark used for evaluation (e.g. SWE-bench Verified - 500 real GitHub issues) | -| **Harness** | The agent framework driving the model. | -| **Model** | The language model being evaluated | -| **Skills** | The set of instructions guiding the agent's behavior | -| **Score** | Outcome of the benchmark, often the fraction of tasks solved correctly (higher is better) | -| **Precision** | Model weight format (e.g. bf16, fp4) - affects speed, memory footprint, and quality | +## About -## Key Concepts +Evaluate and compare Coding Agents. -- **FOSS vs Proprietary** - Filters let you compare fully open-source agents against proprietary ones. A FOSS model with a FOSS harness means anyone can reproduce the result -- **Skills** - Some harnesses augment the model with extra capabilities (tools, retrieval, etc.). Listed in the "skills" column when present -- **Internal results (`*`)** - Benchmarks run by the model provider where the harness and environment were not made public. These are useful reference points but are not independently reproducible +Coding Agent = Model + Harness + Skills. -## Learn More - -Visit the [GitHub repo](https://github.com/redhat-et/coding_agent_bench) for details about the project, methodology, and how to submit your own results. +Visit our [GitHub repo](https://github.com/redhat-et/coding_agent_bench) for more details about the project. """ -HOW_TO_USE_TEXT = """ ---- -## How to interpret these results - -In the absence of enterprise-specific datasets, public benchmarks provide a means of comparing the performance of coding agents across a wide range of tasks. -Better performance on these benchmarks generally translates to better performance on real-world tasks. -All benchmarks are run using Harbor, a sandboxed environment for evaluating coding agents. - -Each benchmark measures the performance of the coding agent on different tasks: +CITATION_BUTTON_TEXT = "TBD" -* **SWE-Bench Verified**: Measures performance on solving GitHub issues in popular Python repositories. -* **SWE-Bench Pro -- Ansible**: Measures performance on solving GitHub issues in the [ansible/ansible](https://github.com/ansible/ansible) repository. - Demonstrates how benchmarking can be used to evaluate coding agents on enterprise-specific tasks. - -Higher scores indicate better performance on the benchmarks. -If an agent scores better on a given benchmark than another, it can be generally considered to be better at those kinds of tasks. -Compare agents within each benchmark rather than relying on a cross-benchmark average. Each benchmark emphasizes different task types, so per-benchmark scores are the clearest way to judge performance. -""" +CITATION_BUTTON_LABEL = "Citation" diff --git a/src/leaderboard.py b/src/leaderboard.py index ce32f832c0bfd50c4cf4fe13050c61dc6652b402..ba7423f05a4274cbef79b3aab8e268ea79a987c2 100644 --- a/src/leaderboard.py +++ b/src/leaderboard.py @@ -1,342 +1,58 @@ -import json from pathlib import Path - +import json import pandas as pd from src.models import Result RESULTS_DIR = Path(__file__).parent.parent / "results" -BENCHMARK_SORT_ORDER = { - "SWE-Bench Verified": 0, - "SWE-Bench Pro -- Ansible": 1, -} - -EFFICIENCY_RESOURCE_METRICS = { - "Total tokens": "Total Tokens Per Task", - "Cost per task": "Cost Per Task", - "Agent time per task": "Agent Time Per Task", -} -ANALYSIS_COLUMNS = [ - "Benchmark", - "Model", - "Harness", - "Run Label", - "Category", - "Score", - "Score (%)", - "Tasks", - "Errors", - "Input Tokens Per Task", - "Cache Tokens Per Task", - "Output Tokens Per Task", - "Total Tokens Per Task", - "Tokens Per Solved Task", - "Cost Per Task", - "Total Time Per Task", - "Agent Time Per Task", - "Token Data Available", +DISPLAY_BY_DEFAULT = [ + "dataset", + "model", + "harness", + "skills", + "environment", + "score", + "costUSD", ] -TOKEN_EFFICIENCY_TABLE_COLUMNS = [ - "Model", - "Harness", - "Benchmark", - "Score (%)", - "Total Tokens Per Task", - "Cost Per Task", - "Agent Time Per Task", - "Input Tokens Per Task", - "Output Tokens Per Task", - "Cache Tokens Per Task", - "Tokens Per Solved Task", - "Tasks", - "Errors", -] - -def benchmark_sort_key(name: str) -> tuple[int, str]: - return (BENCHMARK_SORT_ORDER.get(name, 99), name) +SEARCH_COLUMNS = [ + "dataset", + "model", + "harness", +] -def format_time(seconds: int | None) -> str | None: - if seconds is None: - return None +def format_time(seconds: int): m, s = divmod(seconds, 60) h, m = divmod(m, 60) return f"{h}h{m}m{s}s" -def get_results() -> list[Result]: +def get_leaderboard_df(): results: list[Result] = [] - for file in sorted(RESULTS_DIR.glob("*.json")): + for file in RESULTS_DIR.glob("*.json"): with open(file, "r") as f: data = json.load(f) - results.append(Result(**data)) - return results - - -def get_benchmark_names(results: list[Result] | None = None) -> list[str]: - if results is None: - results = get_results() - return sorted({r.benchmark.name for r in results}, key=benchmark_sort_key) - - -def _positive_number_or_none(value: object) -> float | None: - """Return a positive numeric value, otherwise ``None``.""" - if value is None: - return None - try: - numeric = float(value) - except (TypeError, ValueError): - return None - if pd.isna(numeric) or numeric <= 0: - return None - return numeric - - -def get_efficiency_resource_column(metric_name: str) -> str: - """Resolve an Efficiency UI metric name to its canonical analysis column.""" - if metric_name in EFFICIENCY_RESOURCE_METRICS: - return EFFICIENCY_RESOURCE_METRICS[metric_name] - if metric_name in EFFICIENCY_RESOURCE_METRICS.values(): - return metric_name - raise ValueError(f"Unsupported efficiency resource metric: {metric_name}") - - -def get_analysis_df(results: list[Result] | None = None) -> pd.DataFrame: - """Return one normalized analysis row per benchmark result.""" - if results is None: - results = get_results() + result = Result(**data) + results.append(result) rows = [] for result in results: - metrics = result.metrics - model_label = result.model.repo or result.model.name - total_tokens = _positive_number_or_none(metrics.mean_tokens_per_task) - cost_per_task = _positive_number_or_none(metrics.mean_cost_usd_per_task) - agent_time_per_task = _positive_number_or_none(metrics.mean_agent_time_seconds_per_task) - token_data_available = total_tokens is not None - tokens_per_solved_task = ( - total_tokens / metrics.score - if token_data_available and metrics.score > 0 - else None - ) rows.append( { - "Benchmark": result.benchmark.name, - "Model": model_label, - "Harness": result.harness.name, - "Run Label": f"{model_label} / {result.harness.name}", - "Category": "FOSS" if result.model.is_oss and result.harness.is_oss else "Proprietary", - "Score": metrics.score, - "Score (%)": metrics.score * 100, - "Tasks": metrics.n_tasks, - "Errors": metrics.n_errors, - "Input Tokens Per Task": metrics.mean_input_tokens_per_task, - "Cache Tokens Per Task": metrics.mean_cache_tokens_per_task, - "Output Tokens Per Task": metrics.mean_output_tokens_per_task, - "Total Tokens Per Task": total_tokens, - "Tokens Per Solved Task": tokens_per_solved_task, - "Cost Per Task": cost_per_task, - "Total Time Per Task": metrics.mean_total_time_seconds_per_task, - "Agent Time Per Task": agent_time_per_task, - "Token Data Available": token_data_available, + "dataset": result.dataset.name, + "model": result.model.name, + "harness": result.harness.name, + "skills": str(result.harness.skills) if result.harness.skills else "None", + "environment": result.environment.name, + "score": result.metrics.score, + "costUSD": result.metrics.costUSD, + "time": format_time(result.metrics.time), + "model_is_oss": result.model.is_oss, + "model_num_params": result.model.num_params, } ) - return pd.DataFrame(rows, columns=ANALYSIS_COLUMNS) - - -def get_efficiency_df( - benchmark_name: str, - resource_metric: str = "Total tokens", - analysis_df: pd.DataFrame | None = None, -) -> pd.DataFrame: - """Return valid resource rows for one benchmark-specific Efficiency plot. - - The number of rows excluded because the selected resource is missing or - non-positive is exposed via ``dataframe.attrs["exclusion_count"]``. - """ - metric_column = get_efficiency_resource_column(resource_metric) - dataframe = get_analysis_df() if analysis_df is None else analysis_df.copy() - dataframe = dataframe[dataframe["Benchmark"] == benchmark_name].copy() - - if dataframe.empty: - dataframe.attrs["exclusion_count"] = 0 - return dataframe - - metric_values = pd.to_numeric(dataframe[metric_column], errors="coerce") - valid_mask = metric_values.notna() & (metric_values > 0) - exclusion_count = int((~valid_mask).sum()) - dataframe = dataframe.loc[valid_mask].copy() - dataframe[metric_column] = metric_values.loc[valid_mask] - dataframe = dataframe.sort_values( - [metric_column, "Score (%)", "Model", "Harness"], - ascending=[True, False, True, True], - na_position="last", - ) - dataframe.attrs["exclusion_count"] = exclusion_count - return dataframe - - -def get_token_efficiency_df( - benchmark_name: str, - token_metric: str = "Total tokens", - analysis_df: pd.DataFrame | None = None, -) -> pd.DataFrame: - """Backward-compatible wrapper for :func:`get_efficiency_df`.""" - return get_efficiency_df(benchmark_name, token_metric, analysis_df) - - -def get_token_efficiency_table_df( - benchmark_name: str, - analysis_df: pd.DataFrame | None = None, -) -> pd.DataFrame: - """Return the benchmark-specific Efficiency ranking table for display.""" - dataframe = get_analysis_df() if analysis_df is None else analysis_df.copy() - dataframe = dataframe[dataframe["Benchmark"] == benchmark_name].copy() - if dataframe.empty: - return pd.DataFrame(columns=TOKEN_EFFICIENCY_TABLE_COLUMNS) - - dataframe["Score (%)"] = pd.to_numeric(dataframe["Score (%)"], errors="coerce") - dataframe = dataframe.sort_values( - ["Tokens Per Solved Task", "Score (%)", "Model", "Harness"], - ascending=[True, False, True, True], - na_position="last", - ) - table_df = dataframe.reindex(columns=TOKEN_EFFICIENCY_TABLE_COLUMNS).copy() - table_df["Score (%)"] = table_df["Score (%)"].round(1) - if "Cost Per Task" in table_df: - table_df["Cost Per Task"] = pd.to_numeric(table_df["Cost Per Task"], errors="coerce").round(4) - if "Agent Time Per Task" in table_df: - table_df["Agent Time Per Task"] = pd.to_numeric( - table_df["Agent Time Per Task"], errors="coerce" - ).round(1) - return table_df - - -def get_pareto_frontier_df( - dataframe: pd.DataFrame, - x_column: str, - y_column: str, - *, - lower_x_is_better: bool = True, - higher_y_is_better: bool = True, - require_positive_x: bool = False, -) -> pd.DataFrame: - """Return deterministic non-dominated points for arbitrary metric directions. - - Ties are preserved. A row is excluded only when another row is at least as - good on both axes and strictly better on at least one axis. - """ - if x_column not in dataframe.columns or y_column not in dataframe.columns: - return dataframe.iloc[0:0].copy() - - candidates = dataframe.copy() - candidates[x_column] = pd.to_numeric(candidates[x_column], errors="coerce") - candidates[y_column] = pd.to_numeric(candidates[y_column], errors="coerce") - candidates = candidates.dropna(subset=[x_column, y_column]) - if require_positive_x: - candidates = candidates[candidates[x_column] > 0] - if candidates.empty: - return candidates - - x = candidates[x_column] - y = candidates[y_column] - frontier_mask = pd.Series(True, index=candidates.index) - for idx in candidates.index: - x_at_idx = x.loc[idx] - y_at_idx = y.loc[idx] - x_at_least_as_good = x <= x_at_idx if lower_x_is_better else x >= x_at_idx - y_at_least_as_good = y >= y_at_idx if higher_y_is_better else y <= y_at_idx - x_strictly_better = x < x_at_idx if lower_x_is_better else x > x_at_idx - y_strictly_better = y > y_at_idx if higher_y_is_better else y < y_at_idx - dominated = x_at_least_as_good & y_at_least_as_good & (x_strictly_better | y_strictly_better) - dominated.loc[idx] = False - if dominated.any(): - frontier_mask.loc[idx] = False - - frontier = candidates.loc[frontier_mask].copy() - tie_breakers = [column for column in ("Benchmark", "Model", "Harness", "Run Label") if column in frontier] - return frontier.sort_values( - [x_column, y_column, *tie_breakers], - ascending=[lower_x_is_better, not higher_y_is_better, *([True] * len(tie_breakers))], - kind="mergesort", - ) - - -def get_resource_pareto_frontier_df( - dataframe: pd.DataFrame, - token_metric: str, - score_column: str = "Score (%)", -) -> pd.DataFrame: - """Return deterministic non-dominated points for lower resource use/higher score.""" - return get_pareto_frontier_df( - dataframe, - token_metric, - score_column, - lower_x_is_better=True, - higher_y_is_better=True, - require_positive_x=True, - ) - -def get_token_pareto_frontier_df( - dataframe: pd.DataFrame, - token_metric: str, - score_column: str = "Score (%)", -) -> pd.DataFrame: - """Backward-compatible wrapper for the generic resource Pareto helper.""" - return get_resource_pareto_frontier_df(dataframe, token_metric, score_column) - - -def get_benchmark_run_df(): - results = get_results() - - rows = [] - for result in results: - rows.append( - { - " ": "🟠" if result.model.is_oss and result.harness.is_oss else "🔶", - "Model": f"[{result.model.repo or result.model.name}]({result.model.url})", - "Harness": f"[{result.harness.name}]({result.harness.url})*" - if result.harness.name == "internal" - else f"[{result.harness.name}]({result.harness.url})", - "Benchmark": f"[{result.benchmark.name}]({result.benchmark.url})", - "Base Model": result.model.name, - "Precision": result.model.precision, - "Skills": str(result.harness.skills) if result.harness.skills else "None", - "Score": round(result.metrics.score * 100, 1), - "Avg Cost Per Task (USD)": result.metrics.mean_cost_usd_per_task, - "Avg Seconds Per Task": result.metrics.mean_agent_time_seconds_per_task, - "Avg Input Tokens Per Task": result.metrics.mean_input_tokens_per_task, - "Avg Output Tokens Per Task": result.metrics.mean_output_tokens_per_task, - "Model License": "FOSS" if result.model.is_oss else "Proprietary", - "Harness License": "FOSS" if result.harness.is_oss else "Proprietary", - "Model Num Params (B)": result.model.num_params, - } - ) - - benchmark_run_df = pd.DataFrame(rows) - if benchmark_run_df.empty: - return benchmark_run_df - - benchmark_run_df["_Benchmark Sort"] = benchmark_run_df["Benchmark"].str.extract(r"\[(.*?)\]", expand=False) - benchmark_run_df["_Benchmark Sort Key"] = benchmark_run_df["_Benchmark Sort"].map(benchmark_sort_key) - benchmark_run_df = benchmark_run_df.sort_values( - by=["_Benchmark Sort Key", "Score"], - ascending=[True, False], - ).drop(columns=["_Benchmark Sort", "_Benchmark Sort Key"]) - return benchmark_run_df.fillna("") - - -def get_score_vs_cost_df(): - analysis_df = get_analysis_df() - score_vs_cost_df = analysis_df.dropna(subset=["Cost Per Task"]).copy() - columns = ["Label", "Model", "Harness", "Benchmark", "Category", "Score", "Cost Per Task (USD)"] - if score_vs_cost_df.empty: - return pd.DataFrame(columns=columns) - - score_vs_cost_df["Label"] = score_vs_cost_df["Run Label"] - score_vs_cost_df["Score"] = score_vs_cost_df["Score (%)"].round(1) - score_vs_cost_df["Cost Per Task (USD)"] = score_vs_cost_df["Cost Per Task"].round(2) - return score_vs_cost_df[columns].sort_values(["Benchmark", "Score"], ascending=[True, False]) + leaderboard_df = pd.DataFrame(rows) + return leaderboard_df diff --git a/src/models.py b/src/models.py index 75bda128669ac7b41b7e6880c735b8c33882dd91..117232186c2b9b59f53cdb49c1bf9a9decf9d8f8 100644 --- a/src/models.py +++ b/src/models.py @@ -1,23 +1,17 @@ -from typing import Any, Optional +from typing import Any from pydantic import BaseModel -class Benchmark(BaseModel): +class Dataset(BaseModel): name: str repo: str num_tasks: int - url: str - - def __hash__(self): - return hash(self.name) class Harness(BaseModel): name: str skills: list[str] - is_oss: bool - url: str class Model(BaseModel): @@ -26,63 +20,22 @@ class Model(BaseModel): is_oss: bool num_params: int precision: str - url: str class Environment(BaseModel): name: str - config: Optional[dict[str, Any]] = None - url: str + config: dict[str, Any] class Metrics(BaseModel): - score: float - n_tasks: Optional[int] = None - n_errors: Optional[int] = None - n_input_tokens: Optional[int] = None - n_cache_tokens: Optional[int] = None - n_output_tokens: Optional[int] = None - n_total_tokens: Optional[int] = None - total_time_seconds: Optional[int] = None - agent_time_seconds: Optional[int] = None - cost_usd: Optional[float] = None - mean_input_tokens_per_task: Optional[int] = None - mean_cache_tokens_per_task: Optional[int] = None - mean_output_tokens_per_task: Optional[int] = None - mean_tokens_per_task: Optional[int] = None - mean_cost_usd_per_task: Optional[float] = None - mean_total_time_seconds_per_task: Optional[int] = None - mean_agent_time_seconds_per_task: Optional[int] = None - - -class ContextCompressionConfiguration(BaseModel): - enabled: bool | None = None - method: str | None = None - threshold_tokens: int | None = None - - -class MemoryConfiguration(BaseModel): - enabled: bool | None = None - method: str | None = None - - -class Configuration(BaseModel): - shell_access: bool | None = None - skills: list[str] | None = None - tools: list[str] | None = None - context_compression: ContextCompressionConfiguration | None = None - memory: MemoryConfiguration | None = None - max_context_tokens: int | None = None - reasoning_effort: str | None = None - planning_enabled: bool | None = None - configuration_label: str | None = None + time: int + costUSD: float class Result(BaseModel): - benchmark: Benchmark + dataset: Dataset harness: Harness model: Model environment: Environment metrics: Metrics - configuration: Configuration | None = None diff --git a/src/rankings.py b/src/rankings.py deleted file mode 100644 index 15353dbf910349ca51562e7141b48ee78745c25f..0000000000000000000000000000000000000000 --- a/src/rankings.py +++ /dev/null @@ -1,165 +0,0 @@ -from typing import Any -from pathlib import Path - -import numpy as np -import pandas as pd -import choix - - -EXCLUDED_BENCHMARKS = {"Shellbench"} -EXCLUDED_HARNESSES_FROM_PAIRWISE = {"Codex", "Qwen Code"} - - -def prepare_ranking_data( - df: pd.DataFrame, - catcol: str | list[str], - metcol: str, - descending: bool = False, - eqvcol: str | list[str] = [], -) -> tuple[list[tuple[int, int]], list[Any], dict]: - ndata = df.shape[0] - if ndata < 2: - raise ValueError("Not enough data to prepare ranking comparisons") - catcol = catcol if isinstance(catcol, list) else [catcol] - eqvcol = eqvcol if isinstance(eqvcol, list) else [eqvcol] - ncat = len(catcol) - neqv = len(eqvcol) - tcols = catcol + eqvcol + [metcol] - t = list(df[tcols].itertuples(index=False, name=None)) - metvals = [x[-1] for x in t] - if ncat > 1: - catvals = [x[:ncat] for x in t] - else: - catvals = [x[0] for x in t] - if neqv > 1: - eqvvals = [x[ncat : ncat + neqv] for x in t] - elif neqv == 1: - eqvvals = [x[ncat] for x in t] - else: - eqvvals = ["[ALL]"] * ndata - umap = dict([(y, x) for x, y in enumerate(sorted(set(catvals)))]) - cats = sorted(umap.keys()) - eqvcats = sorted(set(eqvvals)) - eqvdata = {} - for eqv in eqvcats: - eqvdata[eqv] = [[] for _ in range(len(cats))] - compvals = [(umap[c], m, e) for c, m, e in zip(catvals, metvals, eqvvals)] - for category, metric, equivalence in compvals: - eqvdata[equivalence][category].append(metric) - comps = [] - for i in range(ndata): - ic, im, ie = compvals[i] - for j in range(i): - jc, jm, je = compvals[j] - if ie != je: - continue - if im == jm: - continue - iwin = im < jm if descending else im > jm - if iwin: - comps.append((ic, jc)) - else: - comps.append((jc, ic)) - return comps, cats, eqvdata - - -def ranking_dataframe(cats, params, eqvdata) -> pd.DataFrame: - ranking = np.argsort(params)[::-1] - rows = [] - for rank, idx in enumerate(ranking, start=1): - row = { - "Rank": rank, - "Category": cats[idx] if not isinstance(cats[idx], tuple) else " + ".join(cats[idx]), - } - for k in sorted(eqvdata.keys()): - mets = eqvdata[k][idx] - row[k] = round(float(np.mean(mets)), 3) if len(mets) > 0 else "" - rows.append(row) - return pd.DataFrame(rows) - - -RANK_BY_OPTIONS = { - "Benchmark Score": ("metrics.score", False), - "Mean Cost Per Task (USD)": ("metrics.mean_cost_usd_per_task", True), - "Mean Tokens Per Task": ("metrics.mean_tokens_per_task", True), -} - - -def compute_ranking(df, catcol, metcol="metrics.score", descending=False, eqvcol="benchmark.name"): - comps, cats, eqvdata = prepare_ranking_data( - df, catcol, metcol, descending=descending, eqvcol=eqvcol - ) - params = choix.ilsr_pairwise(len(cats), comps, alpha=1e-3) - return ranking_dataframe(cats, params, eqvdata) - - -def rank_harnesses(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame: - return compute_ranking(df, "harness.name", metcol, descending) - - -def rank_models(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame: - return compute_ranking(df, "model.name", metcol, descending) - - -def rank_pairs(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame: - return compute_ranking(df, ["model.name", "harness.name"], metcol, descending) - - -def _load_csv(csv_path: str | Path = "results.csv") -> pd.DataFrame: - df = pd.read_csv(csv_path) - df = df.dropna(subset=["metrics.score"]) - df = df.loc[df["metrics.score"] > 0] - df = df.loc[~df["benchmark.name"].isin(EXCLUDED_BENCHMARKS)] - return df.reset_index(drop=True) - - -def _empty_table(): - return pd.DataFrame({"Rank": [], "Category": []}) - - -def load_and_rank( - csv_path: str | Path = "results.csv", - open_models_only: bool = False, - open_harnesses_only: bool = False, - benchmarks: list[str] | None = None, - models: list[str] | None = None, - harnesses: list[str] | None = None, - rank_by: str = "Benchmark Score", -) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - df = _load_csv(csv_path) - if open_models_only: - df = df.loc[df["model.is_oss"] == True] - if open_harnesses_only: - df = df.loc[df["harness.is_oss"] == True] - if benchmarks is not None: - df = df.loc[df["benchmark.name"].isin(benchmarks)] - if models is not None: - df = df.loc[df["model.name"].isin(models)] - if harnesses is not None: - df = df.loc[df["harness.name"].isin(harnesses)] - - metcol, descending = RANK_BY_OPTIONS.get(rank_by, ("metrics.score", False)) - df = df.dropna(subset=[metcol]) - if descending: - df = df.loc[df[metcol] > 0] - df = df.reset_index(drop=True) - - if len(df) < 2: - empty = _empty_table() - return empty, empty, empty - - pairwise_harness_df = df.loc[ - ~df["harness.name"].isin(EXCLUDED_HARNESSES_FROM_PAIRWISE) - ].reset_index(drop=True) - - results = [] - for rank_fn, ranking_df in ( - (rank_harnesses, pairwise_harness_df), - (rank_models, df), - (rank_pairs, pairwise_harness_df), - ): - try: - results.append(rank_fn(ranking_df, metcol, descending)) - except ValueError: - results.append(_empty_table()) - return results[0], results[1], results[2] diff --git a/src/results_to_csv.py b/src/results_to_csv.py deleted file mode 100644 index 8d95c13e314bd922653ea52b467e16e060b3e3e4..0000000000000000000000000000000000000000 --- a/src/results_to_csv.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -"""Convert result JSON files to CSV and validate data completeness. - -Based on load_dicts_to_df from: -https://github.com/erikerlandson/paired-comparison-ranking/blob/main/nb/paired-comparison-ranking.ipynb -""" - -import ast -import json -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - - -REQUIRED_METRICS = [ - "metrics.n_tasks", - "metrics.n_errors", - "metrics.score", - "metrics.cost_usd", - "metrics.n_input_tokens", - "metrics.n_output_tokens", - "metrics.agent_time_seconds", - "metrics.total_time_seconds", -] - - -def load_dicts_to_df( - directory: str | Path, - pattern: str = "*.json", -) -> pd.DataFrame: - directory = Path(directory) - if not directory.is_dir(): - raise NotADirectoryError(directory) - - rows: list[dict[str, Any]] = [] - for path in sorted(directory.glob(pattern)): - if not path.is_file() or path.name.startswith("."): - continue - suffix = path.suffix.lower() - if suffix == ".json": - with path.open(encoding="utf-8") as f: - data = json.load(f) - else: - text = path.read_text(encoding="utf-8") - data = ast.literal_eval(text) - - if not isinstance(data, dict): - raise TypeError(f"{path} does not contain a dict (got {type(data).__name__})") - rows.append(data) - - return pd.json_normalize(rows) - - -def validate(df: pd.DataFrame) -> bool: - issues = [] - for _, row in df.iterrows(): - label = f"{row.get('benchmark.name', '?')} / {row.get('model.name', '?')} / {row.get('harness.name', '?')}" - missing = [col for col in REQUIRED_METRICS if col not in df.columns or pd.isna(row.get(col))] - if missing: - fields = ", ".join(c.replace("metrics.", "") for c in missing) - issues.append(f" {label}: missing {fields}") - - if issues: - print(f"Validation: {len(issues)} result(s) with missing data:") - for issue in issues: - print(issue) - return False - - print("Validation: all results complete") - return True - - -def main(): - results_dir = Path(__file__).parent.parent / "results" - output_path = Path(__file__).parent.parent / "results.csv" - - if len(sys.argv) > 1: - results_dir = Path(sys.argv[1]) - if len(sys.argv) > 2: - output_path = Path(sys.argv[2]) - - df = load_dicts_to_df(results_dir) - - drop_cols = [c for c in df.columns if c.startswith("environment.config.")] - df = df.drop(columns=drop_cols) - - validate(df) - - df.to_csv(output_path, index=False) - print(f"Wrote {len(df)} rows to {output_path}") - - -if __name__ == "__main__": - main() diff --git a/src/version.py b/src/version.py deleted file mode 100644 index 0a998ccb39fb80141dab96471938b4c8c4dfc0b4..0000000000000000000000000000000000000000 --- a/src/version.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Project version, sourced from the top-level ``VERSION`` file. - -``VERSION`` is the single source of truth. Bump it in a PR (``make bump -VERSION=X.Y.Z``); when the bump lands on ``main``, CI tags the merge commit -``vX.Y.Z`` and deploys it to the production HF Space (see README "Releasing"). -""" - -from pathlib import Path - -VERSION_FILE = Path(__file__).resolve().parent.parent / "VERSION" - - -def read_version() -> str: - try: - return VERSION_FILE.read_text(encoding="utf-8").strip() or "0.0.0" - except OSError: - return "0.0.0" - - -__version__ = read_version() diff --git a/tests/test_configuration.py b/tests/test_configuration.py deleted file mode 100644 index 78ce08c90c8a0451ad167fac94c5e40ad51fe1b4..0000000000000000000000000000000000000000 --- a/tests/test_configuration.py +++ /dev/null @@ -1,54 +0,0 @@ -from src.configuration import ( - changed_configuration_fields, - configuration_fingerprint, - normalize_configuration, -) -from src.models import Configuration, Result - - -def test_existing_result_parses_without_configuration(): - result = Result.model_validate( - { - "benchmark": {"name": "Bench", "repo": "r", "num_tasks": 1, "url": "u"}, - "harness": {"name": "Harness", "skills": [], "is_oss": True, "url": "u"}, - "model": {"name": "Model", "is_oss": True, "num_params": 1, "precision": "fp8", "url": "u"}, - "environment": {"name": "Env", "url": "u"}, - "metrics": {"score": 0.5}, - } - ) - assert result.configuration is None - - -def test_fingerprint_ignores_label_and_collection_order(): - a = Configuration( - shell_access=True, - skills=["planning", "code_search"], - tools=["shell", "editor"], - configuration_label="display A", - ) - b = Configuration( - shell_access=True, - skills=["code_search", "planning"], - tools=["editor", "shell"], - configuration_label="display B", - ) - assert normalize_configuration(a) == normalize_configuration(b) - assert configuration_fingerprint(a) == configuration_fingerprint(b) - - -def test_fingerprint_distinguishes_meaningful_change(): - assert configuration_fingerprint(Configuration(shell_access=True)) != configuration_fingerprint( - Configuration(shell_access=False) - ) - - -def test_missing_values_are_not_guessed(): - normalized = normalize_configuration(Configuration(shell_access=None, planning_enabled=None)) - assert "shell_access" not in normalized - assert "planning_enabled" not in normalized - - -def test_changed_fields_use_structured_paths(): - baseline = Configuration(shell_access=False, planning_enabled=False) - treatment = Configuration(shell_access=True, planning_enabled=False) - assert changed_configuration_fields(baseline, treatment) == ["shell_access"] diff --git a/tests/test_configuration_analysis.py b/tests/test_configuration_analysis.py deleted file mode 100644 index 4d1f7ab785ac23c216e6a198e53c2eddf913a2d5..0000000000000000000000000000000000000000 --- a/tests/test_configuration_analysis.py +++ /dev/null @@ -1,61 +0,0 @@ -from src.configuration_analysis import compare_runs, metric_delta, safe_percent_change -from src.models import Result - - -def make_result(*, shell: bool | None, benchmark="Bench", model="Model", harness="Harness", env="Env", score=0.5, tokens=100, cost=1.0): - configuration = None if shell is None else {"shell_access": shell} - return Result.model_validate( - { - "benchmark": {"name": benchmark, "repo": "r", "num_tasks": 10, "url": "u"}, - "harness": {"name": harness, "skills": [], "is_oss": True, "url": "u"}, - "model": {"name": model, "is_oss": True, "num_params": 1, "precision": "fp8", "url": "u"}, - "environment": {"name": env, "url": "u"}, - "metrics": { - "score": score, - "n_tasks": 10, - "n_errors": 1, - "n_total_tokens": tokens, - "cost_usd": cost, - "mean_tokens_per_task": tokens / 10, - "mean_cost_usd_per_task": cost / 10, - "mean_total_time_seconds_per_task": 5, - }, - "configuration": configuration, - } - ) - - -def test_safe_percent_change_handles_zero_and_missing(): - assert safe_percent_change(0, 10) is None - assert safe_percent_change(None, 10) is None - assert safe_percent_change(100, 80) == -20.0 - - -def test_score_delta_separates_percentage_points_and_relative_change(): - delta = metric_delta(80, 82) - assert delta.absolute == 2.0 - assert delta.percent == 2.5 - - -def test_exact_match_when_only_configuration_changes(): - comparison = compare_runs(make_result(shell=False), make_result(shell=True, score=0.52, tokens=90)) - assert comparison.status == "exact" - assert comparison.changed_fields == ("shell_access",) - assert comparison.metric_deltas["score_percentage_points"].absolute == 2.0 - assert comparison.metric_deltas["total_tokens"].percent == -10.0 - - -def test_partial_match_when_environment_differs(): - comparison = compare_runs(make_result(shell=False), make_result(shell=True, env="Env2")) - assert comparison.status == "partial" - assert "environment" in comparison.unmatched_dimensions - - -def test_unmatched_when_benchmark_or_model_differs(): - comparison = compare_runs(make_result(shell=False), make_result(shell=True, benchmark="Other", model="Other")) - assert comparison.status == "unmatched" - - -def test_missing_configuration_is_observational(): - comparison = compare_runs(make_result(shell=None), make_result(shell=True)) - assert comparison.status == "observational" diff --git a/tests/test_pr2_analytics.py b/tests/test_pr2_analytics.py deleted file mode 100644 index cc149a45ddfd5cfc77c8d296ae6946e395d12b93..0000000000000000000000000000000000000000 --- a/tests/test_pr2_analytics.py +++ /dev/null @@ -1,220 +0,0 @@ -import pandas as pd -import plotly.graph_objects as go - -from src.analytics import ( - BENCHMARK_CATALOG, - benchmark_category, - benchmarks_for_category, - cross_benchmark_ranking_df, - enrich_analysis_df, - filter_category, - matrix_df, - ranking_df, -) -from src.charts import create_coverage_matrix_plot, create_matrix_plot, create_tradeoff_plot - - -def frame(): - return pd.DataFrame( - [ - { - "Benchmark": "SWE-Bench Verified", "Model": "a", "Harness": "h", "Run Label": "a / h", - "Category": "FOSS", "Score": .8, "Score (%)": 80, "Tasks": 10, "Errors": 1, - "Input Tokens Per Task": 50, "Cache Tokens Per Task": 10, "Output Tokens Per Task": 20, - "Total Tokens Per Task": 80, "Tokens Per Solved Task": 100, "Cost Per Task": .2, - "Total Time Per Task": 10, "Agent Time Per Task": 8, "Token Data Available": True, - }, - { - "Benchmark": "SWE-Bench Verified", "Model": "b", "Harness": "h", "Run Label": "b / h", - "Category": "FOSS", "Score": .4, "Score (%)": 40, "Tasks": 10, "Errors": 2, - "Input Tokens Per Task": 90, "Cache Tokens Per Task": None, "Output Tokens Per Task": 30, - "Total Tokens Per Task": 120, "Tokens Per Solved Task": 300, "Cost Per Task": .1, - "Total Time Per Task": 20, "Agent Time Per Task": 15, "Token Data Available": True, - }, - { - "Benchmark": "Terminal Bench 2.0", "Model": "a", "Harness": "h", "Run Label": "a / h", - "Category": "FOSS", "Score": .2, "Score (%)": 20, "Tasks": 10, "Errors": 0, - "Input Tokens Per Task": 30, "Cache Tokens Per Task": 5, "Output Tokens Per Task": 10, - "Total Tokens Per Task": 45, "Tokens Per Solved Task": 225, "Cost Per Task": .3, - "Total Time Per Task": 30, "Agent Time Per Task": 25, "Token Data Available": True, - }, - { - "Benchmark": "Terminal Bench 2.0", "Model": "c", "Harness": "h", "Run Label": "c / h", - "Category": "FOSS", "Score": .9, "Score (%)": 90, "Tasks": 10, "Errors": None, - "Input Tokens Per Task": None, "Cache Tokens Per Task": None, "Output Tokens Per Task": None, - "Total Tokens Per Task": None, "Tokens Per Solved Task": None, "Cost Per Task": None, - "Total Time Per Task": None, "Agent Time Per Task": None, "Token Data Available": False, - }, - { - "Benchmark": "New Benchmark", "Model": "z", "Harness": "h", "Run Label": "z / h", - "Category": "FOSS", "Score": .5, "Score (%)": 50, "Tasks": 10, "Errors": 0, - "Input Tokens Per Task": 1, "Cache Tokens Per Task": 1, "Output Tokens Per Task": 1, - "Total Tokens Per Task": 3, "Tokens Per Solved Task": 6, "Cost Per Task": .01, - "Total Time Per Task": 1, "Agent Time Per Task": 1, "Token Data Available": True, - }, - ] - ) - - -def test_benchmark_catalog_and_unknown_fallback(): - assert benchmark_category("SWE-Bench Verified") == "Coding" - assert benchmark_category("Terminal Bench 2.0") == "Generalist" - assert benchmark_category("Shellbench") == "Generalist" - assert benchmark_category("New Benchmark") == "Other" - assert "SWE-Bench Pro -- Ansible" in BENCHMARK_CATALOG - - -def test_category_filtering_keeps_unknown_visible_as_other(): - df = frame() - assert set(benchmarks_for_category(df, "Coding")) == {"SWE-Bench Verified"} - assert set(filter_category(df, "Generalist")["Benchmark"]) == {"Terminal Bench 2.0"} - assert set(filter_category(df, "Other")["Benchmark"]) == {"New Benchmark"} - - -def test_derived_reliability_and_per_success_metrics(): - df = enrich_analysis_df(frame()) - first = df.iloc[0] - assert first["Execution Error Rate (%)"] == 10 - assert first["Tokens Per Successful Task"] == 100 - assert first["Cost Per Successful Task"] == .25 - assert first["Time Per Successful Task"] == 12.5 - assert pd.isna(df.loc[df["Model"].eq("c"), "Execution Error Rate (%)"]).all() - - -def test_within_benchmark_percentile_and_rank(): - df = enrich_analysis_df(frame()) - coding = df[df["Benchmark"] == "SWE-Bench Verified"].set_index("Model") - assert coding.loc["a", "Within-Benchmark Rank"] == 1 - assert coding.loc["b", "Within-Benchmark Rank"] == 2 - assert coding.loc["a", "Within-Benchmark Percentile"] == 100 - assert coding.loc["b", "Within-Benchmark Percentile"] == 0 - - -def test_cross_benchmark_ordering_uses_normalization_and_coverage_threshold(): - df = frame() - ranked = cross_benchmark_ranking_df(df, minimum_coverage=0.5) - a = ranked[ranked["Model"] == "a"].iloc[0] - assert a["Benchmarks Covered"] == 2 - assert a["Eligible Benchmarks"] == 3 - assert a["Normalized Performance"] == 50 - strict = cross_benchmark_ranking_df(df, minimum_coverage=1.0) - assert strict.empty - - -def test_token_ranking_excludes_missing_and_orders_lower_first(): - ranked = ranking_df(frame(), "Total tokens", benchmark="Terminal Bench 2.0") - assert ranked["Model"].tolist() == ["a"] - - -def test_score_rank_and_metric_matrices_preserve_missing_cells(): - df = frame() - score = matrix_df(df, "Score") - rank = matrix_df(df, "Within-benchmark rank") - cost = matrix_df(df, "Cost") - assert pd.isna(score.loc["b / h", "Terminal Bench 2.0"]) - assert rank.loc["a / h", "SWE-Bench Verified"] == 1 - assert pd.isna(cost.loc["c / h", "Terminal Bench 2.0"]) - - -def test_coverage_matrix_uses_missing_not_zero_score(): - matrix = matrix_df(frame(), "Coverage") - assert matrix.loc["a / h", "SWE-Bench Verified"] == 1 - assert pd.isna(matrix.loc["b / h", "Terminal Bench 2.0"]) - figure = create_coverage_matrix_plot(matrix) - assert isinstance(figure, go.Figure) - assert "Available" in figure.data[0].text[0] or "Missing" in figure.data[0].text[0] - - -def test_generic_tradeoff_and_matrix_charts_construct(): - df = enrich_analysis_df(frame()) - figure = create_tradeoff_plot( - df[df["Benchmark"] == "SWE-Bench Verified"], - "Total Tokens Per Task", "Score (%)", - "Total tokens per task", "Score (%)", - show_pareto_frontier=True, - ) - matrix_figure = create_matrix_plot(matrix_df(df, "Score"), "Score matrix", "Score (%)") - assert isinstance(figure, go.Figure) - assert isinstance(matrix_figure, go.Figure) - assert any(trace.name == "Pareto frontier" for trace in figure.data) - - -def test_tradeoff_pareto_supports_lower_is_better_on_both_axes(): - df = pd.DataFrame( - { - "Model": ["a", "b", "c"], - "Harness": ["h", "h", "h"], - "Benchmark": ["bench", "bench", "bench"], - "Total Tokens Per Task": [100, 200, 300], - "Cost Per Task": [0.3, 0.2, 0.4], - } - ) - figure = create_tradeoff_plot( - df, - "Total Tokens Per Task", - "Cost Per Task", - "Tokens", - "Cost", - show_pareto_frontier=True, - lower_x_is_better=True, - higher_y_is_better=False, - ) - frontier = next(trace for trace in figure.data if trace.name == "Pareto frontier") - assert list(frontier.x) == [100, 200] - assert list(frontier.y) == [0.3, 0.2] - - -def test_matrix_can_color_by_percentile_but_display_raw_values(): - color_matrix = pd.DataFrame([[100.0, 0.0]], index=["a / h"], columns=["b1", "b2"] ) - raw_matrix = pd.DataFrame([[82.5, 41.25]], index=["a / h"], columns=["b1", "b2"] ) - figure = create_matrix_plot( - color_matrix, - "Score matrix", - "Within-benchmark percentile", - display_matrix=raw_matrix, - display_metric_label="Benchmark score (%)", - ) - assert list(figure.data[0].z[0]) == [100.0, 0.0] - assert list(figure.data[0].text[0]) == ["82.5", "41.25"] - assert "Benchmark score (%)" in figure.data[0].hovertemplate - - -def test_ranking_plot_order_can_be_value_or_alphabetical(): - from src.charts import create_ranking_plot - - df = pd.DataFrame( - { - "Model": ["b", "a", "c"], - "Harness": ["h", "h", "h"], - "Benchmark": ["bench", "bench", "bench"], - "Score (%)": [20, 10, 30], - } - ) - largest = create_ranking_plot( - df, "Score (%)", "Score", True, sort_order="Largest value first" - ) - lowest = create_ranking_plot( - df, "Score (%)", "Score", True, sort_order="Lowest value first" - ) - alpha = create_ranking_plot( - df, "Score (%)", "Score", True, sort_order="Alphabetical (A–Z)" - ) - - assert list(largest.layout.yaxis.categoryarray) == ["c / h", "b / h", "a / h"] - assert list(lowest.layout.yaxis.categoryarray) == ["a / h", "b / h", "c / h"] - assert list(alpha.layout.yaxis.categoryarray) == ["a / h", "b / h", "c / h"] - - -def test_matrix_height_scales_with_rows_and_keeps_all_y_labels(): - rows = 18 - matrix = pd.DataFrame( - {"bench-a": range(rows), "bench-b": range(rows)}, - index=[f"model-{i} / harness" for i in range(rows)], - ) - - figure = create_matrix_plot(matrix, "Dense matrix", "Score (%)") - - assert figure.layout.height >= 900 - assert figure.layout.yaxis.tickmode == "array" - assert len(figure.layout.yaxis.tickvals) == rows - assert len(figure.layout.yaxis.ticktext) == rows diff --git a/tests/test_ranking_refinements.py b/tests/test_ranking_refinements.py deleted file mode 100644 index b59672a92ca1d62a9ac8048086483bcddc584cfc..0000000000000000000000000000000000000000 --- a/tests/test_ranking_refinements.py +++ /dev/null @@ -1,135 +0,0 @@ -from pathlib import Path - -import pandas as pd -import pytest - -from src.charts import create_ranking_plot - - -def _rankings_module(): - pytest.importorskip("choix") - from src import rankings - - return rankings - - -def test_paired_comparison_missing_values_render_blank(): - rankings = _rankings_module() - table = rankings.ranking_dataframe( - ["agent-a", "agent-b"], - [1.0, 0.0], - {"Bench A": [[0.8], []]}, - ) - - assert table.loc[0, "Bench A"] == 0.8 - assert table.loc[1, "Bench A"] == "" - - -def test_shellbench_is_excluded_from_paired_comparison_source(tmp_path): - rankings = _rankings_module() - csv_path = tmp_path / "results.csv" - pd.DataFrame( - { - "benchmark.name": ["Shellbench", "SWE-Bench Verified"], - "metrics.score": [0.9, 0.8], - } - ).to_csv(csv_path, index=False) - - loaded = rankings._load_csv(csv_path) - - assert loaded["benchmark.name"].tolist() == ["SWE-Bench Verified"] - - -def test_best_value_first_sorts_lower_is_better_metric_lowest_first(): - df = pd.DataFrame( - { - "Model": ["b", "a", "c"], - "Harness": ["h", "h", "h"], - "Benchmark": ["bench", "bench", "bench"], - "Cost Per Task": [20.0, 10.0, 30.0], - } - ) - - figure = create_ranking_plot( - df, - "Cost Per Task", - "Cost per task", - False, - sort_order="Best first", - ) - - assert list(figure.layout.yaxis.categoryarray) == ["a / h", "b / h", "c / h"] - - -def test_rankings_page_places_paired_comparisons_first_and_has_no_score_all_benchmarks(): - source = Path("app.py").read_text() - rankings = source[source.index('with gr.Tab("Rankings")'):source.index('with gr.Tab("Trade-offs")')] - - assert 'gr.Markdown("### Paired Comparisons")' in rankings - assert rankings.index('gr.Markdown("### Paired Comparisons")') < rankings.index('"### Metric rankings\\n"') - assert 'with gr.Tab("📊 Bradley-Terry Rankings")' not in source - assert 'ranking_sections = [("Score", "Score", BENCHMARK_NAMES, DEFAULT_BENCHMARK)]' in rankings - assert '"Best first"' in rankings - assert '"Best last"' in rankings - - -def test_ranking_plot_height_scales_with_agents_and_keeps_all_y_labels(): - rows = 16 - df = pd.DataFrame( - { - "Model": [f"model-{i}" for i in range(rows)], - "Harness": ["h"] * rows, - "Benchmark": ["bench"] * rows, - "Score (%)": list(range(rows)), - } - ) - - figure = create_ranking_plot(df, "Score (%)", "Score", True, sort_order="Best first") - - assert figure.layout.height >= 340 - assert figure.layout.height < 900 - assert figure.layout.yaxis.tickmode == "array" - assert len(figure.layout.yaxis.tickvals) == rows - assert len(figure.layout.yaxis.ticktext) == rows - - -def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path): - rankings = _rankings_module() - csv_path = tmp_path / "results.csv" - pd.DataFrame( - { - "benchmark.name": [ - "Bench A", "Bench A", "Bench A", "Bench A", - "Bench B", "Bench B", "Bench B", "Bench B", - ], - "model.name": ["m1", "m1", "m1", "m1", "m2", "m2", "m2", "m2"], - "harness.name": [ - "Claude Code", "OpenCode", "Codex", "Qwen Code", - "Claude Code", "OpenCode", "Codex", "Qwen Code", - ], - "metrics.score": [0.8, 0.7, 0.95, 0.9, 0.75, 0.65, 0.93, 0.88], - "model.is_oss": [True] * 8, - "harness.is_oss": [True] * 8, - } - ).to_csv(csv_path, index=False) - - harness_df, model_df, pair_df = rankings.load_and_rank(csv_path) - - assert set(harness_df["Category"]) == {"Claude Code", "OpenCode"} - assert set(pair_df["Category"]) == { - "m1 + Claude Code", - "m1 + OpenCode", - "m2 + Claude Code", - "m2 + OpenCode", - } - assert set(model_df["Category"]) == {"m1", "m2"} - - model_by_category = model_df.set_index("Category") - assert model_by_category.loc["m1", "Bench A"] == pytest.approx(0.838) - - -def test_ranking_minimum_height_is_reduced_without_changing_matrix_minimum(): - source = Path("src/charts.py").read_text() - - assert "RANKING_MIN_HEIGHT_PX = 340" in source - assert "MATRIX_MIN_HEIGHT_PX = 720" in source diff --git a/tests/test_token_efficiency.py b/tests/test_token_efficiency.py deleted file mode 100644 index a2b72009589b3efab1ccd82d9d348c9a099fd3b8..0000000000000000000000000000000000000000 --- a/tests/test_token_efficiency.py +++ /dev/null @@ -1,413 +0,0 @@ -from pathlib import Path -import pandas as pd -import plotly.graph_objects as go - -from src.charts import ( - create_performance_vs_resource_plot, - create_score_vs_cost_plot, - create_score_vs_tokens_plot, -) -from src.leaderboard import ( - ANALYSIS_COLUMNS, - EFFICIENCY_RESOURCE_METRICS, - TOKEN_EFFICIENCY_TABLE_COLUMNS, - get_analysis_df, - get_efficiency_resource_column, - get_efficiency_df, - get_token_efficiency_table_df, - get_resource_pareto_frontier_df, -) -from src.models import Benchmark, Environment, Harness, Metrics, Model, Result - - -def make_result( - *, - benchmark: str = "Benchmark A", - model: str = "model-a", - harness: str = "harness-a", - score: float = 0.5, - n_tasks: int | None = 10, - total_tokens: int | None = 100, - input_tokens: int | None = 60, - cache_tokens: int | None = 10, - output_tokens: int | None = 30, - cost_per_task: float | None = 0.25, - agent_time_per_task: int | None = 10, - cost_usd: float | None = None, - agent_time_seconds: int | None = None, -) -> Result: - return Result( - benchmark=Benchmark( - name=benchmark, - repo="repo", - num_tasks=10, - url="https://example.com/benchmark", - ), - harness=Harness( - name=harness, - skills=[], - is_oss=True, - url="https://example.com/harness", - ), - model=Model( - name=model, - repo=None, - is_oss=True, - num_params=1, - precision="fp16", - url="https://example.com/model", - ), - environment=Environment(name="env", url="https://example.com/env"), - metrics=Metrics( - score=score, - n_tasks=n_tasks, - n_errors=1, - mean_input_tokens_per_task=input_tokens, - mean_cache_tokens_per_task=cache_tokens, - mean_output_tokens_per_task=output_tokens, - mean_tokens_per_task=total_tokens, - mean_cost_usd_per_task=cost_per_task, - mean_total_time_seconds_per_task=12, - mean_agent_time_seconds_per_task=agent_time_per_task, - cost_usd=cost_usd, - agent_time_seconds=agent_time_seconds, - ), - ) - - -def test_cost_vs_performance_tab_removed_and_navigation_is_single_layer(): - app_source = Path("app.py").read_text() - - assert 'gr.Tab("💰 Cost vs Performance")' not in app_source - assert "cost_benchmark" not in app_source - assert "cost_controls" not in app_source - assert "render_score_vs_cost_plot" not in app_source - assert 'with gr.Tab("Overview")' not in app_source - assert app_source.count("with gr.Tabs():") == 1 - assert 'with gr.Tab("Rankings")' in app_source - assert 'with gr.Tab("Trade-offs")' in app_source - assert 'with gr.Tab("Matrices")' in app_source - -def test_analysis_df_columns_and_derived_metrics(): - dataframe = get_analysis_df( - [make_result(score=0.25, total_tokens=200, cost_per_task=0.125, agent_time_per_task=7)] - ) - - assert set(ANALYSIS_COLUMNS).issubset(dataframe.columns) - assert dataframe.loc[0, "Score (%)"] == 25 - assert dataframe.loc[0, "Tokens Per Solved Task"] == 800 - assert dataframe.loc[0, "Cost Per Task"] == 0.125 - assert dataframe.loc[0, "Agent Time Per Task"] == 7 - assert bool(dataframe.loc[0, "Token Data Available"]) is True - - -def test_missing_zero_negative_resource_values_are_unavailable(): - dataframe = get_analysis_df( - [ - make_result(model="missing", total_tokens=None, cost_per_task=None, agent_time_per_task=None), - make_result(model="zero", total_tokens=0, cost_per_task=0, agent_time_per_task=0), - make_result(model="negative", total_tokens=-10, cost_per_task=-1, agent_time_per_task=-3), - ] - ) - - assert dataframe["Token Data Available"].tolist() == [False, False, False] - assert dataframe["Tokens Per Solved Task"].isna().all() - assert dataframe["Cost Per Task"].isna().all() - assert dataframe["Agent Time Per Task"].isna().all() - - -def test_zero_score_does_not_divide_by_zero(): - dataframe = get_analysis_df([make_result(score=0, total_tokens=100)]) - - assert pd.isna(dataframe.loc[0, "Tokens Per Solved Task"]) - assert bool(dataframe.loc[0, "Token Data Available"]) is True - - -def test_invalid_task_denominator_does_not_trigger_total_metric_fallback(): - dataframe = get_analysis_df( - [ - make_result( - n_tasks=0, - cost_per_task=None, - agent_time_per_task=None, - cost_usd=1.5, - agent_time_seconds=30, - ) - ] - ) - - assert pd.isna(dataframe.loc[0, "Cost Per Task"]) - assert pd.isna(dataframe.loc[0, "Agent Time Per Task"]) - - -def test_efficiency_resource_metric_choices_are_exact(): - assert list(EFFICIENCY_RESOURCE_METRICS) == [ - "Total tokens", - "Cost per task", - "Agent time per task", - ] - assert "Tokens Per Solved Task" not in EFFICIENCY_RESOURCE_METRICS - assert get_efficiency_resource_column("Total tokens") == "Total Tokens Per Task" - assert get_efficiency_resource_column("Cost per task") == "Cost Per Task" - assert get_efficiency_resource_column("Agent time per task") == "Agent Time Per Task" - - -def test_efficiency_filtering_is_benchmark_specific_and_counts_exclusions(): - analysis_df = get_analysis_df( - [ - make_result(benchmark="Benchmark A", model="valid", total_tokens=100), - make_result(benchmark="Benchmark A", model="zero", total_tokens=0), - make_result(benchmark="Benchmark A", model="missing", total_tokens=None), - make_result(benchmark="Benchmark B", model="other", total_tokens=100), - ] - ) - - filtered = get_efficiency_df( - benchmark_name="Benchmark A", - resource_metric="Total tokens", - analysis_df=analysis_df, - ) - - assert filtered["Model"].tolist() == ["valid"] - assert filtered["Benchmark"].unique().tolist() == ["Benchmark A"] - assert filtered.attrs["exclusion_count"] == 2 - assert get_efficiency_df( - "All benchmarks", "Total tokens", analysis_df - ).empty - - -def test_cost_and_agent_time_filtering_use_positive_values_only(): - analysis_df = get_analysis_df( - [ - make_result(model="valid", cost_per_task=0.2, agent_time_per_task=9), - make_result(model="invalid", cost_per_task=0, agent_time_per_task=-1), - ] - ) - - cost = get_efficiency_df("Benchmark A", "Cost per task", analysis_df) - agent_time = get_efficiency_df("Benchmark A", "Agent time per task", analysis_df) - - assert cost["Model"].tolist() == ["valid"] - assert agent_time["Model"].tolist() == ["valid"] - assert cost.attrs["exclusion_count"] == 1 - assert agent_time.attrs["exclusion_count"] == 1 - - -def test_efficiency_table_keeps_tokens_per_solved_task_and_expected_order(): - analysis_df = get_analysis_df( - [make_result(score=0.333333333333, total_tokens=120, cost_per_task=0.123456)] - ) - - table = get_token_efficiency_table_df("Benchmark A", analysis_df) - - assert list(table.columns) == TOKEN_EFFICIENCY_TABLE_COLUMNS - assert list(table.columns[:3]) == ["Model", "Harness", "Benchmark"] - assert "Tokens Per Solved Task" in table.columns - assert table.loc[0, "Score (%)"] == 33.3 - assert table.loc[0, "Cost Per Task"] == 0.1235 - - -def test_pareto_frontier_for_all_resource_metrics(): - dataframe = pd.DataFrame( - { - "Run Label": ["a", "b", "c", "d"], - "Model": ["a", "b", "c", "d"], - "Total Tokens Per Task": [100, 200, 300, 400], - "Cost Per Task": [0.1, 0.2, 0.3, 0.4], - "Agent Time Per Task": [10, 20, 30, 40], - "Score (%)": [50, 60, 55, 80], - } - ) - - for metric in ("Total Tokens Per Task", "Cost Per Task", "Agent Time Per Task"): - frontier = get_resource_pareto_frontier_df(dataframe, metric) - assert frontier["Run Label"].tolist() == ["a", "b", "d"] - - -def test_pareto_equal_x_equal_score_ties_are_preserved(): - dataframe = pd.DataFrame( - { - "Run Label": ["z", "a", "dominated", "higher"], - "Model": ["z", "a", "d", "h"], - "Cost Per Task": [0.1, 0.1, 0.1, 0.2], - "Score (%)": [50, 50, 40, 60], - } - ) - - frontier = get_resource_pareto_frontier_df(dataframe, "Cost Per Task") - - assert frontier["Run Label"].tolist() == ["a", "z", "higher"] - assert "dominated" not in frontier["Run Label"].tolist() - - -def test_pareto_excludes_missing_zero_and_negative_resources(): - dataframe = pd.DataFrame( - { - "Run Label": ["valid", "missing", "zero", "negative"], - "Agent Time Per Task": [10, None, 0, -1], - "Score (%)": [50, 100, 100, 100], - } - ) - - frontier = get_resource_pareto_frontier_df(dataframe, "Agent Time Per Task") - - assert frontier["Run Label"].tolist() == ["valid"] - - -def test_performance_resource_charts_construct_with_metric_specific_axes(): - dataframe = get_analysis_df( - [ - make_result(), - make_result(model="model-b", score=0.7, total_tokens=200, cost_per_task=0.4, agent_time_per_task=20), - ] - ) - - expected_titles = { - "Total tokens": "Total tokens per task", - "Cost per task": "Cost per task (USD)", - "Agent time per task": "Agent time per task (seconds)", - } - for metric, title in expected_titles.items(): - figure = create_performance_vs_resource_plot(dataframe, resource_metric=metric) - assert isinstance(figure, go.Figure) - assert figure.layout.xaxis.type == "log" - assert figure.layout.xaxis.title.text == title - assert any(trace.name == "Pareto frontier" for trace in figure.data) - - compatibility = create_score_vs_tokens_plot(dataframe, token_metric="Total tokens") - assert isinstance(compatibility, go.Figure) - - - -def test_performance_resource_chart_rejects_multiple_benchmarks(): - dataframe = get_analysis_df( - [ - make_result(benchmark="Benchmark A"), - make_result(benchmark="Benchmark B", model="model-b"), - ] - ) - figure = create_performance_vs_resource_plot(dataframe, resource_metric="Total tokens") - - assert len(figure.layout.annotations) == 1 - assert "Select one benchmark" in figure.layout.annotations[0].text - -def test_linear_scale_and_point_labels_still_work(): - dataframe = get_analysis_df([make_result()]) - figure = create_performance_vs_resource_plot( - dataframe, - resource_metric="Cost per task", - x_scale="Linear", - show_labels=True, - show_pareto_frontier=False, - ) - - assert figure.layout.xaxis.type == "linear" - assert figure.data[0].mode == "markers+text" - assert list(figure.data[0].text) == ["model-a / harness-a"] - - -def test_color_by_benchmark_is_not_supported_in_efficiency_chart(): - dataframe = get_analysis_df([make_result()]) - figure = create_performance_vs_resource_plot( - dataframe, - resource_metric="Total tokens", - color_by="Benchmark", # type: ignore[arg-type] - ) - - assert len(figure.layout.annotations) == 1 - assert "Color dimension not available" in figure.layout.annotations[0].text - - -def test_empty_and_fully_invalid_resource_chart_data_are_graceful(): - empty = create_performance_vs_resource_plot(pd.DataFrame(), resource_metric="Total tokens") - invalid_df = get_analysis_df([make_result(total_tokens=0)]) - invalid = create_performance_vs_resource_plot(invalid_df, resource_metric="Total tokens") - - assert isinstance(empty, go.Figure) - assert isinstance(invalid, go.Figure) - assert len(empty.layout.annotations) == 1 - assert len(invalid.layout.annotations) == 1 - - -def test_palette_lookup_new_palettes_fallback_and_copy(): - from src.charts import COLOR_PALETTES, get_color_palette - - for palette_name in ("Grayscale", "Viridis", "Plasma", "Cividis"): - assert get_color_palette(palette_name) == COLOR_PALETTES[palette_name] - assert get_color_palette(palette_name) is not COLOR_PALETTES[palette_name] - - fallback = get_color_palette("unknown") - assert fallback == COLOR_PALETTES["Citrus"] - fallback.append("#000000") - assert "#000000" not in COLOR_PALETTES["Citrus"] - - -def test_cost_and_efficiency_scatter_labels_toggle_consistently(): - cost_df = pd.DataFrame( - { - "Benchmark": ["Benchmark A"], - "Model": ["model-a"], - "Harness": ["harness-a"], - "Score": [50.0], - "Cost Per Task (USD)": [0.25], - "Label": ["model-a
harness-a"], - } - ) - efficiency_df = get_analysis_df([make_result()]) - - cost_without = create_score_vs_cost_plot(cost_df, "Benchmark A", show_labels=False) - cost_with = create_score_vs_cost_plot(cost_df, "Benchmark A", show_labels=True) - efficiency_without = create_performance_vs_resource_plot( - efficiency_df, - show_labels=False, - show_pareto_frontier=False, - ) - efficiency_with = create_performance_vs_resource_plot( - efficiency_df, - show_labels=True, - show_pareto_frontier=False, - ) - - assert cost_without.data[0].mode == "markers" - assert cost_with.data[0].mode == "markers+text" - assert efficiency_without.data[0].mode == "markers" - assert efficiency_with.data[0].mode == "markers+text" - assert cost_without.data[0].hovertemplate == cost_with.data[0].hovertemplate - assert efficiency_without.data[0].hovertemplate == efficiency_with.data[0].hovertemplate - -def test_efficiency_figure_uses_responsive_autosizing_without_fixed_width(): - dataframe = get_analysis_df([make_result()]) - figure = create_performance_vs_resource_plot( - dataframe, - resource_metric="Total tokens", - ) - - assert figure.layout.autosize is True - assert figure.layout.width is None - assert figure.layout.height is None - - -def test_shared_plot_container_has_minimum_height_and_scrollable_tables(): - app_source = (Path(__file__).parents[1] / "app.py").read_text() - - assert "RESPONSIVE_PLOT_MIN_HEIGHT_PX = 420" in app_source - assert "min-height: {RESPONSIVE_PLOT_MIN_HEIGHT_PX}px" in app_source - assert "TABLE_MAX_HEIGHT_PX = 720" in app_source - assert "max_height=TABLE_MAX_HEIGHT_PX" in app_source - - -def test_page_tables_are_single_and_below_visualizations_and_plots_resize(): - app_source = Path("app.py").read_text() - - rankings = app_source[app_source.index('with gr.Tab("Rankings")'):app_source.index('with gr.Tab("Trade-offs")')] - tradeoffs = app_source[app_source.index('with gr.Tab("Trade-offs")'):app_source.index('with gr.Tab("Matrices")')] - - assert rankings.count("gr.Dataframe(") == 4 - assert tradeoffs.count("gr.Dataframe(") == 1 - assert rankings.rindex("gr.Dataframe(") > rankings.rindex("gr.Plot(") - assert tradeoffs.rindex("gr.Dataframe(") > tradeoffs.rindex("gr.Plot(") - assert '"Best first"' in rankings - assert '"Best last"' in rankings - assert '"Alphabetical (A–Z)"' in rankings - assert "ResizeObserver" in app_source - assert "window.Plotly.Plots.resize" in app_source