Add CSV export script with data validation and auto-update workflow

- src/results_to_csv.py: converts result JSONs to CSV, validates for missing fields
- results.csv: generated CSV with all 22 current results
- .github/workflows/update-csv.yml: auto-regenerates CSV when results/*.json changes on main
- Makefile: added `make csv` target

.github/workflows/update-csv.yml

@@ -0,0 +1,33 @@
+name: Update results CSV
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "results/*.json"
+
+jobs:
+  update-csv:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pandas
+
+      - name: Generate CSV
+        run: python src/results_to_csv.py
+
+      - name: Commit if changed
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add results.csv
+          git diff --staged --quiet || git commit -m "Auto-update results.csv" && git push

Makefile

@@ -11,3 +11,7 @@ quality:
 	python -m black --check --line-length 119 .
 	python -m isort --check-only .
 	ruff check .
+
+
+csv:
+	python src/results_to_csv.py

results.csv

@@ -0,0 +1,23 @@
+benchmark.name,benchmark.repo,benchmark.num_tasks,benchmark.url,harness.name,harness.skills,harness.is_oss,harness.url,model.name,model.repo,model.is_oss,model.num_params,model.precision,model.url,environment.name,environment.url,metrics.n_tasks,metrics.n_errors,metrics.score,metrics.n_input_tokens,metrics.n_cache_tokens,metrics.n_output_tokens,metrics.n_total_tokens,metrics.agent_time_seconds,metrics.total_time_seconds,metrics.cost_usd,metrics.mean_input_tokens_per_task,metrics.mean_cache_tokens_per_task,metrics.mean_output_tokens_per_task,metrics.mean_tokens_per_task,metrics.mean_cost_usd_per_task,metrics.mean_total_time_seconds_per_task,metrics.mean_agent_time_seconds_per_task
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.6,Opus 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.633,438501505.0,419608736.0,2823496.0,860933737.0,126529.0,243310.0,434.54,1228295.0,1175374.0,7908.0,2411579.0,1.22,681.0,354.0
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.557,500304168.0,479035103.0,4937782.0,984277053.0,150652.0,269008.0,349.77,1401412.0,1341835.0,13831.0,2757078.0,0.98,753.0,421.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.698,192346997.0,186506482.0,2179945.0,381033424.0,32745.0,39030.0,185.66155285,2003614.0,1942775.0,22707.0,3969098.0,1.93,406.0,341.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.781,187217712.0,187209844.0,1280944.0,375708500.0,30686.0,39352.0,151.4104807499999,1950184.0,1950102.0,13343.0,3913630.0,1.58,409.0,319.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.5,190672390.0,184409111.0,1593112.0,376674613.0,40527.0,49734.0,184.42824125000004,1986170.0,1920928.0,16594.0,3923693.0,1.92,518.0,422.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.604,198924339.0,189578624.0,1560836.0,390063799.0,32914.0,39468.0,188.34296700000002,2072128.0,1974777.0,16258.0,4063164.0,1.96,411.0,342.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,6.0,0.458,367897697.0,0.0,1694885.0,369592582.0,39024.0,46758.0,9.64,3832267.0,0.0,17655.0,3849922.0,0.1,487.0,406.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.375,207164679.0,0.0,1598703.0,208763382.0,49450.0,57287.0,12.21,2157965.0,0.0,16653.0,2174618.0,0.13,596.0,515.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,5.0,0.406,0.0,0.0,0.0,0.0,38085.0,50779.0,9.4,0.0,0.0,0.0,0.0,0.1,528.0,396.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.479,742491363.0,0.0,2387609.0,744878972.0,54543.0,62422.0,13.47,7734285.0,0.0,24870.0,7759155.0,0.14,650.0,568.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.438,159198517.0,0.0,972133.0,160170650.0,33610.0,38272.0,9.34,1658317.0,0.0,10126.0,1668444.0,0.1,398.0,350.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.868,402225158.0,388781382.0,4588859.0,795595399.0,104467.0,159566.0,394.7823929,804450.0,777562.0,9177.0,1591190.0,0.79,319.0,208.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.834,347751942.0,347725246.0,3640204.0,699117392.0,114633.0,171540.0,319.53304049999974,695503.0,695450.0,7280.0,1398234.0,0.64,343.0,229.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,,,0.796,,,,,,,,,,,,,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.798,415795756.0,386352384.0,3431224.0,805579364.0,92588.0,141637.0,443.32977200000016,831591.0,772704.0,6862.0,1611158.0,0.89,283.0,185.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.632,1106618897.0,0.0,5733245.0,1112352142.0,122808.0,171897.0,34.11,2213237.0,0.0,11466.0,2224704.0,0.07,343.0,245.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.588,0.0,0.0,0.0,0.0,120399.0,200354.0,33.44,0.0,0.0,0.0,0.0,0.07,400.0,240.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.548,469806650.0,0.0,4937761.0,474744411.0,120473.0,185168.0,29.75,939613.0,0.0,9875.0,949488.0,0.06,370.0,240.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.65,791183735.0,0.0,6333798.0,797517533.0,154531.0,218988.0,38.16,1582367.0,0.0,12667.0,1595035.0,0.08,437.0,309.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.638,609589099.0,0.0,3964252.0,613553351.0,132273.0,178984.0,36.74,1219178.0,0.0,7928.0,1227106.0,0.07,357.0,264.0
+Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.303,47607780.0,0.0,1657188.0,49264968.0,46467.0,58507.0,11.47,534918.0,0.0,18620.0,553538.0,0.13,657.0,522.0
+Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.36,82108716.0,0.0,2056390.0,84165106.0,44991.0,64760.0,11.11,922569.0,0.0,23105.0,945675.0,0.12,727.0,505.0

src/results_to_csv.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Convert result JSON files to CSV and validate data completeness.
+
+Based on load_dicts_to_df from:
+https://github.com/erikerlandson/paired-comparison-ranking/blob/main/nb/paired-comparison-ranking.ipynb
+"""
+
+import ast
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+
+REQUIRED_METRICS = [
+    "metrics.n_tasks",
+    "metrics.n_errors",
+    "metrics.score",
+    "metrics.cost_usd",
+    "metrics.n_input_tokens",
+    "metrics.n_output_tokens",
+    "metrics.agent_time_seconds",
+    "metrics.total_time_seconds",
+]
+
+
+def load_dicts_to_df(
+    directory: str | Path,
+    pattern: str = "*.json",
+) -> pd.DataFrame:
+    directory = Path(directory)
+    if not directory.is_dir():
+        raise NotADirectoryError(directory)
+
+    rows: list[dict[str, Any]] = []
+    for path in sorted(directory.glob(pattern)):
+        if not path.is_file() or path.name.startswith("."):
+            continue
+        suffix = path.suffix.lower()
+        if suffix == ".json":
+            with path.open(encoding="utf-8") as f:
+                data = json.load(f)
+        else:
+            text = path.read_text(encoding="utf-8")
+            data = ast.literal_eval(text)
+
+        if not isinstance(data, dict):
+            raise TypeError(f"{path} does not contain a dict (got {type(data).__name__})")
+        rows.append(data)
+
+    return pd.json_normalize(rows)
+
+
+def validate(df: pd.DataFrame) -> bool:
+    issues = []
+    for _, row in df.iterrows():
+        label = f"{row.get('benchmark.name', '?')} / {row.get('model.name', '?')} / {row.get('harness.name', '?')}"
+        missing = [col for col in REQUIRED_METRICS if col in df.columns and pd.isna(row.get(col))]
+        if missing:
+            fields = ", ".join(c.replace("metrics.", "") for c in missing)
+            issues.append(f"  {label}: missing {fields}")
+
+    if issues:
+        print(f"Validation: {len(issues)} result(s) with missing data:")
+        for issue in issues:
+            print(issue)
+        return False
+
+    print("Validation: all results complete")
+    return True
+
+
+def main():
+    results_dir = Path(__file__).parent.parent / "results"
+    output_path = Path(__file__).parent.parent / "results.csv"
+
+    if len(sys.argv) > 1:
+        results_dir = Path(sys.argv[1])
+    if len(sys.argv) > 2:
+        output_path = Path(sys.argv[2])
+
+    df = load_dicts_to_df(results_dir)
+
+    drop_cols = [c for c in df.columns if c.startswith("environment.config.")]
+    df = df.drop(columns=drop_cols)
+
+    validate(df)
+
+    df.to_csv(output_path, index=False)
+    print(f"Wrote {len(df)} rows to {output_path}")
+
+
+if __name__ == "__main__":
+    main()