Add HF eval-results benchmark registry and API endpoint

- Add canonical EEG_BENCHMARKS registry (8 datasets) in config/benchmarks.py
- Add GET /api/benchmarks endpoint for benchmark metadata
- Refactor leaderboard transform_data to loop over registry instead of hardcoding
- Extract _build_entry_id helper for consistent unique ID generation
- Add push_eval_yaml.py script to register benchmarks on HF Hub
- Add pyyaml dependency

backend/app/api/endpoints/benchmarks.py

@@ -0,0 +1,38 @@
+from dataclasses import asdict
+
+from fastapi import APIRouter, HTTPException
+from typing import Any, Dict, List
+
+from app.config.benchmarks import EEG_BENCHMARKS
+
+router = APIRouter()
+
+# Fields that are internal implementation details, not exposed via API
+_INTERNAL_FIELDS = {"accuracy_field"}
+
+
+def _benchmark_to_dict(key: str, benchmark) -> Dict[str, Any]:
+    """Convert a BenchmarkInfo dataclass to a JSON-serializable dict."""
+    d = asdict(benchmark)
+    for field in _INTERNAL_FIELDS:
+        d.pop(field, None)
+    d["key"] = key
+    return d
+
+
+@router.get("")
+async def list_benchmarks() -> List[Dict[str, Any]]:
+    """Return all registered EEG benchmarks with metadata."""
+    return [_benchmark_to_dict(k, b) for k, b in EEG_BENCHMARKS.items()]
+
+
+@router.get("/{key}")
+async def get_benchmark(key: str) -> Dict[str, Any]:
+    """Return a single benchmark's details by key."""
+    benchmark = EEG_BENCHMARKS.get(key)
+    if not benchmark:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Benchmark '{key}' not found. Available: {list(EEG_BENCHMARKS.keys())}",
+        )
+    return _benchmark_to_dict(key, benchmark)

backend/app/api/router.py

@@ -1,9 +1,10 @@
 from fastapi import APIRouter
 
-from app.api.endpoints import leaderboard, votes, models
+from app.api.endpoints import leaderboard, votes, models, benchmarks
 
 router = APIRouter()
 
 router.include_router(leaderboard.router, prefix="/leaderboard", tags=["leaderboard"])
 router.include_router(votes.router, prefix="/votes", tags=["votes"])
-router.include_router(models.router, prefix="/models", tags=["models"])
+router.include_router(models.router, prefix="/models", tags=["models"])
+router.include_router(benchmarks.router, prefix="/benchmarks", tags=["benchmarks"])

backend/app/config/benchmarks.py

@@ -0,0 +1,102 @@
+"""
+Canonical benchmark registry for the EEG Finetune Arena.
+
+Maps internal benchmark keys to HuggingFace dataset IDs, task IDs,
+and metadata used by both eval.yaml generation and eval results pushing.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class BenchmarkInfo:
+    """Metadata for a single EEG benchmark dataset."""
+
+    dataset_id: str
+    task_id: str
+    display_name: str
+    description: str
+    category: str
+    num_classes: int
+    accuracy_field: str
+    split: str = "test"
+    config: str = "default"
+
+
+EEG_BENCHMARKS: Dict[str, BenchmarkInfo] = {
+    "bcic2a": BenchmarkInfo(
+        dataset_id="braindecode/bcic-2a",
+        task_id="motor_imagery_4class",
+        display_name="BCIC-2a",
+        description="BCI Competition IV Dataset 2a - 4-class motor imagery classification.",
+        category="Motor Imagery",
+        num_classes=4,
+        accuracy_field="bcic2a_accuracy",
+    ),
+    "physionet": BenchmarkInfo(
+        dataset_id="braindecode/physionet-mi",
+        task_id="motor_imagery_4class",
+        display_name="PhysioNet MI",
+        description="PhysioNet Motor Imagery dataset - 4-class motor imagery classification.",
+        category="Motor Imagery",
+        num_classes=4,
+        accuracy_field="physionet_accuracy",
+    ),
+    "isruc_sleep": BenchmarkInfo(
+        dataset_id="braindecode/isruc-sleep",
+        task_id="sleep_staging_5class",
+        display_name="ISRUC-SLEEP",
+        description="ISRUC-SLEEP dataset - 5-class sleep staging classification.",
+        category="Sleep Staging",
+        num_classes=5,
+        accuracy_field="isruc_sleep_accuracy",
+    ),
+    "tuab": BenchmarkInfo(
+        dataset_id="braindecode/tuab",
+        task_id="pathology_binary",
+        display_name="TUAB",
+        description="Temple University Abnormal EEG corpus - binary pathology detection.",
+        category="Pathology Detection",
+        num_classes=2,
+        accuracy_field="tuab_accuracy",
+    ),
+    "tuev": BenchmarkInfo(
+        dataset_id="braindecode/tuev",
+        task_id="event_classification_6class",
+        display_name="TUEV",
+        description="Temple University EEG Events corpus - 6-class event classification.",
+        category="Pathology Detection",
+        num_classes=6,
+        accuracy_field="tuev_accuracy",
+    ),
+    "chbmit": BenchmarkInfo(
+        dataset_id="braindecode/chb-mit",
+        task_id="seizure_detection_binary",
+        display_name="CHB-MIT",
+        description="CHB-MIT Scalp EEG dataset - binary seizure detection.",
+        category="Seizure Detection",
+        num_classes=2,
+        accuracy_field="chbmit_accuracy",
+    ),
+    "faced": BenchmarkInfo(
+        dataset_id="braindecode/faced",
+        task_id="emotion_recognition_9class",
+        display_name="FACED",
+        description="FACED dataset - 9-class emotion recognition from EEG.",
+        category="Emotion Recognition",
+        num_classes=9,
+        accuracy_field="faced_accuracy",
+    ),
+    "seedv": BenchmarkInfo(
+        dataset_id="braindecode/seed-v",
+        task_id="emotion_recognition_5class",
+        display_name="SEED-V",
+        description="SEED-V dataset - 5-class emotion recognition from EEG.",
+        category="Emotion Recognition",
+        num_classes=5,
+        accuracy_field="seedv_accuracy",
+    ),
+}
+
+EVALUATION_FRAMEWORK = "eeg-finetune-arena"

backend/app/services/leaderboard.py

@@ -1,5 +1,4 @@
 from app.core.cache import cache_config
-from datetime import datetime
 from typing import List, Dict, Any
 from pathlib import Path
 import json
@@ -7,6 +6,7 @@ import datasets
 from fastapi import HTTPException
 import logging
 from app.config.base import HF_ORGANIZATION
+from app.config.benchmarks import EEG_BENCHMARKS
 from app.core.formatting import LogFormatter
 
 logger = logging.getLogger(__name__)
@@ -15,6 +15,16 @@ logger = logging.getLogger(__name__)
 SAMPLE_DATA_PATH = Path(__file__).parent.parent / "data" / "sample_results.json"
 
 
+def _build_entry_id(data: Dict[str, Any]) -> str:
+    """Build a unique ID for a leaderboard entry."""
+    return (
+        f"{data.get('fullname', 'Unknown')}"
+        f"_{data.get('adapter', 'Unknown')}"
+        f"_{data.get('Precision', 'Unknown')}"
+        f"_{data.get('Model sha', 'Unknown')}"
+    )
+
+
 class LeaderboardService:
     def __init__(self):
         pass
@@ -145,58 +155,17 @@ class LeaderboardService:
             model_name = data.get("fullname", "Unknown")
             logger.debug(LogFormatter.info(f"Transforming data for model: {model_name}"))
 
-            # Create unique ID combining model name, adapter method and dataset
-            unique_id = (
-                f"{data.get('fullname', 'Unknown')}"
-                f"_{data.get('adapter', 'Unknown')}"
-                f"_{data.get('Precision', 'Unknown')}"
-                f"_{data.get('Model sha', 'Unknown')}"
-            )
-
-            # EEG benchmark evaluations — each dataset is a separate evaluation
-            # Scores are accuracy (0-1 raw) and normalized to 0-100 scale
-            evaluations = {
-                "bcic2a": {
-                    "name": "BCIC-2a",
-                    "value": data.get("bcic2a_accuracy", 0),
-                    "normalized_score": data.get("bcic2a_accuracy", 0) * 100 if data.get("bcic2a_accuracy") else 0,
-                },
-                "physionet": {
-                    "name": "PhysioNet MI",
-                    "value": data.get("physionet_accuracy", 0),
-                    "normalized_score": data.get("physionet_accuracy", 0) * 100 if data.get("physionet_accuracy") else 0,
-                },
-                "isruc_sleep": {
-                    "name": "ISRUC-SLEEP",
-                    "value": data.get("isruc_sleep_accuracy", 0),
-                    "normalized_score": data.get("isruc_sleep_accuracy", 0) * 100 if data.get("isruc_sleep_accuracy") else 0,
-                },
-                "tuab": {
-                    "name": "TUAB",
-                    "value": data.get("tuab_accuracy", 0),
-                    "normalized_score": data.get("tuab_accuracy", 0) * 100 if data.get("tuab_accuracy") else 0,
-                },
-                "tuev": {
-                    "name": "TUEV",
-                    "value": data.get("tuev_accuracy", 0),
-                    "normalized_score": data.get("tuev_accuracy", 0) * 100 if data.get("tuev_accuracy") else 0,
-                },
-                "chbmit": {
-                    "name": "CHB-MIT",
-                    "value": data.get("chbmit_accuracy", 0),
-                    "normalized_score": data.get("chbmit_accuracy", 0) * 100 if data.get("chbmit_accuracy") else 0,
-                },
-                "faced": {
-                    "name": "FACED",
-                    "value": data.get("faced_accuracy", 0),
-                    "normalized_score": data.get("faced_accuracy", 0) * 100 if data.get("faced_accuracy") else 0,
-                },
-                "seedv": {
-                    "name": "SEED-V",
-                    "value": data.get("seedv_accuracy", 0),
-                    "normalized_score": data.get("seedv_accuracy", 0) * 100 if data.get("seedv_accuracy") else 0,
-                },
-            }
+            unique_id = _build_entry_id(data)
+
+            # EEG benchmark evaluations from the canonical registry
+            evaluations = {}
+            for key, benchmark in EEG_BENCHMARKS.items():
+                score = data.get(benchmark.accuracy_field, 0)
+                evaluations[key] = {
+                    "name": benchmark.display_name,
+                    "value": score,
+                    "normalized_score": score * 100 if score else 0,
+                }
 
             features = {
                 "is_not_available_on_hub": data.get("Available on the hub", False),

backend/pyproject.toml

@@ -19,6 +19,7 @@ safetensors = "^0.5.3"
 aiofiles = "^24.1.0"
 fastapi-cache2 = "^0.2.1"
 python-dotenv = "^1.0.1"
+pyyaml = "^6.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.4"

backend/scripts/push_eval_yaml.py

@@ -0,0 +1,131 @@
+"""
+One-time script to push eval.yaml files to benchmark dataset repos on HuggingFace.
+
+This registers each EEG benchmark dataset with HF's decentralized eval system
+by creating the dataset repo (if needed) and uploading an eval.yaml file.
+
+Usage:
+    python -m scripts.push_eval_yaml
+    python -m scripts.push_eval_yaml --dry-run
+    python -m scripts.push_eval_yaml --benchmark bcic2a
+"""
+
+import argparse
+import sys
+import logging
+
+import yaml
+
+from app.config.base import HF_TOKEN
+from app.config.benchmarks import EEG_BENCHMARKS, EVALUATION_FRAMEWORK
+from app.config.hf_config import API as hf_api
+from app.core.formatting import LogFormatter
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def generate_eval_yaml(benchmark) -> str:
+    """Generate eval.yaml content for a benchmark dataset."""
+    eval_config = {
+        "name": f"{benchmark.display_name} {benchmark.category}",
+        "description": benchmark.description,
+        "evaluation_framework": EVALUATION_FRAMEWORK,
+        "tasks": [
+            {
+                "id": benchmark.task_id,
+                "config": benchmark.config,
+                "split": benchmark.split,
+            }
+        ],
+    }
+    return yaml.dump(eval_config, default_flow_style=False, sort_keys=False)
+
+
+def push_eval_yaml(
+    benchmark_key: str = None,
+    dry_run: bool = False,
+):
+    """Push eval.yaml to benchmark dataset repos.
+
+    Args:
+        benchmark_key: If specified, only push for this benchmark.
+        dry_run: If True, print what would happen without pushing.
+    """
+    benchmarks = EEG_BENCHMARKS
+    if benchmark_key:
+        if benchmark_key not in benchmarks:
+            logger.error(f"Unknown benchmark key: {benchmark_key}")
+            logger.info(f"Available keys: {list(benchmarks.keys())}")
+            sys.exit(1)
+        benchmarks = {benchmark_key: benchmarks[benchmark_key]}
+
+    logger.info(LogFormatter.section("PUSHING EVAL.YAML TO BENCHMARK DATASETS"))
+
+    for key, benchmark in benchmarks.items():
+        dataset_id = benchmark.dataset_id
+        logger.info(LogFormatter.subsection(f"Processing: {dataset_id}"))
+
+        eval_yaml_content = generate_eval_yaml(benchmark)
+
+        if dry_run:
+            logger.info(f"[DRY RUN] Would create repo: {dataset_id} (type=dataset)")
+            logger.info(f"[DRY RUN] Would upload eval.yaml:\n{eval_yaml_content}")
+            continue
+
+        # Create the dataset repo if it doesn't exist
+        try:
+            hf_api.create_repo(
+                repo_id=dataset_id,
+                repo_type="dataset",
+                exist_ok=True,
+            )
+            logger.info(LogFormatter.success(f"Repo ready: {dataset_id}"))
+        except Exception as e:
+            logger.error(LogFormatter.error(f"Failed to create repo {dataset_id}", e))
+            continue
+
+        # Upload eval.yaml
+        try:
+            hf_api.upload_file(
+                path_or_fileobj=eval_yaml_content.encode("utf-8"),
+                path_in_repo="eval.yaml",
+                repo_id=dataset_id,
+                repo_type="dataset",
+                commit_message="Add eval.yaml for EEG Finetune Arena benchmark registration",
+            )
+            logger.info(LogFormatter.success(f"Uploaded eval.yaml to {dataset_id}"))
+        except Exception as e:
+            logger.error(
+                LogFormatter.error(f"Failed to upload eval.yaml to {dataset_id}", e)
+            )
+
+    logger.info(LogFormatter.section("DONE"))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Push eval.yaml files to HF benchmark dataset repos."
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview without pushing to HF.",
+    )
+    parser.add_argument(
+        "--benchmark",
+        type=str,
+        default=None,
+        help="Only push for a specific benchmark key (e.g. bcic2a).",
+    )
+    args = parser.parse_args()
+
+    if not HF_TOKEN and not args.dry_run:
+        logger.error("HF_TOKEN environment variable is required (set it or use --dry-run)")
+        sys.exit(1)
+
+    push_eval_yaml(benchmark_key=args.benchmark, dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    main()