Deploy SDR-Arena leaderboard

leaderboard/tabs/upload_tab.py

@@ -2,7 +2,7 @@
 Upload Results Tab - Upload pre-computed agent outputs for evaluation.
 
 Users upload a JSON file containing their agent's outputs for benchmark prompts.
-Submissions are saved server-side for admin review and evaluation.
+Submissions are pushed to a private HF Dataset repo for admin review.
 No submission history is shown publicly.
 """
 
@@ -10,34 +10,67 @@ from __future__ import annotations
 
 import json
 import os
+import tempfile
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Optional
 
 import gradio as gr
 
 from leaderboard.data_loader import LeaderboardDataLoader
 
+SUBMISSIONS_DATASET_REPO = "behavior-in-the-wild/sdr-arena-submissions"
 
-# Directory for uploaded results (server-side only, not publicly visible)
-SUBMISSIONS_DIR = Path(__file__).resolve().parent.parent.parent / "data" / "submissions"
-SUBMISSIONS_DIR.mkdir(parents=True, exist_ok=True)
 
-SUBMISSION_LOG = SUBMISSIONS_DIR / "submissions.json"
+def _get_hf_api():
+    """Return an authenticated HfApi instance, or None if no token is set."""
+    token = os.environ.get("HF_TOKEN", "")
+    if not token:
+        return None
+    from huggingface_hub import HfApi
+    return HfApi(token=token)
 
 
-def _load_submissions_log() -> list[dict]:
-    """Load the submissions log (admin-only)."""
-    if SUBMISSION_LOG.exists():
-        with open(SUBMISSION_LOG) as f:
-            return json.load(f)
-    return []
-
-
-def _save_submissions_log(entries: list[dict]):
-    """Save the submissions log."""
-    with open(SUBMISSION_LOG, "w") as f:
-        json.dump(entries, f, indent=2)
+def _list_existing_submissions() -> set[str]:
+    """Fetch agent names already submitted to the dataset repo."""
+    api = _get_hf_api()
+    if api is None:
+        return set()
+    try:
+        files = api.list_repo_files(
+            repo_id=SUBMISSIONS_DATASET_REPO, repo_type="dataset"
+        )
+        names = set()
+        for f in files:
+            if f.startswith("submissions/") and f.endswith(".json"):
+                stem = f.split("/", 1)[1].rsplit("_", 2)[0]
+                names.add(stem)
+        return names
+    except Exception:
+        return set()
+
+
+def _push_submission(data: dict, filename: str) -> bool:
+    """Push a submission JSON to the private dataset repo. Returns True on success."""
+    api = _get_hf_api()
+    if api is None:
+        return False
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w", suffix=".json", delete=False
+        ) as tmp:
+            json.dump(data, tmp, indent=2)
+            tmp_path = tmp.name
+        api.upload_file(
+            path_or_fileobj=tmp_path,
+            path_in_repo=f"submissions/{filename}",
+            repo_id=SUBMISSIONS_DATASET_REPO,
+            repo_type="dataset",
+            commit_message=f"New submission: {data.get('agent_name', 'unknown')}",
+        )
+        Path(tmp_path).unlink(missing_ok=True)
+        return True
+    except Exception:
+        return False
 
 
 def _validate_results_json(data: dict) -> tuple[bool, str]:
@@ -212,38 +245,22 @@ def build_upload_tab(loader: LeaderboardDataLoader) -> None:
                 "",
             )
 
-        # Check for duplicate agent name
         agent_name = data["agent_name"].strip()
-        log = _load_submissions_log()
-        existing_names = {e["agent_name"] for e in log}
-        if agent_name in existing_names:
-            return (
-                f"<div style='color:var(--dr-danger);font-size:0.9rem;'>"
-                f"&#x274C; An agent named '<strong>{agent_name}</strong>' has already been submitted. "
-                f"Please use a unique agent name.</div>",
-                "",
-            )
+        num_results = len(data.get("results", {}))
 
-        # Save the results file
+        # Push to private HF dataset repo
         safe_name = "".join(c if c.isalnum() or c == "-" else "_" for c in agent_name)
         timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
-        results_path = SUBMISSIONS_DIR / f"{safe_name}_{timestamp}.json"
-        with open(results_path, "w") as f:
-            json.dump(data, f, indent=2)
+        filename = f"{safe_name}_{timestamp}.json"
 
-        # Record in submissions log
-        num_results = len(data.get("results", {}))
-        entry = {
-            "agent_name": agent_name,
-            "agent_author": data.get("agent_author", "").strip() or "Anonymous",
-            "agent_description": data.get("agent_description", "").strip(),
-            "num_results": num_results,
-            "results_file": str(results_path),
-            "submitted_at": datetime.now(timezone.utc).isoformat(),
-            "status": "pending_evaluation",
-        }
-        log.append(entry)
-        _save_submissions_log(log)
+        pushed = _push_submission(data, filename)
+        if not pushed:
+            return (
+                f"<div style='color:var(--dr-danger);font-size:0.9rem;'>"
+                f"&#x274C; Submission storage is temporarily unavailable. "
+                f"Please try again later.</div>",
+                "",
+            )
 
         return (
             f"<div style='color:var(--dr-success);font-size:0.9rem;font-weight:600;'>"