leaderboard: serve renders from the public bucket, not the dataset proxy

Candidate renders (turntables + edit-diff WebP) now live in the public HF
Storage Bucket under renders/<id>/, uploaded once by the eval job. The gallery
grid and the hosted report reference them by anonymous bucket URL, so the Space
is out of the render read path and the binary renders stop bloating the
submissions dataset / its commit queue.

- leaderboard: HF_RENDER_BUCKET + render URL/path/prefix helpers.
- app: render resolvers return public bucket URLs; the two render proxy routes
+ fetchers are retired. Only the private GT render keeps its token-held proxy.
- submit: merge passes render_base_url to generate_html and commits report-only;
dispatched job gets CADGENBENCH_RENDER_BUCKET/HF_ENDPOINT.
- admin: delete also purges the bucket renders/<id>/ prefix (previously orphaned).
- Dockerfile: bump cadgenbench pin to 3d49822 (render_base_url support).

Co-authored-by: Cursor <cursoragent@cursor.com>

Dockerfile

@@ -41,7 +41,7 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt \
 # image rebuild picks up the latest code (pre-v1: always-updated). Lock
 # to a specific commit SHA at the v1 release so published scores are
 # reproducible (see space-setup/post-gt-swap.md Stage F).
-ARG CADGENBENCH_SHA=b3ff8f0
+ARG CADGENBENCH_SHA=3d49822
 # Cache-bust the install below whenever the tracked ref moves: the
 # GitHub commits endpoint's response changes with each new commit on
 # `main`, so BuildKit re-fetches and invalidates the cached pip layer.

admin.py

@@ -34,6 +34,7 @@ import gradio as gr
 from huggingface_hub import cancel_job, list_jobs
 from huggingface_hub.errors import EntryNotFoundError
 
+from leaderboard import HF_RENDER_BUCKET, render_submission_prefix
 from submit import (
     EVAL_JOB_NAMESPACE,
     HF_SUBMISSIONS_REPO,
@@ -194,6 +195,7 @@ def delete_rows(submission_ids: Iterable[str]) -> None:
                     "Failed to delete artifact %s (%s: %s)",
                     path, type(e).__name__, e,
                 )
+        _delete_bucket_renders(sid)
 
     def mutate(rows: list[dict[str, Any]]) -> None:
         rows[:] = [r for r in rows if r.get("submission_id") not in ids]
@@ -203,6 +205,36 @@ def delete_rows(submission_ids: Iterable[str]) -> None:
     )
 
 
+def _delete_bucket_renders(submission_id: str) -> None:
+    """Delete every render for *submission_id* from the public render bucket.
+
+    The renders live under ``renders/<id>/`` in the bucket (uploaded by the eval
+    job). ``batch_bucket_files`` has no recursive prefix delete, so we list the
+    prefix and delete the files in one batch. Best-effort: a bucket failure is
+    logged, never blocks the row deletion (mirrors the dataset-artifact path).
+    """
+    prefix = render_submission_prefix(submission_id)
+    try:
+        paths = [
+            entry.path
+            for entry in _HF_API.list_bucket_tree(
+                HF_RENDER_BUCKET, prefix=prefix, recursive=True,
+            )
+            if getattr(entry, "path", None) and not entry.path.endswith("/")
+        ]
+        if paths:
+            _HF_API.batch_bucket_files(HF_RENDER_BUCKET, delete=paths)
+            logger.info(
+                "Deleted %d render(s) under %s from bucket %s",
+                len(paths), prefix, HF_RENDER_BUCKET,
+            )
+    except Exception as e:  # noqa: BLE001 - bucket failure must not block delete
+        logger.warning(
+            "Failed to delete bucket renders under %s (%s: %s)",
+            prefix, type(e).__name__, e,
+        )
+
+
 def _cancel_jobs_for_submissions(ids: set[str]) -> int:
     """Best-effort cancel every non-terminal eval Job for one of *ids*.
 

app.py

@@ -56,6 +56,7 @@ from leaderboard import (
     build_combined_csv,
     load_admin_table,
     load_leaderboard_split,
+    render_public_url,
 )
 from gallery import render_gallery_page
 from tasks import load_tasks_from_dir, render_tasks_page
@@ -530,54 +531,6 @@ def serve_report(submission_id: str) -> Response:
     return Response(content=content, media_type="text/html; charset=utf-8")
 
 
-def _fetch_render(submission_id: str, fixture: str) -> bytes | None:
-    """Pull a submission's gallery WebP (``renders/<id>/<fixture>/rotating.webp``).
-
-    Deliberately **not** memoized: renders land over time (a submission
-    completes, or an existing row is backfilled) after the Space process
-    booted, so negative-caching a boot-time miss would keep a turntable
-    dashed until the next restart. ``hf_hub_download`` does its own disk
-    caching per revision, so a re-fetch of an unchanged file stays cheap.
-    Returns ``None`` on any failure (the gallery draws the dashed cell).
-    """
-    try:
-        local_path = hf_hub_download(
-            repo_id=HF_SUBMISSIONS_REPO,
-            filename=f"renders/{submission_id}/{fixture}/rotating.webp",
-            repo_type="dataset",
-        )
-        return Path(local_path).read_bytes()
-    except Exception as e:  # noqa: BLE001 - any Hub failure -> 404
-        logger.warning(
-            "Failed to fetch render %s/%s (%s: %s)",
-            submission_id, fixture, type(e).__name__, e,
-        )
-        return None
-
-
-def _fetch_render_diff(submission_id: str, fixture: str) -> bytes | None:
-    """Pull a submission's edit-diff turntable (``renders/<id>/<fixture>/edit_diff.webp``).
-
-    Editing fixtures only; the evaluator writes this alongside the plain
-    ``rotating.webp``. Same no-memoization rationale as :func:`_fetch_render`.
-    Returns ``None`` on any failure, including the expected miss for
-    non-editing fixtures, so the gallery degrades to the dashed cell.
-    """
-    try:
-        local_path = hf_hub_download(
-            repo_id=HF_SUBMISSIONS_REPO,
-            filename=f"renders/{submission_id}/{fixture}/edit_diff.webp",
-            repo_type="dataset",
-        )
-        return Path(local_path).read_bytes()
-    except Exception as e:  # noqa: BLE001 - any Hub failure -> 404
-        logger.warning(
-            "Failed to fetch edit-diff render %s/%s (%s: %s)",
-            submission_id, fixture, type(e).__name__, e,
-        )
-        return None
-
-
 def _fetch_gt_render(fixture: str) -> bytes | None:
     """Pull a fixture's ground-truth GIF from the private GT dataset.
 
@@ -613,66 +566,35 @@ RENDER_CACHE_CONTROL = "public, max-age=31536000, immutable"
 
 
 def _render_proxy_url(submission_id: str, fixture: str) -> str | None:
-    """Resolver returning the cached proxy URL for a submission GIF.
-
-    Returns the route string **without** fetching the bytes (that's the
-    whole point: the browser lazy-fetches on demand, so only the visible
-    tiles load). The gallery only calls this for fixtures whose
-    per-fixture status is ``valid``; an absolute path resolves against
-    the Space origin even inside the iframe ``srcdoc``. A render that
-    404s (valid status but a missing upload) degrades to the dashed cell
-    client-side via the ``<img onerror>`` hook.
-
-    Requires the Space to be **public**: while private, HF's edge 404s
-    in-browser fetches to these custom routes.
+    """Resolver for a submission's plain turntable: a public render-bucket URL.
+
+    The eval job uploads ``renders/<id>/<fixture>/rotating.webp`` to the public
+    bucket, so the browser fetches it straight from object storage (anonymous,
+    no Space proxy hop). The gallery only calls this for ``valid`` fixtures; a
+    missing upload 404s and degrades to the dashed cell via ``<img onerror>``.
     """
-    return f"/render/{submission_id}/{fixture}.webp"
+    return render_public_url(submission_id, fixture, "rotating.webp")
 
 
 def _render_diff_proxy_url(submission_id: str, fixture: str) -> str | None:
-    """Resolver returning the cached proxy URL for a submission's edit-diff WebP.
+    """Resolver for an editing fixture's edit-diff turntable (public bucket URL).
 
     Used by the gallery grid for editing fixtures (see
     ``gallery.build_gallery_payload``). A miss (non-editing fixture, or an edit
-    that never rendered a diff) 404s and degrades to the dashed cell client-side
-    via the ``<img onerror>`` hook, no fallback to the plain turntable.
+    that never rendered a diff) 404s and degrades to the dashed cell, no
+    fallback to the plain turntable.
     """
-    return f"/render-diff/{submission_id}/{fixture}.webp"
+    return render_public_url(submission_id, fixture, "edit_diff.webp")
 
 
 def _gt_proxy_url(fixture: str) -> str | None:
-    """Resolver returning the cached proxy URL for a fixture's GT WebP."""
-    return f"/gt-render/{fixture}.webp"
-
-
-def serve_render(submission_id: str, fixture: str) -> Response:
-    """Stream a submission's per-fixture render WebP with long-lived caching.
+    """Resolver returning the cached proxy URL for a fixture's GT WebP.
 
-    The gallery references ``/render/<id>/<fixture>.webp`` and the browser
-    fetches it lazily. Re-streams the dataset bytes (the Space holds the
-    read token) with an immutable ``Cache-Control`` so the CDN/browser
-    cache it hard.
+    GT renders stay in the **private** GT dataset, so they cannot be public
+    bucket URLs; they are still re-streamed through the Space proxy (which
+    holds the read token).
     """
-    webp = _fetch_render(submission_id, fixture)
-    if webp is None:
-        return Response(status_code=404)
-    return Response(
-        content=webp,
-        media_type="image/webp",
-        headers={"Cache-Control": RENDER_CACHE_CONTROL},
-    )
-
-
-def serve_render_diff(submission_id: str, fixture: str) -> Response:
-    """Stream a submission's edit-diff turntable WebP with long-lived caching."""
-    webp = _fetch_render_diff(submission_id, fixture)
-    if webp is None:
-        return Response(status_code=404)
-    return Response(
-        content=webp,
-        media_type="image/webp",
-        headers={"Cache-Control": RENDER_CACHE_CONTROL},
-    )
+    return f"/gt-render/{fixture}.webp"
 
 
 def serve_gt_render(fixture: str) -> Response:
@@ -1145,16 +1067,9 @@ app.add_api_route(
 # Cached render proxies the gallery's lazy-loaded turntables point at.
 # Registered before the Gradio mount so they're not shadowed by the
 # catch-all sub-app.
-app.add_api_route(
-    "/render/{submission_id}/{fixture}.webp",
-    serve_render,
-    methods=["GET"],
-)
-app.add_api_route(
-    "/render-diff/{submission_id}/{fixture}.webp",
-    serve_render_diff,
-    methods=["GET"],
-)
+# Candidate renders are served directly from the public render bucket (URLs
+# come from the gallery resolvers), so only the private GT render still needs a
+# token-holding Space proxy route.
 app.add_api_route(
     "/gt-render/{fixture}.webp",
     serve_gt_render,

leaderboard.py

@@ -50,10 +50,51 @@ HF_DATA_REPO = os.getenv("HF_DATA_REPO", f"{HF_ORG}/cadgenbench-data")
 # read scope, same token the eval already uses for GT).
 HF_DATA_GT_REPO = os.getenv("HF_DATA_GT_REPO", f"{HF_ORG}/cadgenbench-data-gt")
 
+# Public HF Storage Bucket holding the per-submission gallery/report renders
+# (candidate turntables + edit-diff WebP). Public so the browser can fetch a
+# render straight from object storage with no token and no Space proxy; the
+# eval job is the only writer. Submission renders are public anyway, the GT
+# renders stay in the private GT dataset and are never published here.
+HF_RENDER_BUCKET = os.getenv("HF_RENDER_BUCKET", f"{HF_ORG}/cadgenbench-eval-staging")
+HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://huggingface.co").rstrip("/")
+# Permanent renders live under this prefix; transient shard staging lives under
+# its own prefix and is wiped after merge, so the two never collide.
+RENDER_BUCKET_PREFIX = "renders"
+
 RESULTS_FILENAME = "results.jsonl"
 HUB_FETCH_TIMEOUT_SECONDS = 30
 
 
+def render_object_path(submission_id: str, fixture: str, filename: str) -> str:
+    """Bucket-relative path of one render: ``renders/<id>/<fixture>/<file>``."""
+    return f"{RENDER_BUCKET_PREFIX}/{submission_id}/{fixture}/{filename}"
+
+
+def render_submission_prefix(submission_id: str) -> str:
+    """Bucket-relative prefix holding every render for *submission_id*."""
+    return f"{RENDER_BUCKET_PREFIX}/{submission_id}"
+
+
+def render_public_url(submission_id: str, fixture: str, filename: str) -> str:
+    """Stable anonymous URL for one render (browser follows the 302 to the CDN)."""
+    return (
+        f"{HF_ENDPOINT}/buckets/{HF_RENDER_BUCKET}/resolve/"
+        f"{render_object_path(submission_id, fixture, filename)}"
+    )
+
+
+def render_submission_base_url(submission_id: str) -> str:
+    """Public base URL for *submission_id*'s renders, ``.../resolve/renders/<id>``.
+
+    The report generator appends ``/<fixture>/<file>`` to this; passed to
+    ``cadgenbench``'s ``generate_html`` as the display-only ``render_base_url``.
+    """
+    return (
+        f"{HF_ENDPOINT}/buckets/{HF_RENDER_BUCKET}/resolve/"
+        f"{render_submission_prefix(submission_id)}"
+    )
+
+
 class LeaderboardDataError(RuntimeError):
     """Raised when the live ``results.jsonl`` cannot be read from the Hub.
 

submit.py

@@ -116,7 +116,14 @@ from huggingface_hub import (
 from huggingface_hub.errors import EntryNotFoundError, HfHubHTTPError
 
 import progress
-from leaderboard import HF_DATA_REPO, HF_ORG, HF_SUBMISSIONS_REPO
+from leaderboard import (
+    HF_DATA_REPO,
+    HF_ENDPOINT,
+    HF_ORG,
+    HF_RENDER_BUCKET,
+    HF_SUBMISSIONS_REPO,
+    render_submission_base_url,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -132,7 +139,6 @@ SUBMISSION_ID_SLUG_MAX = 40
 RESULTS_FILENAME = "results.jsonl"
 SUBMISSIONS_DIR = "submissions"
 REPORTS_DIR = "reports"
-RENDERS_DIR = "renders"
 DATA_REV_SHORT_LEN = 12
 FAILURE_REASON_MAX_CHARS = 200
 SHA256_BLOCK_SIZE = 64 * 1024
@@ -1170,6 +1176,9 @@ def _dispatch_eval_command(
     env: dict[str, str] = {
         "HF_SUBMISSIONS_REPO": HF_SUBMISSIONS_REPO,
         "EVAL_WORKER_COUNT": EVAL_JOB_WORKER_COUNT,
+        # The job is the sole render uploader; tell it which public bucket.
+        "CADGENBENCH_RENDER_BUCKET": HF_RENDER_BUCKET,
+        "HF_ENDPOINT": HF_ENDPOINT,
     }
     for key in ("CADGENBENCH_DATA_REPO", "CADGENBENCH_DATA_GT_REPO"):
         value = os.environ.get(key)
@@ -1527,13 +1536,16 @@ def _merge_shards_and_publish(
         report_json = _build_report_json(merged_run)
 
         run_data = discover_run(merged_run)
-        html = generate_html(run_data)
+        # Hosted report references the candidate renders from the public bucket
+        # (uploaded by the shard jobs), keeping the heavy WebP/PNG bytes out of
+        # the committed HTML. GT/input stay inlined (GT is private).
+        html = generate_html(
+            run_data, render_base_url=render_submission_base_url(submission_id),
+        )
         html_path = tmp / f"{submission_id}.html"
         html_path.write_text(html, encoding="utf-8")
 
-        _publish_reports_and_gallery(
-            submission_id, html_path, report_json, merged_run,
-        )
+        _publish_reports_and_gallery(submission_id, html_path, report_json)
         return report_json["run_summary"]
     finally:
         shutil.rmtree(tmp, ignore_errors=True)
@@ -1567,24 +1579,15 @@ def _publish_reports_and_gallery(
     submission_id: str,
     html_path: Path,
     report_json: dict[str, Any],
-    run_dir: Path,
 ) -> None:
-    """Publish report HTML/JSON + every per-fixture gallery render in one commit.
-
-    The merged artifacts land at the exact paths the leaderboard + the
-    row-flip expect: ``reports/<id>.{html,json}`` plus one
-    ``renders/<id>/<fixture>/<filename>`` entry for every PNG/GIF in each
-    fixture's render folder. All of it goes in a single ``create_commit`` rather than
-    one ``upload_file`` per file: a fan-out can stage ~80+ thumbnails,
-    and a commit-per-file both serialises the publish (slow) and hammers
-    the dataset's commit endpoint (the 429 "concurrency queue" failures
-    that stranded earlier runs). One commit is atomic, fast, and
-    rate-limit friendly.
-
-    A fixture with no render folder (missing output / render that never
-    ran) is skipped, matching the single-job behaviour; the gallery
-    draws the dashed "invalid" cell from the row, so an absent thumbnail
-    is not an error.
+    """Publish the merged report HTML + JSON to the submissions dataset.
+
+    Commits ``reports/<id>.{html,json}`` in one ``create_commit``. The gallery
+    renders are **not** committed here: each shard job already uploaded its
+    fixtures' renders to the public render bucket under ``renders/<id>/``, and
+    the report HTML references them by bucket URL. Keeping the binary renders
+    out of the dataset repo is what avoids bloating its git history and the
+    commit-queue 429s the per-file fan-out used to cause.
     """
     operations: list[CommitOperationAdd] = [
         CommitOperationAdd(
@@ -1598,41 +1601,16 @@ def _publish_reports_and_gallery(
             ).encode("utf-8"),
         ),
     ]
-    render_count = 0
-    for fixture_dir in sorted(d for d in run_dir.iterdir() if d.is_dir()):
-        renders_dir = fixture_dir / "renders"
-        if not renders_dir.is_dir():
-            continue
-        for render_path in sorted(renders_dir.iterdir()):
-            if render_path.suffix.lower() not in {".png", ".webp"}:
-                continue
-            operations.append(
-                CommitOperationAdd(
-                    path_in_repo=(
-                        f"{RENDERS_DIR}/{submission_id}/"
-                        f"{fixture_dir.name}/{render_path.name}"
-                    ),
-                    path_or_fileobj=str(render_path),
-                )
-            )
-            render_count += 1
-
     _with_hub_retries(
         lambda: _HF_API.create_commit(
             repo_id=HF_SUBMISSIONS_REPO,
             repo_type="dataset",
             operations=operations,
-            commit_message=(
-                f"publish merged report + {render_count} gallery render(s) "
-                f"for {submission_id}"
-            ),
+            commit_message=f"publish merged report for {submission_id}",
         ),
-        what="merged report+gallery publish",
-    )
-    logger.info(
-        "Published reports/%s.{html,json} + %d gallery render(s) in one commit",
-        submission_id, render_count,
+        what="merged report publish",
     )
+    logger.info("Published reports/%s.{html,json}", submission_id)
 
 
 def _cleanup_shard_artifacts(submission_id: str) -> None:

tests/test_admin.py

@@ -59,6 +59,8 @@ def hub(monkeypatch):
         "rows": [dict(r) for r in SEED_ROWS],
         "uploads": 0,
         "deleted_paths": [],
+        "bucket_listed_prefixes": [],
+        "bucket_deleted_paths": [],
     }
 
     def fake_download() -> str:
@@ -78,9 +80,18 @@ def hub(monkeypatch):
     def fake_delete_file(*, path_in_repo, **kwargs) -> None:
         state["deleted_paths"].append(path_in_repo)
 
+    def fake_list_bucket_tree(bucket_id, *, prefix, recursive=False, **kwargs):
+        state["bucket_listed_prefixes"].append(prefix)
+        return [SimpleNamespace(path=f"{prefix}/101/rotating.webp")]
+
+    def fake_batch_bucket_files(bucket_id, *, add=None, delete=None, **kwargs):
+        state["bucket_deleted_paths"].extend(delete or [])
+
     monkeypatch.setattr(submit, "_download_results_jsonl", fake_download)
     monkeypatch.setattr(submit._HF_API, "upload_file", fake_upload)
     monkeypatch.setattr(submit._HF_API, "delete_file", fake_delete_file)
+    monkeypatch.setattr(submit._HF_API, "list_bucket_tree", fake_list_bucket_tree)
+    monkeypatch.setattr(submit._HF_API, "batch_bucket_files", fake_batch_bucket_files)
     return state
 
 
@@ -180,6 +191,9 @@ def test_delete_rows_removes_rows_and_artifacts(hub):
         "reports/alpha.html",
         "reports/alpha.json",
     ]
+    # Renders for the deleted submission are purged from the public bucket too.
+    assert hub["bucket_listed_prefixes"] == ["renders/alpha"]
+    assert hub["bucket_deleted_paths"] == ["renders/alpha/101/rotating.webp"]
     assert hub["uploads"] == 1
 
 

tests/test_proxy.py

@@ -42,15 +42,20 @@ def test_serve_report_returns_404_when_file_missing(monkeypatch):
     assert "Report not found" in resp.body.decode("utf-8")
 
 
-def test_serve_render_returns_webp_when_file_exists(monkeypatch):
-    """Submission render proxy serves WebP bytes with image/webp media type."""
-    monkeypatch.setattr(app, "_fetch_render", lambda sid, fixture: b"RIFFwebp")
+def test_render_resolvers_return_public_bucket_urls():
+    """Candidate renders are served straight from the public bucket, no proxy.
 
-    resp = app.serve_render("sub-test", "101")
+    The resolvers must return a stable ``/buckets/<id>/resolve/renders/...``
+    URL (anonymous, browser-followable 302) for the plain turntable and the
+    edit-diff WebP.
+    """
+    url = app._render_proxy_url("sub-test", "101")
+    assert url.endswith("/renders/sub-test/101/rotating.webp")
+    assert "/buckets/" in url and "/resolve/" in url
 
-    assert resp.status_code == 200
-    assert resp.media_type == "image/webp"
-    assert resp.body == b"RIFFwebp"
+    diff = app._render_diff_proxy_url("sub-test", "207")
+    assert diff.endswith("/renders/sub-test/207/edit_diff.webp")
+    assert "/buckets/" in diff and "/resolve/" in diff
 
 
 def test_serve_gt_render_returns_webp_when_file_exists(monkeypatch):
@@ -79,22 +84,6 @@ def test_fetch_report_html_returns_none_on_hub_failure(monkeypatch):
     assert app._fetch_report_html("sub-failure-probe-unique-1") is None
 
 
-def test_fetch_render_uses_nested_rotating_webp_path(monkeypatch, tmp_path):
-    """Submission renders are fetched from the nested WebP artifact path."""
-    webp = tmp_path / "rotating.webp"
-    webp.write_bytes(b"RIFFwebp")
-    captured: dict = {}
-
-    def fake_download(**kwargs):
-        captured.update(kwargs)
-        return str(webp)
-
-    monkeypatch.setattr(app, "hf_hub_download", fake_download)
-
-    assert app._fetch_render("sub-test", "101") == b"RIFFwebp"
-    assert captured["filename"] == "renders/sub-test/101/rotating.webp"
-
-
 def test_fetch_gt_render_uses_gt_rotating_webp_path(monkeypatch, tmp_path):
     """GT WebPs are fetched from the GT dataset's per-fixture render folder."""
     webp = tmp_path / "rotating.webp"
@@ -120,7 +109,9 @@ def test_proxy_route_is_registered():
     """
     routes = [getattr(r, "path", None) for r in app.app.routes]
     assert "/reports/{submission_id}.html" in routes
-    assert "/render/{submission_id}/{fixture}.webp" in routes
+    # Candidate renders moved to the public bucket; only the private GT render
+    # still needs a token-holding Space proxy route.
+    assert "/render/{submission_id}/{fixture}.webp" not in routes
     assert "/gt-render/{fixture}.webp" in routes