admin: stop pending jobs before delete

Add an admin action that cancels matching in-flight HF Jobs before deleting submissions, and keep the admin table refreshed so pending rows stay visible.

admin.py

@@ -31,9 +31,11 @@ import os
 from typing import Any, Iterable
 
 import gradio as gr
+from huggingface_hub import cancel_job, list_jobs
 from huggingface_hub.errors import EntryNotFoundError
 
 from submit import (
+    EVAL_JOB_NAMESPACE,
     HF_SUBMISSIONS_REPO,
     REPORTS_DIR,
     SUBMISSIONS_DIR,
@@ -45,6 +47,14 @@ logger = logging.getLogger(__name__)
 
 ADMINS_ENV = "CADGENBENCH_ADMINS"
 
+# HF Job stages that are already finished: cancelling one is a no-op (and
+# usually an error), so the stop step skips them. Mirrors
+# huggingface_hub.JobStage; kept as plain strings so a new terminal
+# stage name added upstream doesn't import-break this module.
+_JOB_TERMINAL_STAGES: frozenset[str] = frozenset(
+    {"COMPLETED", "ERROR", "CANCELED", "DELETED"}
+)
+
 # The evidence types accepted on promotion. Mirrors the
 # `validation_method` enum in cadgenbench-submissions/schema.md and the
 # validation policy doc.
@@ -193,6 +203,78 @@ def delete_rows(submission_ids: Iterable[str]) -> None:
     )
 
 
+def _cancel_jobs_for_submissions(ids: set[str]) -> int:
+    """Best-effort cancel every non-terminal eval Job for one of *ids*.
+
+    Each eval Job is dispatched with its ``submission_id`` baked into the
+    command argv (see :func:`submit._dispatch_eval_command`), so there's
+    no need to persist a ``job_id`` on the row: we list the eval
+    account's jobs and cancel any still-running one whose command
+    mentions a target id. This also catches a submission's shard jobs,
+    since each shard carries the same id in its command.
+
+    Never raises. A job that already finished, a listing failure, or a
+    cancel race must not block the row delete that follows -- the GPU job
+    carries its own ``--timeout`` and self-reaps if a cancel is missed.
+    Returns the count of cancel calls that succeeded (for logging only).
+    """
+    token = os.environ.get("HF_TOKEN")
+    try:
+        jobs = list_jobs(namespace=EVAL_JOB_NAMESPACE, token=token)
+    except Exception as e:  # noqa: BLE001 - listing is best-effort
+        logger.warning(
+            "list_jobs(%s) failed (%s: %s); skipping job cancel, deleting rows",
+            EVAL_JOB_NAMESPACE, type(e).__name__, e,
+        )
+        return 0
+
+    cancelled = 0
+    for job in jobs:
+        stage = getattr(getattr(job, "status", None), "stage", None)
+        if stage in _JOB_TERMINAL_STAGES:
+            continue
+        argv = list(getattr(job, "command", None) or []) + list(
+            getattr(job, "arguments", None) or []
+        )
+        if not any(sid in argv for sid in ids):
+            continue
+        try:
+            cancel_job(
+                job_id=job.id, namespace=EVAL_JOB_NAMESPACE, token=token,
+            )
+            cancelled += 1
+            logger.info(
+                "Cancelled eval job %s (stage %s) before delete", job.id, stage,
+            )
+        except Exception as e:  # noqa: BLE001 - cancel is best-effort
+            logger.warning(
+                "cancel_job(%s) failed (%s: %s); deleting row anyway",
+                job.id, type(e).__name__, e,
+            )
+    return cancelled
+
+
+def stop_and_delete_rows(submission_ids: Iterable[str]) -> None:
+    """Cancel any running eval Job(s) for the listed rows, then delete them.
+
+    The "stop" step (:func:`_cancel_jobs_for_submissions`) is
+    best-effort and never raises; the "delete" step is the existing
+    :func:`delete_rows` (artifacts then row). So this is exactly "stop if
+    needed, then delete", and it is the right action for a stuck/pending
+    submission whose GPU job is still in flight.
+
+    Raises:
+        ValueError: no ids were given.
+    """
+    ids = _clean_id_set(submission_ids)
+    cancelled = _cancel_jobs_for_submissions(ids)
+    logger.info(
+        "stop_and_delete: cancelled %d job(s) for %d submission(s)",
+        cancelled, len(ids),
+    )
+    delete_rows(ids)
+
+
 def _raise_for_missing(requested: set[str], seen: set[str]) -> None:
     """Raise ``LookupError`` if any requested id was not found in the rows."""
     missing = requested - seen

app.py

@@ -64,6 +64,7 @@ from admin import (
     demote_rows,
     is_admin,
     promote_rows,
+    stop_and_delete_rows,
 )
 from submit import handle_submit
 
@@ -297,7 +298,10 @@ def _admin_selection_status(table_df: pd.DataFrame | None) -> str:
 
 def _gate_admin_controls(
     profile: gr.OAuthProfile | None,
-) -> tuple[gr.Dataframe, gr.Radio, gr.Button, gr.Button, gr.Checkbox, gr.Button, str]:
+) -> tuple[
+    gr.Dataframe, gr.Radio, gr.Button, gr.Button, gr.Checkbox, gr.Button,
+    gr.Button, str,
+]:
     """Enable the admin controls only for a logged-in user in the admin set.
 
     Runs on every page load and re-runs on LoginButton auth events, so
@@ -305,8 +309,8 @@ def _gate_admin_controls(
     staying pinned to whatever rows existed when the Space process
     booted. Non-admins and logged-out visitors get the tab with the
     table read-only and every control disabled, mirroring the server-side
-    re-check in each handler. The delete button always loads disarmed:
-    it only enables once the confirm checkbox is ticked.
+    re-check in each handler. The delete + stop-and-delete buttons always
+    load disarmed: they only enable once the confirm checkbox is ticked.
     """
     admin_df, error = _safe_load_admin()
     if error:
@@ -328,15 +332,21 @@ def _gate_admin_controls(
         gr.Button(interactive=admin),
         gr.Checkbox(interactive=admin, value=False),
         gr.Button(interactive=False),
+        gr.Button(interactive=False),
         status,
     )
 
 
 def _arm_delete(
     confirm: bool, profile: gr.OAuthProfile | None,
-) -> gr.Button:
-    """Enable the delete button only when an admin has ticked the confirm box."""
-    return gr.Button(interactive=bool(confirm) and is_admin(profile))
+) -> tuple[gr.Button, gr.Button]:
+    """Arm both destructive buttons once an admin ticks the confirm box.
+
+    The plain delete and the stop-and-delete share the single confirm
+    checkbox, so a deliberate tick is required before either fires.
+    """
+    armed = bool(confirm) and is_admin(profile)
+    return gr.Button(interactive=armed), gr.Button(interactive=armed)
 
 
 def _refresh_admin_table() -> pd.DataFrame:
@@ -352,6 +362,43 @@ def _refresh_admin_table() -> pd.DataFrame:
     return admin_df
 
 
+def _reapply_selection(
+    fresh: pd.DataFrame, selected: set[str],
+) -> pd.DataFrame:
+    """Re-tick the ``select`` column on rows the maintainer had selected.
+
+    A freshly-loaded admin frame comes back all-unchecked; this carries
+    the prior ticks forward by ``submission_id`` so a background refresh
+    doesn't wipe an in-progress selection. Ids that vanished (e.g. a row
+    deleted out from under the table) simply drop out.
+    """
+    if (
+        selected
+        and ADMIN_SELECT_COL in fresh.columns
+        and "submission_id" in fresh.columns
+    ):
+        fresh[ADMIN_SELECT_COL] = (
+            fresh["submission_id"].astype(str).isin(selected)
+        )
+    return fresh
+
+
+def _auto_refresh_admin_table(current_df: pd.DataFrame | None) -> pd.DataFrame:
+    """Timer-tick handler: reload the admin table, preserving ticked rows.
+
+    The leaderboard tables auto-refresh every 10s but the admin table did
+    not, so a pending row submitted after the tab loaded stayed invisible
+    until a manual Refresh. This keeps it current on the same cadence.
+    Unlike the leaderboard handler it stays silent (no per-tick toast)
+    and, on a Hub read failure, returns the current frame unchanged so a
+    transient blip never blanks the table or drops the user's selection.
+    """
+    admin_df, error = _safe_load_admin()
+    if error:
+        return current_df if current_df is not None else admin_df
+    return _reapply_selection(admin_df, set(_selected_ids(current_df)))
+
+
 def _admin_promote(
     table_df: pd.DataFrame | None,
     method: str | None,
@@ -403,11 +450,14 @@ def _admin_delete(
     table_df: pd.DataFrame | None,
     confirm: bool,
     profile: gr.OAuthProfile | None,
-) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, str, gr.Checkbox, gr.Button]:
+) -> tuple[
+    pd.DataFrame, pd.DataFrame, pd.DataFrame, str, gr.Checkbox, gr.Button,
+    gr.Button,
+]:
     """Delete ticked rows, then refresh admin, leaderboard, gallery, and disarm.
 
-    Resets the confirm checkbox and re-disables the delete button on
-    the way out so the next deletion needs a fresh, deliberate confirm.
+    Resets the confirm checkbox and re-disables both destructive buttons
+    on the way out so the next deletion needs a fresh, deliberate confirm.
     """
     if not is_admin(profile):
         raise gr.Error("You are not in the admin set.")
@@ -430,6 +480,47 @@ def _admin_delete(
         _gallery_iframe_html(),
         gr.Checkbox(value=False),
         gr.Button(interactive=False),
+        gr.Button(interactive=False),
+    )
+
+
+def _admin_stop_delete(
+    table_df: pd.DataFrame | None,
+    confirm: bool,
+    profile: gr.OAuthProfile | None,
+) -> tuple[
+    pd.DataFrame, pd.DataFrame, pd.DataFrame, str, gr.Checkbox, gr.Button,
+    gr.Button,
+]:
+    """Stop running eval job(s) for ticked rows, delete them, then disarm.
+
+    Same gating + disarm contract as :func:`_admin_delete`; the only
+    difference is it calls :func:`admin.stop_and_delete_rows`, which
+    best-effort cancels the submissions' in-flight HF Jobs before
+    deleting. Use this for pending rows whose GPU eval is still running.
+    """
+    if not is_admin(profile):
+        raise gr.Error("You are not in the admin set.")
+    if not confirm:
+        raise gr.Error("Tick the confirmation box to enable delete.")
+    ids = _selected_ids(table_df)
+    if not ids:
+        raise gr.Error("Tick at least one row first.")
+    try:
+        stop_and_delete_rows(ids)
+    except ValueError as e:
+        raise gr.Error(str(e))
+    gr.Info(f"Stopped + deleted {len(ids)} submission(s).")
+    validated, unvalidated, _ = _safe_load_split()
+    admin_df, _ = _safe_load_admin()
+    return (
+        admin_df,
+        validated,
+        unvalidated,
+        _gallery_iframe_html(),
+        gr.Checkbox(value=False),
+        gr.Button(interactive=False),
+        gr.Button(interactive=False),
     )
 
 
@@ -601,16 +692,99 @@ def _data_uri(png_bytes: bytes | None) -> str | None:
     return "data:image/png;base64," + base64.b64encode(png_bytes).decode("ascii")
 
 
+# When the Space is **public**, switch the gallery from inlining base64
+# thumbnails to referencing cached proxy URLs (``/render/...`` /
+# ``/gt-render/...``). That lets the browser lazy-fetch only the ~33
+# on-screen tiles (instead of shipping every fixture x row up front) and
+# lets the CDN/browser cache them hard, so fixture-swaps and repeat
+# visits are essentially free. While **private** this must stay off: HF's
+# edge 404s in-browser fetches to our custom routes, so the only thing
+# that renders in the browser is an inlined data URI. Flip
+# ``GALLERY_PUBLIC=1`` in the Space env once the Space is made public.
+GALLERY_PUBLIC = os.getenv("GALLERY_PUBLIC", "").strip().lower() in {
+    "1", "true", "yes", "on",
+}
+
+# Long-lived immutable caching: a (submission, fixture) render never
+# changes (fixed camera + lighting; re-renders would be a new artifact),
+# so the browser/CDN can keep it forever. This is what makes fixture
+# swaps and repeat visits free once the Space is public.
+RENDER_CACHE_CONTROL = "public, max-age=31536000, immutable"
+
+
 def _render_data_uri(submission_id: str, fixture: str) -> str | None:
-    """Resolver for a submission's per-fixture gallery thumbnail."""
+    """Resolver for a submission's per-fixture gallery thumbnail (private mode).
+
+    Inlines the render as a base64 ``data:`` URI: the only mode that
+    works in-browser while the Space is private.
+    """
     return _data_uri(_fetch_render(submission_id, fixture))
 
 
 def _gt_data_uri(fixture: str) -> str | None:
-    """Resolver for a fixture's ground-truth gallery thumbnail."""
+    """Resolver for a fixture's ground-truth gallery thumbnail (private mode)."""
     return _data_uri(_fetch_gt_render(fixture))
 
 
+def _render_proxy_url(submission_id: str, fixture: str) -> str | None:
+    """Resolver returning the cached proxy URL for a submission render.
+
+    Public mode only. Returns the route string **without** fetching the
+    bytes (that's the whole point: the browser lazy-fetches on demand).
+    The gallery only calls this for fixtures whose per-fixture status is
+    ``valid``; an absolute path resolves against the Space origin even
+    inside the iframe ``srcdoc``. A render that 404s (valid status but a
+    missing upload) degrades to the dashed cell client-side via the
+    ``<img onerror>`` hook.
+    """
+    return f"/render/{submission_id}/{fixture}.png"
+
+
+def _gt_proxy_url(fixture: str) -> str | None:
+    """Resolver returning the cached proxy URL for a fixture's GT render."""
+    return f"/gt-render/{fixture}.png"
+
+
+def _gallery_resolvers():
+    """Pick the (render, gt) resolver pair for the current Space mode.
+
+    Public -> lazy cached proxy URLs; private -> base64-inlined data URIs.
+    """
+    if GALLERY_PUBLIC:
+        return _render_proxy_url, _gt_proxy_url
+    return _render_data_uri, _gt_data_uri
+
+
+def serve_render(submission_id: str, fixture: str) -> Response:
+    """Stream a submission's per-fixture render PNG with long-lived caching.
+
+    Used only in public mode (see :data:`GALLERY_PUBLIC`): the gallery
+    references ``/render/<id>/<fixture>.png`` and the browser fetches it
+    lazily. Re-streams the dataset bytes (the Space holds the read token)
+    with an immutable ``Cache-Control`` so the CDN/browser cache it hard.
+    """
+    png = _fetch_render(submission_id, fixture)
+    if png is None:
+        return Response(status_code=404)
+    return Response(
+        content=png,
+        media_type="image/png",
+        headers={"Cache-Control": RENDER_CACHE_CONTROL},
+    )
+
+
+def serve_gt_render(fixture: str) -> Response:
+    """Stream a fixture's ground-truth render PNG with long-lived caching."""
+    png = _fetch_gt_render(fixture)
+    if png is None:
+        return Response(status_code=404)
+    return Response(
+        content=png,
+        media_type="image/png",
+        headers={"Cache-Control": RENDER_CACHE_CONTROL},
+    )
+
+
 def _gallery_iframe_html() -> str:
     """Build the gallery as a self-contained ``srcdoc`` iframe.
 
@@ -625,7 +799,8 @@ def _gallery_iframe_html() -> str:
     except LeaderboardDataError:
         logger.exception("Gallery row load failed; rendering empty gallery")
         rows = []
-    doc = render_gallery_page(rows, _render_data_uri, _gt_data_uri)
+    render_resolver, gt_resolver = _gallery_resolvers()
+    doc = render_gallery_page(rows, render_resolver, gt_resolver)
     escaped = html.escape(doc, quote=True)
     return (
         f'<iframe srcdoc="{escaped}" '
@@ -855,7 +1030,10 @@ to publish the resulting row on the public leaderboard.
             gr.Markdown(
                 "Permanently deletes the ticked rows **and** their uploaded "
                 "zip + report files from the submissions dataset. This cannot "
-                "be undone (only a manual revert of the dataset commit)."
+                "be undone (only a manual revert of the dataset commit).\n\n"
+                "**Stop & delete** additionally cancels any still-running "
+                "evaluation job(s) for the ticked rows before deleting — use "
+                "it for pending submissions whose GPU eval is in flight."
             )
             delete_confirm = gr.Checkbox(
                 label=(
@@ -865,9 +1043,14 @@ to publish the resulting row on the public leaderboard.
                 value=False,
                 interactive=False,
             )
-            delete_btn = gr.Button(
-                "Delete selected", variant="stop", interactive=False,
-            )
+            with gr.Row():
+                delete_btn = gr.Button(
+                    "Delete selected", variant="stop", interactive=False,
+                )
+                stop_delete_btn = gr.Button(
+                    "Stop & delete selected", variant="stop",
+                    interactive=False,
+                )
         admin_refresh_btn = gr.Button("Refresh", size="sm")
 
         admin_table.change(
@@ -886,18 +1069,40 @@ to publish the resulting row on the public leaderboard.
             outputs=[admin_table, validated_view, unvalidated_view, gallery_html],
         )
         delete_confirm.change(
-            fn=_arm_delete, inputs=[delete_confirm], outputs=delete_btn,
+            fn=_arm_delete,
+            inputs=[delete_confirm],
+            outputs=[delete_btn, stop_delete_btn],
         )
         delete_btn.click(
             fn=_admin_delete,
             inputs=[admin_table, delete_confirm],
             outputs=[
                 admin_table, validated_view, unvalidated_view, gallery_html,
-                delete_confirm, delete_btn,
+                delete_confirm, delete_btn, stop_delete_btn,
+            ],
+        )
+        stop_delete_btn.click(
+            fn=_admin_stop_delete,
+            inputs=[admin_table, delete_confirm],
+            outputs=[
+                admin_table, validated_view, unvalidated_view, gallery_html,
+                delete_confirm, delete_btn, stop_delete_btn,
             ],
         )
         admin_refresh_btn.click(fn=_refresh_admin_table, outputs=admin_table)
 
+        # Keep the admin table on the same 10s cadence as the leaderboard
+        # so a row that lands (or a pending row that completes) after the
+        # tab loaded shows up without a manual Refresh. Selection is
+        # preserved across ticks so an in-progress set of checkboxes
+        # survives the reload.
+        admin_auto_refresh_timer = gr.Timer(10)
+        admin_auto_refresh_timer.tick(
+            fn=_auto_refresh_admin_table,
+            inputs=admin_table,
+            outputs=admin_table,
+        )
+
     # gradio_leaderboard.Leaderboard handles its own update path
     # cleanly; bind a Timer to push fresh dataframes every 10 seconds.
     # Single tick runs `_auto_refresh_leaderboard` once and pushes the
@@ -930,6 +1135,7 @@ to publish the resulting row on the public leaderboard.
             demote_btn,
             delete_confirm,
             delete_btn,
+            stop_delete_btn,
             admin_status,
         ],
     )
@@ -945,6 +1151,19 @@ app.add_api_route(
     serve_report,
     methods=["GET"],
 )
+# Cached render proxies the gallery references in public mode (no-ops while
+# private, where the gallery inlines base64 instead). Registered before the
+# Gradio mount so they're not shadowed by the catch-all sub-app.
+app.add_api_route(
+    "/render/{submission_id}/{fixture}.png",
+    serve_render,
+    methods=["GET"],
+)
+app.add_api_route(
+    "/gt-render/{fixture}.png",
+    serve_gt_render,
+    methods=["GET"],
+)
 app = gr.mount_gradio_app(app, blocks, path="/")
 
 

tests/test_admin.py

@@ -181,3 +181,98 @@ def test_delete_rows_removes_rows_and_artifacts(hub):
         "reports/alpha.json",
     ]
     assert hub["uploads"] == 1
+
+
+def _job(job_id: str, stage: str, *args: str) -> SimpleNamespace:
+    """A minimal JobInfo stand-in: id, status.stage, and a command argv."""
+    return SimpleNamespace(
+        id=job_id,
+        status=SimpleNamespace(stage=stage, message=None),
+        command=["python", "/opt/eval_job.py", *args],
+        arguments=None,
+    )
+
+
+@pytest.fixture
+def jobs(monkeypatch):
+    """Mock the Jobs API (``list_jobs`` / ``cancel_job``) admin imports.
+
+    ``state["jobs"]`` is the list ``list_jobs`` returns;
+    ``state["cancelled"]`` records every ``job_id`` a ``cancel_job`` call
+    targeted, so a test can assert exactly which jobs were stopped.
+    """
+    state: dict = {"jobs": [], "cancelled": []}
+
+    def fake_list_jobs(*, namespace=None, token=None):
+        return state["jobs"]
+
+    def fake_cancel_job(*, job_id, namespace=None, token=None):
+        state["cancelled"].append(job_id)
+
+    monkeypatch.setattr(admin, "list_jobs", fake_list_jobs)
+    monkeypatch.setattr(admin, "cancel_job", fake_cancel_job)
+    return state
+
+
+def test_stop_and_delete_cancels_running_then_deletes(hub, jobs):
+    """A running job whose command names the id is cancelled, then the row goes."""
+    jobs["jobs"] = [
+        _job("job-alpha", "RUNNING", "alpha", "https://blob/alpha.zip"),
+        _job("job-beta", "RUNNING", "beta", "https://blob/beta.zip"),
+    ]
+    admin.stop_and_delete_rows(["alpha"])
+    # Only alpha's job was cancelled.
+    assert jobs["cancelled"] == ["job-alpha"]
+    # And alpha's row + artifacts are gone, beta untouched.
+    assert {r["submission_id"] for r in hub["rows"]} == {"beta"}
+    assert "submissions/alpha.zip" in hub["deleted_paths"]
+
+
+def test_stop_and_delete_catches_all_shard_jobs(hub, jobs):
+    """Every shard job for a submission (same id in argv) is cancelled."""
+    jobs["jobs"] = [
+        _job("job-a0", "RUNNING", "alpha", "url", "--shard-id", "shard_000"),
+        _job("job-a1", "RUNNING", "alpha", "url", "--shard-id", "shard_001"),
+    ]
+    admin.stop_and_delete_rows(["alpha"])
+    assert sorted(jobs["cancelled"]) == ["job-a0", "job-a1"]
+
+
+def test_stop_and_delete_skips_terminal_jobs(hub, jobs):
+    """A finished job for the id is not cancelled, but the row still deletes."""
+    jobs["jobs"] = [
+        _job("job-alpha", "COMPLETED", "alpha", "url"),
+    ]
+    admin.stop_and_delete_rows(["alpha"])
+    assert jobs["cancelled"] == []
+    assert {r["submission_id"] for r in hub["rows"]} == {"beta"}
+
+
+def test_stop_and_delete_tolerates_list_jobs_failure(hub, monkeypatch):
+    """A Jobs-API listing failure must not block the row delete."""
+    def boom(*, namespace=None, token=None):
+        raise RuntimeError("jobs API down")
+
+    monkeypatch.setattr(admin, "list_jobs", boom)
+    admin.stop_and_delete_rows(["alpha"])
+    assert {r["submission_id"] for r in hub["rows"]} == {"beta"}
+
+
+def test_stop_and_delete_tolerates_cancel_failure(hub, jobs, monkeypatch):
+    """A cancel that errors is swallowed; the row still deletes."""
+    jobs["jobs"] = [_job("job-alpha", "RUNNING", "alpha", "url")]
+
+    def boom(*, job_id, namespace=None, token=None):
+        raise RuntimeError("cancel rejected")
+
+    monkeypatch.setattr(admin, "cancel_job", boom)
+    admin.stop_and_delete_rows(["alpha"])
+    assert {r["submission_id"] for r in hub["rows"]} == {"beta"}
+
+
+def test_stop_and_delete_empty_selection_raises(hub, jobs):
+    """An empty selection is a caller error, before any job/list work."""
+    with pytest.raises(ValueError):
+        admin.stop_and_delete_rows([])
+    assert jobs["cancelled"] == []
+    assert hub["uploads"] == 0