"""CADGenBench Leaderboard Space - Gradio UI + report-proxy mount. Read path lives in :mod:`leaderboard`. Submit-tab validation lives in :mod:`submit`. Both are wired into the Gradio Blocks below. The Gradio app is mounted under a FastAPI parent so the custom ``/reports/{submission_id}.html`` route can re-serve dataset HTML with ``Content-Type: text/html`` (HF Hub's ``/resolve/`` serves it as ``text/plain`` by policy, which makes the browser show source rather than render). """ from __future__ import annotations import html import logging import os from functools import lru_cache from pathlib import Path import gradio as gr import pandas as pd import uvicorn from fastapi import FastAPI from fastapi.responses import HTMLResponse, Response from gradio_leaderboard import Leaderboard from huggingface_hub import hf_hub_download from leaderboard import ( HF_DATA_REPO, HF_SUBMISSIONS_REPO, LEADERBOARD_DATATYPES, LEADERBOARD_HIDE_COLUMNS, VALIDATED_LEADERBOARD_DATATYPES, _fmt_timestamp, build_combined_csv, load_leaderboard_split, ) from submit import handle_submit logger = logging.getLogger(__name__) # Surface module-level logger.info / logger.warning / logger.exception # calls from leaderboard.py + submit.py in the Space's runtime logs. # Otherwise they go nowhere and any refresh / worker pathology is # silent. Format keeps timestamps + module + level + message. logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s [%(name)s] %(message)s", ) # Canonical policy doc lives in the code repo so contributors reading # the GitHub repo see it without needing to visit the Space. Linked # from both the Leaderboard tab's Validation Guidelines accordion and # the About tab. VALIDATION_DOC_URL = ( "https://github.com/huggingface/cadgenbench/blob/main/docs/benchmark/validation.md" ) ABOUT_MD = f"""## About **CADGenBench** evaluates AI-driven CAD generation: how well a model can turn a description of a mechanical part into a valid, geometrically correct 3D model. - **Reference baseline**: an iterative AI agent that writes build123d Python. - **Submission flow**: upload a zip of per-fixture STEP files; the Space runs the eval and appends a row to the submissions dataset. - **Datasets**: fixture inputs in [`{HF_DATA_REPO}`](https://huggingface.co/datasets/{HF_DATA_REPO}); submissions and computed results in [`{HF_SUBMISSIONS_REPO}`](https://huggingface.co/datasets/{HF_SUBMISSIONS_REPO}). - **Code**: [`huggingface/cadgenbench`](https://github.com/huggingface/cadgenbench). - **Validation policy**: [`docs/benchmark/validation.md`]({VALIDATION_DOC_URL}). """ # Verbatim BibTeX entry locked in space-setup/bundles/1-2-space-ux.md # (Locked decisions section). Shown in the Citation accordion as a # copy-paste handle for anyone citing this benchmark; the About tab # already links the source code via huggingface/cadgenbench so the # Space URL is the right deep-link target for the citation. CITATION_BIBTEX = r"""@misc{cadgenbench2026, author = {Rabinovich, Michael and {Hugging Face}}, title = {{CADGenBench}: a benchmark for {AI}-driven {CAD} generation}, year = {2026}, publisher = {Hugging Face}, howpublished = {\url{https://huggingface.co/spaces/HuggingAI4Engineering/cadgenbench-leaderboard}}, }""" VALIDATION_GUIDELINES_MD = f"""Submissions appear on the **Unvalidated** table the moment evaluation completes. Maintainers promote rows to **Validated** after methodology review, accepting one of four evidence types (`code`, `traces`, `api`, `manual`). Full policy: [`docs/benchmark/validation.md`]({VALIDATION_DOC_URL}).""" DETAIL_PLACEHOLDER = "_Click a row above for details._" def _has(value) -> bool: """True for values that should show up in the detail panel.""" if value is None: return False if isinstance(value, float) and pd.isna(value): return False return str(value).strip() != "" def _build_report_iframe(html_bytes: bytes) -> str: """Wrap a fetched report's HTML bytes into a self-contained iframe. ``srcdoc`` puts the entire report HTML directly inside the iframe attribute. The iframe gets its own document context, so the report's CSS can't collide with Gradio's, and an explicit ``height: 90vh`` keeps the report from being clipped by Gradio's column-flex layout (which was the visible cut-off the user saw in earlier rendering attempts). The Space is private; the FastAPI ``/reports/.html`` route works server-side under Bearer auth but breaks for a logged-in browser user (HF's edge gates same-origin pathname navigations on a JWT that the browser doesn't carry forward). srcdoc sidesteps that entirely by inlining the bytes; no second HTTP request leaves the browser. """ escaped = html.escape( html_bytes.decode("utf-8", errors="replace"), quote=True, ) return ( f'' ) def _refresh_leaderboard_with_toast(): """Manual Refresh button handler: toast + fresh DataFrames. The Timer auto-refresh wires straight to ``load_leaderboard_split`` so it stays silent (a toast every 10s would be noise). Only the explicit click goes through this wrapper. """ gr.Info("Leaderboard refreshed.") return load_leaderboard_split() def _enable_submit_when_logged_in( profile: gr.OAuthProfile | None, ) -> gr.Button: """Flip the Submit button's interactivity based on login state. Runs once per page load via ``blocks.load``. Gradio injects ``gr.OAuthProfile`` automatically (``None`` if the visitor isn't logged in via the LoginButton). The visible-disable mirrors the server-side gate in :func:`submit.handle_submit`; the handler still raises ``gr.Error`` defensively if it ever gets called without a profile. """ return gr.Button(interactive=profile is not None) def _format_detail_and_report( df: pd.DataFrame | None, evt: gr.SelectData, ) -> tuple[str, str]: """Return ``(detail_markdown, report_iframe_html)`` for the clicked row. The detail panel holds metadata (submitter, status, timestamp, notes, links to ZIP and external agent URL). The HTML viewer holds the rendered report when one exists (status == completed AND the row carries the modern-pipeline ``submission_sha256`` sentinel) - the leaderboard reader computes ``report_url`` only for rows that satisfy that gate. Returns ``(DETAIL_PLACEHOLDER, "")`` on a null / out-of-range event so the panel falls back to its initial state. """ if df is None or len(df) == 0 or evt is None or evt.index is None: return DETAIL_PLACEHOLDER, "" idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index if idx < 0 or idx >= len(df): return DETAIL_PLACEHOLDER, "" row = df.iloc[idx] title = row.get("submission_name") or "(unnamed submission)" lines = [f"### {title}", ""] if _has(row.get("submitter_name")): lines.append(f"- **Submitter**: {row['submitter_name']}") if _has(row.get("status")): lines.append(f"- **Status**: {row['status']}") if _has(row.get("submitted_at")): lines.append(f"- **Submitted**: {_fmt_timestamp(row['submitted_at'])}") if _has(row.get("notes")): lines.append(f"- **Notes**: {row['notes']}") lines.append( f"- **Model details (optional)**: " f"{row.get('model details (optional)') or '_None_'}" ) if _has(row.get("submission_blob_url")): lines.append( f"- **Submission ZIP**: [download]({row['submission_blob_url']})" ) if row.get("status") == "failed" and _has(row.get("failure_reason")): lines.append(f"- **Failure reason**: {row['failure_reason']}") detail_md = "\n".join(lines) report_iframe = "" if _has(row.get("report_url")) and _has(row.get("submission_id")): content = _fetch_report_html(str(row["submission_id"])) if content: report_iframe = _build_report_iframe(content) return detail_md, report_iframe @lru_cache(maxsize=128) def _fetch_report_html(submission_id: str) -> bytes | None: """Pull ``reports/.html`` off the submissions dataset. Cached in-process so repeat clicks on the same row don't hit the Hub. Returns ``None`` on any failure so the caller can serve a clean 404 rather than leaking a stack trace. """ try: local_path = hf_hub_download( repo_id=HF_SUBMISSIONS_REPO, filename=f"reports/{submission_id}.html", repo_type="dataset", ) return Path(local_path).read_bytes() except Exception as e: # noqa: BLE001 - any Hub failure -> 404 logger.warning( "Failed to fetch report for %s (%s: %s)", submission_id, type(e).__name__, e, ) return None def serve_report(submission_id: str) -> Response: """Proxy a per-submission HTML report through the Space. HF Hub serves dataset HTML under ``/resolve/`` with ``Content-Type: text/plain`` (security: dataset files can't host live HTML), so a direct dataset link shows source instead of rendering. This route lives on the Space (which can legitimately serve text/html) and re-streams the file's bytes with the right content-type. """ content = _fetch_report_html(submission_id) if content is None: return HTMLResponse( content="

Report not found

", status_code=404, ) return Response(content=content, media_type="text/html; charset=utf-8") with gr.Blocks(title="CADGenBench Leaderboard", theme=gr.themes.Soft()) as blocks: gr.Markdown( "# CADGenBench Leaderboard\n" "_Benchmarking AI-driven CAD generation._" ) with gr.Tab("Leaderboard"): # Collapsed accordions above the tables. Validation guidelines # gives the short two-tier story + link to the full policy # doc; Citation carries the verbatim BibTeX entry. Both start # closed so the leaderboard itself stays above the fold. with gr.Accordion("Validation guidelines", open=False): gr.Markdown(VALIDATION_GUIDELINES_MD) with gr.Accordion("Citation", open=False): # language=None -> plain monospaced render (gr.Code doesn't # ship a BibTeX highlighter); show_line_numbers off because # the entry is meant to be copy-pasted, not annotated. gr.Code( value=CITATION_BIBTEX, language=None, show_line_numbers=False, ) # Two stacked tables, split by `validation_status`. Validated # on top so the curated results are above the fold; unvalidated # below carries every other row (auto-published, awaiting # methodology review). See decisions/validation-policy.md. initial_validated, initial_unvalidated = load_leaderboard_split() validated_view = Leaderboard( value=initial_validated, datatype=VALIDATED_LEADERBOARD_DATATYPES, search_columns=["submission_name", "submitter_name"], hide_columns=LEADERBOARD_HIDE_COLUMNS, label="Validated Leaderboard", interactive=False, ) unvalidated_view = Leaderboard( value=initial_unvalidated, datatype=LEADERBOARD_DATATYPES, search_columns=["submission_name", "submitter_name"], hide_columns=LEADERBOARD_HIDE_COLUMNS, label="Unvalidated Leaderboard", interactive=False, ) with gr.Row(): refresh_btn = gr.Button("Refresh", size="sm") # One file, both tables, `validation_status` discriminator # column. Fresh CSV is generated on every click so the # download reflects the latest data, not a stale snapshot # captured at boot. download_btn = gr.DownloadButton( label="Download CSV", size="sm", ) refresh_btn.click( fn=_refresh_leaderboard_with_toast, outputs=[validated_view, unvalidated_view], ) download_btn.click(fn=build_combined_csv, outputs=download_btn) # Row-click panel: one shared metadata markdown component + # one report viewer below it. The viewer holds an iframe # containing the full per-submission report (srcdoc-inlined # so no second HTTP request needs to leave the browser - the # Space is private and HF's edge would 404 same-origin # pathname navigations that aren't carrying the iframe's # short-lived `__sign` JWT). Both tables share both outputs. detail_panel = gr.Markdown( value=DETAIL_PLACEHOLDER, label="Selected submission", ) report_viewer = gr.HTML(value="", label="Report") for view in (validated_view, unvalidated_view): view.select( fn=_format_detail_and_report, inputs=view, outputs=[detail_panel, report_viewer], ) with gr.Tab("Submit"): gr.Markdown( f""" **Submission format.** A single zip with: - one folder per fixture in `{HF_DATA_REPO}`, each containing `output.step`; - a top-level `meta.json`: ```json {{ "submitter_name": "your name or team", "submission_name": "MyAgent v2.3 (or whatever describes your system)", "agent_url": "https://github.com/... (optional)", "notes": "free text, optional, max 500 chars, single line, plain text", "agree_to_publish": true }} ``` **Submission name.** Free text describing the system being benchmarked, however you choose to describe it. The benchmark is system-agnostic - your submission may use no LLM, one, or many. If you want to disclose your stack, put it here or in `notes`. **Notes field.** Plain text only (no markdown / HTML). Capped at 500 chars and stripped to a single line. Shown in the per-submission detail view, not in the main leaderboard table. **Consent.** `"agree_to_publish": true` in `meta.json` is your consent to publish the resulting row on the public leaderboard. """ ) # OAuth gate. The user must log in via the HF button before # the Submit button becomes interactive; the row gets the # canonical `hf_username` from `gr.OAuthProfile.username` # (not a free-text claim in meta.json). README front-matter # already carries `hf_oauth: true` so HF's OAuth integration # is wired up at the Space level. login_btn = gr.LoginButton() zip_in = gr.File(label="Submission ZIP", file_types=[".zip"]) # Starts disabled; the `blocks.load` handler below flips it # to interactive when an OAuthProfile is present. submit_btn = gr.Button("Submit", variant="primary", interactive=False) # No static markdown output: handle_submit surfaces every # status update via gr.Info / gr.Error toasts. The handler # also reads `gr.OAuthProfile` implicitly via its parameter # type annotation (Gradio's dependency-injection convention). submit_btn.click(fn=handle_submit, inputs=[zip_in]) with gr.Tab("About"): gr.Markdown(ABOUT_MD) # gradio_leaderboard.Leaderboard handles its own update path # cleanly; bind a Timer to push fresh dataframes every 10 seconds. # Single tick runs `load_leaderboard_split` once and pushes the # tuple's two halves into the validated / unvalidated widgets. auto_refresh_timer = gr.Timer(10) auto_refresh_timer.tick( fn=load_leaderboard_split, outputs=[validated_view, unvalidated_view], ) # On page load, read the visitor's OAuth profile (None if not # logged in) and flip the Submit button's interactivity. Runs once # per page load; LoginButton clicks also re-trigger this through # Gradio's auth-event plumbing. blocks.load(fn=_enable_submit_when_logged_in, outputs=submit_btn) # Mount Gradio under a FastAPI parent so the custom proxy route # above lives at the same origin as the UI. Direct routes on `app` # get checked before the Gradio sub-app, so `/reports/.html` # never gets shadowed. app = FastAPI() app.add_api_route( "/reports/{submission_id}.html", serve_report, methods=["GET"], ) app = gr.mount_gradio_app(app, blocks, path="/") if __name__ == "__main__": host = os.getenv("GRADIO_SERVER_NAME", "0.0.0.0") port = int(os.getenv("GRADIO_SERVER_PORT", "7860")) uvicorn.run(app, host=host, port=port)