app.py

"""CADGenBench Leaderboard Space - Gradio UI + report-proxy mount.

Read path lives in :mod:`leaderboard`. Submit-tab validation lives in
:mod:`submit`. Both are wired into the Gradio Blocks below. The
Gradio app is mounted under a FastAPI parent so the custom
``/reports/{submission_id}.html`` route can re-serve dataset HTML
with ``Content-Type: text/html`` (HF Hub's ``/resolve/`` serves it
as ``text/plain`` by policy, which makes the browser show source
rather than render).
"""
from __future__ import annotations

import html
import logging
import os
from functools import lru_cache
from pathlib import Path

import gradio as gr
import pandas as pd
import uvicorn
from fastapi import FastAPI
from fastapi.responses import HTMLResponse, Response
from gradio_leaderboard import Leaderboard
from huggingface_hub import hf_hub_download

from leaderboard import (
    HF_DATA_REPO,
    HF_SUBMISSIONS_REPO,
    LEADERBOARD_DATATYPES,
    LEADERBOARD_HIDE_COLUMNS,
    VALIDATED_LEADERBOARD_DATATYPES,
    _fmt_timestamp,
    build_combined_csv,
    load_leaderboard_split,
)
from submit import handle_submit

logger = logging.getLogger(__name__)

# Surface module-level logger.info / logger.warning / logger.exception
# calls from leaderboard.py + submit.py in the Space's runtime logs.
# Otherwise they go nowhere and any refresh / worker pathology is
# silent. Format keeps timestamps + module + level + message.
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
)


# Canonical policy doc lives in the code repo so contributors reading
# the GitHub repo see it without needing to visit the Space. Linked
# from both the Leaderboard tab's Validation Guidelines accordion and
# the About tab.
VALIDATION_DOC_URL = (
    "https://github.com/huggingface/cadgenbench/blob/main/docs/benchmark/validation.md"
)

ABOUT_MD = f"""## About

**CADGenBench** evaluates AI-driven CAD generation: how well a model can
turn a description of a mechanical part into a valid, geometrically
correct 3D model.

- **Reference baseline**: an iterative AI agent that writes build123d Python.
- **Submission flow**: upload a zip of per-fixture STEP files; the Space
  runs the eval and appends a row to the submissions dataset.
- **Datasets**: fixture inputs in
  [`{HF_DATA_REPO}`](https://huggingface.co/datasets/{HF_DATA_REPO});
  submissions and computed results in
  [`{HF_SUBMISSIONS_REPO}`](https://huggingface.co/datasets/{HF_SUBMISSIONS_REPO}).
- **Code**: [`huggingface/cadgenbench`](https://github.com/huggingface/cadgenbench).
- **Validation policy**: [`docs/benchmark/validation.md`]({VALIDATION_DOC_URL}).
"""

# Verbatim BibTeX entry locked in space-setup/bundles/1-2-space-ux.md
# (Locked decisions section). Shown in the Citation accordion as a
# copy-paste handle for anyone citing this benchmark; the About tab
# already links the source code via huggingface/cadgenbench so the
# Space URL is the right deep-link target for the citation.
CITATION_BIBTEX = r"""@misc{cadgenbench2026,
  author       = {Rabinovich, Michael and {Hugging Face}},
  title        = {{CADGenBench}: a benchmark for {AI}-driven {CAD} generation},
  year         = {2026},
  publisher    = {Hugging Face},
  howpublished = {\url{https://huggingface.co/spaces/HuggingAI4Engineering/cadgenbench-leaderboard}},
}"""

VALIDATION_GUIDELINES_MD = f"""Submissions appear on the **Unvalidated** table the moment evaluation completes. Maintainers promote rows to **Validated** after methodology review, accepting one of four evidence types (`code`, `traces`, `api`, `manual`).

Full policy: [`docs/benchmark/validation.md`]({VALIDATION_DOC_URL})."""

DETAIL_PLACEHOLDER = "_Click a row above for details._"


def _has(value) -> bool:
    """True for values that should show up in the detail panel."""
    if value is None:
        return False
    if isinstance(value, float) and pd.isna(value):
        return False
    return str(value).strip() != ""


def _build_report_iframe(html_bytes: bytes) -> str:
    """Wrap a fetched report's HTML bytes into a self-contained iframe.

    ``srcdoc`` puts the entire report HTML directly inside the iframe
    attribute. The iframe gets its own document context, so the
    report's CSS can't collide with Gradio's, and an explicit
    ``height: 90vh`` keeps the report from being clipped by Gradio's
    column-flex layout (which was the visible cut-off the user saw
    in earlier rendering attempts).

    The Space is private; the FastAPI ``/reports/<id>.html`` route
    works server-side under Bearer auth but breaks for a logged-in
    browser user (HF's edge gates same-origin pathname navigations
    on a JWT that the browser doesn't carry forward). srcdoc
    sidesteps that entirely by inlining the bytes; no second HTTP
    request leaves the browser.
    """
    escaped = html.escape(
        html_bytes.decode("utf-8", errors="replace"), quote=True,
    )
    return (
        f'<iframe srcdoc="{escaped}" '
        'style="width:100%; height:90vh; border:0; display:block;" '
        'title="Submission report"></iframe>'
    )


def _refresh_leaderboard_with_toast():
    """Manual Refresh button handler: toast + fresh DataFrames.

    The Timer auto-refresh wires straight to ``load_leaderboard_split``
    so it stays silent (a toast every 10s would be noise). Only the
    explicit click goes through this wrapper.
    """
    gr.Info("Leaderboard refreshed.")
    return load_leaderboard_split()


def _enable_submit_when_logged_in(
    profile: gr.OAuthProfile | None,
) -> gr.Button:
    """Flip the Submit button's interactivity based on login state.

    Runs once per page load via ``blocks.load``. Gradio injects
    ``gr.OAuthProfile`` automatically (``None`` if the visitor isn't
    logged in via the LoginButton). The visible-disable mirrors the
    server-side gate in :func:`submit.handle_submit`; the handler
    still raises ``gr.Error`` defensively if it ever gets called
    without a profile.
    """
    return gr.Button(interactive=profile is not None)


def _format_detail_and_report(
    df: pd.DataFrame | None, evt: gr.SelectData,
) -> tuple[str, str]:
    """Return ``(detail_markdown, report_iframe_html)`` for the clicked row.

    The detail panel holds metadata (submitter, status, timestamp,
    notes, links to ZIP and external agent URL). The HTML viewer
    holds the rendered report when one exists (status == completed
    AND the row carries the modern-pipeline ``submission_sha256``
    sentinel) - the leaderboard reader computes ``report_url`` only
    for rows that satisfy that gate.

    Returns ``(DETAIL_PLACEHOLDER, "")`` on a null / out-of-range
    event so the panel falls back to its initial state.
    """
    if df is None or len(df) == 0 or evt is None or evt.index is None:
        return DETAIL_PLACEHOLDER, ""
    idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index
    if idx < 0 or idx >= len(df):
        return DETAIL_PLACEHOLDER, ""
    row = df.iloc[idx]

    title = row.get("submission_name") or "(unnamed submission)"
    lines = [f"### {title}", ""]
    if _has(row.get("submitter_name")):
        lines.append(f"- **Submitter**: {row['submitter_name']}")
    if _has(row.get("status")):
        lines.append(f"- **Status**: {row['status']}")
    if _has(row.get("submitted_at")):
        lines.append(f"- **Submitted**: {_fmt_timestamp(row['submitted_at'])}")
    if _has(row.get("notes")):
        lines.append(f"- **Notes**: {row['notes']}")
    lines.append(
        f"- **Model details (optional)**: "
        f"{row.get('model details (optional)') or '_None_'}"
    )
    if _has(row.get("submission_blob_url")):
        lines.append(
            f"- **Submission ZIP**: [download]({row['submission_blob_url']})"
        )
    if row.get("status") == "failed" and _has(row.get("failure_reason")):
        lines.append(f"- **Failure reason**: {row['failure_reason']}")
    detail_md = "\n".join(lines)

    report_iframe = ""
    if _has(row.get("report_url")) and _has(row.get("submission_id")):
        content = _fetch_report_html(str(row["submission_id"]))
        if content:
            report_iframe = _build_report_iframe(content)
    return detail_md, report_iframe


@lru_cache(maxsize=128)
def _fetch_report_html(submission_id: str) -> bytes | None:
    """Pull ``reports/<id>.html`` off the submissions dataset.

    Cached in-process so repeat clicks on the same row don't hit
    the Hub. Returns ``None`` on any failure so the caller can
    serve a clean 404 rather than leaking a stack trace.
    """
    try:
        local_path = hf_hub_download(
            repo_id=HF_SUBMISSIONS_REPO,
            filename=f"reports/{submission_id}.html",
            repo_type="dataset",
        )
        return Path(local_path).read_bytes()
    except Exception as e:  # noqa: BLE001 - any Hub failure -> 404
        logger.warning(
            "Failed to fetch report for %s (%s: %s)",
            submission_id, type(e).__name__, e,
        )
        return None


def serve_report(submission_id: str) -> Response:
    """Proxy a per-submission HTML report through the Space.

    HF Hub serves dataset HTML under ``/resolve/`` with
    ``Content-Type: text/plain`` (security: dataset files can't host
    live HTML), so a direct dataset link shows source instead of
    rendering. This route lives on the Space (which can legitimately
    serve text/html) and re-streams the file's bytes with the right
    content-type.
    """
    content = _fetch_report_html(submission_id)
    if content is None:
        return HTMLResponse(
            content="<h1>Report not found</h1>",
            status_code=404,
        )
    return Response(content=content, media_type="text/html; charset=utf-8")


with gr.Blocks(title="CADGenBench Leaderboard", theme=gr.themes.Soft()) as blocks:
    gr.Markdown(
        "# CADGenBench Leaderboard\n"
        "_Benchmarking AI-driven CAD generation._"
    )

    with gr.Tab("Leaderboard"):
        # Collapsed accordions above the tables. Validation guidelines
        # gives the short two-tier story + link to the full policy
        # doc; Citation carries the verbatim BibTeX entry. Both start
        # closed so the leaderboard itself stays above the fold.
        with gr.Accordion("Validation guidelines", open=False):
            gr.Markdown(VALIDATION_GUIDELINES_MD)
        with gr.Accordion("Citation", open=False):
            # language=None -> plain monospaced render (gr.Code doesn't
            # ship a BibTeX highlighter); show_line_numbers off because
            # the entry is meant to be copy-pasted, not annotated.
            gr.Code(
                value=CITATION_BIBTEX,
                language=None,
                show_line_numbers=False,
            )

        # Two stacked tables, split by `validation_status`. Validated
        # on top so the curated results are above the fold; unvalidated
        # below carries every other row (auto-published, awaiting
        # methodology review). See decisions/validation-policy.md.
        initial_validated, initial_unvalidated = load_leaderboard_split()
        validated_view = Leaderboard(
            value=initial_validated,
            datatype=VALIDATED_LEADERBOARD_DATATYPES,
            search_columns=["submission_name", "submitter_name"],
            hide_columns=LEADERBOARD_HIDE_COLUMNS,
            label="Validated Leaderboard",
            interactive=False,
        )
        unvalidated_view = Leaderboard(
            value=initial_unvalidated,
            datatype=LEADERBOARD_DATATYPES,
            search_columns=["submission_name", "submitter_name"],
            hide_columns=LEADERBOARD_HIDE_COLUMNS,
            label="Unvalidated Leaderboard",
            interactive=False,
        )
        with gr.Row():
            refresh_btn = gr.Button("Refresh", size="sm")
            # One file, both tables, `validation_status` discriminator
            # column. Fresh CSV is generated on every click so the
            # download reflects the latest data, not a stale snapshot
            # captured at boot.
            download_btn = gr.DownloadButton(
                label="Download CSV", size="sm",
            )
        refresh_btn.click(
            fn=_refresh_leaderboard_with_toast,
            outputs=[validated_view, unvalidated_view],
        )
        download_btn.click(fn=build_combined_csv, outputs=download_btn)

        # Row-click panel: one shared metadata markdown component +
        # one report viewer below it. The viewer holds an iframe
        # containing the full per-submission report (srcdoc-inlined
        # so no second HTTP request needs to leave the browser - the
        # Space is private and HF's edge would 404 same-origin
        # pathname navigations that aren't carrying the iframe's
        # short-lived `__sign` JWT). Both tables share both outputs.
        detail_panel = gr.Markdown(
            value=DETAIL_PLACEHOLDER,
            label="Selected submission",
        )
        report_viewer = gr.HTML(value="", label="Report")
        for view in (validated_view, unvalidated_view):
            view.select(
                fn=_format_detail_and_report,
                inputs=view,
                outputs=[detail_panel, report_viewer],
            )

    with gr.Tab("Submit"):
        gr.Markdown(
            f"""
**Submission format.** A single zip with:

- one folder per fixture in `{HF_DATA_REPO}`, each containing `output.step`;
- a top-level `meta.json`:

```json
{{
  "submitter_name": "your name or team",
  "submission_name": "MyAgent v2.3 (or whatever describes your system)",
  "agent_url": "https://github.com/...   (optional)",
  "notes": "free text, optional, max 500 chars, single line, plain text",
  "agree_to_publish": true
}}
```

**Submission name.** Free text describing the system being benchmarked,
however you choose to describe it. The benchmark is system-agnostic - your
submission may use no LLM, one, or many. If you want to disclose your
stack, put it here or in `notes`.

**Notes field.** Plain text only (no markdown / HTML). Capped at 500 chars
and stripped to a single line. Shown in the per-submission detail view,
not in the main leaderboard table.

**Consent.** `"agree_to_publish": true` in `meta.json` is your consent
to publish the resulting row on the public leaderboard.
"""
        )
        # OAuth gate. The user must log in via the HF button before
        # the Submit button becomes interactive; the row gets the
        # canonical `hf_username` from `gr.OAuthProfile.username`
        # (not a free-text claim in meta.json). README front-matter
        # already carries `hf_oauth: true` so HF's OAuth integration
        # is wired up at the Space level.
        login_btn = gr.LoginButton()
        zip_in = gr.File(label="Submission ZIP", file_types=[".zip"])
        # Starts disabled; the `blocks.load` handler below flips it
        # to interactive when an OAuthProfile is present.
        submit_btn = gr.Button("Submit", variant="primary", interactive=False)
        # No static markdown output: handle_submit surfaces every
        # status update via gr.Info / gr.Error toasts. The handler
        # also reads `gr.OAuthProfile` implicitly via its parameter
        # type annotation (Gradio's dependency-injection convention).
        submit_btn.click(fn=handle_submit, inputs=[zip_in])

    with gr.Tab("About"):
        gr.Markdown(ABOUT_MD)

    # gradio_leaderboard.Leaderboard handles its own update path
    # cleanly; bind a Timer to push fresh dataframes every 10 seconds.
    # Single tick runs `load_leaderboard_split` once and pushes the
    # tuple's two halves into the validated / unvalidated widgets.
    auto_refresh_timer = gr.Timer(10)
    auto_refresh_timer.tick(
        fn=load_leaderboard_split,
        outputs=[validated_view, unvalidated_view],
    )

    # On page load, read the visitor's OAuth profile (None if not
    # logged in) and flip the Submit button's interactivity. Runs once
    # per page load; LoginButton clicks also re-trigger this through
    # Gradio's auth-event plumbing.
    blocks.load(fn=_enable_submit_when_logged_in, outputs=submit_btn)


# Mount Gradio under a FastAPI parent so the custom proxy route
# above lives at the same origin as the UI. Direct routes on `app`
# get checked before the Gradio sub-app, so `/reports/<sid>.html`
# never gets shadowed.
app = FastAPI()
app.add_api_route(
    "/reports/{submission_id}.html",
    serve_report,
    methods=["GET"],
)
app = gr.mount_gradio_app(app, blocks, path="/")


if __name__ == "__main__":
    host = os.getenv("GRADIO_SERVER_NAME", "0.0.0.0")
    port = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
    uvicorn.run(app, host=host, port=port)