initial commit: Dockerfile + eval_job.py + README

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.DS_Store
+.venv/
+.env

Dockerfile

@@ -0,0 +1,74 @@
+# syntax=docker/dockerfile:1.7
+#
+# HF Space at HuggingAI4Engineering/cadgenbench-eval-gpu.
+# Provides the Docker image consumed by the leaderboard's HF Jobs
+# eval pipeline (see space-setup/jobs-migration.md). The Space
+# itself is not run as a Gradio app; the image exists only to be
+# pulled by `hf jobs run --image hf.co/spaces/...`. Pause the
+# Space after the first successful build so no idle hardware cost
+# accrues; the built image stays available to Jobs while paused.
+#
+# Local smoke test (slow on Apple Silicon under Rosetta):
+#
+#     docker buildx build --platform linux/amd64 \
+#         -t cadgenbench-eval-gpu-test .
+
+FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    DEBIAN_FRONTEND=noninteractive
+
+# Python 3.12 from deadsnakes (Ubuntu 22.04 ships 3.10 by default)
+# plus the apt runtime deps shared with the leaderboard Dockerfile
+# (OCP / build123d / Pillow / VTK). libegl1 + libegl-mesa0 provide
+# the EGL surface vtk-egl binds to; on this CUDA-base image the
+# NVIDIA driver supplies hardware OpenGL, no Mesa fallback path.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        software-properties-common \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get update && apt-get install -y --no-install-recommends \
+        python3.12 python3.12-venv python3.12-dev \
+        python3-pip \
+        git ca-certificates \
+        libglib2.0-0 libsm6 libxext6 libgomp1 libfontconfig1 \
+        libgl1 libegl1 libegl-mesa0 libxrender1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -sf /usr/bin/python3.12 /usr/local/bin/python \
+    && ln -sf /usr/bin/python3.12 /usr/local/bin/python3
+
+# cadgenbench from the Public GitHub repo, same convention and
+# ARG name as the leaderboard Dockerfile. Bump CADGENBENCH_SHA in
+# lockstep with cadgenbench releases.
+ARG CADGENBENCH_SHA=b22a53c
+RUN python -m pip install --no-cache-dir \
+        "cadgenbench @ git+https://github.com/huggingface/cadgenbench.git@${CADGENBENCH_SHA}"
+
+# The cadgenbench wheel pulls vanilla `vtk` from PyPI (built with
+# vtkXOpenGLRenderWindow, needs an X server). Swap for vtk-egl:
+# same VTK, compiled against EGL so it acquires an off-screen GL
+# context against the NVIDIA driver on this CUDA-base image.
+# PyVista picks up whichever `vtk` dist is installed; no
+# cadgenbench code change. Same shape as the leaderboard's
+# vtk-osmesa swap, just the GPU counterpart.
+RUN python -m pip uninstall -y vtk \
+    && python -m pip install --no-cache-dir \
+        --extra-index-url https://wheels.vtk.org vtk-egl
+
+# In-job entrypoint. Invoked by:
+#
+#     hf jobs run --image hf.co/spaces/HuggingAI4Engineering/cadgenbench-eval-gpu \
+#         --flavor a10g-large --secrets HF_TOKEN \
+#         python /opt/eval_job.py <submission_id> <zip_url>
+COPY eval_job.py /opt/eval_job.py
+
+# Drop privileges. HF Spaces conventionally run as uid 1000.
+RUN useradd -m -u 1000 user
+USER user
+WORKDIR /home/user
+
+# Idle CMD so the Space's runtime starts without restart-flapping.
+# Pause the Space via HF UI or HfApi().pause_space() after the
+# first green build; the cached image stays available to Jobs.
+CMD ["sleep", "infinity"]

README.md

@@ -1,11 +1,44 @@
 ---
-title: Cadgenbench Eval Gpu
-emoji: 👁
-colorFrom: green
-colorTo: indigo
+title: CADGenBench eval (GPU)
+colorFrom: gray
+colorTo: gray
 sdk: docker
 pinned: false
-short_description: GPU infra for eval evaluation
+license: apache-2.0
+short_description: GPU image for the CADGenBench eval HF Jobs pipeline.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# cadgenbench-eval-gpu
+
+Image-only Docker Space. Provides the GPU container that the
+[CADGenBench leaderboard
+Space](https://huggingface.co/spaces/HuggingAI4Engineering/cadgenbench-leaderboard)
+pulls via `hf jobs run` to run the eval pipeline (alignment + render +
+metrics) for each submission.
+
+Not intended to be run as a Gradio / web app. The `CMD ["sleep",
+"infinity"]` only exists so the Space runtime starts without
+restart-flapping after the build; pause the Space after the first
+green build to avoid idle hardware cost. The built image stays cached
+on HF and remains pullable by Jobs while paused.
+
+Design + integration details:
+[`space-setup/jobs-migration.md`](https://github.com/huggingface/cadgenbench)
+(in the umbrella `cadgenbench` working tree). The leaderboard Space's
+worker dispatches `python /opt/eval_job.py <submission_id> <zip_url>`
+against this image on `a10g-large` and polls for completion.
+
+## Image contents
+
+- `nvidia/cuda:12.4.1-runtime-ubuntu22.04` base.
+- Python 3.12 via deadsnakes.
+- Apt runtime deps for OCP / build123d / VTK (shared with the
+  leaderboard Dockerfile) plus `libegl1 libegl-mesa0` for the EGL
+  context.
+- `cadgenbench @ git+https://github.com/huggingface/cadgenbench@<sha>`,
+  pinned via `ARG CADGENBENCH_SHA`.
+- `vtk-egl` swapped in for the PyPI `vtk` wheel (same swap shape as
+  the leaderboard's `vtk-osmesa`; the GPU counterpart). PyVista
+  picks up whichever `vtk` dist is installed; no cadgenbench code
+  change needed.
+- `/opt/eval_job.py` entrypoint script.

eval_job.py

@@ -0,0 +1,245 @@
+"""In-job entrypoint for the CADGenBench eval on HF Jobs.
+
+Invoked by the leaderboard Space's worker (see
+``AI4Engineering/submit.py``) via::
+
+    hf jobs run --image hf.co/spaces/HuggingAI4Engineering/cadgenbench-eval-gpu \\
+        --flavor a10g-large \\
+        --env CADGENBENCH_DATA_REPO=HuggingAI4Engineering/cadgenbench-data \\
+        --env CADGENBENCH_DATA_GT_REPO=HuggingAI4Engineering/cadgenbench-data-gt \\
+        --env HF_SUBMISSIONS_REPO=HuggingAI4Engineering/cadgenbench-submissions \\
+        --env EVAL_WORKER_COUNT=8 \\
+        --secrets HF_TOKEN \\
+        python /opt/eval_job.py <submission_id> <zip_url>
+
+Pipeline, in order. Synchronous, no fallbacks. Any failure raises
+and the container exits non-zero; the Space's poller catches the
+ERROR stage and flips the submission row to ``failed``.
+
+1. Download ``submissions/<id>.zip`` from the submissions dataset
+   via ``hf_hub_download`` (auth via ``HF_TOKEN``).
+2. Unpack into ``/tmp/run/``.
+3. ``cadgenbench evaluate /tmp/run --workers <n>`` (subprocess).
+4. ``cadgenbench report single /tmp/run -o /tmp/<id>.html``
+   (subprocess).
+5. Build ``report.json`` bundling ``run_summary.json`` + every
+   per-fixture ``result.json`` (mirror of submit.py's
+   ``_build_report_json``).
+6. Upload ``reports/<id>.html`` + ``reports/<id>.json`` back to the
+   submissions dataset via ``HfApi.upload_file``.
+7. Exit 0.
+
+The Space-side worker then downloads ``reports/<id>.json``, reads
+``run_summary`` out of it, and flips the row to ``completed``.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi, hf_hub_download
+
+
+RUN_DIR = Path("/tmp/run")
+REPORT_HTML_DIR = Path("/tmp")
+
+EVAL_TIMEOUT_SECONDS = 30 * 60
+REPORT_TIMEOUT_SECONDS = 5 * 60
+
+REPORTS_DIR_IN_REPO = "reports"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Run the CADGenBench eval pipeline on an HF Job.",
+    )
+    parser.add_argument(
+        "submission_id",
+        help="Filesystem-safe slug minted by the Space's submit handler.",
+    )
+    parser.add_argument(
+        "zip_url",
+        help=(
+            "Canonical Hub blob URL of submissions/<id>.zip "
+            "(submission_blob_url from the row)."
+        ),
+    )
+    args = parser.parse_args()
+
+    submission_id: str = args.submission_id
+    zip_url: str = args.zip_url
+
+    token = _require_env("HF_TOKEN")
+    submissions_repo = _require_env("HF_SUBMISSIONS_REPO")
+    worker_count = int(os.environ.get("EVAL_WORKER_COUNT", "8"))
+
+    print(
+        f"[eval_job] submission_id={submission_id} "
+        f"workers={worker_count} repo={submissions_repo}",
+        flush=True,
+    )
+
+    _prepare_run_dir(submission_id, zip_url, submissions_repo, token)
+    _run_eval(RUN_DIR, worker_count)
+    html_path = REPORT_HTML_DIR / f"{submission_id}.html"
+    _run_report(RUN_DIR, html_path)
+    report_json = _build_report_json(RUN_DIR)
+    _upload_reports(
+        submission_id, html_path, report_json, submissions_repo, token,
+    )
+    print(f"[eval_job] done: {submission_id}", flush=True)
+    return 0
+
+
+def _require_env(name: str) -> str:
+    """Return env var *name* or raise with a clear message."""
+    value = os.environ.get(name)
+    if not value:
+        raise RuntimeError(
+            f"Required environment variable {name!r} is unset or empty."
+        )
+    return value
+
+
+def _prepare_run_dir(
+    submission_id: str,
+    zip_url: str,
+    submissions_repo: str,
+    token: str,
+) -> None:
+    """Download the submission zip and unpack into ``RUN_DIR``.
+
+    Derives the in-repo path from *zip_url* and pulls via
+    ``hf_hub_download`` so token auth is handled and the file lands
+    in the Hub cache. *zip_url* is expected to look like
+    ``https://huggingface.co/datasets/<repo>/resolve/main/submissions/<id>.zip``;
+    we accept any URL shape that ends in ``submissions/<id>.zip`` and
+    re-derive the in-repo filename from the *submission_id*.
+    """
+    if RUN_DIR.exists():
+        shutil.rmtree(RUN_DIR)
+    RUN_DIR.mkdir(parents=True)
+
+    in_repo_path = f"submissions/{submission_id}.zip"
+    print(
+        f"[eval_job] downloading {submissions_repo}:{in_repo_path}",
+        flush=True,
+    )
+    local_zip = hf_hub_download(
+        repo_id=submissions_repo,
+        filename=in_repo_path,
+        repo_type="dataset",
+        token=token,
+    )
+
+    # Defensive: matches the validated shape from submit.py's
+    # _extract_zip, but the Space already gate-checked the zip
+    # contents pre-upload so we extract directly without re-
+    # validating zip-slip / symlinks here.
+    with zipfile.ZipFile(local_zip) as zf:
+        zf.extractall(RUN_DIR)
+    print(f"[eval_job] unpacked into {RUN_DIR}", flush=True)
+
+
+def _run_eval(run_dir: Path, workers: int) -> None:
+    """Invoke ``cadgenbench evaluate`` over *run_dir*; raise on non-zero."""
+    cmd = [
+        sys.executable, "-m", "cadgenbench.cli", "evaluate", str(run_dir),
+        "--workers", str(workers),
+    ]
+    print(f"[eval_job] {' '.join(cmd)}", flush=True)
+    proc = subprocess.run(
+        cmd,
+        timeout=EVAL_TIMEOUT_SECONDS,
+        env=os.environ.copy(),
+        check=False,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(
+            f"cadgenbench evaluate exited {proc.returncode}"
+        )
+
+
+def _run_report(run_dir: Path, html_out: Path) -> None:
+    """Invoke ``cadgenbench report single`` for *run_dir*; raise on non-zero."""
+    cmd = [
+        sys.executable, "-m", "cadgenbench.cli", "report", "single",
+        str(run_dir), "-o", str(html_out),
+    ]
+    print(f"[eval_job] {' '.join(cmd)}", flush=True)
+    proc = subprocess.run(
+        cmd,
+        timeout=REPORT_TIMEOUT_SECONDS,
+        env=os.environ.copy(),
+        check=False,
+    )
+    if proc.returncode != 0 or not html_out.is_file():
+        raise RuntimeError(
+            f"cadgenbench report single exited {proc.returncode} "
+            f"(html exists={html_out.is_file()})"
+        )
+
+
+def _build_report_json(run_dir: Path) -> dict[str, Any]:
+    """Bundle ``run_summary.json`` + every per-fixture ``result.json``.
+
+    Identical shape to submit.py's ``_build_report_json``: the
+    Space-side worker reads ``report.json`` after the Job completes
+    and pulls ``run_summary`` out of it to flip the row.
+    """
+    summary_path = run_dir / "run_summary.json"
+    if not summary_path.is_file():
+        raise RuntimeError(
+            f"run_summary.json not produced under {run_dir} (eval issue?)"
+        )
+    summary = json.loads(summary_path.read_text(encoding="utf-8"))
+    per_fixture: dict[str, dict[str, Any]] = {}
+    for fixture_dir in sorted(d for d in run_dir.iterdir() if d.is_dir()):
+        rp = fixture_dir / "result.json"
+        if rp.is_file():
+            per_fixture[fixture_dir.name] = json.loads(
+                rp.read_text(encoding="utf-8")
+            )
+    return {"run_summary": summary, "per_fixture_results": per_fixture}
+
+
+def _upload_reports(
+    submission_id: str,
+    html_path: Path,
+    report_json: dict[str, Any],
+    submissions_repo: str,
+    token: str,
+) -> None:
+    """Upload ``reports/<id>.html`` + ``reports/<id>.json`` to the Hub."""
+    api = HfApi(token=token)
+    api.upload_file(
+        path_or_fileobj=str(html_path),
+        path_in_repo=f"{REPORTS_DIR_IN_REPO}/{submission_id}.html",
+        repo_id=submissions_repo,
+        repo_type="dataset",
+        commit_message=f"add HTML report for {submission_id}",
+    )
+    api.upload_file(
+        path_or_fileobj=json.dumps(
+            report_json, ensure_ascii=False, indent=2,
+        ).encode("utf-8"),
+        path_in_repo=f"{REPORTS_DIR_IN_REPO}/{submission_id}.json",
+        repo_id=submissions_repo,
+        repo_type="dataset",
+        commit_message=f"add JSON report for {submission_id}",
+    )
+    print(
+        f"[eval_job] uploaded reports/{submission_id}.{{html,json}}",
+        flush=True,
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())