Michael Rabinovich Cursor commited on
Commit
c1cb5e4
·
1 Parent(s): 920b1b4

Serve rotating WebP turntables + GT generator

Browse files

Point the gallery and proxy routes at rotating.webp (submissions and
GT), publish the full per-fixture renders folder, and add a reusable
one-time GT turntable generator that renders from the cached GT mesh.

Co-authored-by: Cursor <cursoragent@cursor.com>

Files changed (5) hide show
  1. app.py +30 -37
  2. gallery.py +7 -7
  3. submit.py +14 -14
  4. tests/test_proxy.py +56 -0
  5. tools/generate_gt_turntables.py +215 -0
app.py CHANGED
@@ -528,18 +528,12 @@ def serve_report(submission_id: str) -> Response:
528
  return Response(content=content, media_type="text/html; charset=utf-8")
529
 
530
 
531
- # Single canonical view served as the gallery thumbnail. Matches the
532
- # view uploaded by the eval job (eval_job.py GALLERY_THUMB_VIEW) and the
533
- # GT render the gallery pairs it with, so columns stay comparable.
534
- GALLERY_THUMB_VIEW = "iso"
535
-
536
-
537
  def _fetch_render(submission_id: str, fixture: str) -> bytes | None:
538
- """Pull a submission's gallery thumbnail (``renders/<id>/<fixture>.png``).
539
 
540
  Deliberately **not** memoized: renders land over time (a submission
541
  completes, or an existing row is backfilled) after the Space process
542
- booted, so negative-caching a boot-time miss would keep a thumbnail
543
  dashed until the next restart. ``hf_hub_download`` does its own disk
544
  caching per revision, so a re-fetch of an unchanged file stays cheap.
545
  Returns ``None`` on any failure (the gallery draws the dashed cell).
@@ -547,7 +541,7 @@ def _fetch_render(submission_id: str, fixture: str) -> bytes | None:
547
  try:
548
  local_path = hf_hub_download(
549
  repo_id=HF_SUBMISSIONS_REPO,
550
- filename=f"renders/{submission_id}/{fixture}.png",
551
  repo_type="dataset",
552
  )
553
  return Path(local_path).read_bytes()
@@ -560,13 +554,12 @@ def _fetch_render(submission_id: str, fixture: str) -> bytes | None:
560
 
561
 
562
  def _fetch_gt_render(fixture: str) -> bytes | None:
563
- """Pull a fixture's ground-truth thumbnail from the private GT dataset.
564
 
565
- Path inside the GT repo is ``<fixture>/renders/<view>.png`` (see
566
- ``cadgenbench.common.paths.data_gt_dir``). GT renders are a property
567
- of the data revision, not of any submission, so they're served
568
- straight from the GT repo rather than duplicated per submission.
569
- Not memoized for the same reason as :func:`_fetch_render` (GT
570
  renders can be added/updated on a data revision bump);
571
  ``hf_hub_download`` handles the per-revision disk cache. Needs the
572
  Space ``HF_TOKEN``'s read scope on the private repo.
@@ -574,7 +567,7 @@ def _fetch_gt_render(fixture: str) -> bytes | None:
574
  try:
575
  local_path = hf_hub_download(
576
  repo_id=HF_DATA_GT_REPO,
577
- filename=f"{fixture}/renders/{GALLERY_THUMB_VIEW}.png",
578
  repo_type="dataset",
579
  )
580
  return Path(local_path).read_bytes()
@@ -589,13 +582,13 @@ def _fetch_gt_render(fixture: str) -> bytes | None:
589
  # Long-lived immutable caching: a (submission, fixture) render never
590
  # changes (fixed camera + lighting; re-renders would be a new artifact),
591
  # so the browser/CDN can keep it forever. This is what makes fixture
592
- # swaps and repeat visits free: only the ~33 on-screen thumbnails are
593
  # fetched on first paint, and everything after that is a cache hit.
594
  RENDER_CACHE_CONTROL = "public, max-age=31536000, immutable"
595
 
596
 
597
  def _render_proxy_url(submission_id: str, fixture: str) -> str | None:
598
- """Resolver returning the cached proxy URL for a submission render.
599
 
600
  Returns the route string **without** fetching the bytes (that's the
601
  whole point: the browser lazy-fetches on demand, so only the visible
@@ -608,40 +601,40 @@ def _render_proxy_url(submission_id: str, fixture: str) -> str | None:
608
  Requires the Space to be **public**: while private, HF's edge 404s
609
  in-browser fetches to these custom routes.
610
  """
611
- return f"/render/{submission_id}/{fixture}.png"
612
 
613
 
614
  def _gt_proxy_url(fixture: str) -> str | None:
615
- """Resolver returning the cached proxy URL for a fixture's GT render."""
616
- return f"/gt-render/{fixture}.png"
617
 
618
 
619
  def serve_render(submission_id: str, fixture: str) -> Response:
620
- """Stream a submission's per-fixture render PNG with long-lived caching.
621
 
622
- The gallery references ``/render/<id>/<fixture>.png`` and the browser
623
  fetches it lazily. Re-streams the dataset bytes (the Space holds the
624
  read token) with an immutable ``Cache-Control`` so the CDN/browser
625
  cache it hard.
626
  """
627
- png = _fetch_render(submission_id, fixture)
628
- if png is None:
629
  return Response(status_code=404)
630
  return Response(
631
- content=png,
632
- media_type="image/png",
633
  headers={"Cache-Control": RENDER_CACHE_CONTROL},
634
  )
635
 
636
 
637
  def serve_gt_render(fixture: str) -> Response:
638
- """Stream a fixture's ground-truth render PNG with long-lived caching."""
639
- png = _fetch_gt_render(fixture)
640
- if png is None:
641
  return Response(status_code=404)
642
  return Response(
643
- content=png,
644
- media_type="image/png",
645
  headers={"Cache-Control": RENDER_CACHE_CONTROL},
646
  )
647
 
@@ -649,7 +642,7 @@ def serve_gt_render(fixture: str) -> Response:
649
  def _gallery_iframe_html() -> str:
650
  """Build the gallery as a self-contained ``srcdoc`` iframe.
651
 
652
- Reads the live rows and renders the page (thumbnails referenced as
653
  cached ``/render`` / ``/gt-render`` proxy URLs, lazy-loaded by the
654
  browser), then inlines the whole document into an iframe ``srcdoc``
655
  so it gets its own style context (no Gradio CSS collision). A Hub
@@ -678,7 +671,7 @@ with gr.Blocks(title="CADGenBench Leaderboard", theme=gr.themes.Soft()) as block
678
 
679
  with gr.Tab("Gallery"):
680
  # Visual-first leaderboard. The bespoke surface (sticky GT row,
681
- # fixture picker, thumbnail grid, compare modal) is a
682
  # self-contained HTML doc inlined into an iframe `srcdoc` so it
683
  # keeps its own style context. Thumbnails are lazy-loaded from
684
  # the cached `/render` / `/gt-render` proxy routes (requires the
@@ -1000,16 +993,16 @@ app.add_api_route(
1000
  serve_report,
1001
  methods=["GET"],
1002
  )
1003
- # Cached render proxies the gallery's lazy-loaded thumbnails point at.
1004
  # Registered before the Gradio mount so they're not shadowed by the
1005
  # catch-all sub-app.
1006
  app.add_api_route(
1007
- "/render/{submission_id}/{fixture}.png",
1008
  serve_render,
1009
  methods=["GET"],
1010
  )
1011
  app.add_api_route(
1012
- "/gt-render/{fixture}.png",
1013
  serve_gt_render,
1014
  methods=["GET"],
1015
  )
 
528
  return Response(content=content, media_type="text/html; charset=utf-8")
529
 
530
 
 
 
 
 
 
 
531
  def _fetch_render(submission_id: str, fixture: str) -> bytes | None:
532
+ """Pull a submission's gallery WebP (``renders/<id>/<fixture>/rotating.webp``).
533
 
534
  Deliberately **not** memoized: renders land over time (a submission
535
  completes, or an existing row is backfilled) after the Space process
536
+ booted, so negative-caching a boot-time miss would keep a turntable
537
  dashed until the next restart. ``hf_hub_download`` does its own disk
538
  caching per revision, so a re-fetch of an unchanged file stays cheap.
539
  Returns ``None`` on any failure (the gallery draws the dashed cell).
 
541
  try:
542
  local_path = hf_hub_download(
543
  repo_id=HF_SUBMISSIONS_REPO,
544
+ filename=f"renders/{submission_id}/{fixture}/rotating.webp",
545
  repo_type="dataset",
546
  )
547
  return Path(local_path).read_bytes()
 
554
 
555
 
556
  def _fetch_gt_render(fixture: str) -> bytes | None:
557
+ """Pull a fixture's ground-truth GIF from the private GT dataset.
558
 
559
+ Path inside the GT repo is ``<fixture>/renders/rotating.webp``. GT
560
+ renders are a property of the data revision, not of any submission,
561
+ so they're served straight from the GT repo rather than duplicated
562
+ per submission. Not memoized for the same reason as :func:`_fetch_render` (GT
 
563
  renders can be added/updated on a data revision bump);
564
  ``hf_hub_download`` handles the per-revision disk cache. Needs the
565
  Space ``HF_TOKEN``'s read scope on the private repo.
 
567
  try:
568
  local_path = hf_hub_download(
569
  repo_id=HF_DATA_GT_REPO,
570
+ filename=f"{fixture}/renders/rotating.webp",
571
  repo_type="dataset",
572
  )
573
  return Path(local_path).read_bytes()
 
582
  # Long-lived immutable caching: a (submission, fixture) render never
583
  # changes (fixed camera + lighting; re-renders would be a new artifact),
584
  # so the browser/CDN can keep it forever. This is what makes fixture
585
+ # swaps and repeat visits free: only the ~33 on-screen turntables are
586
  # fetched on first paint, and everything after that is a cache hit.
587
  RENDER_CACHE_CONTROL = "public, max-age=31536000, immutable"
588
 
589
 
590
  def _render_proxy_url(submission_id: str, fixture: str) -> str | None:
591
+ """Resolver returning the cached proxy URL for a submission GIF.
592
 
593
  Returns the route string **without** fetching the bytes (that's the
594
  whole point: the browser lazy-fetches on demand, so only the visible
 
601
  Requires the Space to be **public**: while private, HF's edge 404s
602
  in-browser fetches to these custom routes.
603
  """
604
+ return f"/render/{submission_id}/{fixture}.webp"
605
 
606
 
607
  def _gt_proxy_url(fixture: str) -> str | None:
608
+ """Resolver returning the cached proxy URL for a fixture's GT WebP."""
609
+ return f"/gt-render/{fixture}.webp"
610
 
611
 
612
  def serve_render(submission_id: str, fixture: str) -> Response:
613
+ """Stream a submission's per-fixture render WebP with long-lived caching.
614
 
615
+ The gallery references ``/render/<id>/<fixture>.webp`` and the browser
616
  fetches it lazily. Re-streams the dataset bytes (the Space holds the
617
  read token) with an immutable ``Cache-Control`` so the CDN/browser
618
  cache it hard.
619
  """
620
+ webp = _fetch_render(submission_id, fixture)
621
+ if webp is None:
622
  return Response(status_code=404)
623
  return Response(
624
+ content=webp,
625
+ media_type="image/webp",
626
  headers={"Cache-Control": RENDER_CACHE_CONTROL},
627
  )
628
 
629
 
630
  def serve_gt_render(fixture: str) -> Response:
631
+ """Stream a fixture's ground-truth render WebP with long-lived caching."""
632
+ webp = _fetch_gt_render(fixture)
633
+ if webp is None:
634
  return Response(status_code=404)
635
  return Response(
636
+ content=webp,
637
+ media_type="image/webp",
638
  headers={"Cache-Control": RENDER_CACHE_CONTROL},
639
  )
640
 
 
642
  def _gallery_iframe_html() -> str:
643
  """Build the gallery as a self-contained ``srcdoc`` iframe.
644
 
645
+ Reads the live rows and renders the page (turntables referenced as
646
  cached ``/render`` / ``/gt-render`` proxy URLs, lazy-loaded by the
647
  browser), then inlines the whole document into an iframe ``srcdoc``
648
  so it gets its own style context (no Gradio CSS collision). A Hub
 
671
 
672
  with gr.Tab("Gallery"):
673
  # Visual-first leaderboard. The bespoke surface (sticky GT row,
674
+ # fixture picker, turntable grid, compare modal) is a
675
  # self-contained HTML doc inlined into an iframe `srcdoc` so it
676
  # keeps its own style context. Thumbnails are lazy-loaded from
677
  # the cached `/render` / `/gt-render` proxy routes (requires the
 
993
  serve_report,
994
  methods=["GET"],
995
  )
996
+ # Cached render proxies the gallery's lazy-loaded turntables point at.
997
  # Registered before the Gradio mount so they're not shadowed by the
998
  # catch-all sub-app.
999
  app.add_api_route(
1000
+ "/render/{submission_id}/{fixture}.webp",
1001
  serve_render,
1002
  methods=["GET"],
1003
  )
1004
  app.add_api_route(
1005
+ "/gt-render/{fixture}.webp",
1006
  serve_gt_render,
1007
  methods=["GET"],
1008
  )
gallery.py CHANGED
@@ -17,7 +17,7 @@
17
  Builds a self-contained HTML document (its own CSS + JS) from the live
18
  submission rows. The Space serves it at ``/gallery`` and embeds it in
19
  the Gradio "Gallery" tab via an iframe, so the bespoke visual surface
20
- (sticky ground-truth row, fixture picker, thumbnail grid, report
21
  modal) lives in plain HTML/JS isolated from Gradio's styles rather
22
  than being forced into Gradio components.
23
 
@@ -28,18 +28,18 @@ which the page's JS renders. Render lookups are isolated behind the
28
  pointed at the cached render-proxy URLs the caller injects via the two
29
  resolvers:
30
 
31
- - ``renderFor(sub, fixtureId)`` -> ``/render/<id>/<fixture>.png`` (or
32
  ``null`` when the per-fixture status is invalid/missing, which draws
33
  the dashed "invalid generation" cell).
34
- - ``gtRenderFor(fixtureId)`` -> ``/gt-render/<fixture>.png``.
35
 
36
- Thumbnails are lazy-loaded by the browser, so only the ~33 on-screen
37
  tiles are fetched and CDN/browser caching makes fixture-swaps and
38
  repeat visits essentially free. This requires the Space to be
39
  **public** (HF's edge 404s in-browser fetches to our custom routes
40
  while private).
41
 
42
- Thumbnail clicks open a GT-vs-output compare modal that points at the
43
  existing per-submission detail/report view.
44
  """
45
  from __future__ import annotations
@@ -163,7 +163,7 @@ def render_gallery_page(rows: list[dict], render_resolver, gt_resolver) -> str:
163
 
164
  ``render_resolver`` / ``gt_resolver`` supply the cached render-proxy
165
  URLs (see :func:`build_gallery_payload`); the browser lazy-loads only
166
- the on-screen thumbnails.
167
 
168
  The document is self-contained and uses **system font stacks only**
169
  (no external font CDN fetch) so it never errors inside a sandboxed
@@ -335,7 +335,7 @@ body {
335
  }
336
  .thumb:hover { transform: translateY(-2px); box-shadow: 0 6px 18px rgba(20,22,28,.14); border-color: var(--accent); }
337
  /* Display width is CSS-constrained so the browser downscales the existing
338
- full-res render: no resize step, no new assets. */
339
  .thumb img { width: 100%; height: 100%; display: block; object-fit: contain; }
340
  .thumb .open-hint {
341
  position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
 
17
  Builds a self-contained HTML document (its own CSS + JS) from the live
18
  submission rows. The Space serves it at ``/gallery`` and embeds it in
19
  the Gradio "Gallery" tab via an iframe, so the bespoke visual surface
20
+ (sticky ground-truth row, fixture picker, turntable grid, report
21
  modal) lives in plain HTML/JS isolated from Gradio's styles rather
22
  than being forced into Gradio components.
23
 
 
28
  pointed at the cached render-proxy URLs the caller injects via the two
29
  resolvers:
30
 
31
+ - ``renderFor(sub, fixtureId)`` -> ``/render/<id>/<fixture>.webp`` (or
32
  ``null`` when the per-fixture status is invalid/missing, which draws
33
  the dashed "invalid generation" cell).
34
+ - ``gtRenderFor(fixtureId)`` -> ``/gt-render/<fixture>.webp``.
35
 
36
+ GIFs are lazy-loaded by the browser, so only the ~33 on-screen
37
  tiles are fetched and CDN/browser caching makes fixture-swaps and
38
  repeat visits essentially free. This requires the Space to be
39
  **public** (HF's edge 404s in-browser fetches to our custom routes
40
  while private).
41
 
42
+ Turntable clicks open a GT-vs-output compare modal that points at the
43
  existing per-submission detail/report view.
44
  """
45
  from __future__ import annotations
 
163
 
164
  ``render_resolver`` / ``gt_resolver`` supply the cached render-proxy
165
  URLs (see :func:`build_gallery_payload`); the browser lazy-loads only
166
+ the on-screen turntables.
167
 
168
  The document is self-contained and uses **system font stacks only**
169
  (no external font CDN fetch) so it never errors inside a sandboxed
 
335
  }
336
  .thumb:hover { transform: translateY(-2px); box-shadow: 0 6px 18px rgba(20,22,28,.14); border-color: var(--accent); }
337
  /* Display width is CSS-constrained so the browser downscales the existing
338
+ render artifact: no resize step, no new assets. */
339
  .thumb img { width: 100%; height: 100%; display: block; object-fit: contain; }
340
  .thumb .open-hint {
341
  position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
submit.py CHANGED
@@ -133,11 +133,6 @@ RESULTS_FILENAME = "results.jsonl"
133
  SUBMISSIONS_DIR = "submissions"
134
  REPORTS_DIR = "reports"
135
  RENDERS_DIR = "renders"
136
- # Single canonical view staged per fixture for the leaderboard gallery
137
- # thumbnail (matches eval_job.py's GALLERY_THUMB_VIEW). The merged-shard
138
- # path stages these from the merged run dir, exactly as the single-job
139
- # eval_job does from its in-job run dir.
140
- GALLERY_THUMB_VIEW = "iso"
141
  DATA_REV_SHORT_LEN = 12
142
  FAILURE_REASON_MAX_CHARS = 200
143
  SHA256_BLOCK_SIZE = 64 * 1024
@@ -1456,7 +1451,7 @@ def _merge_shards_and_publish(
1456
  (the proven merge primitive, importable from the Space's own
1457
  ``cadgenbench`` install -- no private-repo dependency), a
1458
  ``report.json`` bundle, an HTML report via the same ``report
1459
- single`` renderer the job uses, and one ``iso`` gallery thumbnail
1460
  per fixture. Uploads ``reports/<id>.{html,json}`` + the gallery
1461
  renders, and returns the merged ``run_summary`` for the row flip.
1462
 
@@ -1574,19 +1569,19 @@ def _publish_reports_and_gallery(
1574
  report_json: dict[str, Any],
1575
  run_dir: Path,
1576
  ) -> None:
1577
- """Publish report HTML/JSON + every gallery thumbnail in one commit.
1578
 
1579
  The merged artifacts land at the exact paths the leaderboard + the
1580
  row-flip expect: ``reports/<id>.{html,json}`` plus one
1581
- ``renders/<id>/<fixture>.png`` per fixture that produced an ``iso``
1582
- thumbnail. All of it goes in a single ``create_commit`` rather than
1583
  one ``upload_file`` per file: a fan-out can stage ~80+ thumbnails,
1584
  and a commit-per-file both serialises the publish (slow) and hammers
1585
  the dataset's commit endpoint (the 429 "concurrency queue" failures
1586
  that stranded earlier runs). One commit is atomic, fast, and
1587
  rate-limit friendly.
1588
 
1589
- A fixture with no ``iso.png`` (missing output / render that never
1590
  ran) is skipped, matching the single-job behaviour; the gallery
1591
  draws the dashed "invalid" cell from the row, so an absent thumbnail
1592
  is not an error.
@@ -1605,14 +1600,19 @@ def _publish_reports_and_gallery(
1605
  ]
1606
  render_count = 0
1607
  for fixture_dir in sorted(d for d in run_dir.iterdir() if d.is_dir()):
1608
- iso_png = fixture_dir / "renders" / f"{GALLERY_THUMB_VIEW}.png"
1609
- if iso_png.is_file():
 
 
 
 
1610
  operations.append(
1611
  CommitOperationAdd(
1612
  path_in_repo=(
1613
- f"{RENDERS_DIR}/{submission_id}/{fixture_dir.name}.png"
 
1614
  ),
1615
- path_or_fileobj=str(iso_png),
1616
  )
1617
  )
1618
  render_count += 1
 
133
  SUBMISSIONS_DIR = "submissions"
134
  REPORTS_DIR = "reports"
135
  RENDERS_DIR = "renders"
 
 
 
 
 
136
  DATA_REV_SHORT_LEN = 12
137
  FAILURE_REASON_MAX_CHARS = 200
138
  SHA256_BLOCK_SIZE = 64 * 1024
 
1451
  (the proven merge primitive, importable from the Space's own
1452
  ``cadgenbench`` install -- no private-repo dependency), a
1453
  ``report.json`` bundle, an HTML report via the same ``report
1454
+ single`` renderer the job uses, and the full gallery render folder
1455
  per fixture. Uploads ``reports/<id>.{html,json}`` + the gallery
1456
  renders, and returns the merged ``run_summary`` for the row flip.
1457
 
 
1569
  report_json: dict[str, Any],
1570
  run_dir: Path,
1571
  ) -> None:
1572
+ """Publish report HTML/JSON + every per-fixture gallery render in one commit.
1573
 
1574
  The merged artifacts land at the exact paths the leaderboard + the
1575
  row-flip expect: ``reports/<id>.{html,json}`` plus one
1576
+ ``renders/<id>/<fixture>/<filename>`` entry for every PNG/GIF in each
1577
+ fixture's render folder. All of it goes in a single ``create_commit`` rather than
1578
  one ``upload_file`` per file: a fan-out can stage ~80+ thumbnails,
1579
  and a commit-per-file both serialises the publish (slow) and hammers
1580
  the dataset's commit endpoint (the 429 "concurrency queue" failures
1581
  that stranded earlier runs). One commit is atomic, fast, and
1582
  rate-limit friendly.
1583
 
1584
+ A fixture with no render folder (missing output / render that never
1585
  ran) is skipped, matching the single-job behaviour; the gallery
1586
  draws the dashed "invalid" cell from the row, so an absent thumbnail
1587
  is not an error.
 
1600
  ]
1601
  render_count = 0
1602
  for fixture_dir in sorted(d for d in run_dir.iterdir() if d.is_dir()):
1603
+ renders_dir = fixture_dir / "renders"
1604
+ if not renders_dir.is_dir():
1605
+ continue
1606
+ for render_path in sorted(renders_dir.iterdir()):
1607
+ if render_path.suffix.lower() not in {".png", ".webp"}:
1608
+ continue
1609
  operations.append(
1610
  CommitOperationAdd(
1611
  path_in_repo=(
1612
+ f"{RENDERS_DIR}/{submission_id}/"
1613
+ f"{fixture_dir.name}/{render_path.name}"
1614
  ),
1615
+ path_or_fileobj=str(render_path),
1616
  )
1617
  )
1618
  render_count += 1
tests/test_proxy.py CHANGED
@@ -42,6 +42,28 @@ def test_serve_report_returns_404_when_file_missing(monkeypatch):
42
  assert "Report not found" in resp.body.decode("utf-8")
43
 
44
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  def test_fetch_report_html_returns_none_on_hub_failure(monkeypatch):
46
  """A Hub-side exception is caught and surfaced as None.
47
 
@@ -57,6 +79,38 @@ def test_fetch_report_html_returns_none_on_hub_failure(monkeypatch):
57
  assert app._fetch_report_html("sub-failure-probe-unique-1") is None
58
 
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  def test_proxy_route_is_registered():
61
  """The mounted FastAPI app exposes ``/reports/{submission_id}.html`` as GET.
62
 
@@ -66,6 +120,8 @@ def test_proxy_route_is_registered():
66
  """
67
  routes = [getattr(r, "path", None) for r in app.app.routes]
68
  assert "/reports/{submission_id}.html" in routes
 
 
69
 
70
 
71
  # --- Boot resilience: no silent fallback, but no crash either -------
 
42
  assert "Report not found" in resp.body.decode("utf-8")
43
 
44
 
45
+ def test_serve_render_returns_webp_when_file_exists(monkeypatch):
46
+ """Submission render proxy serves WebP bytes with image/webp media type."""
47
+ monkeypatch.setattr(app, "_fetch_render", lambda sid, fixture: b"RIFFwebp")
48
+
49
+ resp = app.serve_render("sub-test", "101")
50
+
51
+ assert resp.status_code == 200
52
+ assert resp.media_type == "image/webp"
53
+ assert resp.body == b"RIFFwebp"
54
+
55
+
56
+ def test_serve_gt_render_returns_webp_when_file_exists(monkeypatch):
57
+ """GT render proxy serves generated GT WebP bytes."""
58
+ monkeypatch.setattr(app, "_fetch_gt_render", lambda fixture: b"RIFFwebp")
59
+
60
+ resp = app.serve_gt_render("101")
61
+
62
+ assert resp.status_code == 200
63
+ assert resp.media_type == "image/webp"
64
+ assert resp.body == b"RIFFwebp"
65
+
66
+
67
  def test_fetch_report_html_returns_none_on_hub_failure(monkeypatch):
68
  """A Hub-side exception is caught and surfaced as None.
69
 
 
79
  assert app._fetch_report_html("sub-failure-probe-unique-1") is None
80
 
81
 
82
+ def test_fetch_render_uses_nested_rotating_webp_path(monkeypatch, tmp_path):
83
+ """Submission renders are fetched from the nested WebP artifact path."""
84
+ webp = tmp_path / "rotating.webp"
85
+ webp.write_bytes(b"RIFFwebp")
86
+ captured: dict = {}
87
+
88
+ def fake_download(**kwargs):
89
+ captured.update(kwargs)
90
+ return str(webp)
91
+
92
+ monkeypatch.setattr(app, "hf_hub_download", fake_download)
93
+
94
+ assert app._fetch_render("sub-test", "101") == b"RIFFwebp"
95
+ assert captured["filename"] == "renders/sub-test/101/rotating.webp"
96
+
97
+
98
+ def test_fetch_gt_render_uses_gt_rotating_webp_path(monkeypatch, tmp_path):
99
+ """GT WebPs are fetched from the GT dataset's per-fixture render folder."""
100
+ webp = tmp_path / "rotating.webp"
101
+ webp.write_bytes(b"RIFFwebp")
102
+ captured: dict = {}
103
+
104
+ def fake_download(**kwargs):
105
+ captured.update(kwargs)
106
+ return str(webp)
107
+
108
+ monkeypatch.setattr(app, "hf_hub_download", fake_download)
109
+
110
+ assert app._fetch_gt_render("101") == b"RIFFwebp"
111
+ assert captured["filename"] == "101/renders/rotating.webp"
112
+
113
+
114
  def test_proxy_route_is_registered():
115
  """The mounted FastAPI app exposes ``/reports/{submission_id}.html`` as GET.
116
 
 
120
  """
121
  routes = [getattr(r, "path", None) for r in app.app.routes]
122
  assert "/reports/{submission_id}.html" in routes
123
+ assert "/render/{submission_id}/{fixture}.webp" in routes
124
+ assert "/gt-render/{fixture}.webp" in routes
125
 
126
 
127
  # --- Boot resilience: no silent fallback, but no crash either -------
tools/generate_gt_turntables.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Generate the permanent ground-truth turntable WebPs (one-time work).
3
+
4
+ Each GT fixture in ``cadgenbench-data-gt`` ships a trusted mesh sidecar
5
+ (``ground_truth.mesh.npz``) next to its ``ground_truth.step``. This tool loads
6
+ that **cached mesh** via :class:`cadgenbench.common.artifacts.StepArtifacts`
7
+ (so it never re-tessellates) and renders the same rotating WebP the eval
8
+ pipeline produces for submissions, then commits each to
9
+ ``<fixture>/renders/rotating.webp`` in the GT dataset.
10
+
11
+ GT renders are a property of the data revision, not of any submission, so this
12
+ runs once (and again only when the GT geometry changes). The shared renderer
13
+ (:func:`cadgenbench.common.viewer.render_mesh_turntable_webp`) guarantees GT and
14
+ candidate turntables look identical (same material, lighting, framing, speed).
15
+
16
+ Run locally (against a checkout)::
17
+
18
+ python tools/generate_gt_turntables.py --gt-root ../cadgenbench-data-gt --upload
19
+
20
+ Or on an HF Job (GPU, no checkout — downloads STEP + sidecar from the Hub).
21
+ After the eval-gpu image is rebuilt with the updated ``cadgenbench`` package,
22
+ dispatch this file's contents on that image, e.g. with the Python API::
23
+
24
+ from huggingface_hub import run_job
25
+ run_job(
26
+ image="hf.co/spaces/HuggingAI4Engineering/cadgenbench-eval-gpu",
27
+ command=["python", "-c", Path("generate_gt_turntables.py").read_text()
28
+ + "\nimport sys; sys.exit(main())"],
29
+ flavor="a10g-large",
30
+ namespace="michaelr27",
31
+ secrets={"HF_TOKEN": "<write-token>"},
32
+ timeout="30m",
33
+ )
34
+
35
+ (The job needs a **write**-scoped token for the private GT dataset.)
36
+ """
37
+ from __future__ import annotations
38
+
39
+ import argparse
40
+ import os
41
+ import sys
42
+ import tempfile
43
+ from pathlib import Path
44
+
45
+ from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
46
+
47
+ # Allow running straight from the repo without installing the leaderboard pkg;
48
+ # cadgenbench itself must be importable (installed in the env / eval-gpu image).
49
+ _REPO_ROOT = Path(__file__).resolve().parents[2]
50
+ _SRC = _REPO_ROOT / "cadgenbench" / "src"
51
+ if _SRC.is_dir():
52
+ sys.path.insert(0, str(_SRC))
53
+
54
+ from cadgenbench.common.artifacts import StepArtifacts # noqa: E402
55
+ from cadgenbench.common.viewer import render_mesh_turntable_webp # noqa: E402
56
+
57
+ GT_STEP_NAME = "ground_truth.step"
58
+ GT_SIDECAR_NAME = "ground_truth.mesh.npz"
59
+ RENDER_PATH_IN_FIXTURE = "renders/rotating.webp"
60
+ # One commit per this many files: ~80 GT fixtures fit in a single commit, but
61
+ # chunking keeps an individual commit small and rate-limit friendly.
62
+ COMMIT_CHUNK = 60
63
+
64
+
65
+ def _default_repo_id() -> str:
66
+ return os.getenv(
67
+ "HF_DATA_GT_REPO",
68
+ f"{os.getenv('HF_ORG', 'HuggingAI4Engineering')}/cadgenbench-data-gt",
69
+ )
70
+
71
+
72
+ def _fixture_ids(api: HfApi, repo_id: str, gt_root: Path | None) -> list[str]:
73
+ """Fixture ids that have a ``ground_truth.step`` (local checkout or Hub)."""
74
+ if gt_root is not None:
75
+ ids = [
76
+ p.name for p in gt_root.iterdir()
77
+ if p.is_dir() and (p / GT_STEP_NAME).is_file()
78
+ ]
79
+ else:
80
+ files = api.list_repo_files(repo_id, repo_type="dataset")
81
+ ids = [
82
+ f.split("/", 1)[0] for f in files if f.endswith("/" + GT_STEP_NAME)
83
+ ]
84
+ return sorted(set(ids), key=lambda s: (len(s), s))
85
+
86
+
87
+ def _materialize_fixture(
88
+ api: HfApi,
89
+ repo_id: str,
90
+ fixture: str,
91
+ gt_root: Path | None,
92
+ cache_dir: Path,
93
+ token: str | None,
94
+ ) -> Path:
95
+ """Return a local dir holding this fixture's STEP + trusted mesh sidecar.
96
+
97
+ The sidecar must sit next to the STEP so ``StepArtifacts`` takes the
98
+ trusted-mesh path (no tessellation, no validation).
99
+ """
100
+ if gt_root is not None:
101
+ return gt_root / fixture
102
+ dest = cache_dir / fixture
103
+ dest.mkdir(parents=True, exist_ok=True)
104
+ for name in (GT_STEP_NAME, GT_SIDECAR_NAME):
105
+ local = hf_hub_download(
106
+ repo_id=repo_id,
107
+ filename=f"{fixture}/{name}",
108
+ repo_type="dataset",
109
+ token=token,
110
+ )
111
+ target = dest / name
112
+ if not target.exists():
113
+ target.write_bytes(Path(local).read_bytes())
114
+ return dest
115
+
116
+
117
+ def _render_fixture_webp(fixture_dir: Path) -> bytes:
118
+ """Render the turntable WebP from the fixture's cached GT mesh."""
119
+ art = StepArtifacts(fixture_dir / GT_STEP_NAME, is_ground_truth=True)
120
+ mesh = art.mesh() # trusted sidecar -> no tessellation
121
+ return render_mesh_turntable_webp(mesh)
122
+
123
+
124
+ def _commit_in_chunks(
125
+ api: HfApi, repo_id: str, ops: list[CommitOperationAdd],
126
+ ) -> None:
127
+ for i in range(0, len(ops), COMMIT_CHUNK):
128
+ chunk = ops[i:i + COMMIT_CHUNK]
129
+ api.create_commit(
130
+ repo_id=repo_id,
131
+ repo_type="dataset",
132
+ operations=chunk,
133
+ commit_message=(
134
+ f"add GT turntable webp(s) [{i + 1}-{i + len(chunk)}]"
135
+ ),
136
+ )
137
+ print(f" committed {len(chunk)} file(s)", flush=True)
138
+
139
+
140
+ def main() -> int:
141
+ parser = argparse.ArgumentParser(description=__doc__)
142
+ parser.add_argument(
143
+ "--gt-root", type=Path, default=None,
144
+ help="Local cadgenbench-data-gt checkout. Omit to download from the Hub.",
145
+ )
146
+ parser.add_argument("--repo-id", default=_default_repo_id())
147
+ parser.add_argument(
148
+ "--fixtures",
149
+ help="Comma-separated fixture ids. Omit for every GT fixture.",
150
+ )
151
+ parser.add_argument("--limit", type=int, default=None)
152
+ parser.add_argument(
153
+ "--out-dir", type=Path, default=None,
154
+ help="Also write each webp here (e.g. for local inspection).",
155
+ )
156
+ parser.add_argument(
157
+ "--no-upload", action="store_true",
158
+ help="Render only; do not commit to the GT dataset.",
159
+ )
160
+ args = parser.parse_args()
161
+
162
+ token = os.environ.get("HF_TOKEN")
163
+ api = HfApi(token=token)
164
+ gt_root = args.gt_root.resolve() if args.gt_root else None
165
+ if gt_root is not None and not gt_root.is_dir():
166
+ parser.error(f"--gt-root does not exist: {gt_root}")
167
+
168
+ fixtures = _fixture_ids(api, args.repo_id, gt_root)
169
+ if args.fixtures:
170
+ wanted = {f.strip() for f in args.fixtures.split(",") if f.strip()}
171
+ fixtures = [f for f in fixtures if f in wanted]
172
+ if args.limit is not None:
173
+ fixtures = fixtures[: args.limit]
174
+ if not fixtures:
175
+ parser.error("No GT fixtures matched.")
176
+
177
+ if not args.no_upload and not token:
178
+ parser.error("HF_TOKEN required to upload (or pass --no-upload).")
179
+
180
+ print(
181
+ f"Rendering {len(fixtures)} GT turntable(s) -> {args.repo_id}"
182
+ + ("" if args.no_upload else " (will upload)"),
183
+ flush=True,
184
+ )
185
+ ops: list[CommitOperationAdd] = []
186
+ with tempfile.TemporaryDirectory(prefix="gt-turntable-") as tmp:
187
+ cache_dir = Path(tmp)
188
+ for i, fixture in enumerate(fixtures, start=1):
189
+ print(f"[{i}/{len(fixtures)}] {fixture}", flush=True)
190
+ fixture_dir = _materialize_fixture(
191
+ api, args.repo_id, fixture, gt_root, cache_dir, token,
192
+ )
193
+ webp = _render_fixture_webp(fixture_dir)
194
+ if args.out_dir is not None:
195
+ local_out = args.out_dir / fixture / "rotating.webp"
196
+ local_out.parent.mkdir(parents=True, exist_ok=True)
197
+ local_out.write_bytes(webp)
198
+ ops.append(
199
+ CommitOperationAdd(
200
+ path_in_repo=f"{fixture}/{RENDER_PATH_IN_FIXTURE}",
201
+ path_or_fileobj=webp,
202
+ )
203
+ )
204
+
205
+ if args.no_upload:
206
+ print(f"Rendered {len(ops)} webp(s); upload skipped.", flush=True)
207
+ return 0
208
+ print(f"Uploading {len(ops)} webp(s) to {args.repo_id}…", flush=True)
209
+ _commit_in_chunks(api, args.repo_id, ops)
210
+ print("Done.", flush=True)
211
+ return 0
212
+
213
+
214
+ if __name__ == "__main__":
215
+ raise SystemExit(main())