submit: drop eval workers 4 -> 1 (OOM under parallel osmesa)

submit.py

@@ -72,6 +72,7 @@ import sys
 import tempfile
 import threading
 import zipfile
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
@@ -103,12 +104,14 @@ DATA_REV_SHORT_LEN = 12
 FAILURE_REASON_MAX_CHARS = 200
 EVAL_TIMEOUT_SECONDS = 15 * 60
 REPORT_TIMEOUT_SECONDS = 2 * 60
-# Per-fixture eval workers. The in-process VTK renderer (PyVista off-
-# screen via Mesa software EGL on the Space's cpu-upgrade tier) is
-# cheap enough that one ProcessPoolExecutor worker per fixture no
-# longer oversubscribes the host the way the old Chromium subprocess
-# did. "4" matches the cpu-upgrade vCPU count.
-EVAL_WORKER_COUNT = "4"
+# Per-fixture eval workers. Set to "1" (sequential) because 4 parallel
+# workers OOM-kill on the Space's cpu-upgrade tier: cadquery-ocp +
+# manifold3d + VTK osmesa state per worker, times 4, exceeds the box's
+# RAM and the kernel sends a SIGKILL that surfaces in the parent as
+# concurrent.futures.process.BrokenProcessPool after ~5 min of render.
+# Revisit once we have a hardware tier with more RAM, or once the
+# per-worker footprint shrinks (e.g. lazier OCC imports).
+EVAL_WORKER_COUNT = "1"
 SHA256_BLOCK_SIZE = 64 * 1024
 STUCK_PENDING_THRESHOLD_SECONDS = 30 * 60
 SUBMITTED_AT_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
@@ -137,7 +140,9 @@ class _HubWriteError(Exception):
 
 
 def handle_submit(
-    zip_file, profile: gr.OAuthProfile | None,
+    zip_file,
+    profile: gr.OAuthProfile | None,
+    progress: gr.Progress = gr.Progress(),
 ) -> None:
     """Validate a submission upload; surface progress + outcome via toasts.
 
@@ -148,18 +153,18 @@ def handle_submit(
     defensively if it's called without a profile so a UI mishap
     can't write an anonymous row.
 
-    Side-effect-only (returns ``None``): every status update lands in
-    a ``gr.Info`` / ``gr.Error`` toast instead of a static markdown
-    output below the button. Rejection paths raise ``gr.Error``,
-    which Gradio surfaces as a red toast and aborts the handler; the
-    outer ``try/finally`` still runs to clean up the temp dir.
+    Side-effect-only (returns ``None``): in-flight status drives the
+    Gradio submit button's progress tracker (``progress(fraction,
+    desc=...)``), and the final outcome lands in a ``gr.Info`` /
+    ``gr.Error`` toast. Rejection paths raise ``gr.Error``, which
+    Gradio surfaces as a red toast and aborts the handler; the outer
+    ``try/finally`` still runs to clean up the temp dir.
 
-    Toast sequence on the happy path:
-      1. ``gr.Info("Validating submission...")`` once the form-level
-         check is past.
-      2. ``gr.Info("Submission <id> queued for evaluation...")`` once
-         the pending row + zip are on the Hub and the worker has
-         been spawned.
+    Progress milestones on the happy path: extract zip (0.05),
+    validate meta (0.15), check fixture set (0.30), parse STEPs
+    (0.45), dedup hash + Hub check (0.70), upload zip (0.85), queue
+    eval (0.95), queued (1.0). The single ``gr.Info("Submission <id>
+    queued ...")`` toast fires after the worker is spawned.
 
     On rejection (login-missing, form-level, validation gate, dedup,
     or Hub write), a single ``gr.Error`` toast carries the message;
@@ -181,11 +186,17 @@ def handle_submit(
     run_dir = tmp / "run"
     run_dir.mkdir()
     try:
-        gr.Info("Validating submission...")
+        progress(0.05, desc="Extracting zip")
         try:
             _extract_zip(zip_path, run_dir)
+            progress(0.15, desc="Validating meta.json")
             meta = _load_and_validate_meta(run_dir)
+            progress(0.30, desc="Checking fixture set")
             fixture_names = _validate_fixture_set(run_dir)
+            progress(
+                0.45,
+                desc=f"Parsing {len(fixture_names)} STEP files",
+            )
             _validate_steps_parseable(run_dir, fixture_names)
         except _ValidationError as e:
             raise gr.Error(f"Submission rejected: {e}")
@@ -193,6 +204,7 @@ def handle_submit(
         # Dedup gate: hash the raw zip bytes and reject if an existing
         # row carries the same hash. Runs after validation so a clearly
         # malformed upload still gets the specific validation error.
+        progress(0.70, desc="Checking for duplicates")
         zip_sha256 = _compute_sha256(zip_path)
         existing_id = _find_existing_submission_by_sha256(zip_sha256)
         if existing_id is not None:
@@ -206,17 +218,20 @@ def handle_submit(
             meta["submitter_name"], meta["submission_name"]
         )
         try:
+            progress(0.85, desc="Uploading submission")
             blob_url = _upload_submission_zip(submission_id, zip_path)
             row = _build_pending_row(
                 submission_id, meta, blob_url, zip_sha256,
                 hf_username=profile.username,
             )
+            progress(0.95, desc="Queuing evaluation")
             _append_pending_row(row)
         except _HubWriteError as e:
             raise gr.Error(f"Submission rejected: {e}")
 
         _spawn_worker(submission_id, tmp, run_dir)
         tmp = None  # ownership transferred; skip cleanup below
+        progress(1.0, desc="Queued")
         gr.Info(
             f"Submission {submission_id} queued for evaluation "
             f"({len(fixture_names)} fixtures). Evaluation typically "
@@ -360,7 +375,16 @@ def _validate_fixture_set(unpacked: Path) -> set[str]:
 
 
 def _validate_steps_parseable(unpacked: Path, fixture_names: set[str]) -> None:
-    for name in sorted(fixture_names):
+    # Threads (not processes): OCC's parse_step releases the GIL during
+    # the C++ STEP read, and this gate doesn't touch the VTK renderer
+    # (which is the only piece in the eval pipeline that needs the
+    # ProcessPoolExecutor + spawn dance). Per-fixture I/O + OCC load is
+    # 1-5s, so fanning out a 5+ fixture set across cpu-upgrade vCPUs
+    # cuts wall time roughly linearly. ex.map raises the first child
+    # exception when its iterator is consumed, so wrapping in list()
+    # preserves the same `Fixture <name>` rejection text as the
+    # sequential loop did.
+    def _check_one_step(name: str) -> None:
         step = unpacked / name / "output.step"
         if not step.is_file():
             raise _ValidationError(
@@ -378,6 +402,11 @@ def _validate_steps_parseable(unpacked: Path, fixture_names: set[str]) -> None:
                 f"as STEP geometry: {e}"
             ) from e
 
+    with ThreadPoolExecutor(
+        max_workers=min(8, os.cpu_count() or 1),
+    ) as ex:
+        list(ex.map(_check_one_step, sorted(fixture_names)))
+
 
 def _mint_submission_id(submitter_name: str, submission_name: str) -> str:
     """Build the basename used for ``submissions/<id>.zip`` and ``reports/<id>.*``."""