Isolate CPU fallback in spawn subprocess to prevent CUDA state poisoning

Running torch ops in the main Gradio process during CPU fallback triggers
C-level CUDA runtime queries that partially initialize CUDA state. Since
ZeroGPU uses fork() for GPU workers, all future workers inherit this
corrupted state ("bad fork"), causing permanent "No CUDA GPUs are available"
errors for ALL users until Space restart.

Fix: CPU fallback now runs in a multiprocessing.spawn subprocess with
SPACES_ZERO_GPU="" and CUDA_VISIBLE_DEVICES="" — a clean Python interpreter
with no ZeroGPU patches and no CUDA access. The main process never touches
torch, keeping its CUDA state pristine for future forked workers.

Also converts VAD tensor outputs to numpy for subprocess picklability and
adds _is_in_bad_fork diagnostic logging at GPU worker entry.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/core/cpu_subprocess.py

@@ -0,0 +1,108 @@
+"""Subprocess-isolated CPU inference to prevent CUDA state poisoning.
+
+On HuggingFace Spaces with ZeroGPU, the main Gradio process has PyTorch
+monkey-patched (TorchFunctionMode, fake CUDA availability). Running torch
+operations in the main process can trigger C-level CUDA runtime queries
+that partially initialize CUDA state. Since ZeroGPU uses fork() for GPU
+workers, this corrupted state is inherited by ALL future workers, causing
+permanent "No CUDA GPUs are available" errors.
+
+Solution: run CPU inference in a spawn-context subprocess. spawn creates
+a clean Python interpreter without inherited CUDA state or ZeroGPU patches.
+"""
+
+import importlib
+import multiprocessing
+import os
+import sys
+import traceback
+
+
+def _cpu_worker(func_module, func_name, extra_paths, args, kwargs, result_queue):
+    """Worker function for CPU subprocess. Runs in a clean process.
+
+    Disables ZeroGPU and CUDA so the function runs in a plain CPU PyTorch
+    environment with no monkey patches.
+    """
+    # Add parent's sys.path entries so we can find src/, config, etc.
+    for p in extra_paths:
+        if p and p not in sys.path:
+            sys.path.insert(0, p)
+
+    # Disable ZeroGPU — prevents spaces package from patching torch
+    os.environ["SPACES_ZERO_GPU"] = ""
+    # Disable CUDA — guarantees CPU-only execution
+    os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+    try:
+        module = importlib.import_module(func_module)
+        func = getattr(module, func_name)
+        # Unwrap @gpu_with_fallback decorator to call the raw function.
+        # functools.wraps sets __wrapped__ on each wrapper layer.
+        while hasattr(func, "__wrapped__"):
+            func = func.__wrapped__
+        result = func(*args, **kwargs)
+        result_queue.put(("ok", result))
+    except Exception as e:
+        tb = traceback.format_exc()
+        result_queue.put(("error", (type(e).__name__, str(e), tb)))
+
+
+def run_in_cpu_subprocess(func, args, kwargs, timeout=600):
+    """Run a function in an isolated CPU subprocess.
+
+    Uses 'spawn' context to create a clean Python interpreter that does
+    not inherit the main process's CUDA state or ZeroGPU monkey patches.
+
+    All args, kwargs, and return values must be picklable (numpy arrays,
+    lists, dicts, strings, numbers — no torch tensors or Gradio objects).
+
+    Args:
+        func: The function to call. Must be importable by module + name.
+        args: Positional arguments tuple.
+        kwargs: Keyword arguments dict.
+        timeout: Max seconds to wait (default 600 = 10 min).
+
+    Returns:
+        The function's return value.
+
+    Raises:
+        TimeoutError: If subprocess exceeds timeout.
+        RuntimeError: If subprocess fails or exits without result.
+    """
+    ctx = multiprocessing.get_context("spawn")
+    result_queue = ctx.Queue()
+
+    func_module = func.__module__
+    func_name = func.__qualname__
+    # Pass sys.path so the subprocess can find all modules (app dir, etc.)
+    extra_paths = list(sys.path)
+
+    print(f"[CPU SUBPROCESS] Spawning for {func_module}.{func_name}")
+
+    p = ctx.Process(
+        target=_cpu_worker,
+        args=(func_module, func_name, extra_paths, args, kwargs, result_queue),
+        daemon=True,
+    )
+    p.start()
+    p.join(timeout=timeout)
+
+    if p.is_alive():
+        p.kill()
+        p.join(timeout=5)
+        raise TimeoutError(f"CPU subprocess timed out after {timeout}s")
+
+    if result_queue.empty():
+        raise RuntimeError(
+            f"CPU subprocess exited without result (exit code {p.exitcode})"
+        )
+
+    status, payload = result_queue.get_nowait()
+    if status == "ok":
+        print(f"[CPU SUBPROCESS] {func_name} completed successfully")
+        return payload
+
+    exc_type, exc_msg, exc_tb = payload
+    print(f"[CPU SUBPROCESS] Error traceback:\n{exc_tb}")
+    raise RuntimeError(f"CPU subprocess error ({exc_type}): {exc_msg}")

src/core/zero_gpu.py

@@ -119,6 +119,16 @@ def force_cpu_mode():
 # Model cleanup helpers
 # =========================================================================
 
+def _check_cuda_fork_state(label: str):
+    """Log whether the current process is in a bad-fork CUDA state."""
+    try:
+        import torch
+        bad_fork = torch.cuda._is_in_bad_fork()
+        print(f"[CUDA DIAG] {label}: _is_in_bad_fork={bad_fork}")
+    except Exception as e:
+        print(f"[CUDA DIAG] {label}: check failed: {e}")
+
+
 def _cleanup_after_gpu():
     """Cleanup that runs at the END of a GPU lease.
 
@@ -190,6 +200,7 @@ def gpu_with_fallback(duration=60):
         # to prevent concurrent threads from moving models mid-inference.
         @wraps(func)
         def func_with_cleanup(*args, **kwargs):
+            _check_cuda_fork_state(f"GPU worker entry ({func.__name__})")
             _enter_gpu_lease()
             with model_device_lock:
                 try:
@@ -214,10 +225,21 @@ def gpu_with_fallback(duration=60):
         @wraps(func)
         def wrapper(*args, **kwargs):
             global _models_stale
-            # If user explicitly chose CPU mode, skip GPU entirely
+            # If user explicitly chose CPU mode, skip GPU entirely.
+            # On ZeroGPU, run in an isolated subprocess to prevent CUDA
+            # state poisoning — torch ops in the main process corrupt the
+            # C-level CUDA runtime, making all future forked workers fail.
             if is_user_forced_cpu():
-                print("[CPU] User selected CPU mode")
-                return func(*args, **kwargs)
+                if ZERO_GPU_AVAILABLE:
+                    _check_cuda_fork_state(f"before CPU subprocess ({func.__name__})")
+                    print(f"[CPU] Running {func.__name__} in isolated subprocess")
+                    from .cpu_subprocess import run_in_cpu_subprocess
+                    result = run_in_cpu_subprocess(func, args, kwargs)
+                    _check_cuda_fork_state(f"after CPU subprocess ({func.__name__})")
+                    return result
+                else:
+                    print("[CPU] User selected CPU mode (local dev)")
+                    return func(*args, **kwargs)
 
             # Try GPU
             try:

src/pipeline.py

@@ -679,9 +679,16 @@ def resegment_audio(
     print("[STAGE] Resegmenting...")
 
     # Re-clean speech intervals with new parameters (CPU, no GPU needed)
+    # Convert numpy→torch if needed (VAD returns numpy for picklability)
+    import torch as _torch
+    _intervals_tensor = (
+        _torch.from_numpy(cached_speech_intervals)
+        if isinstance(cached_speech_intervals, np.ndarray)
+        else cached_speech_intervals
+    )
     from recitations_segmenter import clean_speech_intervals
     clean_out = clean_speech_intervals(
-        cached_speech_intervals,
+        _intervals_tensor,
         cached_is_complete,
         min_silence_duration_ms=int(min_silence_ms),
         min_speech_duration_ms=int(min_speech_ms),

src/segmenter/vad.py

@@ -92,11 +92,12 @@ def detect_speech_segments(
 
         raw_speech_intervals = outputs[0].speech_intervals
         raw_is_complete = outputs[0].is_complete
-        # Detach from GPU to prevent CUDA tensor refs escaping the lease
-        if hasattr(raw_speech_intervals, 'cpu'):
-            raw_speech_intervals = raw_speech_intervals.cpu()
-        if hasattr(raw_is_complete, 'cpu'):
-            raw_is_complete = raw_is_complete.cpu()
+        # Convert to numpy: prevents CUDA tensor refs escaping the lease
+        # and ensures picklability for CPU subprocess isolation
+        if hasattr(raw_speech_intervals, 'detach'):
+            raw_speech_intervals = raw_speech_intervals.detach().cpu().numpy()
+        if hasattr(raw_is_complete, 'detach'):
+            raw_is_complete = raw_is_complete.detach().cpu().numpy()
 
         return [(start, end) for start, end in intervals], {"model_load_time": model_load_time, "inference_time": inference_time}, raw_speech_intervals, raw_is_complete