Prevent GPU OOM from crashing the whole Space container

Root cause of "every error triggers a full rebuild": none of the six
@spaces.GPU functions caught CUDA OOM, so an out-of-memory error could
escape as an unrecoverable worker fault instead of a clean exception —
forcing HF to restart the whole container (wiping /tmp, re-downloading
~30GB of checkpoints) instead of just failing one request.

- _catch_oom wraps all 6 @spaces.GPU functions: catches
torch.cuda.OutOfMemoryError, clears the cache, and re-raises as
gr.Error so it renders as a clean UI error instead of a fatal fault.
- Enable HunyuanVideo-Foley's existing (but previously unused)
enable_offload=True path, which swaps SigLIP2/CLAP/Synchformer/DAC-VAE
in and out instead of holding ~14GB of static weights resident for
the whole call — the swap logic was already fully wired in the
vendored model_utils.py/feature_utils.py, just never turned on.
- Add del + empty_cache for MMAudio's net/feature_utils (~9.5GB) at the
end of _mmaudio_gpu_infer and _regen_mmaudio_gpu, which had no
cleanup at all (TARO already does this for its own components).

app.py

@@ -8,6 +8,8 @@ Supported models
   HunyuanFoley  – text-guided foley via SigLIP2 + Synchformer + CLAP (48 kHz, up to 15 s)
 """
 
+import functools
+import gc
 import html as _html
 import math
 import os
@@ -514,7 +516,7 @@ def _load_hunyuan_model(device, model_size):
     hunyuan_weights_dir = str(HUNYUAN_MODEL_DIR / "HunyuanVideo-Foley")
     print(f"[HunyuanFoley] Loading {model_size.upper()} model from {hunyuan_weights_dir}")
     return load_model(hunyuan_weights_dir, config_path, device,
-                      enable_offload=False, model_size=model_size)
+                      enable_offload=True, model_size=model_size)
 
 
 def mux_video_audio(silent_video: str, audio_path: str, output_path: str,
@@ -673,6 +675,29 @@ HUNYUAN_MAX_DUR     = MODEL_CONFIGS["hunyuan"]["window_s"]
 HUNYUAN_SECS_PER_STEP = MODEL_CONFIGS["hunyuan"]["secs_per_step"]
 
 
+def _catch_oom(fn):
+    """Wrap a @spaces.GPU function so a CUDA OOM can't escape and crash the
+    ZeroGPU worker. An escaped OOM doesn't just fail that one request — it can
+    take down the whole Space container, which restarts from scratch (every
+    checkpoint re-downloaded, ~2 minutes) instead of just showing an error.
+    Applied as the innermost decorator (below @spaces.GPU) so spaces.GPU's
+    duration estimator still sees the original call signature via
+    functools.wraps."""
+    @functools.wraps(fn)
+    def _wrapped(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except torch.cuda.OutOfMemoryError as e:
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            gc.collect()
+            raise gr.Error(
+                f"Out of GPU memory during generation ({fn.__name__}). "
+                "Try fewer samples, a shorter clip, or fewer steps."
+            ) from e
+    return _wrapped
+
+
 def _clamp_duration(secs: float, label: str) -> int:
     """Clamp a raw GPU-seconds estimate to [120, GPU_DURATION_CAP] and log it.
     ZeroGPU Pro users get up to 300 s per call; 120 s floor covers cold-disk
@@ -1004,6 +1029,7 @@ def _cpu_preprocess(video_file: str, model_dur: float,
 
 
 @spaces.GPU(duration=_taro_duration)
+@_catch_oom
 def _taro_gpu_infer(video_file, seed_val, cfg_scale, num_steps, mode,
                     crossfade_s, crossfade_db, num_samples):
     """GPU-only TARO inference — model loading + feature extraction + diffusion.
@@ -1176,6 +1202,7 @@ def _mmaudio_duration(video_file, prompt, negative_prompt, seed_val,
 
 
 @spaces.GPU(duration=_mmaudio_duration)
+@_catch_oom
 def _mmaudio_gpu_infer(video_file, prompt, negative_prompt, seed_val,
                        cfg_strength, num_steps, crossfade_s, crossfade_db, num_samples,
                        silent_video, segments_json,
@@ -1273,6 +1300,15 @@ def _mmaudio_gpu_infer(video_file, prompt, negative_prompt, seed_val,
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
 
+    # net + feature_utils (CLIP + Synchformer + VAE) are ~9.5GB combined and,
+    # unlike TARO/HunyuanFoley, were never explicitly freed here — do it before
+    # returning so the ZeroGPU worker isn't left holding peak memory longer
+    # than necessary.
+    del net, feature_utils
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
     return results
 
 
@@ -1343,6 +1379,7 @@ def _hunyuan_duration(video_file, prompt, negative_prompt, seed_val,
 
 
 @spaces.GPU(duration=_hunyuan_duration)
+@_catch_oom
 def _hunyuan_gpu_infer(video_file, prompt, negative_prompt, seed_val,
                        guidance_scale, num_steps, model_size, crossfade_s, crossfade_db,
                        num_samples, silent_video, segments_json, total_dur_s,
@@ -1558,6 +1595,7 @@ def _taro_regen_duration(video_file, seg_idx, seg_meta_json,
 
 
 @spaces.GPU(duration=_taro_regen_duration)
+@_catch_oom
 def _regen_taro_gpu(video_file, seg_idx, seg_meta_json,
                     seed_val, cfg_scale, num_steps, mode,
                     crossfade_s, crossfade_db, slot_id=None):
@@ -1634,6 +1672,7 @@ def _mmaudio_regen_duration(video_file, seg_idx, seg_meta_json,
 
 
 @spaces.GPU(duration=_mmaudio_regen_duration)
+@_catch_oom
 def _regen_mmaudio_gpu(video_file, seg_idx, seg_meta_json,
                        prompt, negative_prompt, seed_val,
                        cfg_strength, num_steps, crossfade_s, crossfade_db,
@@ -1679,6 +1718,14 @@ def _regen_mmaudio_gpu(video_file, seg_idx, seg_meta_json,
             cfg_strength=float(cfg_strength),
         )
     new_wav = audios.float().cpu()[0].numpy()  # full window — _stitch_wavs trims
+
+    # See _mmaudio_gpu_infer — net + feature_utils (~9.5GB) had no explicit
+    # cleanup here before returning to the ZeroGPU worker.
+    del net, feature_utils
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
     return new_wav, sr
 
 
@@ -1719,6 +1766,7 @@ def _hunyuan_regen_duration(video_file, seg_idx, seg_meta_json,
 
 
 @spaces.GPU(duration=_hunyuan_regen_duration)
+@_catch_oom
 def _regen_hunyuan_gpu(video_file, seg_idx, seg_meta_json,
                        prompt, negative_prompt, seed_val,
                        guidance_scale, num_steps, model_size,