Recalibrate GPU duration reservations, drop deprecated warnings
Browse filesLive logs showed all three models finishing (warm) in 18-39s while
every call reserved a blanket 120s of ZeroGPU quota — the hard-coded
120s floor plus generous load_overhead guesses were never tuned
against real GPU timings. ZeroGPU's admission check gates on the
requested duration against remaining quota, so over-reserving causes
premature "quota exceeded" errors well before quota is actually
exhausted.
- Lower the global floor from 120s to 45s.
- Recalibrate each model's load_overhead to ~1.5-2x its measured warm
end-to-end time (TARO 120->35, MMAudio 30->35, HunyuanFoley 90->55,
with extra margin kept for HunyuanFoley's larger cold-disk-read risk).
- Drop the deprecated local_dir_use_symlinks kwarg from hf_hub_download
calls.
- Silence the harmless weight_norm FutureWarning from vendored
HunyuanVideo-Foley DAC VAE loading code.
|
@@ -20,9 +20,18 @@ import tempfile
|
|
| 20 |
import random
|
| 21 |
import threading
|
| 22 |
import time
|
|
|
|
| 23 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 24 |
from pathlib import Path
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
import torch
|
| 27 |
import numpy as np
|
| 28 |
import torchaudio
|
|
@@ -121,22 +130,22 @@ def _dl_taro():
|
|
| 121 |
def _dl_mmaudio():
|
| 122 |
"""Download MMAudio .pth files and return their local paths."""
|
| 123 |
m = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/mmaudio_large_44k_v2.pth",
|
| 124 |
-
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_WEIGHTS_DIR)
|
| 125 |
v = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/v1-44.pth",
|
| 126 |
-
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR)
|
| 127 |
s = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/synchformer_state_dict.pth",
|
| 128 |
-
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR)
|
| 129 |
print("MMAudio checkpoints downloaded.")
|
| 130 |
return m, v, s
|
| 131 |
|
| 132 |
def _dl_hunyuan():
|
| 133 |
"""Download HunyuanVideoFoley .pth files."""
|
| 134 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/hunyuanvideo_foley.pth",
|
| 135 |
-
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR)
|
| 136 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/vae_128d_48k.pth",
|
| 137 |
-
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR)
|
| 138 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/synchformer_state_dict.pth",
|
| 139 |
-
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR)
|
| 140 |
print("HunyuanVideoFoley checkpoints downloaded.")
|
| 141 |
|
| 142 |
def _populate_hf_cache_from_mirror(orig_repo_id, mirror_subpath):
|
|
@@ -743,7 +752,8 @@ MODEL_CONFIGS = {
|
|
| 743 |
"window_s": TARO_MODEL_DUR, # 8.192 s
|
| 744 |
"sr": TARO_SR, # 16000 (output resampled to TARGET_SR)
|
| 745 |
"secs_per_step": 0.025, # measured 0.023 s/step on H200
|
| 746 |
-
"load_overhead":
|
|
|
|
| 747 |
"tab_prefix": "taro",
|
| 748 |
"label": "TARO",
|
| 749 |
"regen_fn": None, # set after function definitions (avoids forward-ref)
|
|
@@ -752,7 +762,8 @@ MODEL_CONFIGS = {
|
|
| 752 |
"window_s": 8.0, # MMAudio's fixed generation window
|
| 753 |
"sr": 48000, # resampled from 44100 in post-processing
|
| 754 |
"secs_per_step": 0.25, # measured 0.230 s/step on H200
|
| 755 |
-
"load_overhead":
|
|
|
|
| 756 |
"tab_prefix": "mma",
|
| 757 |
"label": "MMAudio",
|
| 758 |
"regen_fn": None,
|
|
@@ -761,7 +772,10 @@ MODEL_CONFIGS = {
|
|
| 761 |
"window_s": 15.0, # HunyuanFoley max video duration
|
| 762 |
"sr": 48000,
|
| 763 |
"secs_per_step": 0.50, # wall-time ~0.607s/step (incl. VAE decode + feature extraction)
|
| 764 |
-
"load_overhead":
|
|
|
|
|
|
|
|
|
|
| 765 |
"tab_prefix": "hf",
|
| 766 |
"label": "HunyuanFoley",
|
| 767 |
"regen_fn": None,
|
|
@@ -813,10 +827,12 @@ def _catch_oom(fn):
|
|
| 813 |
|
| 814 |
|
| 815 |
def _clamp_duration(secs: float, label: str) -> int:
|
| 816 |
-
"""Clamp a raw GPU-seconds estimate to [
|
| 817 |
-
ZeroGPU Pro users get up to 300 s per call;
|
| 818 |
-
|
| 819 |
-
|
|
|
|
|
|
|
| 820 |
print(f"[duration] {label}: {secs:.0f}s raw → {result}s reserved")
|
| 821 |
return result
|
| 822 |
|
|
|
|
| 20 |
import random
|
| 21 |
import threading
|
| 22 |
import time
|
| 23 |
+
import warnings
|
| 24 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 25 |
from pathlib import Path
|
| 26 |
|
| 27 |
+
# HunyuanVideo-Foley's DAC VAE loads its weights via the pre-parametrization
|
| 28 |
+
# torch.nn.utils.weight_norm API — harmless, but noisy on every model load.
|
| 29 |
+
warnings.filterwarnings(
|
| 30 |
+
"ignore",
|
| 31 |
+
message="`torch.nn.utils.weight_norm` is deprecated",
|
| 32 |
+
category=FutureWarning,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
import torch
|
| 36 |
import numpy as np
|
| 37 |
import torchaudio
|
|
|
|
| 130 |
def _dl_mmaudio():
|
| 131 |
"""Download MMAudio .pth files and return their local paths."""
|
| 132 |
m = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/mmaudio_large_44k_v2.pth",
|
| 133 |
+
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_WEIGHTS_DIR))
|
| 134 |
v = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/v1-44.pth",
|
| 135 |
+
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR))
|
| 136 |
s = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/synchformer_state_dict.pth",
|
| 137 |
+
cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR))
|
| 138 |
print("MMAudio checkpoints downloaded.")
|
| 139 |
return m, v, s
|
| 140 |
|
| 141 |
def _dl_hunyuan():
|
| 142 |
"""Download HunyuanVideoFoley .pth files."""
|
| 143 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/hunyuanvideo_foley.pth",
|
| 144 |
+
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
|
| 145 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/vae_128d_48k.pth",
|
| 146 |
+
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
|
| 147 |
hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/synchformer_state_dict.pth",
|
| 148 |
+
cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
|
| 149 |
print("HunyuanVideoFoley checkpoints downloaded.")
|
| 150 |
|
| 151 |
def _populate_hf_cache_from_mirror(orig_repo_id, mirror_subpath):
|
|
|
|
| 752 |
"window_s": TARO_MODEL_DUR, # 8.192 s
|
| 753 |
"sr": TARO_SR, # 16000 (output resampled to TARGET_SR)
|
| 754 |
"secs_per_step": 0.025, # measured 0.023 s/step on H200
|
| 755 |
+
"load_overhead": 35, # CAVP+onset+MMDiT+VAE+vocoder load + feature extraction
|
| 756 |
+
# (measured ~18s warm end-to-end on RTX Pro 6000 Blackwell; ~2x margin)
|
| 757 |
"tab_prefix": "taro",
|
| 758 |
"label": "TARO",
|
| 759 |
"regen_fn": None, # set after function definitions (avoids forward-ref)
|
|
|
|
| 762 |
"window_s": 8.0, # MMAudio's fixed generation window
|
| 763 |
"sr": 48000, # resampled from 44100 in post-processing
|
| 764 |
"secs_per_step": 0.25, # measured 0.230 s/step on H200
|
| 765 |
+
"load_overhead": 35, # 15s warm + 15s model init
|
| 766 |
+
# (measured ~23s warm end-to-end on RTX Pro 6000 Blackwell; ~1.5x margin)
|
| 767 |
"tab_prefix": "mma",
|
| 768 |
"label": "MMAudio",
|
| 769 |
"regen_fn": None,
|
|
|
|
| 772 |
"window_s": 15.0, # HunyuanFoley max video duration
|
| 773 |
"sr": 48000,
|
| 774 |
"secs_per_step": 0.50, # wall-time ~0.607s/step (incl. VAE decode + feature extraction)
|
| 775 |
+
"load_overhead": 55, # SigLIP2 + Synchformer + CLAP + main model + DAC VAE load
|
| 776 |
+
# (measured ~39s warm end-to-end on RTX Pro 6000 Blackwell; extra
|
| 777 |
+
# margin kept vs. TARO/MMAudio for first-load cold-disk-cache risk
|
| 778 |
+
# on the ~10 GB XXL checkpoint)
|
| 779 |
"tab_prefix": "hf",
|
| 780 |
"label": "HunyuanFoley",
|
| 781 |
"regen_fn": None,
|
|
|
|
| 827 |
|
| 828 |
|
| 829 |
def _clamp_duration(secs: float, label: str) -> int:
|
| 830 |
+
"""Clamp a raw GPU-seconds estimate to [45, GPU_DURATION_CAP] and log it.
|
| 831 |
+
ZeroGPU Pro users get up to 300 s per call; 45 s floor covers per-call
|
| 832 |
+
scheduling/queue overhead without over-reserving quota on every warm
|
| 833 |
+
call (each model's own load_overhead in MODEL_CONFIGS covers its
|
| 834 |
+
model-load cost above this floor)."""
|
| 835 |
+
result = min(GPU_DURATION_CAP, max(45, int(secs)))
|
| 836 |
print(f"[duration] {label}: {secs:.0f}s raw → {result}s reserved")
|
| 837 |
return result
|
| 838 |
|