Fix RuntimeError: pad onset_feats to truncate_onset=120 when video is shorter than 8.2s

app.py

@@ -3,14 +3,14 @@ import subprocess
 import sys
 
 try:
-    import mmcv
-    print("mmcv already installed")
+        import mmcv
+        print("mmcv already installed")
 except ImportError:
-    print("Installing mmcv with --no-build-isolation...")
-    subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", "mmcv>=2.0.0"])
-    print("mmcv installed successfully")
+        print("Installing mmcv with --no-build-isolation...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", "mmcv>=2.0.0"])
+        print("mmcv installed successfully")
 
-import torch
+    import torch
 import numpy as np
 import random
 import soundfile as sf
@@ -30,31 +30,28 @@ onset_ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="onset_model.ckpt",
 taro_ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="taro_ckpt.pt", cache_dir=CACHE_DIR)
 print("Checkpoints downloaded.")
 
-
 def set_global_seed(seed):
-    np.random.seed(seed % (2**32))
-    random.seed(seed)
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed(seed)
-    torch.backends.cudnn.deterministic = True
-
-
-def strip_audio_from_video(video_path, output_path):
-    """Strip any existing audio from a video file, outputting a silent video."""
-    (
-        ffmpeg
-        .input(video_path)
-        .output(output_path, vcodec="libx264", an=None)
-        .run(overwrite_output=True, quiet=True)
-    )
-
-
-@spaces.GPU(duration=300)
+        np.random.seed(seed % (2**32))
+        random.seed(seed)
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.backends.cudnn.deterministic = True
+
+    def strip_audio_from_video(video_path, output_path):
+            """Strip any existing audio from a video file, outputting a silent video."""""
+            (
+                        ffmpeg
+                        .input(video_path)
+                        .output(output_path, vcodec="libx264", an=None)
+                        .run(overwrite_output=True, quiet=True)
+            )
+
+        @spaces.GPU(duration=300)
 def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
-    seed_val = int(seed_val)
-    if seed_val < 0:
-        seed_val = random.randint(0, 2**32 - 1)
-    set_global_seed(seed_val)
+        seed_val = int(seed_val)
+        if seed_val < 0:
+                    seed_val = random.randint(0, 2**32 - 1)
+                set_global_seed(seed_val)
     torch.set_grad_enabled(False)
     device = "cuda" if torch.cuda.is_available() else "cpu"
     weight_dtype = torch.bfloat16
@@ -70,14 +67,13 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     state_dict = torch.load(onset_ckpt_path, map_location=device, weights_only=False)["state_dict"]
     new_state_dict = {}
     for key, value in state_dict.items():
-        if "model.net.model" in key:
-            new_key = key.replace("model.net.model", "net.model")
-        elif "model.fc." in key:
-            new_key = key.replace("model.fc", "fc")
-        else:
-            new_key = key
-        new_state_dict[new_key] = value
-
+                if "model.net.model" in key:
+                                new_key = key.replace("model.net.model", "net.model")
+                elif "model.fc." in key:
+                                new_key = key.replace("model.fc", "fc")
+                else:
+                                new_key = key
+                            new_state_dict[new_key] = value
     onset_model = VideoOnsetNet(False).to(device)
     onset_model.load_state_dict(new_state_dict)
     onset_model.eval()
@@ -94,7 +90,6 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     vocoder = model_audioldm.vocoder.to(device)
 
     tmp_dir = tempfile.mkdtemp()
-
     silent_video = os.path.join(tmp_dir, "silent_input.mp4")
     strip_audio_from_video(video_file, silent_video)
 
@@ -110,30 +105,43 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     latents_scale = torch.tensor([0.18215] * 8).view(1, 8, 1, 1).to(device)
 
     video_feats = torch.from_numpy(cavp_feats[:truncate_frame]).unsqueeze(0).to(device).to(weight_dtype)
-    onset_feats_t = torch.from_numpy(onset_feats[:truncate_onset]).unsqueeze(0).to(device).to(weight_dtype)
+
+    # Slice onset features and pad to truncate_onset if the video is shorter than expected
+    onset_feats_sliced = onset_feats[:truncate_onset]
+    actual_onset_len = onset_feats_sliced.shape[0]
+    if actual_onset_len < truncate_onset:
+                pad_len = truncate_onset - actual_onset_len
+                onset_feats_sliced = np.pad(
+                                onset_feats_sliced,
+                                ((0, pad_len),),
+                                mode="constant",
+                                constant_values=0,
+                )
+            onset_feats_t = torch.from_numpy(onset_feats_sliced).unsqueeze(0).to(device).to(weight_dtype)
 
     z = torch.randn(len(video_feats), model.in_channels, 204, 16, device=device).to(weight_dtype)
 
     sampling_kwargs = dict(
-        model=model,
-        latents=z,
-        y=onset_feats_t,
-        context=video_feats,
-        num_steps=int(num_steps),
-        heun=False,
-        cfg_scale=float(cfg_scale),
-        guidance_low=0.0,
-        guidance_high=0.7,
-        path_type="linear",
+                model=model,
+                latents=z,
+                y=onset_feats_t,
+                context=video_feats,
+                num_steps=int(num_steps),
+                heun=False,
+                cfg_scale=float(cfg_scale),
+                guidance_low=0.0,
+                guidance_high=0.7,
+                path_type="linear",
     )
 
     with torch.no_grad():
-        if mode == "sde":
-            samples = euler_maruyama_sampler(**sampling_kwargs)
-        else:
-            samples = euler_sampler(**sampling_kwargs)
+                if mode == "sde":
+                                samples = euler_maruyama_sampler(**sampling_kwargs)
+                else:
+                                samples = euler_sampler(**sampling_kwargs)
+
+                        samples = vae.decode(samples / latents_scale).sample
 
-    samples = vae.decode(samples / latents_scale).sample
     # Cast to float32 before vocoder (HiFi-GAN requires float32)
     wav_samples = vocoder(samples.squeeze().float()).detach().cpu().numpy()
 
@@ -141,45 +149,43 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     sf.write(audio_path, wav_samples, sr)
 
     duration = truncate / sr
-
     trimmed_video = os.path.join(tmp_dir, "trimmed.mp4")
     output_video = os.path.join(tmp_dir, "output.mp4")
+
     (
-        ffmpeg
-        .input(silent_video, ss=0, t=duration)
-        .output(trimmed_video, vcodec="libx264", an=None)
-        .run(overwrite_output=True, quiet=True)
+                ffmpeg
+                .input(silent_video, ss=0, t=duration)
+                .output(trimmed_video, vcodec="libx264", an=None)
+                .run(overwrite_output=True, quiet=True)
     )
 
     input_v = ffmpeg.input(trimmed_video)
     input_a = ffmpeg.input(audio_path)
     (
-        ffmpeg
-        .output(input_v, input_a, output_video, vcodec="libx264", acodec="aac", strict="experimental")
-        .run(overwrite_output=True, quiet=True)
+                ffmpeg
+                .output(input_v, input_a, output_video, vcodec="libx264", acodec="aac", strict="experimental")
+                .run(overwrite_output=True, quiet=True)
     )
 
     return output_video, audio_path
 
-
 def get_random_seed():
-    return random.randint(0, 2**32 - 1)
-
-
-demo = gr.Interface(
-    fn=generate_audio,
-    inputs=[
-        gr.Video(label="Input Video"),
-        gr.Number(label="Seed", value=get_random_seed, precision=0),
-        gr.Slider(label="CFG Scale", minimum=1, maximum=15, value=8, step=0.5),
-        gr.Slider(label="Sampling Steps", minimum=10, maximum=50, value=25, step=1),
-        gr.Radio(label="Sampling Mode", choices=["sde", "ode"], value="sde"),
-    ],
-    outputs=[
-        gr.Video(label="Output Video with Audio"),
-        gr.Audio(label="Generated Audio"),
-    ],
-    title="TARO: Video-to-Audio Synthesis (ICCV 2025)",
-    description="Upload a video and generate synchronized audio using TARO. Optimal duration is 8.2s.",
-)
+        return random.randint(0, 2**32 - 1)
+
+    demo = gr.Interface(
+            fn=generate_audio,
+            inputs=[
+                        gr.Video(label="Input Video"),
+                        gr.Number(label="Seed", value=get_random_seed, precision=0),
+                        gr.Slider(label="CFG Scale", minimum=1, maximum=15, value=8, step=0.5),
+                        gr.Slider(label="Sampling Steps", minimum=10, maximum=50, value=25, step=1),
+                        gr.Radio(label="Sampling Mode", choices=["sde", "ode"], value="sde"),
+            ],
+            outputs=[
+                        gr.Video(label="Output Video with Audio"),
+                        gr.Audio(label="Generated Audio"),
+            ],
+            title="TARO: Video-to-Audio Synthesis (ICCV 2025)",
+            description="Upload a video and generate synchronized audio using TARO. Optimal duration is 8.2s.",
+    )
 demo.queue().launch()