Fix vocoder float32 cast error; default seed is now random

app.py

@@ -3,14 +3,14 @@ import subprocess
 import sys
 
 try:
-    import mmcv
-    print("mmcv already installed")
+        import mmcv
+        print("mmcv already installed")
 except ImportError:
-    print("Installing mmcv with --no-build-isolation...")
-    subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", "mmcv>=2.0.0"])
-    print("mmcv installed successfully")
+        print("Installing mmcv with --no-build-isolation...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", "mmcv>=2.0.0"])
+        print("mmcv installed successfully")
 
-import torch
+    import torch
 import numpy as np
 import random
 import soundfile as sf
@@ -30,27 +30,32 @@ onset_ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="onset_model.ckpt",
 taro_ckpt_path = hf_hub_download(repo_id=REPO_ID, filename="taro_ckpt.pt", cache_dir=CACHE_DIR)
 print("Checkpoints downloaded.")
 
+
 def set_global_seed(seed):
-    np.random.seed(seed % (2**32))
-    random.seed(seed)
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed(seed)
-    torch.backends.cudnn.deterministic = True
-
-def strip_audio_from_video(video_path, output_path):
-    """Strip any existing audio from a video file, outputting a silent video."""
-    (
-        ffmpeg
-        .input(video_path)
-        .output(output_path, vcodec="libx264", an=None)
-        .run(overwrite_output=True, quiet=True)
-    )
+        np.random.seed(seed % (2**32))
+        random.seed(seed)
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.backends.cudnn.deterministic = True
+
+
+    def strip_audio_from_video(video_path, output_path):
+            """Strip any existing audio from a video file, outputting a silent video."""""
+            (
+                        ffmpeg
+                        .input(video_path)
+                        .output(output_path, vcodec="libx264", an=None)
+                        .run(overwrite_output=True, quiet=True)
+            )
 
-@spaces.GPU(duration=300)
+
+        @spaces.GPU(duration=300)
 def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
-    set_global_seed(int(seed_val))
+        seed_val = int(seed_val)
+        if seed_val < 0:
+                    seed_val = random.randint(0, 2**32 - 1)
+                set_global_seed(seed_val)
     torch.set_grad_enabled(False)
-
     device = "cuda" if torch.cuda.is_available() else "cpu"
     weight_dtype = torch.bfloat16
 
@@ -65,13 +70,13 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     state_dict = torch.load(onset_ckpt_path, map_location=device, weights_only=False)["state_dict"]
     new_state_dict = {}
     for key, value in state_dict.items():
-        if "model.net.model" in key:
-            new_key = key.replace("model.net.model", "net.model")
-        elif "model.fc." in key:
-            new_key = key.replace("model.fc", "fc")
-        else:
-            new_key = key
-        new_state_dict[new_key] = value
+                if "model.net.model" in key:
+                                new_key = key.replace("model.net.model", "net.model")
+                elif "model.fc." in key:
+                                new_key = key.replace("model.fc", "fc")
+                else:
+                                new_key = key
+                            new_state_dict[new_key] = value
 
     onset_model = VideoOnsetNet(False).to(device)
     onset_model.load_state_dict(new_state_dict)
@@ -111,26 +116,27 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     z = torch.randn(len(video_feats), model.in_channels, 204, 16, device=device).to(weight_dtype)
 
     sampling_kwargs = dict(
-        model=model,
-        latents=z,
-        y=onset_feats_t,
-        context=video_feats,
-        num_steps=int(num_steps),
-        heun=False,
-        cfg_scale=float(cfg_scale),
-        guidance_low=0.0,
-        guidance_high=0.7,
-        path_type="linear"
+                model=model,
+                latents=z,
+                y=onset_feats_t,
+                context=video_feats,
+                num_steps=int(num_steps),
+                heun=False,
+                cfg_scale=float(cfg_scale),
+                guidance_low=0.0,
+                guidance_high=0.7,
+                path_type="linear"
     )
 
     with torch.no_grad():
-        if mode == "sde":
-            samples = euler_maruyama_sampler(**sampling_kwargs)
-        else:
-            samples = euler_sampler(**sampling_kwargs)
+                if mode == "sde":
+                                samples = euler_maruyama_sampler(**sampling_kwargs)
+                else:
+                                samples = euler_sampler(**sampling_kwargs)
 
-    samples = vae.decode(samples / latents_scale).sample
-    wav_samples = vocoder(samples.squeeze()).detach().cpu().numpy()
+                        samples = vae.decode(samples / latents_scale).sample
+    # Cast to float32 before vocoder (HiFi-GAN requires float32)
+    wav_samples = vocoder(samples.squeeze().float()).detach().cpu().numpy()
 
     audio_path = os.path.join(tmp_dir, "output.wav")
     sf.write(audio_path, wav_samples, sr)
@@ -140,40 +146,43 @@ def generate_audio(video_file, seed_val, cfg_scale, num_steps, mode):
     # Trim the silent input video to the target duration (no audio)
     trimmed_video = os.path.join(tmp_dir, "trimmed.mp4")
     output_video = os.path.join(tmp_dir, "output.mp4")
-
     (
-        ffmpeg
-        .input(silent_video, ss=0, t=duration)
-        .output(trimmed_video, vcodec="libx264", an=None)
-        .run(overwrite_output=True, quiet=True)
+                ffmpeg
+                .input(silent_video, ss=0, t=duration)
+                .output(trimmed_video, vcodec="libx264", an=None)
+                .run(overwrite_output=True, quiet=True)
     )
 
     # Combine the trimmed silent video with the generated audio
     input_v = ffmpeg.input(trimmed_video)
     input_a = ffmpeg.input(audio_path)
     (
-        ffmpeg
-        .output(input_v, input_a, output_video, vcodec="libx264", acodec="aac", strict="experimental")
-        .run(overwrite_output=True, quiet=True)
+                ffmpeg
+                .output(input_v, input_a, output_video, vcodec="libx264", acodec="aac", strict="experimental")
+                .run(overwrite_output=True, quiet=True)
     )
 
     return output_video, audio_path
 
 
-demo = gr.Interface(
-    fn=generate_audio,
-    inputs=[
-        gr.Video(label="Input Video"),
-        gr.Number(label="Seed", value=0, precision=0),
-        gr.Slider(label="CFG Scale", minimum=1, maximum=15, value=8, step=0.5),
-        gr.Slider(label="Sampling Steps", minimum=10, maximum=50, value=25, step=1),
-        gr.Radio(label="Sampling Mode", choices=["sde", "ode"], value="sde")
-    ],
-    outputs=[
-        gr.Video(label="Output Video with Audio"),
-        gr.Audio(label="Generated Audio")
-    ],
-    title="TARO: Video-to-Audio Synthesis (ICCV 2025)",
-    description="Upload a video and generate synchronized audio using TARO. Optimal duration is 8.2s."
-)
-demo.queue().launch()
+def get_random_seed():
+        return random.randint(0, 2**32 - 1)
+
+
+    demo = gr.Interface(
+            fn=generate_audio,
+            inputs=[
+                        gr.Video(label="Input Video"),
+                        gr.Number(label="Seed", value=get_random_seed, precision=0),
+                        gr.Slider(label="CFG Scale", minimum=1, maximum=15, value=8, step=0.5),
+                        gr.Slider(label="Sampling Steps", minimum=10, maximum=50, value=25, step=1),
+                        gr.Radio(label="Sampling Mode", choices=["sde", "ode"], value="sde")
+            ],
+            outputs=[
+                        gr.Video(label="Output Video with Audio"),
+                        gr.Audio(label="Generated Audio")
+            ],
+            title="TARO: Video-to-Audio Synthesis (ICCV 2025)",
+            description="Upload a video and generate synchronized audio using TARO. Optimal duration is 8.2s."
+    )
+demo.queue().launch()