vidmuse

README.md

@@ -1,3 +1,66 @@
 ---
 license: cc-by-4.0
 ---
+
+# VidMuse
+
+VidMuse is a framework designed for generating high-fidelity music that is acoustically and semantically aligned with video content.
+
+## Usage
+
+1. First install the [`VidMuse` library](git+https://github.com/ZeyueT/VidMuse)
+```
+pip install git+https://github.com/ZeyueT/VidMuse.git
+```
+
+2. Install ffmpeg:
+Install ffmpeg:
+```bash
+sudo apt-get install ffmpeg
+# Or if you are using Anaconda or Miniconda
+conda install "ffmpeg<5" -c conda-forge
+```
+
+
+3. Run the following Python code:
+
+
+```py
+from video_processor import VideoProcessor, merge_video_audio
+from audiocraft.models import VidMuse
+import scipy
+
+# Path to the video
+video_path = '/data/zeyuet/project/VidMuse/VidMuse-hf/dataset/example/infer/sample.mp4'
+# Initialize the video processor
+processor = VideoProcessor()
+# Process the video to obtain tensors and duration
+local_video_tensor, global_video_tensor, duration = processor.process(video_path)
+
+progress = True
+USE_DIFFUSION = False
+
+# Load the pre-trained VidMuse model
+MODEL = VidMuse.get_pretrained('Zeyue7/VidMuse')
+# Set generation parameters for the model based on video duration
+MODEL.set_generation_params(duration=duration)
+
+try:
+    # Generate outputs using the model
+    outputs = MODEL.generate([local_video_tensor, global_video_tensor], progress=progress, return_tokens=USE_DIFFUSION)
+except RuntimeError as e:
+    print(e)
+
+# Detach outputs from the computation graph and convert to CPU float tensor
+outputs = outputs.detach().cpu().float()
+
+
+sampling_rate = 32000
+output_wav_path = "vidmuse_out.wav"
+# Write the output audio data to a WAV file
+scipy.io.wavfile.write(output_wav_path, rate=sampling_rate, data=outputs[0, 0].numpy())
+
+output_video_path = "output_video.mp4"
+# Merge the original video with the generated music
+merge_video_audio(video_path, output_wav_path, output_video_path)
+```

video_processor.py

@@ -1,4 +1,4 @@
-from moviepy.editor import VideoFileClip
+from moviepy.editor import VideoFileClip, AudioFileClip
 import torch
 from decord import VideoReader, cpu
 import math
@@ -31,7 +31,6 @@ class VideoProcessor:
             video_tensor = torch.cat((video_tensor, last_frame.repeat(1, repeat_times, 1, 1)), dim=1)
         return video_tensor
 
-
     def video_read_global(self, filepath, seek_time=0., duration=-1, target_fps=2, global_mode='average', global_num_frames=32):
         vr = VideoReader(filepath, ctx=cpu(0))
         fps = vr.get_avg_fps()
@@ -67,13 +66,16 @@ class VideoProcessor:
         assert global_video_tensor.shape[1] == global_num_frames, f"the shape of global_video_tensor is {global_video_tensor.shape}"
         return local_video_tensor, global_video_tensor
 
-
-
     def process(self, video_path, target_fps=2, global_mode='average', global_num_frames=32):
         duration = self.get_video_duration(video_path)
         if duration is None:
             raise ValueError("Invalid video path or video file.")
-        
         local_video_tensor, global_video_tensor = self.video_read_global(video_path, duration=duration, target_fps=target_fps, global_mode=global_mode, global_num_frames=global_num_frames)
-        
         return local_video_tensor, global_video_tensor, duration
+
+
+def merge_video_audio(video_path, audio_path, output_path):
+    video = VideoFileClip(video_path).without_audio()
+    audio = AudioFileClip(audio_path)
+    final_video = video.set_audio(audio)
+    final_video.write_videofile(output_path, codec='libx264', audio_codec='aac')