Upload 10 files

.gitattributes

@@ -1,35 +1,39 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/chinese-new-year-dragon.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/photoreal-train.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/train-window.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/big-sur.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,13 @@
----
-title: Video2sfx
-emoji: 👀
-colorFrom: gray
-colorTo: purple
-sdk: gradio
-sdk_version: 4.36.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Video SoundFX
+emoji: 👂🎞️
+colorFrom: blue
+colorTo: pink
+sdk: gradio
+sdk_version: 4.36.0
+app_file: app.py
+pinned: true
+short_description: Generates a sound effect that matches video shot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,279 @@
+import gradio as gr
+from gradio_client import Client
+import os
+import json
+import re
+from moviepy.editor import *
+import cv2
+
+hf_token = os.environ.get("HF_TKN")
+
+def extract_firstframe(video_in):
+    vidcap = cv2.VideoCapture(video_in)
+    success,image = vidcap.read()
+    count = 0
+    while success:
+        if count == 0:
+            cv2.imwrite("first_frame.jpg", image)     # save first extracted frame as jpg file named first_frame.jpg
+        else:
+            break   # exit loop after saving first frame
+        success,image = vidcap.read()
+        print ('Read a new frame: ', success)
+        count += 1
+    print ("Done extracted first frame!")
+    return "first_frame.jpg"
+
+def extract_audio(video_in):
+    input_video = video_in
+    output_audio = 'audio.wav'
+    
+    # Open the video file and extract the audio
+    video_clip = VideoFileClip(input_video)
+    audio_clip = video_clip.audio
+    
+    # Save the audio as a .wav file
+    audio_clip.write_audiofile(output_audio, fps=44100)  # Use 44100 Hz as the sample rate for .wav files  
+    print("Audio extraction complete.")
+
+    return 'audio.wav'
+
+def get_caption_from_kosmos(image_in):
+    kosmos2_client = Client("https://ydshieh-kosmos-2.hf.space/")
+
+    kosmos2_result = kosmos2_client.predict(
+        image_in,	# str (filepath or URL to image) in 'Test Image' Image component
+        "Detailed",	# str in 'Description Type' Radio component
+        fn_index=4
+    )
+
+    print(f"KOSMOS2 RETURNS: {kosmos2_result}")
+
+    with open(kosmos2_result[1], 'r') as f:
+        data = json.load(f)
+    
+    reconstructed_sentence = []
+    for sublist in data:
+        reconstructed_sentence.append(sublist[0])
+
+    full_sentence = ' '.join(reconstructed_sentence)
+    #print(full_sentence)
+
+    # Find the pattern matching the expected format ("Describe this image in detail:" followed by optional space and then the rest)...
+    pattern = r'^Describe this image in detail:\s*(.*)$'
+    # Apply the regex pattern to extract the description text.
+    match = re.search(pattern, full_sentence)
+    if match:
+        description = match.group(1)
+        print(description)
+    else:
+        print("Unable to locate valid description.")
+
+    # Find the last occurrence of "."
+    last_period_index = description.rfind('.')
+
+    # Truncate the string up to the last period
+    truncated_caption = description[:last_period_index + 1]
+
+    # print(truncated_caption)
+    print(f"\n—\nIMAGE CAPTION: {truncated_caption}")
+    
+    return truncated_caption
+
+def get_caption(image_in):
+    client = Client("https://fffiloni-moondream1.hf.space/", hf_token=hf_token)
+    result = client.predict(
+		image_in,	# filepath  in 'image' Image component
+		"Describe precisely the image in one sentence.",	# str  in 'Question' Textbox component
+		#api_name="/answer_question"
+        api_name="/predict"
+    )
+    print(result)
+    return result
+
+def get_magnet(prompt):
+    amended_prompt = f"{prompt}"
+    print(amended_prompt)
+    client = Client("https://fffiloni-magnet.hf.space/")
+    result = client.predict(
+        "facebook/audio-magnet-medium",	# Literal['facebook/magnet-small-10secs', 'facebook/magnet-medium-10secs', 'facebook/magnet-small-30secs', 'facebook/magnet-medium-30secs', 'facebook/audio-magnet-small', 'facebook/audio-magnet-medium']  in 'Model' Radio component
+        "",	# str  in 'Model Path (custom models)' Textbox component
+        amended_prompt,	# str  in 'Input Text' Textbox component
+        3,	# float  in 'Temperature' Number component
+        0.9,	# float  in 'Top-p' Number component
+        10,	# float  in 'Max CFG coefficient' Number component
+        1,	# float  in 'Min CFG coefficient' Number component
+        20,	# float  in 'Decoding Steps (stage 1)' Number component
+        10,	# float  in 'Decoding Steps (stage 2)' Number component
+        10,	# float  in 'Decoding Steps (stage 3)' Number component
+        10,	# float  in 'Decoding Steps (stage 4)' Number component
+        "prod-stride1 (new!)",	# Literal['max-nonoverlap', 'prod-stride1 (new!)']  in 'Span Scoring' Radio component
+        api_name="/predict_full"
+    )
+    print(result)
+    return result[1]
+
+def get_audioldm(prompt):
+    client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
+    result = client.predict(
+        prompt,	# str in 'Input text' Textbox component
+        "Low quality. Music.",	# str in 'Negative prompt' Textbox component
+        10,	# int | float (numeric value between 5 and 15) in 'Duration (seconds)' Slider component
+        3.5,	# int | float (numeric value between 0 and 7) in 'Guidance scale' Slider component
+        45,	# int | float in 'Seed' Number component
+        3,	# int | float (numeric value between 1 and 5) in 'Number waveforms to generate' Slider component
+        fn_index=1
+    )
+    print(result)
+    audio_result = extract_audio(result)
+    return audio_result
+
+def get_audiogen(prompt):
+    client = Client("https://fffiloni-audiogen.hf.space/")
+    result = client.predict(
+        prompt,
+        10,
+        api_name="/infer"
+    )
+    return result
+
+def get_tango(prompt):
+    try:
+        #client = Client("https://declare-lab-tango.hf.space/")
+        client = Client("https://fffiloni-tango.hf.space/", hf_token=hf_token)
+    except:
+        raise gr.Error("Tango space API is not ready, please try again in few minutes ")
+    
+    result = client.predict(
+				prompt,	# str representing string value in 'Prompt' Textbox component
+				100,	# int | float representing numeric value between 100 and 200 in 'Steps' Slider component
+				4,	# int | float representing numeric value between 1 and 10 in 'Guidance Scale' Slider component
+				api_name="/predict"
+    )
+    print(result)
+    return result
+
+def blend_vsfx(video_in, audio_result):
+    audioClip = AudioFileClip(audio_result)
+    print(f"AUD: {audioClip.duration}")
+    clip = VideoFileClip(video_in)
+    print(f"VID: {clip.duration}")
+    if clip.duration < audioClip.duration :
+        audioClip = audioClip.subclip((0.0), (clip.duration))
+    elif clip.duration > audioClip.duration :
+        clip = clip.subclip((0.0), (audioClip.duration))
+    final_clip = clip.set_audio(audioClip)
+    # Set the output codec
+    codec = 'libx264'
+    audio_codec = 'aac'
+    final_clip.write_videofile('final_video_with_sound.mp4', codec=codec, audio_codec=audio_codec)
+    return "final_video_with_sound.mp4"
+
+def infer(video_in, chosen_model):
+    image_in = extract_firstframe(video_in)
+    caption = get_caption(image_in)
+    if chosen_model == "MAGNet" :
+        audio_result = get_magnet(caption)
+    elif chosen_model == "AudioLDM-2" : 
+        audio_result = get_audioldm(caption)
+    elif chosen_model == "AudioGen" :
+        audio_result = get_audiogen(caption)
+    elif chosen_model == "Tango" :
+        audio_result = get_tango(caption)
+    
+    final_res = blend_vsfx(video_in, audio_result)
+    return gr.update(value=caption, interactive=True), gr.update(interactive=True), audio_result, final_res
+
+def retry(edited_prompt, video_in, chosen_model):
+    image_in = extract_firstframe(video_in)
+    caption = edited_prompt
+    if chosen_model == "MAGNet" :
+        audio_result = get_magnet(caption)
+    elif chosen_model == "AudioLDM-2" : 
+        audio_result = get_audioldm(caption)
+    elif chosen_model == "AudioGen" :
+        audio_result = get_audiogen(caption)
+    elif chosen_model == "Tango" :
+        audio_result = get_tango(caption)
+    
+    final_res = blend_vsfx(video_in, audio_result)
+    return audio_result, final_res
+
+def refresh():
+    return gr.update(value=None, interactive=False), gr.update(interactive=False), gr.update(value=None), gr.update(value=None)
+
+css="""
+#col-container{
+    margin: 0 auto;
+    max-width: 800px;
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML("""
+        <h2 style="text-align: center;">
+            Video to SoundFX
+        </h2>
+        <p style="text-align: center;">
+            Get sound effects from a video shot while comparing audio models from image caption.
+        </p>
+        """)
+
+        with gr.Row():
+        
+            with gr.Column():
+                video_in = gr.Video(sources=["upload"], label="Video input")
+                with gr.Row():
+                    chosen_model = gr.Dropdown(label="Choose a model", choices=["MAGNet", "AudioLDM-2", "AudioGen", "Tango"], value="Tango")
+                    submit_btn = gr.Button("Submit", scale=0)
+            with gr.Column():
+                caption_o = gr.Textbox(label="Scene caption", interactive=False)
+                retry_btn = gr.Button("Retry with edited scene caption", interactive=False)
+                audio_o = gr.Audio(label="Audio output")
+        with gr.Column():
+            video_o = gr.Video(label="Video with soundFX")
+
+        gr.Examples(
+            examples = [
+                ["examples/photoreal-train.mp4", "Tango"], 
+                ["examples/train-window.mp4", "Tango"], 
+                ["examples/chinese-new-year-dragon.mp4", "Tango"], 
+                ["examples/big-sur.mp4", "AudioLDM-2"]
+            ],
+            fn=infer,
+            inputs = [video_in, chosen_model],
+            outputs= [caption_o, retry_btn, audio_o, video_o],
+            cache_examples=False
+        )
+
+    video_in.change(
+        fn = refresh,
+        inputs = None,
+        outputs = [caption_o, retry_btn, audio_o, video_o],
+        queue = False,
+        show_progress = False
+    )
+
+    video_in.clear(
+        fn = refresh,
+        inputs = None,
+        outputs = [caption_o, retry_btn, audio_o, video_o],
+        queue = False,
+        show_progress = False
+    )
+    
+    submit_btn.click(
+        fn=infer,
+        inputs=[video_in, chosen_model],
+        outputs=[caption_o, retry_btn, audio_o, video_o],
+        concurrency_limit = 2
+    )
+
+    retry_btn.click(
+        fn=retry,
+        inputs=[caption_o, video_in, chosen_model],
+        outputs=[audio_o, video_o],
+        concurrency_limit = 2
+    )
+
+demo.queue(max_size=10).launch(show_api=False, debug=True, show_error=True)

examples/big-sur.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a088ededf76628fc89e5c738e97ecbc50dc78832fffe3e675e5a937331d1e20e
+size 17049093

examples/chinese-new-year-dragon.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d686ae1b00e6deddbaa637c37d36c21571f9f0e6f6f38cd452ed3470e491a56
+size 15220841

examples/photoreal-train.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a468dbb3d950fcc280b03223169d5c4b76425c9a226fa42b73a055694e156434
+size 5889386

examples/train-window.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82addfe05e21760e1f76efae4345d7d4302d1a8b873dc746f885e139dd789ccb
+size 18462003

requirements.txt

@@ -0,0 +1,2 @@
+moviepy
+opencv-python