Update app.py

app.py

@@ -1,17 +1,31 @@
 import gradio as gr
-
 import torch
 
-from spectro import wav_bytes_from_spectrogram_image
+from scipy.io import wavfile
+import numpy as np
+from PIL import Image
+
+from spectro import wav_bytes_from_spectrogram_image, spectrogram_from_waveform, image_from_spectrogram
+
 from diffusers import StableDiffusionPipeline
+from diffusers import StableDiffusionImg2ImgPipeline
 
 from share_btn import community_icon_html, loading_icon_html, share_js
 
-model_id = "riffusion/riffusion-model-v1"
-pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
+MODEL_ID = "riffusion/riffusion-model-v1"
+pipe = StableDiffusionPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16)
 pipe = pipe.to("cuda")
+pipe2 = StableDiffusionImg2ImgPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16)
+pipe2 = pipe2.to("cuda")
+
+def predict(prompt, audio_input, duration):
+    if audio_input == None:
+        return classic(prompt, duration)
+    else:
+        return audio_transfer(prompt, audio_input)
+    
 
-def predict(prompt, duration):
+def classic(prompt, duration):
     if duration == 5:
         width_duration=512
     else :
@@ -23,6 +37,42 @@ def predict(prompt, duration):
         f.write(wav[0].getbuffer())
     return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
 
+def audio_transfer(prompt, audio):
+    # read uploaded file to wav
+    rate, data = wavfile.read(audio)
+    
+    # convert to mono
+    data = np.mean(data, axis=1)
+    
+    # convert to float32
+    data = data.astype(np.float32)
+    
+    # take a random 7 second slice of the audio
+    data = data[rate*7:rate*14]
+    
+    spectrogram = spectrogram_from_waveform(
+        waveform=data,
+        sample_rate=rate,
+        # width=768,
+        n_fft=8192,
+        hop_length=512,
+        win_length=8192,
+    )
+
+    spec = image_from_spectrogram(spectrogram)
+
+    images = pipe2(
+        prompt=prompt,
+        image=spec,
+        strength=0.5,
+        guidance_scale=7
+    ).images
+
+    wav = wav_bytes_from_spectrogram_image(images[0])
+    with open("output.wav", "wb") as f:
+        f.write(wav[0].getbuffer())
+    return images[0], 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+
 title = """
     <div style="text-align: center; max-width: 500px; margin: 0 auto;">
         <div
@@ -139,6 +189,7 @@ with gr.Blocks(css=css) as demo:
         gr.HTML(title)
         
         prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")
+        audio_input = gr.Audio(label="audio input", type="filepath", source="upload")
         duration_input = gr.Slider(label="Duration in seconds", minimum=5, maximum=10, step=1, value=8, elem_id="duration-slider")
         send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn")
             
@@ -154,7 +205,7 @@ with gr.Blocks(css=css) as demo:
         
         gr.HTML(article)
     
-    send_btn.click(predict, inputs=[prompt_input, duration_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon])
+    send_btn.click(predict, inputs=[prompt_input, audio_input, duration_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon])
     share_button.click(None, [], [], _js=share_js)
 
 demo.queue(max_size=250).launch(debug=True)