Expose AudioToken/Syllable ratio as param to allow manual speed and audio length control

app.py

@@ -27,11 +27,50 @@ def run_asr(audio):
 
     return transcript.text, transcript.text, word_times
 
-def run_inpainter(input_text, output_text, word_times, audio, num_steps, init_temp, init_diversity, guidance, rescale, topk):
-    return inpainter.inpaint(InpaintInput(input_text=input_text, output_text=output_text, input_word_times=word_times, audio=audio, num_steps=num_steps, init_temp=init_temp, init_diversity=init_diversity, guidance=guidance, rescale=rescale, topk=topk))
+def run_inpainter(input_text, output_text, word_times, audio, num_steps, init_temp, init_diversity, guidance, rescale, topk, use_manual_ratio, audio_token_syllable_ratio):
+    if not use_manual_ratio:
+        audio_token_syllable_ratio = None
+    return inpainter.inpaint(InpaintInput(input_text=input_text, output_text=output_text, input_word_times=word_times, audio=audio, num_steps=num_steps,
+                                          init_temp=init_temp, init_diversity=init_diversity, guidance=guidance, rescale=rescale, topk=topk,
+                                          audio_token_syllable_ratio=audio_token_syllable_ratio))
+
+def run_inpainter_tts(input_text, voice_audio, num_steps, init_temp, init_diversity, guidance, rescale, topk, use_manual_ratio, audio_token_syllable_ratio):
+    if not use_manual_ratio:
+        audio_token_syllable_ratio = None
+    return inpainter.tts(TTSInput(output_text=input_text, voice=voice_audio, num_steps=num_steps, init_temp=init_temp,
+                                  init_diversity=init_diversity, guidance=guidance, rescale=rescale, topk=topk,
+                                  audio_token_syllable_ratio=audio_token_syllable_ratio))
+
+def toggle_ratio_input(use_manual):
+    return gr.update(visible=use_manual, interactive=use_manual)
+
+def create_advanced_options_accordion():
+    with gr.Accordion("Advanced options", open=False):
+        num_steps_slider = gr.Slider(1, 100, 30, step=1, label="number of sampling steps codebook")
+        init_temp_slider = gr.Slider(0.5, 10, 1, step=0.1, label="Initial temperature")
+        init_diversity_slider = gr.Slider(0, 10, 1, step=0.1, label="Initial diversity")
+        guidance_slider = gr.Slider(0, 10, 0.5, step=0.1, label="guidance")
+        rescale_slider = gr.Slider(0, 1, 0.7, step=0.1, label="guidance rescale factor")
+        topk_slider = gr.Slider(1, 10000, 25, step=1, label="sampling from top-k logits")
+
+        gr.Markdown("#### Audio Token Syllable Ratio")
+        gr.Markdown("*Automatic calculation (recommended) provides the best results in most cases.*")
+        use_manual_ratio = gr.Checkbox(label="Use manual audio token syllable ratio", value=False)
+        audio_token_syllable_ratio = gr.Number(
+            label="Audio token syllable ratio (manual)",
+            value=12.5, precision=2, minimum=5.0, maximum=25.0,
+            visible=False, interactive=False
+        )
+        use_manual_ratio.change(
+            toggle_ratio_input,
+            inputs=[use_manual_ratio],
+            outputs=[audio_token_syllable_ratio]
+        )
+
+    return (num_steps_slider, init_temp_slider, init_diversity_slider,
+            guidance_slider, rescale_slider, topk_slider,
+            use_manual_ratio, audio_token_syllable_ratio)
 
-def run_inpainter_tts(input_text, voice_audio):
-    return inpainter.tts(TTSInput(output_text=input_text, voice=voice_audio))
 
 if __name__ == '__main__':
     with gr.Blocks(analytics_enabled=False, title="PlayDiffusion") as demo:
@@ -43,13 +82,7 @@ if __name__ == '__main__':
             gr.Markdown("### Run the inpainter to generate the modified audio.")
             gr.Markdown("### Note: The model and demo are currently targeted for English.")
 
-            with gr.Accordion("Advanced options", open=False):
-                num_steps_slider = gr.Slider(minimum=1, maximum=100, step=1, label="number of sampling steps codebook", value=30)
-                init_temp_slider = gr.Slider(minimum=0.5, maximum=10, step=0.1, label="Initial temperature", value=1)
-                init_diversity_slider = gr.Slider(minimum=0, maximum=10, step=0.1, label="Initial diversity", value=1)
-                guidance_slider = gr.Slider(minimum=0, maximum=10, step=0.1, label="guidance", value=0.5)
-                rescale_slider = gr.Slider(minimum=0, maximum=1, step=0.1, label="guidance rescale factor", value=0.7)
-                topk_slider = gr.Slider(minimum=1, maximum=10000, step=1, label="sampling from top-k logits", value=25)
+            inpaint_advanced_options = create_advanced_options_accordion()
 
             with gr.Row():
                 audio_input = gr.Audio(label="Upload audio to be modified", sources=["upload", "microphone"], type="filepath")
@@ -71,10 +104,15 @@ if __name__ == '__main__':
                 audio_output = gr.Audio(label="Output audio")
 
             asr_submit.click(run_asr, inputs=[audio_input], outputs=[text_input, text_output, word_times])
-            inpainter_submit.click(run_inpainter, inputs=[text_input, text_output, word_times, audio_input, num_steps_slider, init_temp_slider, init_diversity_slider, guidance_slider, rescale_slider, topk_slider], outputs=[audio_output])
+            inpainter_submit.click(
+                run_inpainter,
+                inputs=[text_input, text_output, word_times, audio_input] + list(inpaint_advanced_options),
+                outputs=[audio_output])
 
         with gr.Tab("Text to Speech"):
             gr.Markdown("### Text to Speech")
+            tts_advanced_options = create_advanced_options_accordion()
+
             tts_text = gr.Textbox(label="TTS Input", placeholder="Enter text to convert to speech", lines=2)
             tts_voice =  gr.Audio(label="Voice to use for TTS",
                 sources=["upload", "microphone"], type="filepath",
@@ -84,7 +122,7 @@ if __name__ == '__main__':
 
             tts_submit.click(
                 run_inpainter_tts,
-                inputs=[tts_text, tts_voice],
+                inputs=[tts_text, tts_voice] + list(tts_advanced_options),
                 outputs=[tts_output]
             )