updates for voice production and sync

stream_app.py

@@ -3,8 +3,10 @@ import argparse
 import logging
 import gradio as gr
 from openai import OpenAI
-import whisper
+import whisper   # just for local models
 import io
+from pathlib import Path
+import tempfile
 
 import dotenv
 dotenv.load_dotenv()
@@ -29,18 +31,18 @@ def run_gradio(config:dict):
     client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 
     # transcription of audio
-    def audio_transcribe(input_audio_model:str, input_audio:str, input_text:str):
+    def audio_transcribe(audio_input_model:str, audio_input:str, audio_threshold:float, input_text:str):
         global whisper_model
         global logger
 
-        if "offline" in input_audio_model.lower():
+        if "offline" in audio_input_model.lower():
             if whisper_model is None:
                 whisper_model = whisper.load_model("base")
-            audio = whisper.load_audio(input_audio)
+            audio = whisper.load_audio(audio_input)
             result = whisper_model.transcribe(audio)
 
-        elif "online" in input_audio_model.lower():
-            with open(input_audio, 'rb') as file_audio:
+        elif "online" in audio_input_model.lower():
+            with open(audio_input, 'rb') as file_audio:
                 result = client.audio.transcriptions.create(
                     model="whisper-1", file=file_audio, response_format="verbose_json",
                 )
@@ -56,7 +58,7 @@ def run_gradio(config:dict):
             if len(prob_scores) > 0:    # average the probs
                 result["no_speech_prob"] = sum(prob_scores)/len(prob_scores)
 
-        if result["no_speech_prob"] < (1 - config['speech_threshold']):      # threshold to avoid bad output
+        if result["no_speech_prob"] < (1 - audio_threshold):      # threshold to avoid bad output
             return input_text + " " + prompt
         return input_text
 
@@ -64,12 +66,29 @@ def run_gradio(config:dict):
     def audio_reset(input_text):
         # audio = whisper.clear?
         return ""    # return empty
+    
+    # speak input text
+    def audio_speak(input_text, speaker_name, output_complete, auto_speak=None):
+        print(f"OUTPUT: {output_complete}")
+        if not output_complete:
+            return None
+        if auto_speak is not None and auto_speak.lower() == "manual":   # abort if manual
+            return None
+        temp_file = tempfile.NamedTemporaryFile(delete=False)
+        response = client.audio.speech.create(
+            model="tts-1",
+            voice=speaker_name,
+            input=input_text
+        )
+        response.write_to_file(temp_file.name)
+        return temp_file.name
+
 
     # Define Gradio interface
-    def get_ai_response(input_text, input_audio):
+    def get_ai_response(input_text):
         prompt = input_text.strip()
         if not prompt:
-            return "Please enter a prompt for interaction."
+            return "Please enter a prompt for interaction.", False
 
         logger.warning(f"Prompt: {prompt}")
         response = client.chat.completions.create(model=config['model'],
@@ -89,7 +108,8 @@ def run_gradio(config:dict):
             if token is None:
                 break
             partial_response += token
-            yield partial_response
+            yield partial_response, False
+        yield partial_response, True
 
     with gr.Blocks(css="footer{display:none !important}") as demo:
         gr.Markdown("""
@@ -101,47 +121,82 @@ def run_gradio(config:dict):
                 with gr.Group():
                     input_text = gr.Textbox(
                         label="Text Input",
-                        placeholder="Enter your prompt here",
+                        placeholder="Enter your prompt here or use speech recognition to genreate it.",
                         lines=5,
-                        max_lines=10,
+                        max_lines=5,
                     )
                     online_text_model = f"openai-{config['model']} (online)"
-                    input_audio_model = gr.Radio(
-                        label="Textual Model",
+                    audio_input_model = gr.Radio(
+                        label="Textual Model", show_label=False,
                         choices=[online_text_model],
                         value=online_text_model,
                     )
 
                 with gr.Group():
-                    input_audio = gr.Audio(
+                    audio_input = gr.Audio(
                         label="Speech Input",
                         streaming=True,
                         type="filepath",
                     )
-                    input_audio_model = gr.Radio(
-                        label="Audio Model",
+                    audio_threshold = gr.Slider(
+                        label="Speech Threshold", minimum=0.0, maximum=1.0, step=0.01,
+                        value=config['speech_threshold'],
+                    )
+                    audio_input_model = gr.Radio(
+                        label="Audio Model", show_label=False,
                         choices=["whisper (offline)", "openai-whisper (online)"],
                         value="openai-whisper (online)",
                     )
 
             with gr.Column():
-                output_text = gr.Textbox(
-                    label="Output",
-                    interactive=False,
-                    lines=10,
-                )
-        submit_button = gr.Button("Submit", variant='primary')
-        input_audio.stream(audio_transcribe,
-            inputs=[input_audio_model, input_audio, input_text],
+                with gr.Group():
+                    with gr.Row():
+                        output_text = gr.Textbox(
+                            label="Output",
+                            interactive=False,
+                            lines=10, max_lines=15,
+                        )
+                    with gr.Row():
+                        combo_speaker = gr.Dropdown(
+                            choices=["alloy", "echo", "fable", "onyx", "nova", "shimmer"], 
+                            show_label=False, value="nova", interactive=True,
+                        )
+                        combo_autospeak = gr.Radio(
+                            choices=["Auto-speak", "Manual"], show_label=False,
+                            value="Manual", interactive=True,
+                        )
+                    with gr.Row():
+                        speak_button = gr.Button("Speak!", variant='secondary', interactive=True)
+                    with gr.Row():
+                        audio_playback = gr.Audio(
+                            label="Speech", autoplay=True,
+                            streaming=False,
+                            type="filepath", sources=None,
+                        )
+
+        with gr.Row():
+            submit_button = gr.Button("Submit", variant='primary')
+            output_complete = gr.State(False)
+                            
+                
+        audio_input.stream(audio_transcribe,
+            inputs=[audio_input_model, audio_input, audio_threshold, input_text],
             outputs=input_text)
-        input_audio.clear(audio_reset, inputs=input_text, outputs=input_text)
-        input_audio.start_recording(audio_reset, inputs=input_text, outputs=input_text)
-        input_audio.stop_recording(get_ai_response, 
-            inputs=[input_text, input_audio], 
-            outputs=output_text)
+        audio_input.clear(audio_reset, inputs=input_text, outputs=input_text)
+        audio_input.start_recording(audio_reset, inputs=input_text, outputs=input_text)
+        audio_input.stop_recording(get_ai_response, 
+            inputs=[input_text], 
+            outputs=[output_text, output_complete])
         submit_button.click(get_ai_response, 
-            inputs=[input_text, input_audio], 
-            outputs=output_text)
+            inputs=[input_text], 
+            outputs=[output_text, output_complete])
+        output_text.change(audio_speak, 
+            inputs=[output_text, combo_speaker, output_complete, combo_autospeak],
+            outputs=audio_playback)
+        speak_button.click(audio_speak, 
+            inputs=[output_text, combo_speaker, output_complete],
+            outputs=audio_playback)
+        
 
     # demo.set_api_mode(enabled=False)   # Disable API exposure
     # demo.set_footer(enabled=False)   # Disable Gradio footers
@@ -152,7 +207,7 @@ def run_gradio(config:dict):
 
 def parse_args() -> dict:
     parser = argparse.ArgumentParser()
-    opt_group = parser.add_argument_group("Model  Configuration")
+    opt_group = parser.add_argument_group("Model Configuration")
     opt_group.add_argument("--model", type=str, default="gpt-4o",
         help="Model to use for chat completion.")
     opt_group.add_argument("--temperature", type=float, default=1.0,
@@ -161,7 +216,7 @@ def parse_args() -> dict:
         help="Maximum number of tokens to generate in chat completion.")
     
     opt_group = parser.add_argument_group("Speech Processing")
-    opt_group.add_argument("--speech_threshold", type=float, default=0.10,
+    opt_group.add_argument("--speech_threshold", type=float, default=0.15,
         help="Speech threshold (probability) for recognition to add text to a prompt. ")
 
     opt_group = parser.add_argument_group("App Settings")