updated ui to focus on chatbot interface

planning/Prompts Planning.txt

@@ -65,3 +65,6 @@ Let's start with Warm-up Begin by greeting the student and briefly discussing th
 # CEFR levels:
 
 ## A1
+
+# Conversation based on photo
+Act as a Spanish dialogue writer for student textbooks. The student is B1. Generate a dialogue with as many or as few characters as you need. Keep the conversation only in Spanish. Discuss the photo. Make it interesting.

requirements.txt

@@ -2,4 +2,5 @@ gradio>=4.1
 openai>=1.0.0
 openai-whisper
 pyautogen
-ollama
+ollama
+ffmpeg

stream_app.py

@@ -41,8 +41,8 @@ def run_gradio(config:dict):
 
     def get_teacher_prompt(language_input, cefr_level_input, is_initial_image):
         global teacher_prompt
-        teacher_prompt = f"Act as a {language_input} teacher only speaking in {language_input}. The student is still learning {language_input}, so explain topics in simple words and ask questions to continue the conversation. Repeat and restate what the student says when they respond. Keep it highly conversational because you're talking with the student. The student is just starting to learn, so keep it simple. The speaker is able to speak and understand at {cefr_level_input}." 
-        teacher_image_prompt = f"Your job is to understand the following image and create a {language_input} lesson around it. Describe the exercise or situation that you see. Provide the student with instructions, a simple example, and then ask the student to participate. Only speak in {language_input} at a {cefr_level_input} level."
+        teacher_prompt = f"Act as a {language_input} teacher only speaking in {language_input}. Help me learn spanish." 
+        teacher_image_prompt = f"Here is a photo to start the conversation."
         if is_initial_image is True:
             return teacher_prompt+teacher_image_prompt
         return teacher_prompt
@@ -88,22 +88,32 @@ def run_gradio(config:dict):
         # audio = whisper.clear?
         return "", None    # return empty, clear prior file
     
-    def clear_inputs(input_audio, input_text):
-        return None, None
+    def reset_inputs(input_audio,input_audio_2,input_text):
+        return None, gr.Audio(interactive=True),gr.Text(visible=False)
+    def reset_audio_generate(audio_generate_done):
+        return False
+    def hide_image_input(image_input):
+        return gr.Image(visible=False)
+    def show_chatbot(chatbot,audio_input,submit_button):
+        return gr.Chatbot(visible=True),gr.Audio(visible=True)
+    def stop_recording(audio_input, text_input):
+        return gr.Audio(interactive=False),gr.Text(visible=True)
     
     # speak input text
-    def audio_speak(input_text, speaker_name, input_done=True, offset_prior=0, path_prior=None, auto_speak=None):
+    def audio_speak(input_text, speaker_name, input_done=True, offset_prior=0, path_prior=None, auto_speak=None, audio_generate_done=False):
         # alternate on-device? - https://github.com/suno-ai/bark?tab=readme-ov-file
         # print(f"Speak: {input_text}, {offset_prior} of {len(input_text)}")
 
+        logger.info(f"Speak: {input_text}, {offset_prior} of {len(input_text)}")
+
 
         if not input_text:                          # empty string on conclusion (when streaming)
-            return gr.Audio(), None, 0
+            return gr.Audio(), None, 0, False
         elif auto_speak is not None:
             if "manual" in auto_speak.lower():      # don't proceed if manual
-                return gr.Audio(), None, 0
+                return gr.Audio(), None, 0, False
             elif (not input_done) and ("stream" not in auto_speak.lower()):    # stream, not done
-                return gr.Audio(), None, 0
+                return gr.Audio(), None, 0, False
         
         if (path_prior is None) or (offset_prior > len(input_text)):
             temp_file = tempfile.NamedTemporaryFile(delete=False)
@@ -120,7 +130,10 @@ def run_gradio(config:dict):
         with open(path_prior, 'ab') as file_append:
             for chunk in response.iter_bytes(chunk_size=4096):
                 file_append.write(chunk)
-        return path_prior, path_prior, offset_prior
+        
+        logger.info(f"audio processed")
+
+        return path_prior, path_prior, offset_prior, True
 
     # Define Gradio interface
     def start_initial_conversation(language_input, cefr_level_input, input_image, model_target=None):
@@ -183,15 +196,10 @@ def run_gradio(config:dict):
     # def initial_upload_complete():
     #     return gr.update(visible=True), gr.update(visible=True)
     def add_message(history, message, ai_response=False):
-        # type is either AI or human
-        # if type not in ["AI","human"]:
-        #     raise ValueError("type must be either AI or human")
-        # if type == "human":
-        message_type = str(type(message))
-        logger.info(f"message input type: {message_type}")
 
-        if "PIL.Image.Image" in message_type:
-            # TODO: Figure out image upload
+        logger.info(f"adding message to chat")
+        logger.info(f"message: {message}")
+
 
             # # Save the image to a buffer
             # buffer = io.BytesIO()
@@ -202,7 +210,10 @@ def run_gradio(config:dict):
             # history.append((input_image_base64,None))
             # return history
 
-            message = "Initial image uploaded"
+        if ".wav" in str(message):
+            message = gr.Audio(message,autoplay=True,label="Speech", streaming=False, type="filepath", sources=None,)
+        if "PIL.Image.Image" in str(message):
+            message = gr.Image(message)
 
         if ai_response is True:
             history[-1][1] = message
@@ -226,7 +237,6 @@ def run_gradio(config:dict):
 
         logger.info(f"Prompt: {prompt}")
         
-        # TODO: Add in full chat history again
         messages=[
             {"role": "system", "content": system_prompt+teacher_prompt},
             {"role": "user", "content": prompt},
@@ -269,49 +279,43 @@ def run_gradio(config:dict):
             generate_done = gr.State(False)             # is last genai content chunked?
             path_prior = gr.State(None)                 # retain prior file for audio playback
             offset_prior = gr.State(0)                  # track textual offset in genrated content
+            audio_generate_done = gr.State(False)                  # track textual offset in genrated content
             # initial_image_uploaded = gr.State(False)                  # visibility of chat sections
 
         gr.Markdown("""
-        # Capture an image, and our AI guides you through a conversation in the language of your choice at your level
+        # Capture an image to start a conversation with our AI language tutor.
         """)
         with gr.Row():
             with gr.Column():
-                with gr.Group():
-                    with gr.Row():
-                        language_input = gr.Dropdown(
-                            ["English","French","Mandarin","Spanish","German","Italian"], value="Spanish", label="Target Language", info="Select the language you're learning", interactive=True
-                        )
-                        cefr_level_input = gr.Dropdown(
-                            ["A0 - brand new","A1 - basic phrases","A2 - basic interactions","B1 - basic conversation","B2 - conversational"], value="A0 - brand new", label="Your CEFR Level", info="Your currently ability in the language", interactive=True
-                        )
-
-                with gr.Group():
-                    image_input = gr.Image(
-                        label="Image Input",
-                        type="pil",
-                    )
-                    # image_submit_button = gr.Button("Start conversation", variant='primary') # trigger automatically instead of trigger
 
+                image_input = gr.Image(
+                    label="Image Input",
+                    type="pil",
+                )
+                # image_submit_button = gr.Button("Start conversation", variant='primary') # trigger automatically instead of trigger
 
                 with gr.Group() as chat_response_section:
                     chatbot = gr.Chatbot(
                         elem_id="chatbot",
                         bubble_full_width=True,
                         scale=1,
+                        visible=False
                     )
                     audio_input = gr.Audio(
                         label="Speech Input",
                         # streaming=True,   # true for stream to text
                         sources="microphone",
                         type="filepath",
+                        visible=False
                     )
                     input_text = gr.Textbox(
                         label="Text Input",
                         placeholder="Enter your prompt here or use speech recognition to generate it.",
                         lines=5,
                         max_lines=5,
+                        visible=False
                     )
-                    submit_button = gr.Button("Send to teacher", variant='primary')
+                    # submit_button = gr.Button("Send Response", variant='primary',visible=False)
                 
                 # with gr.Group():
                 #     chat_interface = gr.ChatInterface(yes_man,
@@ -322,9 +326,9 @@ def run_gradio(config:dict):
 
                 with gr.Row() as input_details_section:
                     with gr.Group():
-                        with gr.Accordion("Teacher Response Details", open=True):
+                        with gr.Accordion("Transcription and Audio Details", open=False):
                             audio_playback = gr.Audio(
-                                label="Speech", autoplay=True, streaming=False,
+                                label="Speech", autoplay=False, streaming=False,
                                 type="filepath", sources=None,
                             )
                             output_text = gr.Textbox(
@@ -336,12 +340,19 @@ def run_gradio(config:dict):
 
                 with gr.Group():
                     with gr.Accordion("Settings", open=False):
-                        # teacher_text = gr.Textbox(
-                        #     label="Teacher Prompt",
-                        #     lines=5,
-                        #     max_lines=5,
-                        #     interactive=False
-                        # )
+                        with gr.Row():
+                            language_input = gr.Dropdown(
+                                ["English","French","Mandarin","Spanish","German","Italian"], value="Spanish", label="Target Language", info="Select the language you're learning", interactive=True
+                            )
+                            cefr_level_input = gr.Dropdown(
+                                ["A0 - brand new","A1 - basic phrases","A2 - basic interactions","B1 - basic conversation","B2 - conversational"], value="A0 - brand new", label="Your CEFR Level", info="Your currently ability in the language", interactive=True
+                            )
+                        teacher_text = gr.Textbox(
+                            label="Teacher Prompt",
+                            lines=5,
+                            max_lines=5,
+                            interactive=False
+                        )
                         prompt_model = gr.Radio(
                             label="Textual Model", show_label=False,
                             choices=[online_text_model, offline_text_model],
@@ -364,72 +375,64 @@ def run_gradio(config:dict):
                         with gr.Row():
                             combo_autospeak = gr.Radio(
                                 choices=["Auto-speak", "Auto-speak (stream)", "Manual"], show_label=False,
-                                value="Auto-speak", interactive=True,
+                                value="Auto-speak", interactive=False,
                             )
 
      
         
         # language_input.change()   # can update the teacher prompt
         # cefr_level_input.change()   # can update the teacher prompt
+
         initial_image_uploaded = image_input.upload(add_message,     # uploaded image, add to chat
             inputs=[chatbot, image_input],
             outputs=[chatbot])
+        initial_image_uploaded.then(show_chatbot,
+            inputs=[chatbot,audio_input],
+            outputs=[chatbot,audio_input])
+        initial_image_uploaded.then(hide_image_input,image_input,image_input)
         text_response_generate = initial_image_uploaded.then(start_initial_conversation,     # uploaded image, start response
             inputs=[language_input,cefr_level_input, image_input, prompt_model],
             outputs=[output_text, generate_done])
 
-        # TODO: Be able to log audio response to chat
-        # audio_response_generate.then(add_message,     # generated response, add to chat
-        #     inputs=[chatbot, audio_playback],
-        #     outputs=[chatbot])
-
-
-        # audio_input.stream(audio_transcribe,            # started streaming to transcribe
-        #     inputs=[audio_input_model, audio_input, audio_threshold, input_text],
-        #     outputs=input_text)
         audio_input.clear(audio_reset,                  # cleared audio
             inputs=[input_text, path_prior], 
             outputs=[input_text, path_prior])
-        audio_input.start_recording(audio_reset,        # started a new speech->text
+        audio_input.start_recording(audio_reset,        # started a new speech recording
             inputs=[input_text, path_prior], 
             outputs=[input_text, path_prior])
-        # TODO: Can I just send the audio file to the chatbot component
-        audio_input.stop_recording(audio_transcribe,            # stop recording, create text
+        stop_input_recording = audio_input.stop_recording(stop_recording,            # stop recording, create text
+            inputs=[audio_input,input_text],
+            outputs=[audio_input,input_text])
+        stop_input_recording.then(audio_transcribe,            # stop recording, create text
             inputs=[audio_input_model, audio_input, audio_threshold, input_text],
             outputs=input_text)
-        # audio_input.stop_recording(get_ai_response,     # stopped recording, start response
-        #     inputs=[input_text, full_chat_context, prompt_model], 
-        #     outputs=[output_text, full_chat_context, generate_done])
 
-        # TODO: This needs to take in the audio file
-        student_input = submit_button.click(add_message,            # send in sound / text file
-            inputs=[chatbot, input_text], 
-            outputs=[chatbot])
+        #TODO: Submit button before text generation complete
+        #TODO: Handle submit button press still recording
+        # input_text.change(get_ai_response,  # transcription done, submit to bot
+        #     inputs=[input_text, chatbot, prompt_model], 
+        #     outputs=[output_text, generate_done]))
 
-        student_input.then(clear_inputs,            # send in sound / text file
-            inputs=[audio_input, input_text], 
-            outputs=[audio_input, input_text])
+        # output_text_logged = output_text.change(add_message,     # generated response, add to chat
+        #     inputs=[chatbot, output_text, gr.State(value=True)],
+        #     outputs=[chatbot])
 
-        student_input.then(get_ai_response,            # send in sound / text file
+        student_submit = input_text.change(add_message,            # submit, update chatbot
+            inputs=[chatbot, audio_input], 
+            outputs=[chatbot])
+        student_submit.then(reset_inputs,            # then clear speech input
+            inputs=[audio_input,audio_input,input_text], 
+            outputs=[audio_input,audio_input,input_text])
+        student_submit.then(get_ai_response,            # then get ai response
             inputs=[input_text, chatbot, prompt_model], 
             outputs=[output_text, generate_done])
-            
-        speak_button.click(audio_speak,                 # click for speak trigger
-            inputs=[output_text, combo_speaker],
-            outputs=[audio_playback, path_prior, offset_prior])
-
-        output_text_logged = output_text.change(add_message,     # generated response, add to chat
-            inputs=[chatbot, output_text, gr.State(value=True)],
-            outputs=[chatbot])
 
-        output_text_logged.then(audio_speak,               # streaming response from generate
-            inputs=[output_text, combo_speaker, generate_done, offset_prior, path_prior, combo_autospeak],
-            outputs=[audio_playback, path_prior, offset_prior])
-
-        # output_text_logged.then(audio_speak,               # streaming response from generate
-        #     inputs=[output_text, combo_speaker, generate_done, offset_prior, path_prior, combo_autospeak],
-        #     outputs=[audio_playback, path_prior, offset_prior])
-        
+        output_text_generated = output_text.change(audio_speak,               # streaming response from generate
+            inputs=[output_text, combo_speaker, generate_done, offset_prior, path_prior, combo_autospeak, audio_generate_done],
+            outputs=[audio_playback, path_prior, offset_prior, audio_generate_done])
+        audio_playback.change(add_message,     # generated audio, add to chat
+            inputs=[chatbot, audio_playback, gr.State(value=True)],
+            outputs=[chatbot]).then(reset_audio_generate,audio_generate_done,audio_generate_done)        
 
     # demo.set_api_mode(enabled=False)   # Disable API exposure
     # demo.set_footer(enabled=False)   # Disable Gradio footers
@@ -478,5 +481,4 @@ if __name__ == "__main__":
         raise ValueError("OPENAI_API_KEY environment variable not set as environment variable or as a setting in `.env`.  (see https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key)")
 
     config = parse_args()
-    run_gradio(config)
-
+    run_gradio(config)