Update app.py

app.py

@@ -91,7 +91,7 @@ def validate_api_key(api_key):
         return False
 
 
-def init_openai_api_key(api_key=""):
+def init_openai_api_key(gpt_state,api_key=""):
     text_refiner = None
     visual_chatgpt = None
     if api_key and len(api_key) > 30:
@@ -112,12 +112,13 @@ def init_openai_api_key(api_key=""):
     print(text_refiner)
     openai_available = text_refiner is not None
     if openai_available:
-        return [gr.update(visible=True)]*7 + [gr.update(visible=False)]*2 + [text_refiner, visual_chatgpt, None]
+        gpt_state.set(1)
+        return [gr.update(visible=True)]+[gr.update(visible=False)]+[gr.update(visible=True)]*3+[gr.update(visible=False)]+ [gr.update(visible=True)]+ [gr.update(visible=False)]*2 + [text_refiner, visual_chatgpt, None]
     else:
         return [gr.update(visible=False)]*7 + [gr.update(visible=True)]*2 + [text_refiner, visual_chatgpt, 'Your OpenAI API Key is not available']
         
 def init_wo_openai_api_key():
-        return  [gr.update(visible=False)]*4 + [gr.update(visible=True)]*3 + [gr.update(visible=False)]*2 + [None, None, None]
+        return  [gr.update(visible=False)]*4 + [gr.update(visible=True)]+ [gr.update(visible=False)]+[gr.update(visible=True)]+[gr.update(visible=False)]*2 + [None, None, None]
 
 def get_click_prompt(chat_input, click_state, click_mode):
     inputs = json.loads(chat_input)
@@ -304,21 +305,20 @@ def submit_caption(image_input, state, generated_caption, text_refiner, visual_c
                                                       input_points=input_points, input_labels=input_labels)
             try:
                 waveform_visual, audio_output = tts.predict(new_cap, input_language, input_audio, input_mic, use_mic, agree)
-                print("error tts")
-                yield state, state, refined_image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, waveform_visual, audio_output
+                return state, state, refined_image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, waveform_visual, audio_output
             except Exception as e:
                 state = state + [(None, f"Error during TTS prediction: {str(e)}")]
                 print(f"Error during TTS prediction: {str(e)}")
-                yield state, state, refined_image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, None, None
+                return state, state, refined_image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, None, None
 
         else:
             try:
                 waveform_visual, audio_output = tts.predict(generated_caption, input_language, input_audio, input_mic, use_mic, agree)
-                yield state, state, image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, waveform_visual, audio_output
+                return state, state, image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, waveform_visual, audio_output
             except Exception as e:
                 state = state + [(None, f"Error during TTS prediction: {str(e)}")]
                 print(f"Error during TTS prediction: {str(e)}")
-                yield state, state, image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, None, None
+                return state, state, image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, None, None
 
 
 
@@ -421,7 +421,7 @@ def clear_chat_memory(visual_chatgpt, keep_global=False):
             visual_chatgpt.current_image = None
             visual_chatgpt.global_prompt = ""
 
-def cap_everything(image_input, visual_chatgpt, text_refiner):
+def cap_everything(image_input, visual_chatgpt, text_refiner,input_language, input_audio, input_mic, use_mic, agree):
     
     model = build_caption_anything_with_models(
         args,
@@ -438,7 +438,8 @@ def cap_everything(image_input, visual_chatgpt, text_refiner):
     AI_prompt = "Received."
     visual_chatgpt.global_prompt = Human_prompt + 'AI: ' + AI_prompt
     visual_chatgpt.agent.memory.buffer = visual_chatgpt.agent.memory.buffer + visual_chatgpt.global_prompt 
-    return paragraph
+    waveform_visual, audio_output=tts.predict(paragraph, input_language, input_audio, input_mic, use_mic, agree)
+    return paragraph,waveform_visual, audio_output
 
     
 def get_style():
@@ -494,6 +495,7 @@ def create_ui():
         input_mask_state = gr.State(np.zeros((1, 1)))
         input_points_state = gr.State([])
         input_labels_state = gr.State([])
+        gpt_state = gr.State(0)
 
         gr.Markdown(title)
         gr.Markdown(description)
@@ -501,7 +503,13 @@ def create_ui():
         with gr.Row():
             with gr.Column(scale=1.0):
                 with gr.Column(visible=False) as modules_not_need_gpt:
-                    with gr.Tab("Click"):
+                    with gr.Tab("Base(GPT Power)",visible=False) as base_tab:
+                        image_input_base = gr.Image(type="pil", interactive=True, elem_id="image_upload")
+                        example_image = gr.Image(type="pil", interactive=False, visible=False)
+            
+                    
+                    with gr.Tab("Click") as click_tab:
+                        modules_not_need_gpt2=True
                         image_input = gr.Image(type="pil", interactive=True, elem_id="image_upload")
                         example_image = gr.Image(type="pil", interactive=False, visible=False)
                         with gr.Row(scale=1.0):
@@ -578,10 +586,13 @@ def create_ui():
                                                         variant='primary')
                 with gr.Column(visible=False) as module_notification_box:
                     notification_box = gr.Textbox(lines=1, label="Notification", max_lines=5, show_label=False)
-                with gr.Column(visible=False) as modules_need_gpt2: 
-                    paragraph_output = gr.Textbox(lines=7, label="Describe Everything", max_lines=7)
-                with gr.Column(visible=False) as modules_need_gpt0:
-                    cap_everything_button = gr.Button(value="Caption Everything in a Paragraph", interactive=True)
+                
+                with gr.Column():
+                    with gr.Column(visible=False) as modules_need_gpt2: 
+                        paragraph_output = gr.Textbox(lines=7, label="Describe Everything", max_lines=7)
+                    with gr.Column(visible=False) as modules_need_gpt0:
+                        cap_everything_button = gr.Button(value="Caption Everything in a Paragraph", interactive=True)
+                
                 with gr.Column(visible=False) as modules_not_need_gpt2: 
                     chatbot = gr.Chatbot(label="Chatbox", ).style(height=550, scale=0.5)
                     with gr.Column(visible=False) as modules_need_gpt3:
@@ -590,7 +601,8 @@ def create_ui():
                         with gr.Row():
                             clear_button_text = gr.Button(value="Clear Text", interactive=True)
                             submit_button_text = gr.Button(value="Submit", interactive=True, variant="primary")
-                
+            
+            with gr.Column(scale=0.5):
                 # TTS interface hidden initially
                 with gr.Column(visible=False) as tts_interface:
                     input_text = gr.Textbox(label="Text Prompt", value="Hello, World !, here is an example of light voice cloning. Try to upload your best audio samples quality")
@@ -600,7 +612,7 @@ def create_ui():
                     use_mic = gr.Checkbox(label="Check to use Microphone as Reference", value=False)
                     agree = gr.Checkbox(label="Agree", value=True)
                     output_waveform = gr.Video(label="Waveform Visual")
-                    output_audio = gr.Audio(label="Synthesised Audio")
+                    output_audio = gr.Audio(label="Synthesised Audio",autoplay=True)
 
                     with gr.Row():
                         submit_tts = gr.Button(value="Submit", interactive=True)
@@ -625,10 +637,10 @@ def create_ui():
         )
 
         
-        openai_api_key.submit(init_openai_api_key, inputs=[openai_api_key],
+        openai_api_key.submit(init_openai_api_key, inputs=[gpt_state,openai_api_key],
                               outputs=[modules_need_gpt0, modules_need_gpt1, modules_need_gpt2, modules_need_gpt3, modules_not_need_gpt,
                                        modules_not_need_gpt2, tts_interface,module_key_input ,module_notification_box, text_refiner, visual_chatgpt, notification_box])
-        enable_chatGPT_button.click(init_openai_api_key, inputs=[openai_api_key],
+        enable_chatGPT_button.click(init_openai_api_key, inputs=[gpt_state,openai_api_key],
                                     outputs=[modules_need_gpt0, modules_need_gpt1, modules_need_gpt2, modules_need_gpt3,
                                              modules_not_need_gpt,
                                              modules_not_need_gpt2, tts_interface,module_key_input,module_notification_box, text_refiner, visual_chatgpt, notification_box])
@@ -652,7 +664,8 @@ def create_ui():
             show_progress=False
         )
 
-        cap_everything_button.click(cap_everything, [origin_image, visual_chatgpt, text_refiner], [paragraph_output])
+        cap_everything_button.click(cap_everything, [origin_image, visual_chatgpt, text_refiner,input_language, input_audio, input_mic, use_mic, agree], 
+                                    [paragraph_output,output_waveform, output_audio])
         
         clear_button_click.click(
             lambda x: ([[], [], []], x),
@@ -688,6 +701,10 @@ def create_ui():
         )
 
         image_input.clear(clear_chat_memory, inputs=[visual_chatgpt])
+
+        image_input_base.upload(upload_callback, [image_input_base, state, visual_chatgpt],
+                           [chatbot, state, origin_image, click_state, image_input_base, sketcher_input,
+                            image_embedding, original_size, input_size])
         
 
         image_input.upload(upload_callback, [image_input, state, visual_chatgpt],
@@ -705,8 +722,36 @@ def create_ui():
         example_image.change(upload_callback, [example_image, state, visual_chatgpt],
                              [chatbot, state, origin_image, click_state, image_input, sketcher_input,
                               image_embedding, original_size, input_size])
+        example_image.change(upload_callback, [example_image, state, visual_chatgpt],
+                             [chatbot, state, origin_image, click_state, image_input_base, sketcher_input,
+                              image_embedding, original_size, input_size])
         example_image.change(clear_chat_memory, inputs=[visual_chatgpt])
 
+        def on_click_tab_selected():
+            if gpt_state ==1:
+                print(gpt_state)
+                print("using gpt")
+                return [gr.update(visible=True)]*2+[gr.update(visible=False)]*2
+            else: 
+                print("no gpt")
+                return [gr.update(visible=False)]+[gr.update(visible=True)]+[gr.update(visible=False)]*2
+        
+        def on_base_selected():
+            if gpt_state ==1:
+                print(gpt_state)
+                print("using gpt")
+                return [gr.update(visible=True)]*2+[gr.update(visible=False)]*2
+            else: 
+                print("no gpt")
+                return [gr.update(visible=False)]*4
+        
+
+        click_tab.select(on_click_tab_selected, outputs=[modules_need_gpt1,modules_not_need_gpt2,modules_need_gpt0,modules_need_gpt2])
+        base_tab.select(on_base_selected, outputs=[modules_need_gpt0,modules_need_gpt2,modules_not_need_gpt2,modules_need_gpt1])
+
+  
+  
+
         image_input.select(
             inference_click,
             inputs=[
@@ -746,6 +791,11 @@ def create_ui():
             show_progress=False, queue=True
         )
 
+        def update_output_audio():
+            return gr.update(autoplay=True)
+        
+        output_audio.change(update_output_audio,outputs=[output_audio])
+