Update app.py

app.py

@@ -34,7 +34,7 @@ def encode_image(image):
 
 def respond(
     message,
-    images, # New parameter for uploaded images
+    image_files,  # Changed parameter name and structure
     history: list[tuple[str, str]],
     system_message,
     max_tokens,
@@ -49,7 +49,7 @@ def respond(
     selected_model
 ):
     print(f"Received message: {message}")
-    print(f"Received {len(images) if images else 0} images")
+    print(f"Received {len(image_files) if image_files else 0} images")
     print(f"History: {history}")
     print(f"System message: {system_message}")
     print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}")
@@ -77,7 +77,7 @@ def respond(
         seed = None
 
     # Create multimodal content if images are present
-    if images and any(images):
+    if image_files and len(image_files) > 0:
         # Process the user message to include images
         user_content = []
         
@@ -89,16 +89,20 @@ def respond(
             })
         
         # Add image parts
-        for img in images:
+        for img in image_files:
             if img is not None:
-                encoded_image = encode_image(img)
-                if encoded_image:
-                    user_content.append({
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{encoded_image}"
-                        }
-                    })
+                # Get raw image data from path
+                try:
+                    encoded_image = encode_image(img)
+                    if encoded_image:
+                        user_content.append({
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/jpeg;base64,{encoded_image}"
+                            }
+                        })
+                except Exception as e:
+                    print(f"Error encoding image: {e}")
     else:
         # Text-only message
         user_content = message
@@ -112,8 +116,36 @@ def respond(
         user_part = val[0]
         assistant_part = val[1]
         if user_part:
-            messages.append({"role": "user", "content": user_part})
+            # Handle both text-only and multimodal messages in history
+            if isinstance(user_part, tuple) and len(user_part) == 2:
+                # This is a multimodal message with text and images
+                history_content = []
+                if user_part[0]:  # Text
+                    history_content.append({
+                        "type": "text",
+                        "text": user_part[0]
+                    })
+                
+                for img in user_part[1]:  # Images
+                    if img:
+                        try:
+                            encoded_img = encode_image(img)
+                            if encoded_img:
+                                history_content.append({
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:image/jpeg;base64,{encoded_img}"
+                                    }
+                                })
+                        except Exception as e:
+                            print(f"Error encoding history image: {e}")
+                
+                messages.append({"role": "user", "content": history_content})
+            else:
+                # Regular text message
+                messages.append({"role": "user", "content": user_part})
             print(f"Added user message to context (type: {type(user_part)})")
+        
         if assistant_part:
             messages.append({"role": "assistant", "content": assistant_part})
             print(f"Added assistant message to context: {assistant_part}")
@@ -190,19 +222,15 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo:
     print("Chatbot interface created.")
     
     with gr.Row():
-        # Text input for messages
-        msg = gr.Textbox(
-            placeholder="Type a message...",
+        # Multimodal textbox for messages (combines text and file uploads)
+        msg = gr.MultimodalTextbox(
+            placeholder="Type a message or upload images...",
             show_label=False,
             container=False,
-            scale=9
-        )
-        
-        # Image upload button
-        image_upload = gr.Image(
-            type="filepath", 
-            label="Upload Image",
-            scale=1
+            scale=12,
+            file_types=["image"],
+            file_count="multiple",
+            sources=["upload"]
         )
 
     # Send button for messages
@@ -367,37 +395,55 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo:
         return selected
 
     # Function for the chat interface
-    def user(user_message, image, history):
-        if user_message == "" and image is None:
+    def user(user_message, history):
+        # Skip if message is empty (no text and no files)
+        if (not user_message["text"] or user_message["text"].strip() == "") and not user_message["files"]:
             return history
         
-        # Format image reference for display
-        img_placeholder = ""
-        if image is not None:
-            img_placeholder = f"![Image]({image})"
+        # Process images and text into a display message
+        display_message = ""
         
-        # Combine text and image reference for display
-        display_message = f"{user_message}\n{img_placeholder}" if img_placeholder else user_message
+        # Add text if present
+        if user_message["text"] and user_message["text"].strip() != "":
+            display_message += user_message["text"]
         
-        # Return updated history
-        return history + [[display_message, None]]
+        # Add image references if present
+        file_displays = []
+        for file in user_message["files"]:
+            file_displays.append(file)
+        
+        # Return updated history with display message
+        if file_displays:
+            return history + [[(display_message, file_displays), None]]
+        else:
+            return history + [[display_message, None]]
     
     # Define chat interface
-    def bot(history, images, system_msg, max_tokens, temperature, top_p, freq_penalty, seed, provider, api_key, custom_model, search_term, selected_model):
+    def bot(history, system_msg, max_tokens, temperature, top_p, freq_penalty, seed, provider, api_key, custom_model, search_term, selected_model):
         # Extract the last user message
-        user_message = history[-1][0] if history and len(history) > 0 else ""
+        if not history or len(history) == 0:
+            return history
+        
+        user_message = history[-1][0]
         
-        # Clean up the user message to remove image reference
-        if "![Image]" in user_message:
-            text_parts = user_message.split("![Image]")[0].strip()
+        # Determine if the message is multimodal or text-only
+        is_multimodal = False
+        text_parts = ""
+        image_files = []
+        
+        # Process text and images from the message
+        if isinstance(user_message, tuple):
+            text_parts = user_message[0]
+            image_files = user_message[1]
+            is_multimodal = True
         else:
             text_parts = user_message
         
         # Process message through respond function
         history[-1][1] = ""
         for response in respond(
-            text_parts,  # Send only the text part
-            [images],    # Send images separately
+            text_parts,               # Text part
+            image_files if is_multimodal else None,  # Image part
             history[:-1],
             system_msg,
             max_tokens,
@@ -417,32 +463,36 @@ with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo:
     # Event handlers
     msg.submit(
         user,
-        [msg, image_upload, chatbot],
+        [msg, chatbot],
         [chatbot],
         queue=False
     ).then(
         bot,
-        [chatbot, image_upload, system_message_box, max_tokens_slider, temperature_slider, top_p_slider, 
+        [chatbot, system_message_box, max_tokens_slider, temperature_slider, top_p_slider, 
          frequency_penalty_slider, seed_slider, provider_radio, byok_textbox, custom_model_box, 
          model_search_box, featured_model_radio],
         [chatbot]
+    ).then(
+        lambda: {"text": "", "files": []},  # Clear inputs after submission
+        None,
+        [msg]
     )
     
     submit_btn.click(
         user,
-        [msg, image_upload, chatbot],
+        [msg, chatbot],
         [chatbot],
         queue=False
     ).then(
         bot,
-        [chatbot, image_upload, system_message_box, max_tokens_slider, temperature_slider, top_p_slider, 
+        [chatbot, system_message_box, max_tokens_slider, temperature_slider, top_p_slider, 
          frequency_penalty_slider, seed_slider, provider_radio, byok_textbox, custom_model_box, 
          model_search_box, featured_model_radio],
         [chatbot]
     ).then(
-        lambda: (None, "", None),  # Clear inputs after submission
+        lambda: {"text": "", "files": []},  # Clear inputs after submission
         None,
-        [msg, msg, image_upload]
+        [msg]
     )
     
     # Connect the model filter to update the radio choices