Update app.py

app.py

@@ -22,18 +22,19 @@ class MultimodalChatbot:
             base_url="https://openrouter.ai/api/v1",
             api_key=api_key,
         )
-        self.model = "google/gemma-3n-e2b-it:free"
+        self.model = "google/gemma-2-9b-it:free"  # Updated to a valid text model
         self.conversation_history = []
         # Initialize the pipeline for image-text-to-text processing
         try:
             self.pipe = pipeline(
-                "image-text-to-text",
-                model="google/gemma-3n-e2b",
+                "image-captioning",
+                model="Salesforce/blip-image-captioning-base",
                 device="cpu",  # Optimized for CPU in HF Spaces
                 torch_dtype=torch.float32,  # Use float32 for CPU compatibility
             )
+            print("Image captioning pipeline initialized successfully")
         except Exception as e:
-            print(f"Error initializing pipeline: {e}")
+            print(f"Error initializing image captioning pipeline: {str(e)}")
             self.pipe = None
     
     def encode_image_to_base64(self, image) -> str:
@@ -130,21 +131,28 @@ class MultimodalChatbot:
                 
             cap = cv2.VideoCapture(video_path)
             if not cap.isOpened():
-                return None
+                return None, "Could not open video file"
             
             total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+            if total_frames <= 0:
+                cap.release()
+                return None, "Video has no frames"
+                
             if frame_number is None:
                 frame_number = total_frames // 2  # Extract middle frame
+            if frame_number >= total_frames:
+                frame_number = total_frames - 1
+                
             cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number)
             ret, frame = cap.read()
             cap.release()
             if ret:
                 frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-                return Image.fromarray(frame)
+                return Image.fromarray(frame), f"Extracted frame {frame_number} of {total_frames}"
             else:
-                return None
+                return None, "Failed to extract frame"
         except Exception as e:
-            return None
+            return None, f"Error extracting video frame: {str(e)}"
     
     def create_multimodal_message(self, 
                                 text_input: str = "",
@@ -175,11 +183,13 @@ class MultimodalChatbot:
                     image = Image.open(image_file)
                 else:
                     image = image_file
-                # Use user's text input as prompt, or default if none provided
-                prompt = f"<image_soft_token> {text_input}" if text_input else "<image_soft_token> Describe this image"
-                output = self.pipe(image, text=prompt)
-                description = output[0]['generated_text']
-                content_parts.append({"type": "text", "text": f"Image TwitchImage analysis: {description}"})
+                # Use BLIP model for image captioning
+                output = self.pipe(image)
+                description = output[0]['generated_caption']
+                if text_input:
+                    content_parts.append({"type": "text", "text": f"Image analysis (based on '{text_input}'): {description}"})
+                else:
+                    content_parts.append({"type": "text", "text": f"Image analysis: {description}"})
                 processing_info.append("🖼️ Image analyzed")
             except Exception as e:
                 content_parts.append({"type": "text", "text": f"Error analyzing image: {str(e)}"})
@@ -189,20 +199,21 @@ class MultimodalChatbot:
             processing_info.append("🖼️ Image received (analysis failed)")
         
         if video_file is not None and self.pipe is not None:
-            frame = self.extract_video_frame(video_file)
+            frame, frame_info = self.extract_video_frame(video_file)
             if frame:
                 try:
-                    # Use user's text input with context, or default for frame
-                    prompt = f"<image_soft_token> This is a frame from the video. {text_input}" if text_input else "<image_soft_token> Describe this frame from the video"
-                    output = self.pipe(frame, text=prompt)
-                    description = output[0]['generated_text']
-                    content_parts.append({"type": "text", "text": f"Video frame analysis: {description}. Please describe the video for further assistance."})
+                    output = self.pipe(frame)
+                    description = output[0]['generated_caption']
+                    if text_input:
+                        content_parts.append({"type": "text", "text": f"Video frame analysis (based on '{text_input}'): {description}. Frame info: {frame_info}. Please describe the video for further assistance."})
+                    else:
+                        content_parts.append({"type": "text", "text": f"Video frame analysis: {description}. Frame info: {frame_info}. Please describe the video for further assistance."})
                     processing_info.append("🎥 Video frame analyzed")
                 except Exception as e:
-                    content_parts.append({"type": "text", "text": f"Error analyzing video frame: {str(e)}"})
+                    content_parts.append({"type": "text", "text": f"Error analyzing video frame: {str(e)}. Frame info: {frame_info}"})
                     processing_info.append("🎥 Video frame analysis failed")
             else:
-                content_parts.append({"type": "text", "text": "Could not extract frame from video. Please describe the video."})
+                content_parts.append({"type": "text", "text": f"Could not extract frame from video: {frame_info}. Please describe the video."})
                 processing_info.append("🎥 Video processing failed")
         elif video_file is not None:
             content_parts.append({"type": "text", "text": "Video uploaded. Analysis failed due to model initialization error."})
@@ -246,7 +257,7 @@ class MultimodalChatbot:
             completion = self.client.chat.completions.create(
                 extra_headers={
                     "HTTP-Referer": "https://multimodal-chatbot.local",
-                    "X-Title": "Multimodal Chatbot",
+                    "X-Title": "Mult Mosaic Chatbot",
                 },
                 model=self.model,
                 messages=messages,
@@ -264,16 +275,16 @@ class MultimodalChatbot:
 
 def create_interface():
     """Create the Gradio interface"""
-    with gr.Blocks(title="Multimodal Chatbot with Gemma 3n", theme=gr.themes.Soft()) as demo:
+    with gr.Blocks(title="Multimodal Chatbot with BLIP and Gemma", theme=gr.themes.Soft()) as demo:
         gr.Markdown("""
-        # 🤖 Multimodal Chatbot with Gemma 3n
+        # 🤖 Multimodal Chatbot with BLIP and Gemma
         
         This chatbot can process multiple types of input:
-        - **Text**: Regular text messages
+        - **Text**: Regular text messages using Gemma
         - **PDF**: Extract and analyze document content  
         - **Audio**: Transcribe speech to text (supports WAV, MP3, M4A, FLAC)
-        - **Images**: Upload images for analysis using Gemma 3n
-        - **Video**: Upload videos for basic frame analysis using Gemma 3n
+        - **Images**: Upload images for analysis using BLIP
+        - **Video**: Upload videos for basic frame analysis using BLIP
         
         **Setup**: Enter your OpenRouter API key below to get started
         """)
@@ -514,14 +525,18 @@ def create_interface():
         api_key_input.change(
             validate_api_key,
             inputs=[api_key_input],
-            outputs=[api_status, text_submit_btn, pdf_submit_btn, audio_submit_btn, 
+            F
+            outputs=[api_status, text_submit_btn, pdf pimodalChatbotdf_submit_btn, audio_submit_btn, 
                     image_submit_btn, video_submit_btn, combined_submit_btn]
         )
         
         text_submit_btn.click(
             process_text_input,
             inputs=[api_key_input, text_input, text_chatbot],
-            outputs=[text_chatbot, text_input]
+            outputs=[text_chatbot, text upset_btn.click(
+            process_pdf_input,
+            inputs=[api_key_input, pdf_input, pdf_text_input, pdf_chatbot],
+            outputs=[pdf_chatbot, pdf_text_input]
         )
         text_input.submit(
             process_text_input,
@@ -571,7 +586,7 @@ def create_interface():
         gr.Markdown("""
         ### 🎯 How to Use Each Tab:
         
-        **💬 Text Chat**: Simple text conversations with the AI
+        **💬 Text Chat**: Simple text conversations with the AI using Gemma
         
         **📄 PDF Chat**: Upload a PDF and ask questions about its content
         
@@ -579,10 +594,10 @@ def create_interface():
         - Supports: WAV, MP3, M4A, FLAC, OGG formats
         - Best results with clear speech and minimal background noise
         
-        **🖼️ Image Chat**: Upload images for analysis using Gemma 3n
+        **🖼️ Image Chat**: Upload images for analysis using BLIP
         - Provide a text prompt to guide the analysis (e.g., "What is in this image?")
         
-        **🎥 Video Chat**: Upload videos for basic frame analysis using Gemma 3n
+        **🎥 Video Chat**: Upload videos for basic frame analysis using BLIP
         - Analysis is based on a single frame; provide a text description for full video context
         
         **🌟 Combined Chat**: Use multiple input types together for comprehensive analysis
@@ -598,6 +613,7 @@ def create_interface():
         - Image and video analysis may be slow on CPU in Hugging Face Spaces
         - Video analysis is limited to a single frame due to CPU constraints
         - Large files may take longer to process
+        - BLIP model may provide basic captions; detailed video descriptions require additional user input
         """)
     
     return demo
@@ -616,7 +632,7 @@ if __name__ == "__main__":
         "torch"
     ]
     
-    print("🚀 Multimodal Chatbot with Gemma 3n")
+    print("🚀 Multimodal Chatbot with BLIP and Gemma")
     print("=" * 50)
     print("Required packages:", ", ".join(required_packages))
     print("\n📦 To install: pip install " + " ".join(required_packages))