Update app.py

app.py

@@ -4,22 +4,83 @@ from langdetect import detect
 from huggingface_hub import login
 import os
 import base64
+from PIL import Image
+from transformers import BlipProcessor, BlipForConditionalGeneration
+import requests
+from bs4 import BeautifulSoup
+import cv2
+from io import BytesIO
 
 login(token=os.getenv("chatbot"))
 generator = pipeline("text-generation", model="mistralai/Mistral-7B-Instruct-v0.1")
 bg_to_en = pipeline("translation", model="Helsinki-NLP/opus-mt-bg-en")
 en_to_bg = pipeline("translation", model="Helsinki-NLP/opus-mt-en-bg")
+# Load BLIP for image captioning
+blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
+blip_model.to("cuda" if torch.cuda.is_available() else "cpu")
 
-def generate_response(user_input, top_p, temperature, chat_counter, chatbot, history, request: gr.Request):
+def generate_response(user_input, top_p, temperature, chat_counter, chatbot, history, image=None, video=None, url=None, request: gr.Request = None):
     lang = detect(user_input)
     print(f"Detected language: {lang}")
 
     # Translate if needed
     if lang == "bg":
         user_input_translated = bg_to_en(user_input)[0]["translation_text"]
+
+            prompt = ""
+        
+    # Multimodal additions
+    if image is not None:
+        try:
+            img_desc = describe_image(image)
+            prompt += f"[Image Description]: {img_desc}\n"
+        except Exception as e:
+            prompt += f"[Image Error]: {str(e)}\n"
+
+    if video is not None:
+        try:
+            vid_desc = describe_video(video)
+            prompt += f"[Video Description]: {vid_desc}\n"
+        except Exception as e:
+            prompt += f"[Video Error]: {str(e)}\n"
+
+    if url:
+        url_content = extract_text_from_url(url)
+        prompt += f"[URL Content]: {url_content}\n"
+
+    # Append user message to prompt
+    prompt += f"User said:\n{user_input_translated}\n\nGive simple, friendly, clear advice:\n"
+
     else:
         user_input_translated = user_input
 
+def describe_image(image):
+    inputs = blip_processor(images=image, return_tensors="pt").to(blip_model.device)
+    out = blip_model.generate(**inputs)
+    caption = blip_processor.decode(out[0], skip_special_tokens=True)
+    return caption
+
+def describe_video(video_path):
+    cap = cv2.VideoCapture(video_path)
+    success, frame = cap.read()
+    cap.release()
+    if success:
+        img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+        return describe_image(img)
+    else:
+        return "Could not read video frame."
+
+def extract_text_from_url(url):
+    try:
+        response = requests.get(url, timeout=5)
+        soup = BeautifulSoup(response.text, 'html.parser')
+        paragraphs = soup.find_all('p')
+        text = ' '.join([p.text for p in paragraphs])
+        return text[:2000]  # keep it short
+    except Exception as e:
+        return f"Failed to fetch URL: {str(e)}"
+
     # Create prompt
     prompt = f"""
 Comprehensive E-commerce Bot Prompt:
@@ -301,6 +362,9 @@ with gr.Blocks(theme=masterbrand_theme, css=custom_css, title="MasterBrand AI As
 
             inputs = gr.Textbox(
                 placeholder="Ask me anything about your e-commerce business... 💡",
+                image_input = gr.Image(label="Upload an Image (optional)", type="pil", optional=True)
+                video_input = gr.Video(label="Upload a Video (optional)", optional=True)
+                url_input = gr.Textbox(label="Paste a URL (optional)", placeholder="https://example.com", optional=True)
                 label="Your Question",
                 lines=3,
                 elem_classes=["input-container"],
@@ -359,9 +423,19 @@ with gr.Blocks(theme=masterbrand_theme, css=custom_css, title="MasterBrand AI As
 
     # Event Handlers
     inputs.submit(reset_textbox, [], [inputs, submit_btn], queue=False)
-    inputs.submit(generate_response, [inputs, top_p, temperature, chat_counter, chatbot, state], [chatbot, state, chat_counter, status_box, inputs, submit_btn])
-    submit_btn.click(reset_textbox, [], [inputs, submit_btn], queue=False)
-    submit_btn.click(generate_response, [inputs, top_p, temperature, chat_counter, chatbot, state], [chatbot, state, chat_counter, status_box, inputs, submit_btn])
+inputs.submit(
+    generate_response,
+    [inputs, top_p, temperature, chat_counter, chatbot, state, image_input, video_input, url_input],
+    [chatbot, state, chat_counter, status_box, inputs, submit_btn]
+)
+
+submit_btn.click(reset_textbox, [], [inputs, submit_btn], queue=False)
+submit_btn.click(
+    generate_response,
+    [inputs, top_p, temperature, chat_counter, chatbot, state, image_input, video_input, url_input],
+    [chatbot, state, chat_counter, status_box, inputs, submit_btn]
+)
+
 
 # ==== Launch ====
 if __name__ == "__main__":