Update tools.py

tools.py

@@ -6,8 +6,17 @@ import pandas as pd
 import whisper
 import os
 import tempfile
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+from PIL import Image
+import torch
 
 whisper_model = whisper.load_model("base")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Load model and processor once
+processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
+model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b", device_map="auto").to(device)
+
 
 @tool
 def add(a: int, b: int) -> int:
@@ -55,6 +64,25 @@ def divide(a: int, b: int) -> int:
         raise ValueError("Cannot divide by zero.")
     return a / b
 
+
+@tool
+def analyze_image(image_path: str, question: str = "What’s in this image?") -> str:
+    """
+    Analyzes an image and answers a question about it using BLIP-2.
+    
+    Args:
+        image_path (str): Path to the image file.
+        question (str): A natural language question about the image.
+    """
+    try:
+        image = Image.open(image_path).convert("RGB")
+        inputs = processor(images=image, text=question, return_tensors="pt").to(device)
+        generated_ids = model.generate(**inputs, max_new_tokens=100)
+        response = processor.decode(generated_ids[0], skip_special_tokens=True)
+        return response
+    except Exception as e:
+        return f"Error analyzing image: {str(e)}"
+
 @tool
 def read_excel_summary(file_path: str) -> str:
     """