Update app.py

app.py

@@ -4,7 +4,7 @@ import requests
 import inspect
 import pandas as pd
 from smolagents import CodeAgent, InferenceClientModel, DuckDuckGoSearchTool,Tool,tool,VisitWebpageTool,FinalAnswerTool
-
+import base64
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -79,7 +79,7 @@ class BasicAgent:
                 df = pd.read_excel(file_path)
             return df.to_string()
 
-        """class ReadPDFTool(Tool):
+        class ReadPDFTool(Tool):
             name = "read_pdf"
             description = "Extracts text from a PDF file."
             inputs = {
@@ -93,7 +93,56 @@ class BasicAgent:
             def forward(self, file_path: str) -> str:
                 from pypdf import PdfReader
                 reader = PdfReader(file_path)
-                return "\n".join(page.extract_text() or "" for page in reader.pages)"""
+                return "\n".join(page.extract_text() or "" for page in reader.pages)
+
+        @tool
+        def analyze_image(file_path: str, question: str) -> str:
+            """
+            Analyzes an image and answers a question about its contents.
+            
+            Args: 
+                 file_path: Local path to the image file.
+                 question: What to look for or answer about the image.
+
+            Returns: 
+                 A text answer describing what's found in the image.
+                  """
+            client = InferenceClient(token=os.environ["HF_TOKEN"])
+
+            with open(file_path, "rb") as f:
+                image_bytes = f.read()
+            image_b64 = base64.b64encode(image_bytes).decode("utf-8")
+
+             result = client.chat_completion(
+                 model="Qwen/Qwen2.5-VL-72B-Instruct",
+                  messages=[
+                      {
+                          "role": "user",
+                          "content": [
+                              {"type": "text", "text": question},
+                              {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
+                          ]
+                      }
+                  ]
+             )
+            return result.choices[0].message.content
+        @tool
+        def transcribe_audio(file_path: str) -> str:
+            """
+            Transcribes speech from an audio file to text.
+            
+            Args:
+                file_path: Local path to the audio file.
+                
+            Returns:
+                The transcribed text.
+                """
+            client = InferenceClient(token=os.environ["HF_TOKEN"])
+            result = client.automatic_speech_recognition(
+                file_path,
+                model="openai/whisper-large-v3",
+            )
+            return result.text
         #pdf_tool = load_tool(
             #"matterattetatte/pdf-upload-extractor-tool", 
            # trust_remote_code = True
@@ -110,11 +159,15 @@ class BasicAgent:
                 #read_pdf,
                 read_spreadsheet,
                 get_youtube_transcript,
+                analyze_image,
+                transcribe_audio,
                 ],
             model= InferenceClientModel("Qwen/Qwen2.5-Coder-32B-Instruct") ,
-            #additional_authorized_imports=["pandas", "requests", "re"],
+            additional_authorized_imports=["pandas", "requests", "re"],
             instructions = ("You are an advanced CodeAgent that will show your capabilities to work in the real world by being tested in GAIA, the agent testing platform. If the question includes a task_id and mentions a file, call fetch_task_file first; route YouTube URLs to get_youtube_transcript, other URLs to visit_webpage, PDFs to pdf_tool, and spreadsheets to read_spreadsheet, using web_search only when no URL or file is given,then respond with only the exact final answer value, no explanation, no prefix."
-                           "Use the Thought Action observation to produce high quality results and only answer when you are sure you have performed the necessary steps for the task and question"),
+                           "Use the Thought Action observation to produce high quality results and only answer when you are sure you have performed the necessary steps for the task and question"
+                           "If the file is an image, use analyze_image with a specific question about "
+                           "what to find. If the file is audio, use transcribe_audio first, then reason over the transcribed text "),
             max_steps=20,
         )
 #answering questions