Update app.py

app.py

@@ -95,18 +95,40 @@ class BasicAgent:
         )
         print("BasicAgent initialized.")
         # print("Available tools:", [tool.name for tool in self.agent.tools])
+    # def __call__(self, question):
+    #     if isinstance(question, dict):
+    #         text = question.get("question", "")
+    #         image = question.get("image", None)
+    #         if image:
+    #             question["image"] = {"type": "pil", "data": image}
+    #     else:
+    #         text = question
+    
+    #     print(f"Agent received question (first 50 chars): {text[:50]}...")
+    #     answer = self.agent.run(text)
+    #     return answer.strip()
     def __call__(self, question):
         if isinstance(question, dict):
             text = question.get("question", "")
             image = question.get("image", None)
-            if image:
-                question["image"] = {"type": "pil", "data": image}
         else:
             text = question
+            image = None
     
         print(f"Agent received question (first 50 chars): {text[:50]}...")
-        answer = self.agent.run(text)
-        return answer.strip()
+    
+        prompt = system_prompt + "\n\nUser: " + text.strip()
+        inputs = {}
+    
+        if image:
+            try:
+                image_caption = image_captioner(image=image, question=text)
+                prompt += f"\n\nThe image contains: {image_caption}"
+                inputs["image"] = image
+            except Exception as e:
+                print(f"Image captioning failed: {e}")
+    
+        return self.agent.run(prompt, inputs=inputs).strip()
 
 
 
@@ -180,7 +202,7 @@ def ask_agent(question):
                 image = download_image(image_url)
                 if image:
                     # Use the ImageCaptioningTool to get a caption
-                    image_captioner = [tool for tool in tools if tool.name == "image-captioning"][0] 
+                    image_captioner = [tool for tool in tools if tool.name == "image_captioner "][0] 
                     image_caption = image_captioner(image=image, question=question)
                     #Append the caption to the user's original question
                     prompt +=f"\n\nThe image contains: {image_caption}"
@@ -269,7 +291,8 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
                 except Exception as e:
                     print(f"Failed to decode image for task {task_id}: {e}")
 
-            submitted_answer = agent(question_input)
+            # submitted_answer = agent(question_input)
+            submitted_answer = ask_agent(question_text)
     
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
@@ -385,6 +408,10 @@ if __name__ == "__main__":
 
     print("Launching Gradio Interface for Basic Agent Evaluation...")
     demo.launch(debug=True, share=False)
+
+# DEBUG TEST
+agent = BasicAgent(model, tools)
+print(agent("What is the capital of France?"))
 # import os
 # import pandas as pd
 # import requests