fix web search, pass image url to llm

Agents/agent.py

@@ -19,7 +19,7 @@ from langchain_core.messages import (
     convert_to_messages,
 )
 from pydantic import BaseModel
-from Tools.tools import webSearch, arxivSearch, wikiSearch,add,multiply,divide,substract, modulus
+from Tools.tools import webSearch, youtubeVideoTranscript, arxivSearch, wikiSearch,add,multiply,divide,substract,modulus,power,count_substring
 from langchain_core.messages import SystemMessage, HumanMessage
 from dotenv import load_dotenv
 from supabase.client import Client, create_client
@@ -120,7 +120,10 @@ tools = [
     add,
     substract,
     divide,
-    modulus
+    modulus,
+    power,
+    count_substring,
+    youtubeVideoTranscript
 ]
 
 def tools_condition1(

Tools/tools.py

@@ -93,16 +93,17 @@ def arxivSearch(searchQuery:str) -> str:
         ])
     return {"arxiv_result": formatted_results}
 
+
 @tool
 def webSearch(searchQuery:str) -> str:
     """
-      search the web using Tavily to get three matching results
+      search the web using Tavily to get 2 matching results
 
       args: 
         searchQuery: search query
     """
-    print("web_search")
-    search_results = TavilySearchResults(max_results=3).invoke(query=searchQuery)
+    print("web_search: {searchQuery}")
+    search_results = TavilySearchResults(max_results=2).invoke(input=searchQuery)
     formatted_results = "\n\n--\n\n".join(
         [
             f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page","")}"/>\n{doc.page_content}\n</Document>'
@@ -112,9 +113,9 @@ def webSearch(searchQuery:str) -> str:
     return {"web_search": formatted_results}
 
 @tool
-def youtubeTranscript(youtubeURL:str) -> str:
+def youtubeVideoTranscript(youtubeURL:str) -> str:
     """
-      obtain youtube transcript by passing in the youtube url
+      Get youtube video transcript by passing in the youtube url
 
       args: 
         youtubeURL: youtube url to pull out the transcript
@@ -129,5 +130,25 @@ def youtubeTranscript(youtubeURL:str) -> str:
     formatted_results = "\n\n".join(map(repr, loader.load()))
     return {"Youtube transcript":formatted_results}
 
+@tool
+def power(a: float, b: float) -> float:
+    """
+    Get the power of two numbers.
+    Args:
+        a (float): the first number
+        b (float): the second number
+    """
+    return a**b
+
+@tool
+def count_substring(substring:str, text:str) -> int:
+    """
+    Get the number of occurences of a substring within some text. Useful for 'How many (substring) are in (text)?'
+    Args:
+        substring (str): the substring to check for.
+        text (str): the text to search through.
+    """
+    return text.count(substring)
+
 
     

app.py

@@ -16,6 +16,7 @@ langfuse_handler = CallbackHandler()
 testMode = bool(os.getenv("TestMode"))
 langFuseOn = bool(os.getenv("LangFuseOn"))
 agentType = os.getenv("AgentType")
+gaiaValidationURL = os.getenv("GaiaValidationURL")
 
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
@@ -39,9 +40,13 @@ class BasicAgent:
     def __init__(self):
         print("BasicAgent initialized.")
         self.graph = build_graph()
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, imageURL: str="") -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
-        messages = [HumanMessage(content=question)]
+        if(imageURL.strip == ""):
+             messages = [HumanMessage(content=question)]
+        else:
+            formattedImageURL = gaiaValidationURL + imageURL
+            messages = [HumanMessage(content=question, additional_kwargs={imageURL:formattedImageURL })]
 
         if(not langFuseOn):
             print("no langfuse")
@@ -140,13 +145,14 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         print(f"Running agent on {len(questions_data)} questions...")
         for item in questions_data:
             task_id = item.get("task_id")
+            file_name = item.get("file_name")
             question_text = item.get("question")
             print(f"running on Question data {question_text}")
             if not task_id or question_text is None:
                 print(f"Skipping item with missing task_id or question: {item}")
                 continue
             try:
-                submitted_answer = agent(question_text)
+                submitted_answer = agent(question_text,file_name)
                 answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
                 results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
             except Exception as e: