Multiple updates refining the design with updated prompt , updated tool set

app.py

@@ -23,12 +23,52 @@ from langchain_openai import ChatOpenAI
 from openai import OpenAI
 
 # tools imported from helper.py
-from helper import repl_tool, get_travily_api_search_tool,audio_transcriber_tool,wikipedia_search_tool,file_saver_tool,wikipedia_full_content_tool,serpapi_Google_Search_tool,gemini_multimodal_tool
+from helper import repl_tool, get_travily_api_search_tool,audio_transcriber_tool,wikipedia_search_tool,gemini_multimodal_tool, load_text_file_tool
 
 # (Keep Constants as is)
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+from PIL import Image
+import PIL
+from io import BytesIO
+from openai import OpenAI
+
+def download_file(task_id):
+    file_response = requests.get(f"{DEFAULT_API_URL}/files/{task_id}")
+    return BytesIO(file_response.content)
+
+
+def save_local(buffer, fname):
+    os.makedirs('file_cache/', exist_ok=True)
+    path = os.path.join('file_cache', fname)
+    with open(path, 'wb') as outfile:
+        outfile.write(buffer.read())
+    return path
+
+
+def download_image(task_id: str):
+    return Image.open(download_file(task_id))
+
+
+def attachment_kwargs(task_id, fname):
+    kwargs = {}
+    ext = os.path.splitext(fname)[1].lower()
+    if ext in ['.jpg', '.jpeg', '.png']:
+        kwargs['images'] = [download_image(task_id)]
+    elif ext:
+        buffer = download_file(task_id)
+        path = save_local(buffer, fname)
+        kwargs['additional_args'] = {
+            "file": {
+                "name": fname,
+                "type": ext,
+                "path": path
+            }
+        }
+    return kwargs
+
+
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
 class BasicAgent:
@@ -63,12 +103,12 @@ class BasicAgent:
         # Adjust this check if your error type is different
         return isinstance(e, genai.errors.APIError) and getattr(e, "code", None) in {429, 503}
 
-    def invoke_with_retry(self,question: str, max_retries: int = 5, initial_delay: float = 10.0) -> str:
+    def invoke_with_retry(self,question: str, max_retries: int = 5, initial_delay: float = 10.0, kwargs=None) -> str:
         current_delay = initial_delay
         for attempt in range(max_retries):
             try:
                 result = self.agent_obj.invoke(
-                    {"input": question},
+                    {"input": question, **kwargs} if kwargs else {"input": question},
                     config={"configurable": {"session_id": "test-session"}},
                 )
                 return result['output']
@@ -91,11 +131,11 @@ class BasicAgent:
         # If all retries fail, raise a RuntimeError
         raise RuntimeError(f"Max retries ({max_retries}) exceeded due to persistent quota errors or other retriable issues.")
 
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, additional_args=None) -> str:
         """
         Allows the instance to be called directly to get an AgentExecutor.
         """
-        return self.invoke_with_retry(question)
+        return self.invoke_with_retry(question,additional_args)
 
 def run_and_submit_all( profile: gr.OAuthProfile | None):
     """
@@ -158,172 +198,64 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     print(f"Using Tavily API key: {tavily_api_key[:4]}... (truncated for security)")
 
     travily_api_search_tool = get_travily_api_search_tool(tavily_api_key)
-    tools = [ repl_tool, file_saver_tool,audio_transcriber_tool,travily_api_search_tool, gemini_multimodal_tool]
-
-    EX5_OBSERVATION_STRING = (
-    "[{{'title': '1977 New York Yankees Hitting Stats - Baseball-Reference.com', "
-    "'url': 'https://www.baseball-reference.com/teams/NYY/1977.shtml', "
-    "'content': '| Rk | Player | Age | Pos | WAR | W | L | W-L% | ERA | G | GS | GF | CG | SHO | SV | IP | H | R | ER | HR | BB | IBB | SO | HBP | BK | WP | BF | ERA+ | FIP | WHIP | H9 | HR9 | BB9 | SO9 | SO/BB | Awards | All logos are the trademark & property of their owners and not Sports Reference LLC. Copyright © 2000-2025 Sports Reference LLC. Sports Info Solutions logo Sports Info Solutions logo Sports Info Solutions logo'}}]"
-    )
+    tools = [ repl_tool,audio_transcriber_tool,travily_api_search_tool, gemini_multimodal_tool, wikipedia_search_tool, load_text_file_tool]
 
     prompt = PromptTemplate(
-    input_variables=["input", "agent_scratchpad", "chat_history", "tool_names"],
+    input_variables=["input", "agent_scratchpad", "tool_names"],
     template="""
-    You are a smart and helpful AI Agent/Assistant that excels at fact-based reasoning. You are allowed and encouraged to use one or more tools as needed to answer complex questions and perform tasks.
-    It is CRUCIAL that you ALWAYS follow the exact format below. Do not deviate.
-    NOTE: it is MANDATORY for you to be precise and concise in your response. Respond directly with ONLY the answer, without any introductory phrases or additional details.
-    For example, if asked for the number of letters in the English alphabet, respond with '26'. Do NOT say "The number of letters is 26."
-
-    You have access to the following tools:
-    {tools}
-
-    To use a tool, you MUST follow this precise format:
-
-    Thought: I need to use a tool to find the answer.
-    Action: [tool_name] # This will be one of [{tool_names}]
-    Action Input: [input_for_the_tool]
-    Observation: [result_from_the_tool]
-
-    IMPORTANT NOTE ON TOOL USAGE:
-    - If an 'Observation' from a tool, especially `tavily_search` or `serpapi_Google Search_tool`, contains a list, table, or structured text that might hold the answer, your next step should be to use `python_repl` to parse and extract the required information from that observation's content. Do NOT search again unless the content is genuinely insufficient or irrelevant.
-    - If an 'Observation' from a tool does NOT directly contain the specific answer to your question, you MUST refine your query or switch to a different, more suitable tool (e.g., 'tavily_search' for broader or more current information if 'wikipedia_search_tool' was insufficient). Do NOT get stuck repeatedly using the same tool if it's not yielding the direct answer.
-    - If the input contains the exact phrase "Attachment '{{file_name}}' available at: {{attachment_url}}" (where '{{file_name}}' and '{{attachment_url}}' are placeholders for actual values), consider the file type:
-      - If the file type is binary/text (e.g., .xlsx, .docx, .mp3, .jpg, .pdf,.png), you MUST use the 'file_saver' tool to download and save it.
-        For 'file_saver', the Action Input must be a JSON string like: '{{"url": "the_attachment_url", "local_filename": "the_file_name_from_attachment"}}'
-        example: for input, Attachment '1f975693-876d-457b-a649-393859e79bf3.mp3' available at EXACT URL: https://agents-course-unit4-scoring.hf.space/files/1f975693-876d-457b-a649-393859e79bf3, Action Input for file_saver would be '{{"url": "https://agents-course-unit4-scoring.hf.space/files/1f975693-876d-457b-a649-393859e79bf3", "local_filename": "1f975693-876d-457b-a649-393859e79bf3.mp3"}}'
-
-    IMPORTANT: When processing audio files (like .mp3) that have been saved using 'file_saver', the 'audio_transcriber_tool' MUST be used with the 'local_filename' of the saved audio file as its Action Input. Do NOT pass URLs or remote paths directly to 'audio_transcriber_tool'.
-
-    **For image files (like .jpg, .png) that have been saved using 'file_saver', the 'gemini_multimodal_tool' MUST be used to analyze their content and answer questions based on the image. The Action Input for 'gemini_multimodal_tool' must be a JSON string like: '{{"image_path": "the_local_filename", "question": "the_user_question"}}'**
-
-    Example: given a chess board image and asked to predict the next best move, if Multi-modal LLM is available, you can use it to answer the question.
-
-    If you have sufficient information and can provide a CONCISE response, or if no tool is needed, you MUST use this precise format:
-
-    Thought: I have enough information, or no tool is needed.
-    Final Answer: [your concise/short response here]
-
-    NOTE: it is MANDATORY for you to be precise and concise in your response. Respond directly with ONLY the answer, without any introductory phrases or additional details.
-    For example, if asked for the number of letters in the English alphabet, respond with '26'. Do NOT say "The number of letters is 26."
-    VERY IMPORTANT: Your response MUST always start with 'Thought:'.
-
-    Here are some examples of how you should respond:
-
-    Example 1:
-    Question: What is the capital of France?
-    Thought: I need to use a tool to find the capital of France.
-    Action: tavily_search
-    Action Input: capital of France
-    Observation: The capital of France is Paris.
-    Thought: I have found the answer.
-    Final Answer: Paris
-
-    Example 2:
-    Question: What is 2 + 2?
-    Thought: This is a simple arithmetic question, no tool is needed.
-    Final Answer: 4
-
-    Example 3:
-    Question: How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.
-    Thought: The user is asking for specific information about discography, which might be found with a search tool. The `serpapi_Google Search_tool` can fetch detailed sections. After getting the content, I will need to parse it using `python_repl` to count the albums within the specified years.
-    Action: serpapi_Google Search
-    Action Input: Mercedes Sosa discography
-    Observation: [Discography text content from search result]
-    Thought: I have retrieved discography text. Now I need to parse this text to identify and count studio albums released between 2000 and 2009. I will use the `python_repl` tool for this.
-    Action: python_repl
-    Action Input:
-    ```python
-    import re
-    text = "[Discography text content from previous observation]" # Replace with actual text
-    albums_2000_2009 = []
-    pattern = r"\((\d{{4}})\)\s*(.*?)(?:\[|\n|$)" # Ensures year is captured. Double braces {{}} to escape regex literal braces
-    for match in re.finditer(pattern, text):
-        year = int(match.group(1))
-        if 2000 <= year <= 2009:
-            albums_2000_2009.append(match.group(2).strip())
-    print(len(albums_2000_2009))
-    ```
-    Observation: 3
-    Thought: I have parsed the discography and counted the albums. I have found the answer.
-    Final Answer: 3
-
-    **Example 4: (Crucial new example for image processing)**
-    Question: What is the next best move in this chess position? Attachment 'chess_board.png' available at EXACT URL: https://agents-course-unit4-scoring.hf.space/files/cca530fc-4052-43b2-b130-b30968d8aa44
-    Thought: The user is asking a question about a chess position and has provided an image. I need to first save the image locally using the 'file_saver' tool, and then use the 'gemini_multimodal_tool' to analyze the image and answer the question.
-    Action: file_saver
-    Action Input: {{"url": "https://agents-course-unit4-scoring.hf.space/files/cca530fc-4052-43b2-b130-b30968d8aa44", "local_filename": "cca530fc-4052-43b2-b130-b30968d8aa44.png"}}
-    Observation: File downloaded successfully to cca530fc-4052-43b2-b130-b30968d8aa44.png
-    Thought: The image has been successfully downloaded. Now I need to analyze its content to determine the next best chess move using the 'gemini_multimodal_tool'.
-    Action: gemini_multimodal_tool
-    Action Input: {{"image_path": "cca530fc-4052-43b2-b130-b30968d8aa44.png", "question": "What is the next best move in this chess position?"}}
-    Observation: The next best move is e4.
-    Thought: I have used the 'gemini_multimodal_tool' to get the best move based on the image.
-    Final Answer: e4
-
-    Example 5: (Crucial negative example for conciseness)
-    Question: What is the opposite of up?
-    Thought: The question asks for the opposite of up. This is a direct knowledge question.
-    Final Answer: down
-    DO NOT RESPOND LIKE THIS: The opposite of up is down. or The answer is down.
-
-    Example 6: (New example for parsing baseball stats)
-    Question: How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?
-    Thought: I need to find the Yankee player with the most walks in 1977 and then find their at bats. This will require searching for Yankees 1977 stats and then parsing the results to extract the relevant player and their at-bats. I will use 'tavily_search' first to find the stats. After getting the search results, I will examine their content for a list or table of players and their stats. If found, I will use 'python_repl' to parse it.
-    Action: tavily_search
-    Action Input: New York Yankees 1977 batting stats
-    Observation: {{EX5_OBSERVATION_STRING}}
-    Thought: I have received an observation from `tavily_search`. I need to examine its `content` to determine if it contains the necessary data (e.g., a list or table of players/stats/winners). If so, my next step is to use `python_repl` to parse this content to extract the specific information needed to answer the question. I should only consider another `tavily_search` if the current observation's content is clearly insufficient.
-    Action: python_repl
-    Action Input:
-    ```python
-    # Example: Parse the text content from the tavily_search observation.
-    # This is a placeholder for the actual parsing logic you would write.
-    # For the Malko question, you would parse the list of winners and their nationalities.
-    # For instance, if the observation content contains:
-    # "1983 | Claus Peter Flor | East Germany"
-    # You would extract this and apply your filtering logic.
-    ```
-    Observation: 519
-    Thought: I have parsed the data and identified Roy White as the Yankee with the most walks (75) in 1977, and his at-bats were 519. I have found the answer.
-    Final Answer: 519
-
-    Example 7: (Parsing a table for minimum value)
-    Question: What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.
-    Thought: I need to find a table of athlete counts by country for the 1928 Olympics. I will use 'tavily_search' to find the data. After getting the search results, I will examine their content for a list or table of countries and their athlete counts. If found, I will use 'python_repl' to parse it.
-    Action: tavily_search
-    Action Input: 1928 Summer Olympics athlete count by country
-    Observation: [Table or HTML/text with country and athlete counts]
-    Thought: I have found the table in the `tavily_search` observation. Now I need to parse it to find the country with the least athletes, and if there is a tie, pick the first alphabetically. I will use 'python_repl' for this.
-    Action: python_repl
-    Action Input:
-    ```python
-    # Example: python code to parse the text table and find the IOC country code with the least athletes.
-    # (Replace this with actual code as needed)
-    ```
-    Observation: LUX Thought: I have found the country with the least athletes.
-    Final Answer: LUX
-
-    ---
-    Previous conversation history:
-    {chat_history}
-
-    New input: {input}
-    ---
-    {agent_scratchpad}
+        You are a highly capable and autonomous agent designed to solve complex, multi-step problems accurately and efficiently. Your goal is to answer the user's question based on the provided information and tools.
+
+        **GUIDING PRINCIPLES:**
+
+        1.  **Deconstruct the Goal:** Carefully analyze the user's question (`input`). Break it down into a logical sequence of smaller, manageable steps.
+        2.  **Think Step-by-Step:** For each step, reason about the best tool to use to gather the necessary information or perform the required action.
+        3.  **Pay Attention to Detail:** Meticulously follow all constraints in the user's prompt, especially regarding the final answer's format (e.g., "comma-separated list", "alphabetical order", "algebraic notation", "first name only").
+        4.  **Be Self-Reliant:** The user's request is complete. Do not ask for clarification. Use your tools to find the answer. If a file is mentioned, use a tool to read it.
+        5.  **Synthesize:** Once all necessary information is gathered, synthesize it to formulate the final, precise answer.
+
+        **TOOLS:**
+
+        You have access to the following tools. You must use them one at a time to find the information you need.
+
+        {{tools}}
+
+        **RESPONSE FORMAT:**
+
+        You must always use the following format for your responses. The `Thought`, `Action`, `Action Input` cycle can repeat multiple times.
+
+        **Thought:** Your internal monologue. Analyze the user's request, break it down into steps, and decide which tool to use next. Criticize your own plan.
+        **Action:** The name of the single tool to use from the list above.
+        **Action Input:** The specific input to send to that tool.
+        **Observation:** [The result of the tool will be inserted here by the system]
+
+        ... (This Thought/Action/Action Input/Observation cycle can repeat N times) ...
+
+        **Thought:** I have now gathered all the necessary information and can confidently answer the user's original question. I will now format the final answer exactly as requested.
+        **Final Answer:** [The final, direct, and precisely formatted answer to the user's question]
+        NOTE: Final Answer format: YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+        Remember that you can open xlsx files using pandas
+        Remember that you are smart and a perceptive reader! If you need to extract information from a piece of text, you should be able to do that just by reading the text and without writing code to extract entities.
+        Begin!
+
+        **User's Question:** {{input}}
+        **Scratchpad:** {{agent_scratchpad}}
+
     """
     )
 
-    summary_memory = ConversationSummaryMemory(llm=llm_client, memory_key="chat_history")
+    #summary_memory = ConversationSummaryMemory(llm=llm_client, memory_key="chat_history")
 
 
     # Initialize gemini model with streaming enabled
     
+    '''
     summary_llm = ChatGoogleGenerativeAI(
         model=gemini_model,
         google_api_key=google_api_key,
         temperature=0,
         streaming=True
     )
+    '''
     
 
     #summary_llm = ChatOpenAI(model='gpt-4o', temperature=0, streaming=False,api_key=openai_api_key)
@@ -331,15 +263,15 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
 
 
     # Create a ReAct agent
-    summary_react_agent = create_react_agent(
-        llm=summary_llm,
+    my_react_agent = create_react_agent(
+        llm=llm_client,
         tools=tools,
         prompt=prompt
     )
 
     # 1. Instantiate Agent ( modify this part to create your agent)
     try:
-        agent = BasicAgent(summary_react_agent, tools, True, True, 5, summary_memory)
+        agent = BasicAgent(my_react_agent, tools, True, True, 30,None)
     except Exception as e:
         print(f"Error instantiating agent: {e}")
         return f"Error initializing agent: {e}", None
@@ -374,35 +306,32 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
-        file_name = item.get("file_name") # Get the file_name if it exists
-
-        # Construct the question string that your LLM will see,
-        # including the attachment URL if present.
-        full_question_for_agent = question_text
-        if file_name:
-            attachment_url = f"https://agents-course-unit4-scoring.hf.space/files/{task_id}"
-            full_question_for_agent += f"\n\nAttachment '{file_name}' available at EXACT URL: {attachment_url}"
-            print(f"Running agent on task {task_id}: {full_question_for_agent}",flush=True)
+        #file_name = item.get("file_name") # Get the file_name if it exists
+
+
+
         
-        '''
         allowed_ids = {
             #"cca530fc-4052-43b2-b130-b30968d8aa44",
             #"a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
-            "3f57289b-8c60-48be-bd80-01f8099ca449",
+            #"3f57289b-8c60-48be-bd80-01f8099ca449",
             #"2d83110e-a098-4ebb-9987-066c06fa42d0",
             #"cf106601-ab4f-4af9-b045-5295fe67b37d",
             #"7bd855d8-463d-4ed5-93ca-5fe35145f733",
-            "5a0c1adf-205e-4841-a666-7c3ef95def9d",
+            #"5a0c1adf-205e-4841-a666-7c3ef95def9d",
             "f918266a-b3e0-4914-865d-4faa564f1aef",
         }
         if task_id not in allowed_ids:
             continue
-        '''
+        
         try:
-            submitted_answer = agent(full_question_for_agent)
+            kwargs = attachment_kwargs(item['task_id'], item['file_name'])
+            submitted_answer = agent(question=question_text, **kwargs)
+
+            #submitted_answer = agent(full_question_for_agent)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
-            time.sleep(61)  # Add a 1 min delay before running the agent
+            #time.sleep(61)  # Add a 1 min delay before running the agent
         except Exception as e:
              print(f"Error running agent on task {task_id}: {e}")
              results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})