Integrate Project work to HF template

.gitattributes

@@ -1,37 +1,2 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -38,7 +38,6 @@ notebooks/tools_poc/*.pt
 *.npy
 *.npz
 *.joblib
-*.png
 
 ms-playwright/
 .playwright/

README.md

@@ -1,15 +1,110 @@
----
-title: Template Final Assignment
-emoji: 🕵🏻‍♂️
-colorFrom: indigo
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.25.2
-app_file: app.py
-pinned: false
-hf_oauth: true
-# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
-hf_oauth_expiration_minutes: 480
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Chappie
+emoji: 🤖
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "4.16.0"
+app_file: app.py
+pinned: false
+---
+
+# **🤖 Chappi-e**
+
+Chappi-e is a lightweight AI assistant agent built using the ********React architecture******** with [__LangGraph__](__https://docs.langgraph.dev/__). It is designed to reason, plan, and execute actions using tools in a flexible and modular workflow.
+
+---
+
+
+https://github.com/user-attachments/assets/8b1c4ffb-f1c3-426d-9ac7-77a81d04d991
+
+
+## **🛠️ Setup**
+
+This project uses [__Poetry__](__https://python-poetry.org/__) for dependency management.
+
+### **1. Clone the repository**
+
+```bash
+git clone https://github.com/yourusername/chappie.git
+cd chappie
+````
+
+_### 2. Install dependencies_
+
+Make sure you have Poetry installed. Then run:
+
+```bash
+env varibles
+sudo apt install ffmpeg
+poetry install
+```
+
+### **3. Activate the environment**
+
+```bash
+poetry shell # Then activate the shown environment with conda or pyenv
+```
+
+---
+
+## **🔐 API Keys Required**
+
+To run Chappie, you need valid API keys for the following services:
+
+* [__OpenAI__](__https://platform.openai.com/__)
+* [__LangGraph__](__https://www.langgraph.dev/__)
+* [__Tavily Search__](__https://docs.tavily.com/__)
+* [__LangChain__](__https://www.langchain.com/__) (used for tool wrappers/utilities)
+
+You can provide these via environment variables or a `.env` file.
+
+---
+
+## **🧠 Architecture**
+
+Chappie is a ********React-style agent********, meaning it follows a loop of:
+
+1. ********Reason******** about the query
+2. ********Act******** by calling a tool if needed
+3. ********Observe******** the result
+4. ********Repeat or respond********
+
+The logic is built using ********LangGraph********, which provides a composable and stateful way to define agent workflows.
+
+---
+
+## **🚀 Run the agent**
+
+You can invoke Chappie using:
+
+```bash
+python -m src.agents.react
+```
+
+---
+
+## **📄 License**
+
+MIT License.
+
+
+--- 
+
+# TODO
+
+1. Include tutorial to download stockfish ->
+```bash
+mkdir -p external/chess-modules
+curl -L https://github.com/official-stockfish/Stockfish/releases/latest/download/stockfish-ubuntu-x86-64-avx2.tar -o external/chess-modules/stockfish-ubuntu-x86-64-avx2.tar
+mkdir -p external/chess-modules/stockfish
+tar -xvf external/chess-modules/stockfish-ubuntu-x86-64-avx2.tar -C external/chess-modules/stockfish --strip-components=1
+rm external/chess-modules/stockfish-ubuntu-x86-64-avx2.tar
+```
+2. Change repo name from chappie to chappi-e
+Probably use an `init.sh` script
+
+---
+
+# References
+- For the Chess tool, we leveraged [board_to_fen](https://github.com/mcdominik/board_to_fen) and [Stockfish 17.1](https://github.com/official-stockfish/Stockfish)

app.py

@@ -1,196 +0,0 @@
-import os
-import gradio as gr
-import requests
-import inspect
-import pandas as pd
-
-# (Keep Constants as is)
-# --- Constants ---
-DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
-
-# --- Basic Agent Definition ---
-# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
-class BasicAgent:
-    def __init__(self):
-        print("BasicAgent initialized.")
-    def __call__(self, question: str) -> str:
-        print(f"Agent received question (first 50 chars): {question[:50]}...")
-        fixed_answer = "This is a default answer."
-        print(f"Agent returning fixed answer: {fixed_answer}")
-        return fixed_answer
-
-def run_and_submit_all( profile: gr.OAuthProfile | None):
-    """
-    Fetches all questions, runs the BasicAgent on them, submits all answers,
-    and displays the results.
-    """
-    # --- Determine HF Space Runtime URL and Repo URL ---
-    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
-
-    if profile:
-        username= f"{profile.username}"
-        print(f"User logged in: {username}")
-    else:
-        print("User not logged in.")
-        return "Please Login to Hugging Face with the button.", None
-
-    api_url = DEFAULT_API_URL
-    questions_url = f"{api_url}/questions"
-    submit_url = f"{api_url}/submit"
-
-    # 1. Instantiate Agent ( modify this part to create your agent)
-    try:
-        agent = BasicAgent()
-    except Exception as e:
-        print(f"Error instantiating agent: {e}")
-        return f"Error initializing agent: {e}", None
-    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
-    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
-    print(agent_code)
-
-    # 2. Fetch Questions
-    print(f"Fetching questions from: {questions_url}")
-    try:
-        response = requests.get(questions_url, timeout=15)
-        response.raise_for_status()
-        questions_data = response.json()
-        if not questions_data:
-             print("Fetched questions list is empty.")
-             return "Fetched questions list is empty or invalid format.", None
-        print(f"Fetched {len(questions_data)} questions.")
-    except requests.exceptions.RequestException as e:
-        print(f"Error fetching questions: {e}")
-        return f"Error fetching questions: {e}", None
-    except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
-    except Exception as e:
-        print(f"An unexpected error occurred fetching questions: {e}")
-        return f"An unexpected error occurred fetching questions: {e}", None
-
-    # 3. Run your Agent
-    results_log = []
-    answers_payload = []
-    print(f"Running agent on {len(questions_data)} questions...")
-    for item in questions_data:
-        task_id = item.get("task_id")
-        question_text = item.get("question")
-        if not task_id or question_text is None:
-            print(f"Skipping item with missing task_id or question: {item}")
-            continue
-        try:
-            submitted_answer = agent(question_text)
-            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
-        except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
-
-    if not answers_payload:
-        print("Agent did not produce any answers to submit.")
-        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
-
-    # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
-    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
-    print(status_update)
-
-    # 5. Submit
-    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
-    try:
-        response = requests.post(submit_url, json=submission_data, timeout=60)
-        response.raise_for_status()
-        result_data = response.json()
-        final_status = (
-            f"Submission Successful!\n"
-            f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', 'N/A')}% "
-            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
-            f"Message: {result_data.get('message', 'No message received.')}"
-        )
-        print("Submission successful.")
-        results_df = pd.DataFrame(results_log)
-        return final_status, results_df
-    except requests.exceptions.HTTPError as e:
-        error_detail = f"Server responded with status {e.response.status_code}."
-        try:
-            error_json = e.response.json()
-            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
-        except requests.exceptions.JSONDecodeError:
-            error_detail += f" Response: {e.response.text[:500]}"
-        status_message = f"Submission Failed: {error_detail}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.Timeout:
-        status_message = "Submission Failed: The request timed out."
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.RequestException as e:
-        status_message = f"Submission Failed: Network error - {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except Exception as e:
-        status_message = f"An unexpected error occurred during submission: {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-
-
-# --- Build Gradio Interface using Blocks ---
-with gr.Blocks() as demo:
-    gr.Markdown("# Basic Agent Evaluation Runner")
-    gr.Markdown(
-        """
-        **Instructions:**
-
-        1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
-        2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
-        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
-
-        ---
-        **Disclaimers:**
-        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
-        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
-        """
-    )
-
-    gr.LoginButton()
-
-    run_button = gr.Button("Run Evaluation & Submit All Answers")
-
-    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
-    # Removed max_rows=10 from DataFrame constructor
-    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
-
-    run_button.click(
-        fn=run_and_submit_all,
-        outputs=[status_output, results_table]
-    )
-
-if __name__ == "__main__":
-    print("\n" + "-"*30 + " App Starting " + "-"*30)
-    # Check for SPACE_HOST and SPACE_ID at startup for information
-    space_host_startup = os.getenv("SPACE_HOST")
-    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
-
-    if space_host_startup:
-        print(f"✅ SPACE_HOST found: {space_host_startup}")
-        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
-    else:
-        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
-
-    if space_id_startup: # Print repo URLs if SPACE_ID is found
-        print(f"✅ SPACE_ID found: {space_id_startup}")
-        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
-    else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
-
-    print("-"*(60 + len(" App Starting ")) + "\n")
-
-    print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)

data/agent_experiments/iterations/1_initial_experiment.csv

@@ -0,0 +1,47 @@
+Question,Agent response,Final answer,is_correct
+How many slides in this PowerPoint presentation mention crustaceans?,"I cannot access or analyze the content of a PowerPoint presentation directly. Please provide the relevant information or data, and I can assist you with any analysis or calculations needed.",4,0
+"If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.","To solve this problem, we need to follow these steps:
+
+1. **Determine the distance from Earth to the Moon at its closest approach (perigee).**
+   - According to Wikipedia, the minimum perigee distance is approximately 356,500 km.
+
+2. **Find Eliud Kipchoge's marathon pace.**
+   - Kipchoge's marathon world record is 2 hours, 1 minute, and 39 seconds for 42.195 km.
+   - Convert this time to hours: \(2 + \frac{1}{60} + \frac{39}{3600} \approx 2.0275\) hours.
+   - Calculate his speed: \( \frac{42.195}{2.0275} \approx 20.82 \) km/h.
+
+3. **Calculate the time it would take to run the distance to the Moon.**
+   - Use the formula: \( \text{Time} = \frac{\text{Distance}}{\text{Speed}} \).
+   - \( \text{Time} = \frac{356,500}{20.82} \approx 17120.65 \) hours.
+
+4. **Convert the time to thousand hours and round to the nearest 1000.**
+   - \( \frac{17120.65}{1000} \approx 17.12065 \).
+   - Rounded to the nearest 1000 hours: 17000.
+
+The answer is:
+
+17000",17,0
+"According to Girls Who Code, how long did it take in years for the percentage of computer scientists that were women to change by 13% from a starting point of 37%?",The percentage of computer scientists that were women changed by 13% from a starting point of 37% over a period of 30 years.,22,0
+"In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",`BaseForest`,BaseLabelPropagation,0
+"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",The work performed by R. G. Arendt was supported by NASA award number 80GSFC21M0002.,80GSFC21M0002,0
+What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,Gianluigi,Claus,0
+"Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?","""Japan""",Guatemala,0
+"You are Van Helsing, a renowned vampire hunter. A Count of Moldova, Lațcu IV, son of  Costea, has tasked you with investigating the village of Șirnea in neighboring Wallachia. The Count's advisors have reported that a vampire was spotted crossing the border near the village, and would like you to investigate it.
+
+You travel to the village of Șirnea, and you begin your investigation. One night, just before dawn, you catch a glimpse of a man in a long black cape with red lining leaping from roof-top to roof-top with superhuman agility. It's a vampire! You try to chase the creature back to its home, but the creature is too fast. However, because of the remoteness of the village, you know with absolute certainty that the vampire must be a resident of the village. You decide that your best course of action will be to visit all 100 residents of the town during the day. You know something about vampires and humans that will make your investigation possible; humans always tell the truth, but vampires always lie.
+
+In the afternoon, you go from house to house, speaking with all 100 residents of Șirnea. You ask everyone the same question: ""How many vampires are living in Șirnea"". Everyone in the village gives the same response, ""At least one of us is a human.""
+
+How many residents of Șirnea have been turned into vampires?",99,100,0
+"You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, ""-"", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",2,3,0
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",0.5,0.1777,0
+"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.","Tamai, Tanaka","Yoshida, Uehara",0
+"Here's a fun riddle that I think you'll enjoy.
+
+You have been selected to play the final round of the hit new game show ""Pick That Ping-Pong"". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.
+
+A device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.
+
+The ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.
+
+Which ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.","To maximize your odds of winning the big prize, you should choose ball number `1`.",3,0

data/agent_experiments/iterations/2_search_and_code_tools_integrated.csv

@@ -0,0 +1,74 @@
+Question,Agent response,Final answer,is_correct
+"In the year 2022, and before December, what does ""R"" stand for in the three core policies of the type of content that was violated in the public logs on the Legume Wikipedia page?","The search did not provide specific information about the ""R"" in the three core policies related to content violations on the Legume Wikipedia page. You may need to refer directly to Wikipedia's content policies or the page's edit history for detailed information.",research,0
+"In Audre Lorde’s poem “Father Son and Holy Ghost”, what is the number of the stanza in which some lines are indented?",3,2,0
+"In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",`BaseBagging`,BaseLabelPropagation,0
+How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.,I couldn't find the specific number of studio albums published by Mercedes Sosa between 2000 and 2009 from the search. Please check the latest version of English Wikipedia for detailed discography information.,3,0
+"In Emily Midkiff's June 2014 article in a journal named for the one of Hreidmar's sons that guarded his house, what word was quoted from two different authors in distaste for the nature of dragon depictions?","""grotesque""",fluffy,0
+"An office held a Secret Santa gift exchange where each of its twelve employees was assigned one other employee in the group to present with a gift. Each employee filled out a profile including three likes or hobbies. On the day of the gift exchange, only eleven gifts were given, each one specific to one of the recipient's interests. Based on the information in the document, who did not give a gift?","To determine who did not give a gift, we need to identify the employee who did not present a gift to their assigned recipient. Since there are twelve employees and only eleven gifts were given, one employee did not fulfill their Secret Santa obligation.
+
+Given that each gift was specific to the recipient's interests, we can deduce that the missing gift corresponds to the employee whose interests were not matched with a gift. Therefore, the employee who did not give a gift is the one whose assigned recipient did not receive a gift.
+
+Without specific information about the interests and the gifts given, we cannot directly identify the employee. However, if you provide the list of employees, their interests, and the gifts given, I can help determine who did not give a gift.",Fred,0
+"Bob was invited to participate in a game show, and he advanced to the final round. The final round offered Bob the chance to win a large sum by playing a game against the host. The host has 30 shiny prop coins, each of which is worth $1,000 if Bob manages to win them by playing the game. The host hides the coins in three different prize boxes and then shuffles their order. The only rule restricting the host's coin placement is that one box must contain at least 2 coins, and one box must contain 6 more coins than another box. In order to play, Bob must submit three guesses, one guess for the number of coins in each box. The box is then opened and the number of coins is revealed. If Bob's guess is a number greater than the number of coins in the box, Bob earns no coins. If Bob guesses a number equal to or less than the number of coins in the box, Bob wins a number of coins equal to his guess.
+
+If Bob plays uses the optimal strategy, what's the minimum amount of money he can win from the game?","To solve this problem, we need to determine the distribution of coins in the boxes that satisfies the given conditions and then find Bob's optimal strategy to maximize his winnings.
+
+### Conditions:
+1. Total coins = 30
+2. One box must contain at least 2 coins.
+3. One box must contain 6 more coins than another box.
+
+### Analysis:
+Let's denote the number of coins in the three boxes as \( x \), \( y \), and \( z \) such that \( x + y + z = 30 \).
+
+- Without loss of generality, assume \( x \leq y \leq z \).
+- From the conditions, we have:
+  - \( x \geq 2 \)
+  - \( z = x + 6 \)
+
+### Solving the Equations:
+Substitute \( z = x + 6 \) into the total:
+\[ x + y + (x + 6) = 30 \]
+\[ 2x + y + 6 = 30 \]
+\[ 2x + y = 24 \]
+\[ y = 24 - 2x \]
+
+Since \( x \leq y \), we have:
+\[ x \leq 24 - 2x \]
+\[ 3x \leq 24 \]
+\[ x \leq 8 \]
+
+### Possible Values for \( x \):
+- \( x \) can be 2, 3, 4, 5, 6, 7, or 8.
+
+### Calculate \( y \) and \( z \) for each \( x \):
+- For \( x = 2 \): \( y = 20 \), \( z = 8 \) (invalid as \( y \geq z \))
+- For \( x = 3 \): \( y = 18 \), \( z = 9 \) (invalid as \( y \geq z \))
+- For \( x = 4 \): \( y = 16 \), \( z = 10 \) (invalid as \( y \geq z \))
+- For \( x = 5 \): \( y = 14 \), \( z = 11 \) (valid)
+- For \( x = 6 \): \( y = 12 \), \( z = 12 \) (valid)
+- For \( x = 7 \): \( y = 10 \), \( z = 13 \) (valid)
+- For \( x = 8 \): \( y = 8 \), \( z = 14 \) (valid)
+
+### Optimal Strategy:
+Bob should guess the minimum number of coins in each box to ensure he wins at least that many coins. The optimal strategy is to guess the minimum number of coins in each box configuration.
+
+### Minimum Winnings:
+- For \( (x, y, z) = (5, 14, 11) \), Bob guesses 5, 11, 11 → Wins 5 + 11 + 11 = 27 coins
+- For \( (x, y, z) = (6, 12, 12) \), Bob guesses 6, 12, 12 → Wins 6 + 12 + 12 = 30 coins
+- For \( (x, y, z) = (7, 10, 13) \), Bob guesses 7, 10, 10 → Wins 7 + 10 + 10 = 27 coins
+- For \( (x, y, z) = (8, 8, 14) \), Bob guesses 8, 8, 8 → Wins 8 + 8 + 8 = 24 coins
+
+The minimum amount of money Bob can win, using the optimal strategy, is when the distribution is \( (8, 8, 14) \), and he wins 24 coins.
+
+### Conclusion:
+The minimum amount of money Bob can win is \( 24,000 \) dollars.",16000,0
+The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.,"I currently don't have the capability to access or analyze files directly. Please provide the relevant data or details from the Excel file, and I can assist you with the calculations.",89706.00,0
+Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.,Saint Petersburg,Saint Petersburg,1
+".rewsna eht sa ""tfel"" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",right,Right,1
+"Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.
+
+In your response, please only list the ingredients, not any measurements. So if the recipe calls for ""a pinch of salt"" or ""two cups of ripe strawberries"" the ingredients on the list would be ""salt"" and ""ripe strawberries"".
+
+Please format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.","I'm unable to listen to audio files. However, if you can provide a transcription of the voice memo, I can help you extract and alphabetize the ingredients for the filling.","cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",0
+"In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.",graphene,diamond,0

data/agent_experiments/iterations/3_integrate_whisper_audio_transcriber.csv

@@ -0,0 +1,23 @@
+Question,file_path,Agent response,Final answer,is_correct
+What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?,,Agnew,Louvrier,0
+What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,,Dmitry,Claus,0
+"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",,"""BUL""",CUB,0
+"You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, ""-"", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt,7,3,0
+"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
+
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/1f975693-876d-457b-a649-393859e79bf3.mp3,"132, 133, 134, 197, 245","132, 133, 134, 197, 245",1
+Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.,,Please provide the text or content of the attached files so I can assist you further.,Saint Petersburg,0
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",,"I currently don't have the capability to access or retrieve content from attached files. You might want to check the paper directly for the specific information regarding the volume of the fish bag. If you have any other questions or need further assistance, feel free to ask!",0.1777,0
+"Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?",,Switzerland,Guatemala,0
+"In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",,`semi_supervised.BaseLabelPropagation`,BaseLabelPropagation,0
+"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",,The work performed by R. G. Arendt was supported by NASA award number 80NSSC21M0056.,80GSFC21M0002,0
+"In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.",,"I can't access or retrieve specific files or documents. You might want to check the official script or a reliable source for the exact setting name as it appears in the first scene heading of Series 9, Episode 11 of Doctor Who.",THE CASTLE,0
+".rewsna eht sa ""tfel"" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",,"""right""",Right,1
+How many slides in this PowerPoint presentation mention crustaceans?,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/a3fbeb63-0e8c-4a11-bff6-0e3b484c3e9c.pptx,"I'm unable to directly analyze PowerPoint files. Please provide the text content or a summary of the slides, and I can assist you further.",4,0
+"On Cornell Law School website's legal information institute, under the fifth section of federal rules alphabetically, what word was deleted in the last amendment to the first rule in the article that has ""witnesses"" in the most titles as of 2021?",,"The last amendment to Rule 601 of the Federal Rules of Evidence was a stylistic restyling in 2011, which did not involve the deletion of any specific word.",inference,0
+The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx,"I'm unable to directly access or process Excel files. Please provide the relevant data or information from the file, and I'll be happy to assist you with your request.",89706.00,0
+"You are given this Excel file as a map. You start on the START cell and move toward the END cell. You are allowed to move two cells per turn, and you may move up, down, left, or right. You may not move fewer than two cells, and you may not move backward. You must avoid moving onto any blue cells. On the eleventh turn, what is the 6-digit hex code (without prefix) of the color of the cell where you land after moving?",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/65afbc8a-89ca-4ad5-8d62-355bb401f61d.xlsx,"I currently don't have the capability to directly read or process Excel files. You can provide the relevant data or information from the Excel file, and I can assist you with the problem using that information.",F478A7,0
+"In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",,I can't access or analyze video content directly. You might want to watch the video and count the number of bird species visible at the same time.,3,0
+"As a comma separated list with no whitespace, using the provided image provide all the fractions that use / as the fraction line and the answers to the sample problems. Order the list by the order in which the fractions appear.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/9318445f-fe6a-4e1b-acbf-c68228c9906a.png,"I'm unable to view or analyze images directly. Please provide the text or details from the image, and I'll assist you accordingly.","3/4,1/4,3/4,3/4,2/4,1/2,5/35,7/21,30/5,30/5,3/4,1/15,1/3,4/9,1/8,32/23,103/170",0
+What was the complete title of the book in which two James Beard Award winners recommended the restaurant where Ali Khan enjoyed a New Mexican staple in his cost-conscious TV show that started in 2015? Write the numbers in plain text if there are some in the title.,,"""Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them""",Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them,1
+What is the final numeric output from the attached Python code?,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/f918266a-b3e0-4914-865d-4faa564f1aef.py,"I currently don't have the capability to access or read files directly from a file path. Please provide the content of the Python code here, and I can help determine the final numeric output.",0,0

data/agent_experiments/iterations/4_test_workflow.csv

@@ -0,0 +1,36 @@
+Question,file_path,Agent response,Final answer,is_correct
+"You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, ""-"", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt,4,3,0
+"In the year 2022, and before December, what does ""R"" stand for in the three core policies of the type of content that was violated in the public logs on the Legume Wikipedia page?",,"""R"" stands for ""No original research"" in the three core policies of Wikipedia.",research,0
+The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx,"I can't directly access or read Excel files. Please provide the relevant data or information from the file, and I'll be happy to assist you with your request.",89706.00,0
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",,"I currently don't have the capability to access or retrieve content from attached files. You might want to check the paper directly for the specific information regarding the volume of the fish bag. If you have any other questions or need further assistance, feel free to ask!",0.1777,0
+What was the complete title of the book in which two James Beard Award winners recommended the restaurant where Ali Khan enjoyed a New Mexican staple in his cost-conscious TV show that started in 2015? Write the numbers in plain text if there are some in the title.,,"""Cheap Eats: The Restaurant Guide by James Beard Award Winners""",Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them,0
+"An office held a Secret Santa gift exchange where each of its twelve employees was assigned one other employee in the group to present with a gift. Each employee filled out a profile including three likes or hobbies. On the day of the gift exchange, only eleven gifts were given, each one specific to one of the recipient's interests. Based on the information in the document, who did not give a gift?",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/cffe0e32-c9a6-4c52-9877-78ceb4aaa9fb.docx,"To determine who did not give a gift, I need to extract and analyze the information from the provided document. However, I currently do not have the capability to directly read or extract content from `.docx` files. You can provide the text content or key details from the document, and I can assist you further in solving the problem.",Fred,0
+"In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",,I can't access or analyze video content directly. You might want to watch the video and count the number of bird species visible at the same time.,3,0
+"Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?",,Switzerland,Guatemala,0
+How many more blocks (also denoted as layers) in BERT base encoder than the encoder from the architecture proposed in Attention is All You Need?,,"The BERT base encoder has 12 layers, while the encoder from the ""Attention is All You Need"" architecture, also known as the original Transformer model, has 6 layers. Therefore, the BERT base encoder has 6 more layers than the original Transformer encoder.",6,0
+"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",,The work performed by R. G. Arendt was supported by NASA award number 80GSFC21M0002.,80GSFC21M0002,0
+".rewsna eht sa ""tfel"" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",,"""right""",Right,1
+What is the final numeric output from the attached Python code?,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/f918266a-b3e0-4914-865d-4faa564f1aef.py,"I couldn't access the file directly, so I simulated a simple Python script that calculates the sum of numbers from 1 to 10. The final numeric output from this simulated code is `55`. If the actual file content differs, please provide the code for accurate execution.",0,0
+"Pull out the sentence in the following 5x7 block of text. Read from left to right and use all of the letters in order:
+
+THESE
+AGULL
+GLIDE
+DPEAC
+EFULL
+YTOMY
+CHAIR",,"""THE SEAGULL GLIDED PEACEFULLY TO MY CHAIR""",The seagull glided peacefully to my chair.,1
+Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.,,Mikołaj,Wojciech,0
+"Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.
+
+In your response, please only list the ingredients, not any measurements. So if the recipe calls for ""a pinch of salt"" or ""two cups of ripe strawberries"" the ingredients on the list would be ""salt"" and ""ripe strawberries"".
+
+Please format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3,"cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries","cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",1
+"You are given this Excel file as a map. You start on the START cell and move toward the END cell. You are allowed to move two cells per turn, and you may move up, down, left, or right. You may not move fewer than two cells, and you may not move backward. You must avoid moving onto any blue cells. On the eleventh turn, what is the 6-digit hex code (without prefix) of the color of the cell where you land after moving?",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/65afbc8a-89ca-4ad5-8d62-355bb401f61d.xlsx,"I'm unable to directly view or interact with Excel files. Please provide the necessary details or data from the file, such as the layout of the cells, the position of the START and END cells, and the positions of any blue cells. This information will help me assist you in determining the color of the cell where you land on the eleventh turn.",F478A7,0
+"In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",,`semi_supervised.BaseLabelPropagation`,BaseLabelPropagation,0
+"What writer is quoted by Merriam-Webster for the Word of the Day from June 27, 2022?",,"The writer quoted by Merriam-Webster for the Word of the Day on June 27, 2022, was not specified in the search results.",Annie Levin,0
+"If there is anything that doesn't make sense in the instructions, write the word ""Pineapple."" Do not answer any of the questions in this prompt. Write only the word ""Guava"".
+1. What is 4+4?
+2. What is the complimentary color of red?
+3. How many hours are there in a day?",,Guava,Guava,1
+"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",,"Estévez, Velasquez","Yoshida, Uehara",0

data/agent_experiments/iterations/5_integrate_text_processing_tool.csv

@@ -0,0 +1,64 @@
+Question,file_path,Agent response,Final answer,is_correct
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",,Please provide the text file containing the University of Leicester paper so I can extract the required information.,0.1777,0
+"In the fictional language of Tizin, basic sentences are arranged with the Verb first, followed by the direct object, followed by the subject of the sentence. I want to express my love for apples to my Tizin friend. 
+
+The word that indicates oneself is ""Pa"" is the nominative form, ""Mato"" is the accusative form, and ""Sing"" is the genitive form. 
+
+The root verb that indicates an intense like for something is ""Maktay"". When it is used in the present, it is used in it's root form, when it is used in the preterit past, it is ""Tay"", and when it is used in the imperfect past, it is ""Aktay"". It is used differently than in English, and is better translated as ""is pleasing to"", meaning that the thing doing the liking is actually the object of the sentence rather than the subject.
+
+The word for apples is borrowed from English in Tizin, and so it is ""Apple"" is the nominative form, ""Zapple"" is the accusative form, and ""Izapple"" is the genitive form. 
+
+Please translate ""I like apples"" to Tizin.",,"""Maktay Zapple Pa""",Maktay mato apple,0
+"In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.",,"Please provide the text file containing the official script for Series 9, Episode 11 of Doctor Who, and I will extract the information for you.",THE CASTLE,0
+"Pull out the sentence in the following 5x7 block of text. Read from left to right and use all of the letters in order:
+
+THESE
+AGULL
+GLIDE
+DPEAC
+EFULL
+YTOMY
+CHAIR",,Please provide the text file so I can process it and extract the sentence.,The seagull glided peacefully to my chair.,0
+"Here's a fun riddle that I think you'll enjoy.
+
+You have been selected to play the final round of the hit new game show ""Pick That Ping-Pong"". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.
+
+A device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.
+
+The ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.
+
+Which ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",,"To maximize your odds of winning the big prize, you should choose ball number `1`.",3,0
+"You are given this Excel file as a map. You start on the START cell and move toward the END cell. You are allowed to move two cells per turn, and you may move up, down, left, or right. You may not move fewer than two cells, and you may not move backward. You must avoid moving onto any blue cells. On the eleventh turn, what is the 6-digit hex code (without prefix) of the color of the cell where you land after moving?",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/65afbc8a-89ca-4ad5-8d62-355bb401f61d.xlsx,"I currently cannot directly process Excel files. Please convert the Excel file to a text format or provide the necessary data in text form, and I will assist you with the task.",F478A7,0
+"In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",,I can't access or analyze video content directly. You might want to watch the video and count the number of bird species visible at the same time.,3,0
+What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,,"""Claus""",Claus,1
+"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",,"I couldn't find the specific paper or the NASA award number from the search results. You may need to visit the Universe Today website and locate the article by Carolyn Collins Petersen published on June 6, 2023, to find the link to the paper and the NASA award number.",80GSFC21M0002,0
+"Each cell in the attached spreadsheet represents a plot of land. The color of the cell indicates who owns that plot. Green cells are plots owned by Earl Smith. Can Earl walk through every plot he owns (and no other plots) and return to his starting plot without backtracking? For this question, consider backtracking to be any instance where Earl would enter a plot of land he had already entered since leaving his starting plot.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/5cfb274c-0207-4aa7-9575-6ac0bd95d9b2.xlsx,"I'm unable to directly view or process spreadsheet files. Please provide the data in a text format or describe the layout of the plots, and I can help determine if Earl can walk through every plot he owns without backtracking.",No,0
+"You are Van Helsing, a renowned vampire hunter. A Count of Moldova, Lațcu IV, son of  Costea, has tasked you with investigating the village of Șirnea in neighboring Wallachia. The Count's advisors have reported that a vampire was spotted crossing the border near the village, and would like you to investigate it.
+
+You travel to the village of Șirnea, and you begin your investigation. One night, just before dawn, you catch a glimpse of a man in a long black cape with red lining leaping from roof-top to roof-top with superhuman agility. It's a vampire! You try to chase the creature back to its home, but the creature is too fast. However, because of the remoteness of the village, you know with absolute certainty that the vampire must be a resident of the village. You decide that your best course of action will be to visit all 100 residents of the town during the day. You know something about vampires and humans that will make your investigation possible; humans always tell the truth, but vampires always lie.
+
+In the afternoon, you go from house to house, speaking with all 100 residents of Șirnea. You ask everyone the same question: ""How many vampires are living in Șirnea"". Everyone in the village gives the same response, ""At least one of us is a human.""
+
+How many residents of Șirnea have been turned into vampires?",,99,100,0
+"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",,LUX,CUB,0
+"A standard Rubik’s cube has been broken into cubes making up its sides. The cubes are jumbled, and one is removed. There are 6 cubes with one colored face, 12 edge cubes with two colored faces, and 8 corner cubes with three colored faces. All blue cubes have been found. All cubes directly left, right, above, and below the orange center cube have been found, along with the center cube. The green corners have all been found, along with all green that borders yellow. For all orange cubes found, the opposite face’s cubes have been found. The removed cube has two colors on its faces. What are they? Answer using a comma separated list, with the colors ordered alphabetically.",,"The removed cube has two colors on its faces: orange, red.","green, white",0
+"As a comma separated list with no whitespace, using the provided image provide all the fractions that use / as the fraction line and the answers to the sample problems. Order the list by the order in which the fractions appear.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/9318445f-fe6a-4e1b-acbf-c68228c9906a.png,I can't directly view images. Please provide the text or details from the image for further assistance.,"3/4,1/4,3/4,3/4,2/4,1/2,5/35,7/21,30/5,30/5,3/4,1/15,1/3,4/9,1/8,32/23,103/170",0
+"Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.
+
+What does Teal'c say in response to the question ""Isn't that hot?""",,"I can't access or analyze video content directly from YouTube or any other external websites. You can watch the video and listen for Teal'c's response to the question ""Isn't that hot?""",Extremely,0
+"On Cornell Law School website's legal information institute, under the fifth section of federal rules alphabetically, what word was deleted in the last amendment to the first rule in the article that has ""witnesses"" in the most titles as of 2021?",,"""Rule 611""",inference,0
+"Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?",,Please provide the attached files so I can assist you further.,Guatemala,0
+"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
+
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/1f975693-876d-457b-a649-393859e79bf3.mp3,"132, 133, 134, 197, 245","132, 133, 134, 197, 245",1
+"In Audre Lorde’s poem “Father Son and Holy Ghost”, what is the number of the stanza in which some lines are indented?",,"Please provide the text file containing Audre Lorde's poem ""Father Son and Holy Ghost"" so I can analyze it for you.",2,0
+How many more blocks (also denoted as layers) in BERT base encoder than the encoder from the architecture proposed in Attention is All You Need?,,"To determine the difference in the number of blocks (or layers) between the BERT base encoder and the encoder from the ""Attention is All You Need"" architecture, we need to know the number of layers in each model:
+
+1. **BERT Base Encoder**: It has 12 layers.
+2. **Attention is All You Need (Transformer) Encoder**: It has 6 layers.
+
+Now, we can calculate the difference:
+
+12 (BERT Base) - 6 (Transformer) = 6
+
+Therefore, the BERT base encoder has 6 more layers than the encoder from the ""Attention is All You Need"" architecture.",6,0

data/agent_experiments/iterations/6_integrate_text_handler_tool.csv

@@ -0,0 +1,66 @@
+Question,file_path,Agent response,Final answer,is_correct
+"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
+
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/1f975693-876d-457b-a649-393859e79bf3.mp3,"132, 133, 134, 197, 245","132, 133, 134, 197, 245",1
+"In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.",,"Please provide the text file containing the official script for Series 9, Episode 11 of Doctor Who, so I can extract the required information.",THE CASTLE,0
+How many slides in this PowerPoint presentation mention crustaceans?,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/a3fbeb63-0e8c-4a11-bff6-0e3b484c3e9c.pptx,I can only process text files directly. Please convert the PowerPoint presentation to a text format and provide the file for analysis.,4,0
+"You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, ""-"", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt,2,3,0
+"In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.",,"Please provide the text file containing the conference proceedings from the 2012 Nature journal's Scientific Reports, so I can process it and find the information you need.",diamond,0
+"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",,"""BUL""",CUB,0
+Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?,,FunkMonk,FunkMonk,1
+"Here's a fun riddle that I think you'll enjoy.
+
+You have been selected to play the final round of the hit new game show ""Pick That Ping-Pong"". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.
+
+A device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.
+
+The ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.
+
+Which ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",,"To maximize your odds of winning the big prize, you should choose ball number **1**.",3,0
+"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",,NNH16ZDA001N,80GSFC21M0002,0
+What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,,"To find the first name of the Malko Competition recipient from the 20th Century (after 1977) whose nationality was from a country that no longer exists, I would need to access the relevant data or list of recipients. If you have a file with this information, please provide it, and I can assist you further.",Claus,0
+Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.,,"The Polish-language version of ""Everybody Loves Raymond"" retains the original English cast, so there is no separate Polish actor for Ray.",Wojciech,0
+"According to Girls Who Code, how long did it take in years for the percentage of computer scientists that were women to change by 13% from a starting point of 37%?",,"The percentage of computer scientists that were women changed by 13% from a starting point of 37% over a period of 28 years, from 1995 to 2023.",22,0
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",,Please provide the text file containing the University of Leicester paper so I can extract the required information.,0.1777,0
+What was the complete title of the book in which two James Beard Award winners recommended the restaurant where Ali Khan enjoyed a New Mexican staple in his cost-conscious TV show that started in 2015? Write the numbers in plain text if there are some in the title.,,"""Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them""",Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them,1
+How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.,,3,3,1
+"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",,Please provide the file containing the relevant information so I can assist you further.,"Yoshida, Uehara",0
+"I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:
+
+milk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts
+
+I need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.",,"bell pepper, broccoli, celery, fresh basil, green beans, lettuce, sweet potatoes, zucchini","broccoli, celery, fresh basil, lettuce, sweet potatoes",0
+"Of the authors (First M. Last) that worked on the paper ""Pie Menus or Linear Menus, Which Is Better?"" in 2015, what was the title of the first paper authored by the one that had authored prior papers?",,"Please provide the text file containing the paper ""Pie Menus or Linear Menus, Which Is Better?"" from 2015, so I can extract the necessary information.",Mapping Human Oriented Information to Software Agents for Online Systems Usage,0
+"If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.",,"Eliud Kipchoge's marathon world record pace is approximately 2 hours, 1 minute, and 39 seconds for 42.195 kilometers. To find his pace in kilometers per hour, we first convert his marathon time to hours:
+
+2 hours + (1/60) hours + (39/3600) hours = 2.0275 hours
+
+Now, calculate his speed in kilometers per hour:
+
+42.195 km / 2.0275 hours ≈ 20.82 km/h
+
+Next, calculate the time it would take to run the minimum perigee distance to the Moon:
+
+363,396 km / 20.82 km/h ≈ 17,454.28 hours
+
+Finally, convert this time to thousand hours and round to the nearest 1000:
+
+17,454.28 hours ≈ 17,000 hours
+
+Therefore, it would take approximately 17000 hours.",17,0
+"¬(A ∧ B) ↔ (¬A ∨ ¬B)
+¬(A ∨ B) ↔ (¬A ∧ ¬B)
+(A → B) ↔ (¬B → ¬A)
+(A → B) ↔ (¬A ∨ B)
+(¬A → B) ↔ (A ∨ ¬B)
+¬(A → B) ↔ (A ∧ ¬B)
+
+Which of the above is not logically equivalent to the rest? Provide the full statement that doesn't fit.",,"I am unable to evaluate the logical equivalence of the statements due to technical issues. However, I can provide a general approach to solve this problem:
+
+1. **Understand Logical Equivalence**: Logical equivalence means that two statements have the same truth value in every possible scenario.
+
+2. **Evaluate Each Statement**: Use truth tables or logical laws to determine if each statement is equivalent to the others.
+
+3. **Identify the Odd One Out**: The statement that does not have the same truth value as the others in all scenarios is the one that doesn't fit.
+
+If you have access to a tool or software that can evaluate logical expressions, you can use it to perform these steps. Alternatively, you can manually create truth tables for each statement to determine their equivalence.",(¬A → B) ↔ (A ∨ ¬B),0

data/agent_experiments/iterations/7_test_agent_performance_against_tasks_with_attached_txt_files.csv

@@ -0,0 +1,2 @@
+Question,file_path,Agent response,Final answer,is_correct
+"You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, ""-"", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt,2,3,0

data/agent_experiments/iterations/8_integrate_chess_tool.csv

@@ -0,0 +1,56 @@
+Question,file_path,Agent response,Final answer,is_correct
+"If there is anything that doesn't make sense in the instructions, write the word ""Pineapple."" Do not answer any of the questions in this prompt. Write only the word ""Guava"".
+1. What is 4+4?
+2. What is the complimentary color of red?
+3. How many hours are there in a day?",,Guava,Guava,1
+"Here's a fun riddle that I think you'll enjoy.
+
+You have been selected to play the final round of the hit new game show ""Pick That Ping-Pong"". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.
+
+A device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.
+
+The ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.
+
+Which ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",,"To maximize your odds of winning the grand prize, you should choose ball number **1**.",3,0
+"Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.
+
+What does Teal'c say in response to the question ""Isn't that hot?""",,"I can't access or analyze video content directly from YouTube or any other external websites. You can watch the video and listen for Teal'c's response to the question ""Isn't that hot?""",Extremely,0
+How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.,,I couldn't find the specific number of studio albums released by Mercedes Sosa between 2000 and 2009. Please check the latest version of her discography on Wikipedia for detailed information.,3,0
+"In the fictional language of Tizin, basic sentences are arranged with the Verb first, followed by the direct object, followed by the subject of the sentence. I want to express my love for apples to my Tizin friend. 
+
+The word that indicates oneself is ""Pa"" is the nominative form, ""Mato"" is the accusative form, and ""Sing"" is the genitive form. 
+
+The root verb that indicates an intense like for something is ""Maktay"". When it is used in the present, it is used in it's root form, when it is used in the preterit past, it is ""Tay"", and when it is used in the imperfect past, it is ""Aktay"". It is used differently than in English, and is better translated as ""is pleasing to"", meaning that the thing doing the liking is actually the object of the sentence rather than the subject.
+
+The word for apples is borrowed from English in Tizin, and so it is ""Apple"" is the nominative form, ""Zapple"" is the accusative form, and ""Izapple"" is the genitive form. 
+
+Please translate ""I like apples"" to Tizin.",,"""Maktay Zapple Pa""",Maktay mato apple,0
+Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/cca530fc-4052-43b2-b130-b30968d8aa44.png,Rd5,Rd5,1
+"In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",,Please provide the file path to the Scikit-Learn July 2017 changelog so I can access the information.,BaseLabelPropagation,0
+"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",,"To find the country with the least number of athletes at the 1928 Summer Olympics, I need data on the number of athletes each country sent to the event. Please provide the relevant data file or information.",CUB,0
+"In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.",,"Please provide the file containing the conference proceedings from the Nature journal's Scientific Reports in 2012, so I can assist you in finding the information you need.",diamond,0
+"What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?""",,Please provide the file containing the University of Leicester paper so I can extract the required information.,0.1777,0
+"Of the authors (First M. Last) that worked on the paper ""Pie Menus or Linear Menus, Which Is Better?"" in 2015, what was the title of the first paper authored by the one that had authored prior papers?",,"To assist you, I need the attached files that contain the relevant information about the paper and its authors. Please provide the files so I can proceed.",Mapping Human Oriented Information to Software Agents for Online Systems Usage,0
+"Pull out the sentence in the following 5x7 block of text. Read from left to right and use all of the letters in order:
+
+THESE
+AGULL
+GLIDE
+DPEAC
+EFULL
+YTOMY
+CHAIR",,Please provide the file containing the 5x7 block of text.,The seagull glided peacefully to my chair.,0
+How many slides in this PowerPoint presentation mention crustaceans?,/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/a3fbeb63-0e8c-4a11-bff6-0e3b484c3e9c.pptx,I can only process text files. Please convert the PowerPoint presentation to a text format and provide it again.,4,0
+"In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.",,"Please provide the file path to the script of Series 9, Episode 11 of Doctor Who, so I can retrieve the information for you.",THE CASTLE,0
+Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.,,Please provide the file containing Nedoshivina's 2010 paper so I can extract the required information.,Saint Petersburg,0
+"In Audre Lorde’s poem “Father Son and Holy Ghost”, what is the number of the stanza in which some lines are indented?",,"Please provide the text of the poem ""Father Son and Holy Ghost"" by Audre Lorde, and I will help you identify the stanza with indented lines.",2,0
+"A standard Rubik’s cube has been broken into cubes making up its sides. The cubes are jumbled, and one is removed. There are 6 cubes with one colored face, 12 edge cubes with two colored faces, and 8 corner cubes with three colored faces. All blue cubes have been found. All cubes directly left, right, above, and below the orange center cube have been found, along with the center cube. The green corners have all been found, along with all green that borders yellow. For all orange cubes found, the opposite face’s cubes have been found. The removed cube has two colors on its faces. What are they? Answer using a comma separated list, with the colors ordered alphabetically.",,"The removed cube has two colors on its faces: `red, white`.","green, white",0
+"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
+
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/1f975693-876d-457b-a649-393859e79bf3.mp3,"132, 133, 134, 197, 245","132, 133, 134, 197, 245",1
+"Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.
+
+In your response, please only list the ingredients, not any measurements. So if the recipe calls for ""a pinch of salt"" or ""two cups of ripe strawberries"" the ingredients on the list would be ""salt"" and ""ripe strawberries"".
+
+Please format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.",/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3.mp3,"cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries","cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",1
+"Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?",,Please provide the attached files so I can assist you further.,Guatemala,0

data/agent_experiments/summary.csv

@@ -0,0 +1,9 @@
+iteration,experiment,agent,tools,accuracy
+1,Implement Calculator Tool,React agent,Aritmetic,0.0
+2,Implement Search and Code tools,React agent,"Aritmetic, Search, Code",0.17
+3,Integrate Whisper Audio Transcriber,React agent,"Aritmetic, Search, Code, Audio Transcriber",0.15
+4,Test Workflow,React agent,"Aritmetic, Search, Code, Audio Transcriber, ",0.2
+5,Integrate Text processing tool,React agent,"Aritmetic, Search, Code, Audio Transcriber, Text Handler",0.1
+6,Integrate Text Handler tool,React agent,"Aritmetic, Search, Code, Audio Transcriber, Text Handler, Text Handler",0.2
+7,Test Agent performance against tasks with attached .txt files,React agent,"Aritmetic, Search, Code, Audio Transcriber, Text Handler, Text Handler, , ",0.0
+8,Integrate Chess Tool,React agent,"Aritmetic, Search, Code, Audio Transcriber, Text Handler, Text Handler, , , chess_tool",0.2

data/temp-data/temp_json.json

@@ -0,0 +1 @@
+{"key1": 1, "key2": 2, "key3": 30}

data/temp-data/temp_text.txt

@@ -0,0 +1,2 @@
+Hi, this is a sample text
+that's all:)

data/temp/results_web_search.csv

@@ -0,0 +1,39 @@
+task_id,answer
+5d0080cb-90d7-4712-bc33-848150e917d3,"It seems there is an issue with accessing the webpage directly. You can try visiting the [University of Leicester's journal page](https://journals.le.ac.uk/ojs1/index.php/jist/article/view/2983) to find the paper ""Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?"" and check the volume of the fish bag mentioned in the paper. If you have any other questions or need further assistance, feel free to ask!"
+46719c30-f4c3-4cad-be07-d5cb21eee6bb,"It seems there's an issue with accessing external websites directly. However, you can manually search for the paper ""Pie Menus or Linear Menus, Which Is Better?"" on Google Scholar or a similar academic database. Once you find the paper, you can check the list of authors and then search for their previous works to identify the first paper authored by the one with prior publications. If you need further assistance with this process, feel free to ask!"
+b816bfce-3d80-4913-a07d-69b752ce6377,"It seems I can't access external websites directly to retrieve the information. However, I can guide you on how to find this information:
+
+1. **Journal Name**: The journal named for one of Hreidmar's sons is likely ""Fafnir – Nordic Journal of Science Fiction and Fantasy Research,"" as Fafnir is a son of Hreidmar in Norse mythology.
+
+2. **Finding the Article**:
+   - Visit a database like JSTOR or your institution's library portal.
+   - Search for Emily Midkiff's article from June 2014 in ""Fafnir.""
+
+3. **Reading the Article**:
+   - Once you have access to the article, look for sections where Midkiff discusses dragon depictions.
+   - Pay attention to any quotes from other authors that express distaste for how dragons are depicted.
+
+If you have access to JSTOR or another academic database, you should be able to find the article and the specific word you're looking for. If you need further assistance, feel free to ask!"
+b415aba4-4b68-4fc6-9b89-2c812e55a3e1,"It seems I'm unable to access external websites directly at the moment. However, you can visit the [Nature Scientific Reports conference proceedings for 2012](https://www.nature.com/srep/conference-proceedings/2012) and search for the article that does not mention plasmons or plasmonics. Look for the specific nano-compound studied in that article. If you have any other questions or need further assistance, feel free to ask!"
+935e2cff-ae78-4218-b3f5-115589b19dae,"It seems that I'm unable to navigate to the webpage directly. However, you can find the information you're looking for by visiting the Legume Wikipedia page and checking the public logs for content violations. The three core policies often referred to in Wikipedia content violations are ""Neutral point of view,"" ""Verifiability,"" and ""No original research."" The ""R"" in this context likely stands for ""No original research,"" but you should verify this by checking the specific logs on the Wikipedia page."
+5188369a-3bbe-43d8-8b94-11558f909a08,"It seems there is an issue with accessing the Merriam-Webster website directly. You can visit [Merriam-Webster's Word of the Day Archive](https://www.merriam-webster.com/word-of-the-day/calendar) and navigate to June 27, 2022, to find the writer quoted for that day's word."
+7673d772-ef80-4f0f-a602-1bf4485c9b43,"It seems there is an issue with navigating to the website directly. However, I can guide you on how to find the information you need:
+
+1. **Visit the Cornell Law School's Legal Information Institute website**: Go to [https://www.law.cornell.edu/](https://www.law.cornell.edu/).
+
+2. **Access the Federal Rules**: Look for a section or link related to ""Federal Rules"" on the homepage or use the search function.
+
+3. **Find the Fifth Section Alphabetically**: Once you are in the Federal Rules section, identify the fifth section alphabetically. This could be something like ""Federal Rules of Civil Procedure"" or similar.
+
+4. **Locate Articles with ""Witnesses"" in the Title**: Within the identified section, look for articles or rules that have ""witnesses"" in their titles.
+
+5. **Check the First Rule for Amendments**: Open the first rule in the article with the most titles containing ""witnesses"" and look for any amendment notes. These notes often indicate what changes were made, including any words that were deleted.
+
+If you follow these steps, you should be able to find the information you are looking for. If you need further assistance, please let me know!"
+c365c1c7-a3db-4d5e-a9a1-66f56eae7865,"Honolulu, Norfolk"
+7d4a7d1d-cac6-44a8-96e8-ea9584a70825,"I am unable to access the Girls Who Code website directly at the moment. However, you can visit their website and look for statistics or reports related to the percentage of women in computer science. If you have any other questions or need further assistance, feel free to ask!"
+3f57289b-8c60-48be-bd80-01f8099ca449,"The New York Yankee with the most walks in the 1977 regular season was Reggie Jackson, who had 86 walks. During that same season, Reggie Jackson had 582 at-bats."
+23dd907f-1261-4488-b21c-e9185af91d5e,"It seems there is an issue with accessing the webpage directly. However, you can visit the [Poetry Foundation's website](https://www.poetryfoundation.org/poems/42576/father-son-and-holy-ghost) and search for Audre Lorde's poem ""Father Son and Holy Ghost"" to find the stanza with indented lines. If you have any other questions or need further assistance, feel free to ask!"
+840bfca7-4f7b-481a-8794-c560c340185d,"It seems there is an issue with navigating to the Universe Today website. Unfortunately, I can't directly access the web to retrieve the article and the linked paper. However, you can visit the Universe Today website and search for the article by Carolyn Collins Petersen published on June 6, 2023. At the bottom of the article, you should find a link to the paper. Once you access the paper, you can look for the NASA award number associated with R. G. Arendt's work in the acknowledgments or funding section of the paper."
+a0068077-79f4-461a-adfe-75c1a4148545,"It seems there is an issue with accessing the website directly. You can visit [ClinicalTrials.gov](https://clinicaltrials.gov) and search for the clinical trial on H. pylori in acne vulgaris patients to find the actual enrollment count for the period of January to May 2018. If you need further assistance with navigating the site or specific search tips, feel free to ask!"
+a0c07678-e491-4bbc-8f0b-07405144218f,"It seems I'm unable to access the website directly to find the information. However, you can visit the [official NPB website](https://npb.jp/bis/eng/teams/index_d.html) and navigate to the team roster for the Orix Buffaloes to find Taishō Tamai's number. From there, you can identify the pitchers with the numbers immediately before and after his. If you need further assistance, feel free to ask!"

external/chess-modules/board_to_fen/.github/workflows/workflow.yml

@@ -0,0 +1,51 @@
+name: workflow
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - uses: actions/cache@v3
+      id: cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.*') }}
+        restore-keys: | 
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt     
+        pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu   
+    - name: Run pytest
+      run: | 
+        pytest
+  deploy: 
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10.6
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.6
+      - run: python3 setup.py sdist
+      - name: Publish a Python distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

external/chess-modules/board_to_fen/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.egg/
+.egg
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+.venv
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+board_to_fen.egg-info/
+.DS_Store
+DS_Store
+board_to_fen/.DS_Store
+*/__pycache__
+*/*/__pycache__

external/chess-modules/board_to_fen/CHANGELOG.txt

@@ -0,0 +1,26 @@
+===============================
+november 2022
+    version 0.0.0-0.0.13
+     - fix problems related with publishing on pypi
+================================
+december 2022
+    version 0.0.14
+    - add automatic resizing (now image can have any size)
+===============================
+january 2023
+    version 0.0.14-0.0.16
+     - fix problems related with github actions
+================================
+january 2023
+    version 0.0.17-25
+     - add simple board validation
+     - bug fixes
+================================
+================================
+february 2023
+    version 0.1.0-0.1.1
+     - migratation from cv2 to PIL
+     - new function for direct image object load
+     - add simple tests
+     - bug fixes
+================================

external/chess-modules/board_to_fen/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 mcdominik
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

external/chess-modules/board_to_fen/MANIFEST.in

@@ -0,0 +1 @@
+global-include *.txt *.py *.jpg *png saved_models/* *

external/chess-modules/board_to_fen/README.md

@@ -0,0 +1,86 @@
+# board_to_fen
+
+
+Python package that converts digital chessboard image into Forsyth-Edwards notation (FEN) notation
+
+[![Downloads](https://static.pepy.tech/personalized-badge/board-to-fen?period=total&units=none&left_color=purple&right_color=blue&left_text=downloads)](https://pepy.tech/project/board-to-fen)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![PyPI](https://img.shields.io/pypi/v/board_to_fen)](https://pypi.org/project/board_to_fen/)
+[![GitHub last commit](https://img.shields.io/github/last-commit/mcdominik/board_to_fen)](https://github.com/mcdominik/board_to_fen)
+
+### Installation
+board_to_fen is available on PyPI:
+```
+pip3 install board_to_fen
+```
+
+### Quick Start
+```python
+from board_to_fen.predict import get_fen_from_image_path
+
+print(get_fen_from_image_path(PATH_TO_CHESSBOARD_IMAGE))
+```
+
+or, if you want you can load image object by yourself:
+
+```python
+from PIL import Image
+from board_to_fen.predict import get_fen_from_image
+
+img = Image.open(PATH_TO_CHESSBOARD_IMAGE)
+
+print(get_fen_from_image(img))
+```
+
+
+**Note:** *The package uses tensorflow+keras API. 
+They are pretty heavy.*
+
+### Customization
+
+get_fen_from_image_path takes has 3 arguments:
+
+- image_path [required]
+- end_of_row '/' by default 
+- black_view False by default -> set True if chessboard is provided from black player perspective
+
+
+### Web version (currently may not work)
+Available at: https://board2fen.bieda.it
+
+
+### Training
+For training You would probably want to download the source code by cloning the repository:
+```
+$ git clone https://github.com/mcdominik/board_to_fen.git
+```
+Download training data from:<br>
+I will supply url for data in the future
+
+In the main repository dir, run
+```
+$ python3 ./board_to_fen/train_model.py
+```
+
+### Version history
+
+- january 2023
+     - versions 0.0.17-25
+     - added simple board validation
+     - bug fixes
+
+- february 2023
+    version 0.1.0-0.1.1
+     - migratation from cv2 to PIL
+     - new function for direct image object load
+     - add simple tests
+     - bug fixes
+
+## Warnings
+- Image has to be provided in neutral angle (white or black player's perspective).
+- Image has to be square (~3% tolerance depending on image resolution).
+- Image can't contain paddings, board borders etc. other than 64 squares (with pieces) itself.
+
+
+### References:
+https://www.kaggle.com/datasets/koryakinp/chess-positions

external/chess-modules/board_to_fen/board_to_fen/KerasNeuralNetwork.py

@@ -0,0 +1,52 @@
+import os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"  # Ignore warnings (E.g. CUDA drivers, tf, etc.)
+from contextlib import redirect_stderr
+from keras import models, layers
+import numpy as np
+
+INPUT_SHAPE = (50, 50, 3)
+
+class KerasNeuralNetwork:
+    def __init__(self) -> None:
+        self.CATEGORIES = ["bishop_black", "bishop_white","empty","king_black","king_white","knight_black", "knight_white","pawn_black", "pawn_white", "queen_black","queen_white", "rook_black","rook_white"]
+        self.predictions = []
+        self.model = models.Sequential()
+        self.model.add(layers.Input(shape=INPUT_SHAPE))
+        #self.model.add(layers.Conv2D(50, (3,3), activation='relu', input_shape=(50,50,3)))  # DEPRECATED
+        self.model.add(layers.Conv2D(50, (3,3), activation='relu'))
+        self.model.add(layers.MaxPooling2D((2,2)))
+        self.model.add(layers.Conv2D(100,(3,3), activation='relu'))
+        self.model.add(layers.MaxPooling2D((2,2)))
+        self.model.add(layers.Conv2D(100, (3,3), activation='relu'))
+        self.model.add(layers.Flatten())
+        self.model.add(layers.Dense(100, activation='relu'))
+        self.model.add(layers.Dense(13, activation='softmax'))
+        self.model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
+
+    def fit(self, train_images, train_labels, test_images, test_labels, num_of_epochs=2, batch_size=32):
+        self.model.fit(train_images, train_labels, epochs=num_of_epochs, verbose=1, validation_data=(test_images, test_labels), batch_size=batch_size)
+
+    def save(self, path='.') -> None:
+        self.model.save(path)
+
+    def evaluate(self, test_images, test_labels):
+        loss, accuracy = self.model.evaluate(test_images, test_labels)
+        print(f"accuracy:{accuracy}")
+        print(f"loss:{loss}")
+
+    def load_model_from_weights(self, path):
+        self.model.load_weights(path)
+    
+    def load_model(self, path):
+        self.model = models.load_model(path)
+    
+    def predict(self, tiles) -> list:
+        for image in tiles:
+            image = np.array(image)
+            image = np.reshape(image, (INPUT_SHAPE))
+            image = np.expand_dims(image, axis=0)
+            prediction = self.model.predict(image, verbose=0)
+            index = np.argmax(prediction)
+            self.predictions.append(self.CATEGORIES[index])
+        self.predictions = np.array(self.predictions)
+        return self.predictions

external/chess-modules/board_to_fen/board_to_fen/predict.py

@@ -0,0 +1,102 @@
+import io
+from PIL import Image
+from .KerasNeuralNetwork import KerasNeuralNetwork
+from .utils import Decoder_FEN, Tiler
+import os
+import tensorflow as tf
+from board_to_fen import saved_models
+import numpy as np
+
+# try:
+#     import importlib.resources as pkg_resources
+# except ImportError:
+#     # Try backported to PY<37 `importlib_resources`.
+#     import importlib_resources as pkg_resources
+# deprecated
+
+os.sys.path.append("board_to_fen/KerasNeuralNetwork")
+from .KerasNeuralNetwork import KerasNeuralNetwork
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+MODELS_DIR = os.path.join(CURRENT_DIR, "saved_models")
+PATH_TO_MODEL = os.path.join(MODELS_DIR, "november_model")
+PATH_TO_MODEL_WEIGHTS = os.path.join(MODELS_DIR, "november_model_weights.h5")
+# MODELS_DIR = "board_to_fen/saved_models/"
+# PATH_TO_MODEL_WEIGHTS = os.path.join(MODELS_DIR, "november_model_weights.h5")
+
+model = KerasNeuralNetwork()
+model.load_model_from_weights(path=PATH_TO_MODEL_WEIGHTS)
+# model = tf.keras.models.load_model("board_to_fen/saved_models/november_model_weights.h5")
+
+def get_fen_from_image_path(image_path, end_of_row='/', black_view=False) -> str:
+    """
+    Predict FEN position from path to a chess image.
+
+    Parameters
+    ----------
+    image : type
+        Description
+    end_of_row : str
+        Indicate how to process end of chess row
+    black_view : bool
+        Set to True iff the board is viewed from Black's perspective.
+    
+    Returns:
+        str: Predicted chess position in FEN notation
+    """
+    image = Image.open(image_path)
+    decoder = Decoder_FEN()
+    # net = KerasNeuralNetwork()
+    tiler = Tiler()
+    tiles = tiler.get_tiles(img=image)
+
+    # f = pkg_resources.open_text(saved_models, 'november_model')  # Deprecated
+    #net.load_model(f.name)
+    #predictions = net.predict(tiles=tiles) # Deprecated
+
+    # convert tiles object to tf tensor before compute predictions
+    tiles_array = np.array([np.array(tile) for tile in tiles], dtype=np.float32)
+    tiles_tensor = tf.convert_to_tensor(tiles_array, dtype=tf.float32)
+    predictions = model.predict(tiles_tensor)
+    fen = decoder.fen_decode(squares=predictions, end_of_row=end_of_row, black_view=black_view)
+    return fen
+
+def get_fen_from_image(image, end_of_row='/', black_view=False) -> str:
+    """
+    Predict FEN position from Image-like object, which represents a chess board.
+
+    Parameters
+    ----------
+    image : PIL.Image
+        image Object 
+    end_of_row : str
+        Indicate how to process end of chess row
+    black_view : bool
+        Set to True if the board is viewed from Black's perspective.
+
+    
+    Returns:
+        str: Predicted chess position in FEN notation
+    """
+    decoder = Decoder_FEN()
+    tiler = Tiler()
+    tiles = tiler.get_tiles(img=image)
+
+    tiles_array = np.array([np.array(tile) for tile in tiles], dtype=np.float32)
+    tiles_tensor = tf.convert_to_tensor(tiles_array, dtype=tf.float32)
+
+    # Predict
+    predictions = model.predict(tiles_tensor)
+
+    # Decode predictions to FEN
+    fen = decoder.fen_decode(
+        squares=predictions,
+        end_of_row=end_of_row,
+        black_view=black_view
+    )
+    return fen
+
+if __name__ == "__main__":
+    img_path = "./board_to_fen/test_image.jpeg"
+    fen = get_fen_from_image_path(img_path)
+    print(fen)

external/chess-modules/board_to_fen/board_to_fen/train_model.py

@@ -0,0 +1,22 @@
+import os
+# silence tensorflow and keras debugging information
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
+from .utils import DataFetcher
+from .KerasNeuralNetwork import KerasNeuralNetwork
+
+
+if __name__ == "__main__":
+    #fetch and prepare data
+    fetcher = DataFetcher()
+    fetcher.fetch_and_shuffle(data_dir='./test_splitted/')
+    train_images, train_labels, test_images, test_labels = fetcher.get_train_test()
+
+    #call and feed the neural network
+    net = KerasNeuralNetwork()
+    net.fit(train_images, train_labels, test_images, test_labels)
+    net.evaluate(test_images, test_labels)
+    net.save(path='./saved_models/another_model')
+
+
+
+

external/chess-modules/board_to_fen/board_to_fen/utils.py

@@ -0,0 +1,126 @@
+import os
+import numpy as np
+import random
+import re
+from PIL import Image
+from itertools import product
+
+
+LEGEND = {
+    'pawn_white' : 'P',
+    'pawn_black' : 'p',
+    'knight_white' : 'N',
+    'knight_black' : 'n',
+    'bishop_white' : 'B',
+    'bishop_black' : 'b',
+    'rook_white' : 'R',
+    'rook_black' : 'r',
+    'queen_white' : 'Q',
+    'queen_black' : 'q',
+    'king_white' : 'K',
+    'king_black' : 'k',
+    'empty':'1'
+}
+
+class Decoder_FEN():
+    def __init__(self) -> None:
+        pass
+
+    def _squeeze(self, input) -> str:
+        filtered = re.sub('11111111', '8', input)
+        filtered = re.sub('1111111', '7', filtered)
+        filtered = re.sub('111111', '6', filtered)
+        filtered = re.sub('11111', '5', filtered)
+        filtered = re.sub('1111', '4', filtered)
+        filtered = re.sub('111', '3', filtered)
+        squeezed = re.sub('11', '2', filtered)
+        return squeezed
+
+    def _simple_validator(self, squares):
+        ''' ensures that model:
+            - found only one white king
+            - found only one black king
+            - number of pieces is not greater than 32
+
+        Args:
+        figures: list of figures before decoding
+        '''
+        count_king_white = 0
+        count_king_black = 0
+        figures = 0
+
+        for square in squares:
+            if square == 'king_white':
+                count_king_white+=1
+            if square == 'king_black':
+                count_king_black+=1
+            if square != 'empty':
+                figures +=1
+        if ((count_king_black != 1) or (count_king_white != 1) or ((figures) > 32)):
+            return 'invalid'
+
+    def fen_decode(self, squares, end_of_row='/', black_view=False) -> str:
+        if (self._simple_validator(squares) == 'invalid'):
+            return 'Model can\'t find valid chessboard layout'
+        long_fen = ''
+        for i, square in enumerate(squares):
+            if i % 8 == 0 and i>0:
+                long_fen+=end_of_row
+            long_fen+=LEGEND[square]
+            fen = self._squeeze(long_fen)
+            if black_view:
+                fen = fen[::-1]
+        return fen
+
+
+class DataFetcher:
+    def __init__(self) -> None: 
+        self.CATEGORIES = ["bishop_black", "bishop_white","empty","king_black","king_white","knight_black", "knight_white","pawn_black", "pawn_white", "queen_black","queen_white", "rook_black","rook_white"]        
+        self.data = []
+        self.images = []
+        self.labels = []
+
+    def fetch_and_shuffle(self, data_dir):
+        for category in self.CATEGORIES:
+            path = os.path.join(data_dir, category)
+            class_num = self.CATEGORIES.index(category)
+            for filename in os.listdir(path):
+                try:
+                    img = Image.open(os.path.join(path,filename)).convert('RGB')
+                    ima = np.array(img)
+                    #consider division /255
+                    self.data.append([img/255, class_num])
+                except Exception as e:
+                    pass
+        random.shuffle(self.data)
+        self._create_labels_and_images()
+
+    def _create_labels_and_images(self):
+        for features, label in self.data:
+            self.images.append(features)
+            self.labels.append(label)
+        self.images = np.array(self.images).reshape(-1, 50, 50, 3)
+        self.labels = np.array(self.labels)
+
+    def get_train_test(self, split=0.85):
+        pivot = int(split * len(self.images))
+        train_images = self.images[:pivot]
+        train_labels = self.labels[:pivot]
+        test_images = self.images[pivot:]
+        test_labels = self.labels[pivot:]
+        return train_images, train_labels, test_images, test_labels
+
+
+class Tiler:
+    def __init__(self) -> None:
+        self.tile_images = []
+        pass
+
+    def get_tiles(self, img, d=50) -> list:
+        img = img.resize((400,400), Image.NEAREST).convert('RGB')
+        w, h = img.size
+        grid = product(range(0, h-h%d, d), range(0, w-w%d, d))
+        for i, j in grid:
+            box = (j, i, j+d, i+d)
+            self.tile_images.append(img.crop(box))
+        return self.tile_images

external/chess-modules/board_to_fen/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "board_to_fen"
+version = "0.1.2"
+description = "Convert chessboard images to FEN notation using a CNN model."
+authors = ["Dominik Maćkiewicz <https://github.com/mcdominik>"]
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.13"
+tensorflow = "2.17.1"
+keras = "3.2.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "7.1.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

external/chess-modules/board_to_fen/requirements.txt

@@ -0,0 +1,3 @@
+tensorflow==2.17.1
+keras==3.2.1
+pytest==7.1.1

external/chess-modules/board_to_fen/setup.py

@@ -0,0 +1,27 @@
+from setuptools import setup, find_packages
+
+classifiers = [
+  'Development Status :: 5 - Production/Stable',
+  'Intended Audience :: Education',
+  'License :: OSI Approved :: MIT License',
+  'Programming Language :: Python :: 3'
+]
+
+
+
+setup(
+   name='board_to_fen',
+   version='0.1.1',
+   author='Dominik Maćkiewicz',
+   author_email='dominik.mackiewicz@icloud.com',
+   packages=find_packages(),
+   scripts=[],
+   url='http://pypi.python.org/pypi/board_to_fen/',
+   license='MIT',
+   description='An package that converts digial chessboard image into Forsyth-Edwards notation',
+   long_description=open('README.md').read(),
+   include_package_data=True,
+    long_description_content_type='text/markdown',
+   python_requires='>=3.7',
+   install_requires=['pillow']
+)

external/chess-modules/board_to_fen/test_predict.py

@@ -0,0 +1,13 @@
+import pytest
+from board_to_fen.predict import get_fen_from_image, get_fen_from_image_path
+from PIL import Image
+
+
+img_path = "./test_images/test_image1.png"
+
+class TestNetwork:
+    def test_path_prediction(self):
+        assert get_fen_from_image_path(img_path) == "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR"
+    def test_object_prediction(self):
+        img = Image.open(img_path)
+        assert get_fen_from_image(img) == "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR"

langgraph.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": ["./"],
+  "graphs": {
+    "mvp_agent": "src/agents/react.py:graph"
+  }
+}

notebooks/final_submition.ipynb

@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a9bbf8e8",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ce6fb6a",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Imports\n",
+    "\n",
+    "\n",
+    "# Local modules\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "747d91b7",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# HF settings"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "049ae61d",
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "source": [
+    "### Get GAIA test questions"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7ca1548",
+   "metadata": {},
+   "source": [
+    "### Make inferences on test questions"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bd0bd958",
+   "metadata": {},
+   "source": [
+    "### Submit results"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/gaia_level_one_questions_validation.ipynb

@@ -0,0 +1,1311 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c545435f",
+   "metadata": {},
+   "source": [
+    "This Notebook is made for experimentation purposes, so we can track the changes vs performance of our agent"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8a78bd29",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f7c489ed",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/santiagoal/.cache/pypoetry/virtualenvs/chappie-CHLGiFC_-py3.12/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Libraries\n",
+    "from typing import Literal\n",
+    "from datasets import load_dataset\n",
+    "import pandas as pd\n",
+    "import os\n",
+    "from random import sample\n",
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import snapshot_download, login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "585346bc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Fetching 114 files: 100%|██████████| 114/114 [00:00<00:00, 2483.91it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import tasks attached files \n",
+    "load_dotenv()\n",
+    "login(token=os.getenv(key=\"HF_TOKEN_CHAPPIE\"))\n",
+    "gaia_repo_dir = snapshot_download(repo_id=\"gaia-benchmark/GAIA\", repo_type=\"dataset\")\n",
+    "print(gaia_repo_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "2fbda084",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Langfuse client is disabled since no public_key was provided as a parameter or environment variable 'LANGFUSE_PUBLIC_KEY'. See our docs: https://langfuse.com/docs/sdk/python/low-level-sdk#initialize-client\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Local Modules\n",
+    "os.sys.path.append(\"../src/\")\n",
+    "os.sys.path.append(\"../src/agents/\")\n",
+    "os.sys.path.append(\"../src/utils/\")\n",
+    "\n",
+    "import react  # My AI assistant\n",
+    "import gaia_eval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "73bcdc75",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Paths\n",
+    "output_results_path = \"/home/santiagoal/current-projects/chappie/data/agent_experiments/\"\n",
+    "experiment_iterations_path = os.path.join(output_results_path, \"iterations/\")\n",
+    "summary_experiments_path = os.path.join(output_results_path, \"summary.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "53df192e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# GAIA dataset\n",
+    "gaia_dataset = load_dataset(\"gaia-benchmark/GAIA\", \"2023_level1\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c4faa863",
+   "metadata": {},
+   "source": [
+    "Docs suggest to focus on validation set for dev purposes, let's explore it"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "6fed73d9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['task_id', 'Question', 'Level', 'Final answer', 'file_name', 'file_path', 'Annotator Metadata'],\n",
+       "    num_rows: 53\n",
+       "})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dev_set = gaia_dataset[\"validation\"]\n",
+    "dev_set"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ce03df90",
+   "metadata": {},
+   "source": [
+    "### Explore GAIA Lvl. 1 Questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "540838d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_dev = pd.DataFrame(dev_set)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "70dcbd6a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "task_id",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Level",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Annotator Metadata",
+         "rawType": "object",
+         "type": "unknown"
+        }
+       ],
+       "ref": "3d225c1c-ded5-478b-9b86-dc72eed15534",
+       "rows": [
+        [
+         "0",
+         "e1fc63a2-da7a-432f-be78-7c4a95598703",
+         "If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.",
+         "1",
+         "17",
+         "",
+         "",
+         "{'Steps': '1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.', 'Number of steps': '4', 'How long did this take?': '20 Minutes', 'Tools': '1. A web browser.\\n2. A search engine.\\n3. A calculator.', 'Number of tools': '3'}"
+        ],
+        [
+         "1",
+         "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",
+         "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': '1. I did a search for Mercedes Sosa\\n2. I went to the Wikipedia page for her\\n3. I scrolled down to \"Studio albums\"\\n4. I counted the ones between 2000 and 2009', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. web browser\\n2. google search', 'Number of tools': '2'}"
+        ],
+        [
+         "2",
+         "ec09fa32-d03f-4bf8-84b0-1f16922c3ae4",
+         "Here's a fun riddle that I think you'll enjoy.\n\nYou have been selected to play the final round of the hit new game show \"Pick That Ping-Pong\". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.\n\nA device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.\n\nThe ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.\n\nWhich ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': 'Step 1: Evaluate the problem statement provided in my user\\'s prompt\\nStep 2: Consider the probability of any ball on the platform earning the prize.\\nStep 3: Evaluate the ball in position one. The probability of it earning the prize, P1, is 1/3\\nStep 4: Using a calculator, evaluate the ball in position two. The probability of it earning the prize, P2, is the difference between 1 and the product of the complementary probabilities for each trial\\nP2 = 1 - (2/3)(2/3)\\nP2 = 5/9\\nStep 5: Using a calculator, evaluate the ball in position three. The probability of it earning the prize, P3, is the difference between 1 and the product of the complementary probabilities for each trial\\nP3 = 1 - (2/3)(2/3)(2/3)\\nP3 = 19/27\\nStep 6: Consider the possible outcomes of numbers higher than 3.\\nStep 7: For each trial, either 1 or 2 balls from the ramp will advance to the platform. For any given selection, there is a 50% chance that the ball advances to position 2 or position 3.\\nStep 8: As position three holds the highest chance of earning the prize, select the only ball known to occupy position three with certainty, ball 3.\\nStep 9: Report the correct answer to my user, \"3\"', 'Number of steps': '9', 'How long did this take?': '1 minute', 'Tools': 'None', 'Number of tools': '0'}"
+        ],
+        [
+         "3",
+         "5d0080cb-90d7-4712-bc33-848150e917d3",
+         "What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"",
+         "1",
+         "0.1777",
+         "",
+         "",
+         "{'Steps': '1. Searched \\'\"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"\\' on Google.\\n2. Opened \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\" at https://journals.le.ac.uk/ojs1/index.php/jist/article/view/733.\\n3. Clicked \"PDF\".\\n4. Found the calculations for the volume of the fish bag and noted them.', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. PDF access', 'Number of tools': '3'}"
+        ],
+        [
+         "4",
+         "a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
+         "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': '1. Navigate to the YouTube link.\\n2. Watch the video to see the highest number of bird species.\\n3. Note the number.', 'Number of steps': '3', 'How long did this take?': '3 minutes', 'Tools': '1. Web browser\\n2. Video parsing', 'Number of tools': '2'}"
+        ]
+       ],
+       "shape": {
+        "columns": 7,
+        "rows": 5
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>task_id</th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Level</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>file_name</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Annotator Metadata</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>e1fc63a2-da7a-432f-be78-7c4a95598703</td>\n",
+       "      <td>If Eliud Kipchoge could maintain his record-ma...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>17</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Googled Eliud Kipchoge marathon ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>8e867cd7-cff9-4e6c-867a-ff5ddc2550be</td>\n",
+       "      <td>How many studio albums were published by Merce...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. I did a search for Mercedes Sosa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ec09fa32-d03f-4bf8-84b0-1f16922c3ae4</td>\n",
+       "      <td>Here's a fun riddle that I think you'll enjoy....</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': 'Step 1: Evaluate the problem statem...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>5d0080cb-90d7-4712-bc33-848150e917d3</td>\n",
+       "      <td>What was the volume in m^3 of the fish bag tha...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.1777</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched '\"Can Hiccup Supply Eno...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>a1e91b78-d3d8-4675-bb8d-62741b4b68a6</td>\n",
+       "      <td>In the video https://www.youtube.com/watch?v=L...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Navigate to the YouTube link.\n",
+       "2....</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                task_id  \\\n",
+       "0  e1fc63a2-da7a-432f-be78-7c4a95598703   \n",
+       "1  8e867cd7-cff9-4e6c-867a-ff5ddc2550be   \n",
+       "2  ec09fa32-d03f-4bf8-84b0-1f16922c3ae4   \n",
+       "3  5d0080cb-90d7-4712-bc33-848150e917d3   \n",
+       "4  a1e91b78-d3d8-4675-bb8d-62741b4b68a6   \n",
+       "\n",
+       "                                            Question Level Final answer  \\\n",
+       "0  If Eliud Kipchoge could maintain his record-ma...     1           17   \n",
+       "1  How many studio albums were published by Merce...     1            3   \n",
+       "2  Here's a fun riddle that I think you'll enjoy....     1            3   \n",
+       "3  What was the volume in m^3 of the fish bag tha...     1       0.1777   \n",
+       "4  In the video https://www.youtube.com/watch?v=L...     1            3   \n",
+       "\n",
+       "  file_name file_path                                 Annotator Metadata  \n",
+       "0                      {'Steps': '1. Googled Eliud Kipchoge marathon ...  \n",
+       "1                      {'Steps': '1. I did a search for Mercedes Sosa...  \n",
+       "2                      {'Steps': 'Step 1: Evaluate the problem statem...  \n",
+       "3                      {'Steps': '1. Searched '\"Can Hiccup Supply Eno...  \n",
+       "4                      {'Steps': '1. Navigate to the YouTube link.\n",
+       "2....  "
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_dev.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5eef07a0",
+   "metadata": {},
+   "source": [
+    "In order to start our dev phase, let's observe how does our React Agent perform of a single question"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "be110221",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "# Experiment with only .txt like tasts\n",
+    "# Temporal cell\n",
+    "df_dev = df_dev[df_dev[\"file_path\"].apply(lambda row: row.endswith(\".txt\"))]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "b3632592",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_samples = min(20, df_dev.shape[0])\n",
+    "sample_questions = df_dev.sample(n_samples)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0d10258b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Dataset copy just for eval\n",
+    "\n",
+    "results_df = sample_questions.copy()[[\"Question\", \"Final answer\", \"file_path\"]]\n",
+    "results_df[\"Agent response\"] = None\n",
+    "results_df[\"is_correct\"] = None  # 1 if it is correct, 0 otherwise\n",
+    "\n",
+    "results_df = results_df[[\"Question\", \"file_path\", \"Agent response\", \"Final answer\", \"is_correct\"]]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ef5f9a65",
+   "metadata": {},
+   "source": [
+    "### Experiment and Track performance on dev set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "ef1a21ec",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get XP history\n",
+    "old_experiments_data = pd.read_csv(summary_experiments_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "58541f5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Form\n",
+    "\n",
+    "# Get last information \n",
+    "latest_experiment = old_experiments_data.iloc[-1]\n",
+    "#latest_xp_name = latest_experiment[\"experiment\"]\n",
+    "latest_agent = latest_experiment[\"agent\"]\n",
+    "latest_tools = latest_experiment[\"tools\"]\n",
+    "latest_iteration = latest_experiment[\"iteration\"]\n",
+    "current_iteration = latest_iteration + 1\n",
+    "\n",
+    "# Get XP name\n",
+    "msg = \"Type the experiment name (E.g. Integrate Whisper Transcriber)\"\n",
+    "usr_response = \"\"\n",
+    "while usr_response.lower() == \"\":\n",
+    "    usr_response = input(msg + \": \")\n",
+    "xp_name = usr_response\n",
+    "\n",
+    "\n",
+    "# Get Agent Archictecure\n",
+    "usr_response = \"\"\n",
+    "msg = f\"Is your agent different from '{latest_agent}'? [y/N]\"\n",
+    "usr_response = input(msg + \": \")\n",
+    "warning_msg = f\"Oops! '{usr_response}' is not a valid response, pls try again. \"\n",
+    "while usr_response.lower() not in (\"y\", \"n\"):\n",
+    "    usr_response = input(warning_msg.format(usr_response) + msg + \": \")\n",
+    "\n",
+    "if usr_response.lower() == \"n\":\n",
+    "    agent_architecture = latest_agent\n",
+    "elif usr_response.lower() == \"y\":\n",
+    "    agent_architecture = input(\"Please type the new agent architecture to track\" + \": \")\n",
+    "\n",
+    "# Get Tools\n",
+    "usr_response = \"\"\n",
+    "msg = \"Are there new tools to track? [y/N]\"\n",
+    "usr_response = input(msg + \": \")\n",
+    "while usr_response.lower() not in (\"y\", \"n\"):\n",
+    "    usr_response = input(warning_msg.format(usr_response) + msg + \": \")\n",
+    "\n",
+    "if usr_response.lower() == \"n\":\n",
+    "    new_tools = \"\"\n",
+    "elif usr_response.lower() == \"y\":\n",
+    "    new_tools = input(\"Please type the new tools list to track (comma separated)\" + \": \")\n",
+    "\n",
+    "new_tools_list = latest_tools + \", \" + new_tools\n",
+    "\n",
+    "# Format xp name and path\n",
+    "xp_name_snake = str(current_iteration) + \"_\" + xp_name.replace(\" \", \"_\").replace(\",\", \"\").lower()\n",
+    "xp_path = os.path.join(experiment_iterations_path, xp_name_snake + \".csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "7e4c15d8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Agent response",
+         "rawType": "object",
+         "type": "unknown"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "is_correct",
+         "rawType": "object",
+         "type": "unknown"
+        }
+       ],
+       "ref": "8ca69d17-0092-4421-a86c-d63edb41dea2",
+       "rows": [
+        [
+         "22",
+         "You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, \"-\", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",
+         "/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt",
+         null,
+         "3",
+         null
+        ]
+       ],
+       "shape": {
+        "columns": 5,
+        "rows": 1
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Question</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Agent response</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>is_correct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>You are a telecommunications engineer who want...</td>\n",
+       "      <td>/home/santiagoal/.cache/huggingface/hub/datase...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Question  \\\n",
+       "22  You are a telecommunications engineer who want...   \n",
+       "\n",
+       "                                            file_path Agent response  \\\n",
+       "22  /home/santiagoal/.cache/huggingface/hub/datase...           None   \n",
+       "\n",
+       "   Final answer is_correct  \n",
+       "22            3       None  "
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "2b48dcc8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "attached_files: /home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Agent response",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "is_correct",
+         "rawType": "int64",
+         "type": "integer"
+        }
+       ],
+       "ref": "cc4b4ed7-e0af-4c13-8682-95d79797ea08",
+       "rows": [
+        [
+         "22",
+         "You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, \"-\", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",
+         "/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt",
+         "2",
+         "3",
+         "0"
+        ]
+       ],
+       "shape": {
+        "columns": 5,
+        "rows": 1
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Question</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Agent response</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>is_correct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>You are a telecommunications engineer who want...</td>\n",
+       "      <td>/home/santiagoal/.cache/huggingface/hub/datase...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Question  \\\n",
+       "22  You are a telecommunications engineer who want...   \n",
+       "\n",
+       "                                            file_path Agent response  \\\n",
+       "22  /home/santiagoal/.cache/huggingface/hub/datase...              2   \n",
+       "\n",
+       "   Final answer  is_correct  \n",
+       "22            3           0  "
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Compute and save agent responses and their evaluation\n",
+    "\n",
+    "results_df[\"Agent response\"] = results_df.apply(func=gaia_eval.get_agent_response, axis=1)\n",
+    "results_df[\"is_correct\"] = results_df.apply(func=gaia_eval.evaluate_response, axis=1)\n",
+    "results_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0bfdb7b8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "89f8f584",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " Experiment Accuracy: 0.00 %\n"
+     ]
+    }
+   ],
+   "source": [
+    "accuracy = results_df[\"is_correct\"].mean()\n",
+    "print(f\" Experiment Accuracy: {(100 * accuracy):.2f} %\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "a326127a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Agent response",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "is_correct",
+         "rawType": "int64",
+         "type": "integer"
+        }
+       ],
+       "ref": "74bb579d-4727-47ec-af34-50c669e56ee9",
+       "rows": [
+        [
+         "22",
+         "You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, \"-\", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",
+         "/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/389793a7-ca17-4e82-81cb-2b3a2391b4b9.txt",
+         "2",
+         "3",
+         "0"
+        ]
+       ],
+       "shape": {
+        "columns": 5,
+        "rows": 1
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Question</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Agent response</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>is_correct</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>You are a telecommunications engineer who want...</td>\n",
+       "      <td>/home/santiagoal/.cache/huggingface/hub/datase...</td>\n",
+       "      <td>2</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Question  \\\n",
+       "22  You are a telecommunications engineer who want...   \n",
+       "\n",
+       "                                            file_path Agent response  \\\n",
+       "22  /home/santiagoal/.cache/huggingface/hub/datase...              2   \n",
+       "\n",
+       "   Final answer  is_correct  \n",
+       "22            3           0  "
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results_df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c826fde0",
+   "metadata": {},
+   "source": [
+    "### Save Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "06eb6132",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save current experiment\n",
+    "results_df.to_csv(xp_path, index=False)\n",
+    "\n",
+    "# Update experimentation history\n",
+    "\n",
+    "xp_results = {\n",
+    "    \"iteration\": current_iteration,\n",
+    "    \"experiment\": xp_name,\n",
+    "    \"agent\": agent_architecture,\n",
+    "    \"tools\": new_tools_list,\n",
+    "    \"accuracy\": round(results_df.copy()[\"is_correct\"].mean(), 2),\n",
+    "}\n",
+    "\n",
+    "updated_experiments_data = pd.concat([old_experiments_data, pd.DataFrame([xp_results])], ignore_index=True)\n",
+    "updated_experiments_data.drop_duplicates(inplace=True)\n",
+    "updated_experiments_data.to_csv(summary_experiments_path, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "84074f8d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "iteration",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "experiment",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "agent",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "tools",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "accuracy",
+         "rawType": "float64",
+         "type": "float"
+        }
+       ],
+       "ref": "42db43c4-0a26-4465-9ab4-0a3a9b103b8a",
+       "rows": [
+        [
+         "0",
+         "1",
+         "Implement Calculator Tool",
+         "React agent",
+         "Aritmetic",
+         "0.0"
+        ],
+        [
+         "1",
+         "2",
+         "Implement Search and Code tools",
+         "React agent",
+         "Aritmetic, Search, Code",
+         "0.17"
+        ],
+        [
+         "2",
+         "3",
+         "Integrate Whisper Audio Transcriber",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber",
+         "0.15"
+        ],
+        [
+         "3",
+         "4",
+         "Test Workflow",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber, ",
+         "0.2"
+        ],
+        [
+         "4",
+         "5",
+         "Integrate Text processing tool",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber, , Text Handler",
+         "0.1"
+        ],
+        [
+         "5",
+         "6",
+         "Integrate Text Handler tool",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber, , Text Handler, Text Handler",
+         "0.2"
+        ],
+        [
+         "6",
+         "7",
+         "Evaluate Agent Performance against tasks with .txt files",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber, , Text Handler, Text Handler, ",
+         null
+        ],
+        [
+         "7",
+         "8",
+         "Test Agent performance against tasks with attached .txt files",
+         "React agent",
+         "Aritmetic, Search, Code, Audio Transcriber, , Text Handler, Text Handler, , ",
+         "0.0"
+        ]
+       ],
+       "shape": {
+        "columns": 5,
+        "rows": 8
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>iteration</th>\n",
+       "      <th>experiment</th>\n",
+       "      <th>agent</th>\n",
+       "      <th>tools</th>\n",
+       "      <th>accuracy</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Implement Calculator Tool</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic</td>\n",
+       "      <td>0.00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Implement Search and Code tools</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code</td>\n",
+       "      <td>0.17</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Integrate Whisper Audio Transcriber</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber</td>\n",
+       "      <td>0.15</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Test Workflow</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber,</td>\n",
+       "      <td>0.20</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5</td>\n",
+       "      <td>Integrate Text processing tool</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber, , ...</td>\n",
+       "      <td>0.10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>6</td>\n",
+       "      <td>Integrate Text Handler tool</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber, , ...</td>\n",
+       "      <td>0.20</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>7</td>\n",
+       "      <td>Evaluate Agent Performance against tasks with ...</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber, , ...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>8</td>\n",
+       "      <td>Test Agent performance against tasks with atta...</td>\n",
+       "      <td>React agent</td>\n",
+       "      <td>Aritmetic, Search, Code, Audio Transcriber, , ...</td>\n",
+       "      <td>0.00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   iteration                                         experiment        agent  \\\n",
+       "0          1                          Implement Calculator Tool  React agent   \n",
+       "1          2                    Implement Search and Code tools  React agent   \n",
+       "2          3                Integrate Whisper Audio Transcriber  React agent   \n",
+       "3          4                                      Test Workflow  React agent   \n",
+       "4          5                     Integrate Text processing tool  React agent   \n",
+       "5          6                        Integrate Text Handler tool  React agent   \n",
+       "6          7  Evaluate Agent Performance against tasks with ...  React agent   \n",
+       "7          8  Test Agent performance against tasks with atta...  React agent   \n",
+       "\n",
+       "                                               tools  accuracy  \n",
+       "0                                          Aritmetic      0.00  \n",
+       "1                            Aritmetic, Search, Code      0.17  \n",
+       "2         Aritmetic, Search, Code, Audio Transcriber      0.15  \n",
+       "3       Aritmetic, Search, Code, Audio Transcriber,       0.20  \n",
+       "4  Aritmetic, Search, Code, Audio Transcriber, , ...      0.10  \n",
+       "5  Aritmetic, Search, Code, Audio Transcriber, , ...      0.20  \n",
+       "6  Aritmetic, Search, Code, Audio Transcriber, , ...       NaN  \n",
+       "7  Aritmetic, Search, Code, Audio Transcriber, , ...      0.00  "
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "updated_experiments_data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f705469a",
+   "metadata": {},
+   "source": [
+    "### Evaluation Summary\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "71648634",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "good_responses = results_df[results_df[\"is_correct\"] == 1].copy()\n",
+    "good_extensions = (\n",
+    "    good_responses[\"file_path\"]\n",
+    "    .fillna(\"No files\")\n",
+    "    .apply(lambda row: row.split(\".\")[-1] if \".\" in row else \"No files\")\n",
+    "    .unique()\n",
+    ")\n",
+    "good_file_management = \", \".join(sorted(good_extensions))\n",
+    "\n",
+    "bad_responses = results_df[results_df[\"is_correct\"] == 0].copy()\n",
+    "bad_extensions = (\n",
+    "    bad_responses[\"file_path\"]\n",
+    "    .fillna(\"No files\")\n",
+    "    .apply(lambda row: row.split(\".\")[-1] if \".\" in row else \"No files\")\n",
+    "    .unique()\n",
+    ")\n",
+    "bad_file_management = \", \".join(sorted(bad_extensions))\n",
+    "\n",
+    "performance_no_attached = results_df[results_df[\"file_path\"].apply(lambda row: len(row))==0][\"is_correct\"].mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "1d29a298",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Insights\n",
+      "\n",
+      " -------------------------------------------------- \n",
+      "\n",
+      "1. The Agent has an overall accuracy of 0.0%\n",
+      "2. The Agent succeded at questions with the following files types:  (0/1) \n",
+      "3. The Agent failed at questions with the following files types: txt (1/1) \n",
+      "4. The Agent has an Accuracy of nan% at tasks with no attached files\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\n",
+    "    \"Insights\\n\\n\",\n",
+    "    \"-\" * 50,\n",
+    "    f\"\\n\\n1. The Agent has an overall accuracy of {100 * accuracy:.1f}%\"\n",
+    "    f\"\\n2. The Agent succeded at questions with the following files types: {good_file_management} ({good_responses.is_correct.shape[0]}/{results_df.shape[0]})\",\n",
+    "    f\"\\n3. The Agent failed at questions with the following files types: {bad_file_management} ({bad_responses.is_correct.shape[0]}/{results_df.shape[0]})\",\n",
+    "    f\"\\n4. The Agent has an Accuracy of {100 * performance_no_attached:.1f}% at tasks with no attached files\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c4c9f4ff",
+   "metadata": {},
+   "source": [
+    "Conclusions\n",
+    "\n",
+    "1. The Agent has a 0% of accuracy. Showing poor performance at tasks with attached .txt files\n",
+    "4. Reasoning task has a big margin of improvement (0% of success on reasoning tasks). There are some questions that ask for watching internet videos, so it might be necessary to implement tools to adress this kind of tasks\n",
+    "\n",
+    "Next steps\n",
+    "\n",
+    "1. Build tools to deal with .png, .pptx, .py .xlsx files\n",
+    "2. Investigate improvements for pure-resoning tasks\n",
+    "3. Build tools to enable the agent watch / process youtube videos"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9e69ee21",
+   "metadata": {},
+   "source": [
+    "\n",
+    "### Main Questions to solve\n",
+    "\n",
+    "$\\square$ Which are the core tools for each level of questions \n",
+    "  - Level 1:\n",
+    "  - Level 2:\n",
+    "  - Level 3:\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "chappie-CHLGiFC_-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/tools_poc/audio_tool.ipynb

@@ -0,0 +1,688 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bf9121e7",
+   "metadata": {},
+   "source": [
+    "In this Notebook we create and sketch tools for the Agent. Building proofs of concept (PoC)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c29d8861",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94d01490",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Libraries\n",
+    "\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f09b59ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Local Modules\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f32d7959",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import Whisper\n",
+    "\n",
+    "import whisper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e131840",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Agent\n",
+    "\n",
+    "os.sys.path.append(\"../src\")\n",
+    "os.sys.path.append(\"../src/agents\")\n",
+    "\n",
+    "import react  # My AI assistant"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc02cd0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEBUG\n",
+    "\n",
+    "os.sys.path.append(\"../src/tools\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ca7a88a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Import GAIA Questions\n",
+    "from datasets import load_dataset\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a46c1c67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import pandas as pd\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a5cf241",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import snapshot_download, login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64a3d4d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load Hugging face credentials\n",
+    "\n",
+    "#load_dotenv()\n",
+    "#login(os.getenv(key=\"HF_TOKEN_CHAPPIE\"))  # Replace with your hf api key name\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fda05cb1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions_path = snapshot_download(repo_id=\"gaia-benchmark/GAIA\", repo_type=\"dataset\")\n",
+    "#gaia_questions = load_dataset(path=\"gaia-benchmark/GAIA\", name=\"2023_level1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9ebf52c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_index_dir = \"../../../.cache/huggingface/datasets/gaia-benchmark___gaia/2023_level1/0.0.1/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005\"\n",
+    "#gaia_index_dir = os.path.abspath(gaia_index_dir)\n",
+    "gaia_data_path = os.path.join(gaia_index_dir, \"gaia-validation.arrow\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "101f8c73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Temporal cell: wrote because of hugging face api calls limit\n",
+    "\n",
+    "import pyarrow.ipc as ipc\n",
+    "\n",
+    "with open(gaia_data_path, \"rb\") as f:\n",
+    "    reader = ipc.RecordBatchStreamReader(f)\n",
+    "    table = reader.read_all()\n",
+    "\n",
+    "gaia_df = table.to_pandas()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe9a5e51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions = gaia_questions[\"validation\"]  # Filter for dev purposes\n",
+    "#gaia_df = pd.DataFrame(gaia_questions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30c08c4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4826e755",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "# temp filter to get chess images\n",
+    "gaia_df[gaia_df[\"file_path\"].map(lambda f: f.endswith(\"44.png\"))]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f2d34bca",
+   "metadata": {},
+   "source": [
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0150df47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filetypes = {d[1].file_path.split(\".\")[-1] for d in gaia_df.iterrows()}\n",
+    "filetypes"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9c9208da",
+   "metadata": {},
+   "source": [
+    "### Read Historical XPs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82dd866a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "xp_paths = \"../data/agent_experiments/iterations/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5649c3c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Index all XPs\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "xp_dir = \"../data/agent_experiments/iterations\"\n",
+    "xp_list = []\n",
+    "# FIXME: from the 10th XP iteration, the sorted method is unuseful\n",
+    "\n",
+    "for i, xp_path in enumerate(sorted(os.listdir(xp_paths))):  \n",
+    "    xp_path = os.path.join(xp_dir, xp_path)\n",
+    "    temp_xp_df = pd.read_csv(xp_path)\n",
+    "    xp_list.append((i, temp_xp_df))\n",
+    "    del temp_xp_df "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e62498d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "xp[xp[\"Question\"] == xp_question][[\"is_correct\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ca5b1fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join latest XP result to each question\n",
+    "\n",
+    "index = gaia_df.index.tolist()  # Index for all the questions\n",
+    "questions = gaia_df[\"Question\"].tolist()\n",
+    "answers = list()\n",
+    "\n",
+    "\n",
+    "def filter_condition(xp_data: list) -> bool:\n",
+    "    i, xp = xp_data\n",
+    "    for question in questions:\n",
+    "        xp_addressed_questions = xp[\"Question\"].tolist()\n",
+    "        if question in xp_addressed_questions:\n",
+    "            return True\n",
+    "    return False\n",
+    "\n",
+    "filtered_xps = list(filter(filter_condition, xp_list))\n",
+    "filtered_xps_reversed = filtered_xps\n",
+    "filtered_xps_reversed.reverse()\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "for question in questions:\n",
+    "    answer_result = 0  # Assume wrong answer by default\n",
+    "    for i, xp in filtered_xps_reversed:\n",
+    "        for xp_question in xp[\"Question\"].tolist():\n",
+    "            \n",
+    "            if question == xp_question:\n",
+    "                answer_result_row = list(xp[xp[\"Question\"] == xp_question][\"is_correct\"])#.loc[0]\n",
+    "                answer_result_temp = answer_result_row[0]                    \n",
+    "                if answer_result_temp >= answer_result:\n",
+    "                    answer_result = answer_result_temp\n",
+    "        del xp\n",
+    "    answers.append(answer_result)\n",
+    "\n",
+    "# Join answers\n",
+    "historical_xp_results = gaia_df.copy()\n",
+    "historical_xp_results[\"is_correct\"] = pd.Series(answers)\n",
+    "del answers, questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca756f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd11615a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results.is_correct.mean()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3ddaede",
+   "metadata": {},
+   "source": [
+    "Let's study first which are the most common wrong tasks. I.e. How can we increase accuracy with a single next step (e.g. implement a new tool, modify sys message, etc.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0613fc4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df = historical_xp_results[historical_xp_results[\"is_correct\"]==0]\n",
+    "wrong_ans_df[\"fp_extension\"] = wrong_ans_df[\"file_path\"].map(lambda path: path.split(\".\")[-1])\n",
+    "wrong_ans_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43925ec4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Summary\n",
+    "wrong_ans_df.groupby(\"fp_extension\")[\"is_correct\"].count()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f77dabb",
+   "metadata": {},
+   "source": [
+    "The vast majority of remaining tasks do not include files to read. So we sould study them at first"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1379da15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension = wrong_ans_df[wrong_ans_df[\"file_path\"].map(lambda fp: len(fp)==0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e40e190",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b4cbc05d",
+   "metadata": {},
+   "source": [
+    "## Audio Tool"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "57b579ef",
+   "metadata": {},
+   "source": [
+    "Let's study where does our current Agent fail, especially on audio-like questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b7a1444",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "60d3af35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fe9e932b",
+   "metadata": {},
+   "source": [
+    "### Identify Audio-like tasks"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9e8c9a12",
+   "metadata": {},
+   "source": [
+    "The first questions is to understand which questions need audio? \n",
+    "\n",
+    "* Hypothesis: Just look at such questions that have a .mp3 file attached\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23516201",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Gather a sample file from any task\n",
+    "\n",
+    "audio_tasks = gaia_df[(gaia_df[\"file_path\"].str.len()>0) & (gaia_df[\"file_path\"].str.endswith(\".mp3\"))]\n",
+    "sample_task = audio_tasks.iloc[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa03a8a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(f\"{(audio_tasks.shape[0] / gaia_df.shape[0]) * 100:.1f}% of tasks need audio processing\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b30a0394",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_task"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d47c9bde",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get the sample file\n",
+    "\n",
+    "filepath = sample_task[\"file_path\"]\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0d07cab8",
+   "metadata": {},
+   "source": [
+    "Let's import the mp3 file with [ffmpeg](https://stackoverflow.com/questions/9458480/read-mp3-in-python-3#:~:text=%24%20ffmpeg%20%2Di%20foo.mp3%20%2Dvn%20%2Dacodec%20pcm_s16le%20%2Dac%201%20%2Dar%2044100%20%2Df%20wav%20foo.wav)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a82f6246",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "temp_data_path = \"/home/santiagoal/current-projects/chappie/data/temp-data/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c8e255c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if run:\n",
+    "    os.system(f\"ffmpeg -i {filepath} -vn -acodec pcm_s16le -ac 1 -ar 44100 -f wav {temp_data_path}sample_audio.wav\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f9d0ff9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if run:\n",
+    "    use_gpu = False\n",
+    "    model_size = \"tiny\"\n",
+    "\n",
+    "    model = (\n",
+    "        whisper.load_model(model_size).cuda()\n",
+    "        if use_gpu\n",
+    "        else whisper.load_model(model_size)\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "df808df1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(filepath)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9ce2c352",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if run: \n",
+    "    raw_transcript = model.transcribe(\n",
+    "            filepath,\n",
+    "            word_timestamps=False,\n",
+    "            no_speech_threshold=0.5,\n",
+    "            condition_on_previous_text=True,\n",
+    "            compression_ratio_threshold=2.0,\n",
+    "        )\n",
+    "\n",
+    "    transcript = raw_transcript[\"text\"]\n",
+    "    print(transcript)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0654ee48",
+   "metadata": {},
+   "source": [
+    "### Tool POC\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83335239",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "if run: \n",
+    "    def transcriber(audio_path: str, ai_model = model) -> str:\n",
+    "        \"\"\"\n",
+    "        Transcribes an audio file\n",
+    "\n",
+    "        Parameters\n",
+    "        ----------\n",
+    "        audio_path : str\n",
+    "            Path to the audio file\n",
+    "        ai_model\n",
+    "            audio-to-text AI model \n",
+    "\n",
+    "        Returns:\n",
+    "            str: Text of the transcript \n",
+    "        \"\"\"\n",
+    "        raw_transcript = ai_model.transcribe(\n",
+    "            audio_path,\n",
+    "            word_timestamps=False,\n",
+    "            no_speech_threshold=0.5,\n",
+    "            condition_on_previous_text=True,\n",
+    "            compression_ratio_threshold=2.0,\n",
+    "        )\n",
+    "\n",
+    "        transcript = raw_transcript[\"text\"]\n",
+    "\n",
+    "        return transcript"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aa9d4e58",
+   "metadata": {},
+   "source": [
+    "### Integrate Tool POC and experiment\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8dee4276",
+   "metadata": {},
+   "source": [
+    "The changes have been integrated, now we will experiment with the new version of the agent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "811185e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "audio_tasks[\"Agent response\"] = audio_tasks.apply(func=get_agent_response, axis=1)\n",
+    "audio_tasks[\"is_correct\"] = audio_tasks.apply(func=eval_answer, axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c2ab6ae6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "audio_tasks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76e9241c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Next steps: Update model... "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "chappie-CHLGiFC_-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/tools_poc/chess_tool.ipynb

@@ -0,0 +1,863 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bf9121e7",
+   "metadata": {},
+   "source": [
+    "In this Notebook we create and sketch tools for the Agent. Building proofs of concept (PoC)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c29d8861",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94d01490",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Libraries\n",
+    "\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f09b59ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Local Modules\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f32d7959",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import Whisper\n",
+    "\n",
+    "import whisper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e131840",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Agent\n",
+    "\n",
+    "os.sys.path.append(\"../src\")\n",
+    "os.sys.path.append(\"../src/agents\")\n",
+    "\n",
+    "import react  # My AI assistant"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc02cd0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEBUG\n",
+    "\n",
+    "os.sys.path.append(\"../src/tools\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ca7a88a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Import GAIA Questions\n",
+    "from datasets import load_dataset\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a46c1c67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import pandas as pd\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a5cf241",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import snapshot_download, login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64a3d4d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load Hugging face credentials\n",
+    "\n",
+    "#load_dotenv()\n",
+    "#login(os.getenv(key=\"HF_TOKEN_CHAPPIE\"))  # Replace with your hf api key name\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fda05cb1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions_path = snapshot_download(repo_id=\"gaia-benchmark/GAIA\", repo_type=\"dataset\")\n",
+    "#gaia_questions = load_dataset(path=\"gaia-benchmark/GAIA\", name=\"2023_level1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9ebf52c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_index_dir = \"../../../.cache/huggingface/datasets/gaia-benchmark___gaia/2023_level1/0.0.1/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005\"\n",
+    "#gaia_index_dir = os.path.abspath(gaia_index_dir)\n",
+    "gaia_data_path = os.path.join(gaia_index_dir, \"gaia-validation.arrow\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "101f8c73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Temporal cell: wrote because of hugging face api calls limit\n",
+    "\n",
+    "import pyarrow.ipc as ipc\n",
+    "\n",
+    "with open(gaia_data_path, \"rb\") as f:\n",
+    "    reader = ipc.RecordBatchStreamReader(f)\n",
+    "    table = reader.read_all()\n",
+    "\n",
+    "gaia_df = table.to_pandas()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe9a5e51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions = gaia_questions[\"validation\"]  # Filter for dev purposes\n",
+    "#gaia_df = pd.DataFrame(gaia_questions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30c08c4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4826e755",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "# temp filter to get chess images\n",
+    "gaia_df[gaia_df[\"file_path\"].map(lambda f: f.endswith(\"44.png\"))]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f2d34bca",
+   "metadata": {},
+   "source": [
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0150df47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filetypes = {d[1].file_path.split(\".\")[-1] for d in gaia_df.iterrows()}\n",
+    "filetypes"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "94422aae",
+   "metadata": {},
+   "source": [
+    "### Read Historical XPs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82dd866a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "xp_paths = \"../data/agent_experiments/iterations/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5649c3c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Index all XPs\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "xp_dir = \"../data/agent_experiments/iterations\"\n",
+    "xp_list = []\n",
+    "# FIXME: from the 10th XP iteration, the sorted method is unuseful\n",
+    "\n",
+    "for i, xp_path in enumerate(sorted(os.listdir(xp_paths))):  \n",
+    "    xp_path = os.path.join(xp_dir, xp_path)\n",
+    "    temp_xp_df = pd.read_csv(xp_path)\n",
+    "    xp_list.append((i, temp_xp_df))\n",
+    "    del temp_xp_df "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e62498d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "xp[xp[\"Question\"] == xp_question][[\"is_correct\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ca5b1fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join latest XP result to each question\n",
+    "\n",
+    "index = gaia_df.index.tolist()  # Index for all the questions\n",
+    "questions = gaia_df[\"Question\"].tolist()\n",
+    "answers = list()\n",
+    "\n",
+    "\n",
+    "def filter_condition(xp_data: list) -> bool:\n",
+    "    i, xp = xp_data\n",
+    "    for question in questions:\n",
+    "        xp_addressed_questions = xp[\"Question\"].tolist()\n",
+    "        if question in xp_addressed_questions:\n",
+    "            return True\n",
+    "    return False\n",
+    "\n",
+    "filtered_xps = list(filter(filter_condition, xp_list))\n",
+    "filtered_xps_reversed = filtered_xps\n",
+    "filtered_xps_reversed.reverse()\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "for question in questions:\n",
+    "    answer_result = 0  # Assume wrong answer by default\n",
+    "    for i, xp in filtered_xps_reversed:\n",
+    "        for xp_question in xp[\"Question\"].tolist():\n",
+    "            \n",
+    "            if question == xp_question:\n",
+    "                answer_result_row = list(xp[xp[\"Question\"] == xp_question][\"is_correct\"])#.loc[0]\n",
+    "                answer_result_temp = answer_result_row[0]                    \n",
+    "                if answer_result_temp >= answer_result:\n",
+    "                    answer_result = answer_result_temp\n",
+    "        del xp\n",
+    "    answers.append(answer_result)\n",
+    "\n",
+    "# Join answers\n",
+    "historical_xp_results = gaia_df.copy()\n",
+    "historical_xp_results[\"is_correct\"] = pd.Series(answers)\n",
+    "del answers, questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca756f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd11615a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results.is_correct.mean()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3ddaede",
+   "metadata": {},
+   "source": [
+    "Let's study first which are the most common wrong tasks. I.e. How can we increase accuracy with a single next step (e.g. implement a new tool, modify sys message, etc.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0613fc4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df = historical_xp_results[historical_xp_results[\"is_correct\"]==0]\n",
+    "wrong_ans_df[\"fp_extension\"] = wrong_ans_df[\"file_path\"].map(lambda path: path.split(\".\")[-1])\n",
+    "wrong_ans_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43925ec4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Summary\n",
+    "wrong_ans_df.groupby(\"fp_extension\")[\"is_correct\"].count()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f77dabb",
+   "metadata": {},
+   "source": [
+    "The vast majority of remaining tasks do not include files to read. So we sould study them at first"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1379da15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension = wrong_ans_df[wrong_ans_df[\"file_path\"].map(lambda fp: len(fp)==0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e40e190",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad4ce0bc",
+   "metadata": {},
+   "source": [
+    "### Identify Image-like tasks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "387e0aa7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "img_filetypes = ('png', 'jpg')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ec59bb6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_tasks_df = gaia_df[gaia_df[\"file_path\"].apply(lambda row: row.split(\".\")[-1] in img_filetypes)]\n",
+    "image_tasks_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "180294ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_tasks_df.iloc[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49d48c65",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chess_task = image_tasks_df.iloc[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73d7605e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.path.isfile(chess_task.file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "672fec32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cv2\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "image_path = chess_task.file_path\n",
+    "image_bgr = cv2.imread(image_path)\n",
+    "\n",
+    "image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)\n",
+    "\n",
+    "plt.imshow(image_rgb)\n",
+    "plt.title(\"Chess Image\")\n",
+    "plt.axis(\"off\")\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1d60868",
+   "metadata": {},
+   "source": [
+    "### Chess Tool"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a0738cd7",
+   "metadata": {},
+   "source": [
+    "In this section, our methodology is to \n",
+    "\n",
+    "1. Use a CV model to get the FEN position of a 2D chess image\n",
+    "2. Usea another model (likely stockfish) to predict the best move from a FEN position"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c16f891",
+   "metadata": {},
+   "source": [
+    "#### Tool POC"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d531e68",
+   "metadata": {},
+   "source": [
+    "To extract the FEN position from the board image, we used [board_to_fen](https://github.com/mcdominik/board_to_fen)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de3a51d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cairosvg\n",
+    "from io import BytesIO\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "01860727",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image\n",
+    "from board_to_fen.predict import get_fen_from_image\n",
+    "\n",
+    "img = Image.open(chess_task.file_path)\n",
+    "fen = get_fen_from_image(img, black_view=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e07228d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fen"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6819c048",
+   "metadata": {},
+   "source": [
+    "We observed this position has no enough information about the current position, so we need to post process it:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02ba9e63",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fen_cleaned = fen + \" b - - 0 1\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ac810f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fen_cleaned"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "38f81960",
+   "metadata": {},
+   "source": [
+    "Looking forward, we will need to a warn the AI agent to post-process the FEN before passing to stockfish. However, we can validate the prected FEN "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e6365917",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# try out with pychess\n",
+    "\n",
+    "import chess as c\n",
+    "import chess.engine as ce\n",
+    "\n",
+    "board = c.Board(fen_cleaned)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bc3963d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "board"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96037282",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "svg_data = c.svg.board(board=board, flipped=True)\n",
+    "png_data = BytesIO()\n",
+    "cairosvg.svg2png(bytestring=svg_data.encode('utf-8'), write_to=png_data)\n",
+    "png_data.seek(0)\n",
+    "img_array = np.asarray(bytearray(png_data.read()), dtype=np.uint8)\n",
+    "img = cv2.imdecode(img_array, cv2.COLOR_BGR2RGB)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0fbe36a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(25, 10))\n",
+    "\n",
+    "ax[0].imshow(cv2.imread(chess_task.file_path))\n",
+    "ax[0].axis(\"off\")\n",
+    "ax[0].set_title(\"Original Chess Board\", fontsize=18)\n",
+    "\n",
+    "ax[1].imshow(img)\n",
+    "ax[1].axis(\"off\")\n",
+    "ax[1].set_title(f\"Predicted Chess Board\\nFEN: {fen_cleaned}\", fontsize=18)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1037165",
+   "metadata": {},
+   "source": [
+    "Once we extracted the FEN chess position, we pass it to Stockfish in order to get the best next move"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67f340fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Stockfish \n",
+    "stockfish_path = \"../external/chess-engines/stockfish/stockfish-ubuntu-x86-64-avx2\"\n",
+    "chess_engine = ce.SimpleEngine.popen_uci(stockfish_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "901f4ad7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Predict next best move \n",
+    "stockfish_player = chess_engine.play(board=board, limit=ce.Limit(time=10))\n",
+    "best_move_uci = stockfish_player.move\n",
+    "best_move = board.san(move=best_move_uci) # Convert from Universal Chess interface to Standard Algebraic Notation\n",
+    "best_move"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8b2bffbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\n",
+    "    \"Results\\n\",\n",
+    "    \"=\" * 30,\n",
+    "    f\"\\nPredicted best next move: {best_move}\",\n",
+    "    f\"\\nCorrect Answer: {chess_task['Final answer']}\"\n",
+    "    f\"\\nIs the prediction correct?: {'yes' if best_move==chess_task['Final answer'] else 'no'}\"\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7d56f6c1",
+   "metadata": {},
+   "source": [
+    "There we go! our model finds the correct answer"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4eb6d948",
+   "metadata": {},
+   "source": [
+    "As a bonus, we might consider the case the board view is set from black or white pieces' perspective, so it worths considering to extract text from the board image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d48f84bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import cv2 \n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "chess_img_path = \"/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/cca530fc-4052-43b2-b130-b30968d8aa44.png\"\n",
+    "cropped_chess_board_path = \"../data/images/cropped_chess_board.png\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df75df03",
+   "metadata": {},
+   "source": [
+    "Our method is to crop the bottom-left chess square and extract the text. \n",
+    "1. If the text is a1 -> The board has white view\n",
+    "2. If the text is h8 -> The board has black view"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c5e520a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Read Chess Board Img and Crop the bottom-left \n",
+    "img_bgr = cv2.imread(chess_img_path)\n",
+    "img_bgr = cv2.imread(chess_img_path)\n",
+    "img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n",
+    "\n",
+    "img_np = np.array(img_rgb)\n",
+    "img_np_shape = img_np.shape\n",
+    "img_rows, img_cols = img_np_shape[:2]\n",
+    "img_cropped = img_np[img_rows // 8 * 7:, :img_cols // 8 * 1, :]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a332c910",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig, ax = plt.subplots(ncols=2, nrows=1, figsize=(20, 10))\n",
+    "\n",
+    "ax[0].imshow(img_rgb)\n",
+    "ax[1].imshow(img_cropped)\n",
+    "\n",
+    "ax[0].set_title(\"Original Image\")\n",
+    "ax[1].set_title(\"Image Cropped\")\n",
+    "\n",
+    "ax[0].axis(\"off\")\n",
+    "ax[1].axis(\"off\")\n",
+    "\n",
+    "fig.suptitle(\"Crop image to identify board orientation\", y=0.92, fontsize=15)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8723e6fd",
+   "metadata": {},
+   "source": [
+    "Once we crop the board image, we might streamline a simple OCR model to extract the board text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d60d5248",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import easyocr\n",
+    "\n",
+    "reader = easyocr.Reader(['en'], gpu=False)  \n",
+    "\n",
+    "results = reader.readtext(cropped_chess_board_path)\n",
+    "results.reverse()\n",
+    "\n",
+    "# Grab text\n",
+    "left_bottom_cell = \"\"\n",
+    "for _, text_temp, _ in results:\n",
+    "    left_bottom_cell += text_temp\n",
+    "    \n",
+    "print(f\"The left-bottom chess board square is {left_bottom_cell}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "54fadcf2",
+   "metadata": {},
+   "source": [
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "57b579ef",
+   "metadata": {},
+   "source": [
+    "Let's study where does our current Agent fail, especially on audio-like questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b7a1444",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "60d3af35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_df"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "chappie-CHLGiFC_-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/tools_poc/template.ipynb

@@ -0,0 +1,475 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bf9121e7",
+   "metadata": {},
+   "source": [
+    "In this Notebook we create and sketch tools for the Agent. Building proofs of concept (PoC)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c29d8861",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94d01490",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Libraries\n",
+    "\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f09b59ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Local Modules\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f32d7959",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import Whisper\n",
+    "\n",
+    "import whisper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e131840",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import Agent\n",
+    "\n",
+    "os.sys.path.append(\"../src\")\n",
+    "os.sys.path.append(\"../src/agents\")\n",
+    "\n",
+    "import react  # My AI assistant"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc02cd0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEBUG\n",
+    "\n",
+    "os.sys.path.append(\"../src/tools\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ca7a88a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Import GAIA Questions\n",
+    "from datasets import load_dataset\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a46c1c67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import pandas as pd\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a5cf241",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import snapshot_download, login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64a3d4d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load Hugging face credentials\n",
+    "\n",
+    "#load_dotenv()\n",
+    "#login(os.getenv(key=\"HF_TOKEN_CHAPPIE\"))  # Replace with your hf api key name\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fda05cb1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions_path = snapshot_download(repo_id=\"gaia-benchmark/GAIA\", repo_type=\"dataset\")\n",
+    "#gaia_questions = load_dataset(path=\"gaia-benchmark/GAIA\", name=\"2023_level1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9ebf52c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_index_dir = \"../../../.cache/huggingface/datasets/gaia-benchmark___gaia/2023_level1/0.0.1/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005\"\n",
+    "#gaia_index_dir = os.path.abspath(gaia_index_dir)\n",
+    "gaia_data_path = os.path.join(gaia_index_dir, \"gaia-validation.arrow\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "101f8c73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Temporal cell: wrote because of hugging face api calls limit\n",
+    "\n",
+    "import pyarrow.ipc as ipc\n",
+    "\n",
+    "with open(gaia_data_path, \"rb\") as f:\n",
+    "    reader = ipc.RecordBatchStreamReader(f)\n",
+    "    table = reader.read_all()\n",
+    "\n",
+    "gaia_df = table.to_pandas()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe9a5e51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions = gaia_questions[\"validation\"]  # Filter for dev purposes\n",
+    "#gaia_df = pd.DataFrame(gaia_questions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30c08c4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4826e755",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "# temp filter to get chess images\n",
+    "gaia_df[gaia_df[\"file_path\"].map(lambda f: f.endswith(\"44.png\"))]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f2d34bca",
+   "metadata": {},
+   "source": [
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0150df47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filetypes = {d[1].file_path.split(\".\")[-1] for d in gaia_df.iterrows()}\n",
+    "filetypes"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c41c99d7",
+   "metadata": {},
+   "source": [
+    "### Read Historical XPs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82dd866a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "xp_paths = \"../data/agent_experiments/iterations/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5649c3c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Index all XPs\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "xp_dir = \"../data/agent_experiments/iterations\"\n",
+    "xp_list = []\n",
+    "# FIXME: from the 10th XP iteration, the sorted method is unuseful\n",
+    "\n",
+    "for i, xp_path in enumerate(sorted(os.listdir(xp_paths))):  \n",
+    "    xp_path = os.path.join(xp_dir, xp_path)\n",
+    "    temp_xp_df = pd.read_csv(xp_path)\n",
+    "    xp_list.append((i, temp_xp_df))\n",
+    "    del temp_xp_df "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e62498d0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEL\n",
+    "xp[xp[\"Question\"] == xp_question][[\"is_correct\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ca5b1fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join latest XP result to each question\n",
+    "\n",
+    "index = gaia_df.index.tolist()  # Index for all the questions\n",
+    "questions = gaia_df[\"Question\"].tolist()\n",
+    "answers = list()\n",
+    "\n",
+    "\n",
+    "def filter_condition(xp_data: list) -> bool:\n",
+    "    i, xp = xp_data\n",
+    "    for question in questions:\n",
+    "        xp_addressed_questions = xp[\"Question\"].tolist()\n",
+    "        if question in xp_addressed_questions:\n",
+    "            return True\n",
+    "    return False\n",
+    "\n",
+    "filtered_xps = list(filter(filter_condition, xp_list))\n",
+    "filtered_xps_reversed = filtered_xps\n",
+    "filtered_xps_reversed.reverse()\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "for question in questions:\n",
+    "    answer_result = 0  # Assume wrong answer by default\n",
+    "    for i, xp in filtered_xps_reversed:\n",
+    "        for xp_question in xp[\"Question\"].tolist():\n",
+    "            \n",
+    "            if question == xp_question:\n",
+    "                answer_result_row = list(xp[xp[\"Question\"] == xp_question][\"is_correct\"])#.loc[0]\n",
+    "                answer_result_temp = answer_result_row[0]                    \n",
+    "                if answer_result_temp >= answer_result:\n",
+    "                    answer_result = answer_result_temp\n",
+    "        del xp\n",
+    "    answers.append(answer_result)\n",
+    "\n",
+    "# Join answers\n",
+    "historical_xp_results = gaia_df.copy()\n",
+    "historical_xp_results[\"is_correct\"] = pd.Series(answers)\n",
+    "del answers, questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca756f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd11615a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results.is_correct.mean()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3ddaede",
+   "metadata": {},
+   "source": [
+    "Let's study first which are the most common wrong tasks. I.e. How can we increase accuracy with a single next step (e.g. implement a new tool, modify sys message, etc.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0613fc4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df = historical_xp_results[historical_xp_results[\"is_correct\"]==0]\n",
+    "wrong_ans_df[\"fp_extension\"] = wrong_ans_df[\"file_path\"].map(lambda path: path.split(\".\")[-1])\n",
+    "wrong_ans_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43925ec4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Summary\n",
+    "wrong_ans_df.groupby(\"fp_extension\")[\"is_correct\"].count()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f77dabb",
+   "metadata": {},
+   "source": [
+    "The vast majority of remaining tasks do not include files to read. So we sould study them at first"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1379da15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension = wrong_ans_df[wrong_ans_df[\"file_path\"].map(lambda fp: len(fp)==0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e40e190",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d25a6fd5",
+   "metadata": {},
+   "source": [
+    "### Build Tool"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "116ee0cb",
+   "metadata": {},
+   "source": [
+    "#### 1. FIlter tasks to solve with the tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95a3b948",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6a68fb6e",
+   "metadata": {},
+   "source": [
+    "#### 2. Tool PoC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2a9a59a6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "30a22395",
+   "metadata": {},
+   "source": [
+    "#### 3. Test Tool"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "069ffda9",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "chappie-CHLGiFC_-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/tools_poc/web_search.ipynb

@@ -0,0 +1,2011 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bf9121e7",
+   "metadata": {},
+   "source": [
+    "In this Notebook we create and sketch tools for the Agent. Building proofs of concept (PoC)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c29d8861",
+   "metadata": {},
+   "source": [
+    "### Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "94d01490",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Libraries\n",
+    "\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f32d7959",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import Whisper\n",
+    "\n",
+    "import whisper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5e131840",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Langfuse client is disabled since no public_key was provided as a parameter or environment variable 'LANGFUSE_PUBLIC_KEY'. See our docs: https://langfuse.com/docs/sdk/python/low-level-sdk#initialize-client\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import Agent\n",
+    "\n",
+    "os.sys.path.append(\"../../src\")\n",
+    "os.sys.path.append(\"../../src/agents\")\n",
+    "\n",
+    "import react  # My AI assistant"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "dc02cd0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DEBUG\n",
+    "\n",
+    "os.sys.path.append(\"../../src/tools\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "3ca7a88a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/santiagoal/.cache/pypoetry/virtualenvs/chappie-CHLGiFC_-py3.11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "# Import GAIA Questions\n",
+    "from datasets import load_dataset\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "a46c1c67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import pandas as pd\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "5a5cf241",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import snapshot_download, login"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "64a3d4d1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load Hugging face credentials\n",
+    "\n",
+    "#load_dotenv()\n",
+    "#login(os.getenv(key=\"HF_TOKEN_CHAPPIE\"))  # Replace with your hf api key name\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "fda05cb1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions_path = snapshot_download(repo_id=\"gaia-benchmark/GAIA\", repo_type=\"dataset\")\n",
+    "#gaia_questions = load_dataset(path=\"gaia-benchmark/GAIA\", name=\"2023_level1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e9ebf52c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaia_index_dir = \"../../../../.cache/huggingface/datasets/gaia-benchmark___gaia/2023_level1/0.0.1/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005\"\n",
+    "#gaia_index_dir = os.path.abspath(gaia_index_dir)\n",
+    "gaia_data_path = os.path.join(gaia_index_dir, \"gaia-validation.arrow\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "101f8c73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Temporal cell: wrote because of hugging face api calls limit\n",
+    "\n",
+    "import pyarrow.ipc as ipc\n",
+    "\n",
+    "with open(gaia_data_path, \"rb\") as f:\n",
+    "    reader = ipc.RecordBatchStreamReader(f)\n",
+    "    table = reader.read_all()\n",
+    "\n",
+    "gaia_df = table.to_pandas()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "fe9a5e51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#gaia_questions = gaia_questions[\"validation\"]  # Filter for dev purposes\n",
+    "#gaia_df = pd.DataFrame(gaia_questions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "30c08c4d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "task_id",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Level",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Annotator Metadata",
+         "rawType": "object",
+         "type": "unknown"
+        }
+       ],
+       "ref": "46594f81-f33f-486c-8c5c-005503176990",
+       "rows": [
+        [
+         "0",
+         "e1fc63a2-da7a-432f-be78-7c4a95598703",
+         "If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.",
+         "1",
+         "17",
+         "",
+         "",
+         "{'Steps': '1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.', 'Number of steps': '4', 'How long did this take?': '20 Minutes', 'Tools': '1. A web browser.\\n2. A search engine.\\n3. A calculator.', 'Number of tools': '3'}"
+        ],
+        [
+         "1",
+         "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",
+         "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': '1. I did a search for Mercedes Sosa\\n2. I went to the Wikipedia page for her\\n3. I scrolled down to \"Studio albums\"\\n4. I counted the ones between 2000 and 2009', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. web browser\\n2. google search', 'Number of tools': '2'}"
+        ],
+        [
+         "2",
+         "ec09fa32-d03f-4bf8-84b0-1f16922c3ae4",
+         "Here's a fun riddle that I think you'll enjoy.\n\nYou have been selected to play the final round of the hit new game show \"Pick That Ping-Pong\". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.\n\nA device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.\n\nThe ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.\n\nWhich ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': 'Step 1: Evaluate the problem statement provided in my user\\'s prompt\\nStep 2: Consider the probability of any ball on the platform earning the prize.\\nStep 3: Evaluate the ball in position one. The probability of it earning the prize, P1, is 1/3\\nStep 4: Using a calculator, evaluate the ball in position two. The probability of it earning the prize, P2, is the difference between 1 and the product of the complementary probabilities for each trial\\nP2 = 1 - (2/3)(2/3)\\nP2 = 5/9\\nStep 5: Using a calculator, evaluate the ball in position three. The probability of it earning the prize, P3, is the difference between 1 and the product of the complementary probabilities for each trial\\nP3 = 1 - (2/3)(2/3)(2/3)\\nP3 = 19/27\\nStep 6: Consider the possible outcomes of numbers higher than 3.\\nStep 7: For each trial, either 1 or 2 balls from the ramp will advance to the platform. For any given selection, there is a 50% chance that the ball advances to position 2 or position 3.\\nStep 8: As position three holds the highest chance of earning the prize, select the only ball known to occupy position three with certainty, ball 3.\\nStep 9: Report the correct answer to my user, \"3\"', 'Number of steps': '9', 'How long did this take?': '1 minute', 'Tools': 'None', 'Number of tools': '0'}"
+        ],
+        [
+         "3",
+         "5d0080cb-90d7-4712-bc33-848150e917d3",
+         "What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"",
+         "1",
+         "0.1777",
+         "",
+         "",
+         "{'Steps': '1. Searched \\'\"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"\\' on Google.\\n2. Opened \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\" at https://journals.le.ac.uk/ojs1/index.php/jist/article/view/733.\\n3. Clicked \"PDF\".\\n4. Found the calculations for the volume of the fish bag and noted them.', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. PDF access', 'Number of tools': '3'}"
+        ],
+        [
+         "4",
+         "a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
+         "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",
+         "1",
+         "3",
+         "",
+         "",
+         "{'Steps': '1. Navigate to the YouTube link.\\n2. Watch the video to see the highest number of bird species.\\n3. Note the number.', 'Number of steps': '3', 'How long did this take?': '3 minutes', 'Tools': '1. Web browser\\n2. Video parsing', 'Number of tools': '2'}"
+        ]
+       ],
+       "shape": {
+        "columns": 7,
+        "rows": 5
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>task_id</th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Level</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>file_name</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Annotator Metadata</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>e1fc63a2-da7a-432f-be78-7c4a95598703</td>\n",
+       "      <td>If Eliud Kipchoge could maintain his record-ma...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>17</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Googled Eliud Kipchoge marathon ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>8e867cd7-cff9-4e6c-867a-ff5ddc2550be</td>\n",
+       "      <td>How many studio albums were published by Merce...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. I did a search for Mercedes Sosa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ec09fa32-d03f-4bf8-84b0-1f16922c3ae4</td>\n",
+       "      <td>Here's a fun riddle that I think you'll enjoy....</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': 'Step 1: Evaluate the problem statem...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>5d0080cb-90d7-4712-bc33-848150e917d3</td>\n",
+       "      <td>What was the volume in m^3 of the fish bag tha...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.1777</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched '\"Can Hiccup Supply Eno...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>a1e91b78-d3d8-4675-bb8d-62741b4b68a6</td>\n",
+       "      <td>In the video https://www.youtube.com/watch?v=L...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Navigate to the YouTube link.\n",
+       "2....</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                task_id  \\\n",
+       "0  e1fc63a2-da7a-432f-be78-7c4a95598703   \n",
+       "1  8e867cd7-cff9-4e6c-867a-ff5ddc2550be   \n",
+       "2  ec09fa32-d03f-4bf8-84b0-1f16922c3ae4   \n",
+       "3  5d0080cb-90d7-4712-bc33-848150e917d3   \n",
+       "4  a1e91b78-d3d8-4675-bb8d-62741b4b68a6   \n",
+       "\n",
+       "                                            Question Level Final answer  \\\n",
+       "0  If Eliud Kipchoge could maintain his record-ma...     1           17   \n",
+       "1  How many studio albums were published by Merce...     1            3   \n",
+       "2  Here's a fun riddle that I think you'll enjoy....     1            3   \n",
+       "3  What was the volume in m^3 of the fish bag tha...     1       0.1777   \n",
+       "4  In the video https://www.youtube.com/watch?v=L...     1            3   \n",
+       "\n",
+       "  file_name file_path                                 Annotator Metadata  \n",
+       "0                      {'Steps': '1. Googled Eliud Kipchoge marathon ...  \n",
+       "1                      {'Steps': '1. I did a search for Mercedes Sosa...  \n",
+       "2                      {'Steps': 'Step 1: Evaluate the problem statem...  \n",
+       "3                      {'Steps': '1. Searched '\"Can Hiccup Supply Eno...  \n",
+       "4                      {'Steps': '1. Navigate to the YouTube link.\n",
+       "2....  "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gaia_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "4826e755",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "task_id",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Level",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Annotator Metadata",
+         "rawType": "object",
+         "type": "unknown"
+        }
+       ],
+       "ref": "682031dd-71c0-4a11-a29b-48f858136885",
+       "rows": [
+        [
+         "16",
+         "cca530fc-4052-43b2-b130-b30968d8aa44",
+         "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
+         "1",
+         "Rd5",
+         "cca530fc-4052-43b2-b130-b30968d8aa44.png",
+         "/home/santiagoal/.cache/huggingface/hub/datasets--gaia-benchmark--GAIA/snapshots/897f2dfbb5c952b5c3c1509e648381f9c7b70316/2023/validation/cca530fc-4052-43b2-b130-b30968d8aa44.png",
+         "{'Steps': 'Step 1: Evaluate the position of the pieces in the chess position\\nStep 2: Report the best move available for black: \"Rd5\"', 'Number of steps': '2', 'How long did this take?': '10 minutes', 'Tools': '1. Image recognition tools', 'Number of tools': '1'}"
+        ]
+       ],
+       "shape": {
+        "columns": 7,
+        "rows": 1
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>task_id</th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Level</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>file_name</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Annotator Metadata</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>cca530fc-4052-43b2-b130-b30968d8aa44</td>\n",
+       "      <td>Review the chess position provided in the imag...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Rd5</td>\n",
+       "      <td>cca530fc-4052-43b2-b130-b30968d8aa44.png</td>\n",
+       "      <td>/home/santiagoal/.cache/huggingface/hub/datase...</td>\n",
+       "      <td>{'Steps': 'Step 1: Evaluate the position of th...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 task_id  \\\n",
+       "16  cca530fc-4052-43b2-b130-b30968d8aa44   \n",
+       "\n",
+       "                                             Question Level Final answer  \\\n",
+       "16  Review the chess position provided in the imag...     1          Rd5   \n",
+       "\n",
+       "                                   file_name  \\\n",
+       "16  cca530fc-4052-43b2-b130-b30968d8aa44.png   \n",
+       "\n",
+       "                                            file_path  \\\n",
+       "16  /home/santiagoal/.cache/huggingface/hub/datase...   \n",
+       "\n",
+       "                                   Annotator Metadata  \n",
+       "16  {'Steps': 'Step 1: Evaluate the position of th...  "
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# DEL\n",
+    "# temp filter to get chess images\n",
+    "gaia_df[gaia_df[\"file_path\"].map(lambda f: f.endswith(\"44.png\"))]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f2d34bca",
+   "metadata": {},
+   "source": [
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "0150df47",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'', 'docx', 'mp3', 'png', 'pptx', 'py', 'txt', 'xlsx'}"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "filetypes = {d[1].file_path.split(\".\")[-1] for d in gaia_df.iterrows()}\n",
+    "filetypes"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c41c99d7",
+   "metadata": {},
+   "source": [
+    "### Read Historical XPs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82dd866a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "xp_paths = \"../data/agent_experiments/iterations/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5649c3c2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Index all XPs\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "xp_dir = \"../../data/agent_experiments/iterations\"\n",
+    "xp_list = []\n",
+    "# FIXME: from the 10th XP iteration, the sorted method is unuseful\n",
+    "\n",
+    "for i, xp_path in enumerate(sorted(os.listdir(xp_dir))):  \n",
+    "    xp_path = os.path.join(xp_dir, xp_path)\n",
+    "    temp_xp_df = pd.read_csv(xp_path)\n",
+    "    xp_list.append((i, temp_xp_df))\n",
+    "    del temp_xp_df "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ca5b1fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join latest XP result to each question\n",
+    "\n",
+    "index = gaia_df.index.tolist()  # Index for all the questions\n",
+    "questions = gaia_df[\"Question\"].tolist()\n",
+    "answers = list()\n",
+    "\n",
+    "\n",
+    "def filter_condition(xp_data: list) -> bool:\n",
+    "    i, xp = xp_data\n",
+    "    for question in questions:\n",
+    "        xp_addressed_questions = xp[\"Question\"].tolist()\n",
+    "        if question in xp_addressed_questions:\n",
+    "            return True\n",
+    "    return False\n",
+    "\n",
+    "filtered_xps = list(filter(filter_condition, xp_list))\n",
+    "filtered_xps_reversed = filtered_xps\n",
+    "filtered_xps_reversed.reverse()\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "# Join latest answer result (i.e. result of the latest xp)\n",
+    "\n",
+    "for question in questions:\n",
+    "    answer_result = 0  # Assume wrong answer by default\n",
+    "    for i, xp in filtered_xps_reversed:\n",
+    "        for xp_question in xp[\"Question\"].tolist():\n",
+    "            \n",
+    "            if question == xp_question:\n",
+    "                answer_result_row = list(xp[xp[\"Question\"] == xp_question][\"is_correct\"])#.loc[0]\n",
+    "                answer_result_temp = answer_result_row[0]                    \n",
+    "                if answer_result_temp >= answer_result:\n",
+    "                    answer_result = answer_result_temp\n",
+    "        del xp\n",
+    "    answers.append(answer_result)\n",
+    "\n",
+    "# Join answers\n",
+    "historical_xp_results = gaia_df.copy()\n",
+    "historical_xp_results[\"is_correct\"] = pd.Series(answers)\n",
+    "del answers, questions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca756f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd11615a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "historical_xp_results.is_correct.mean()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3ddaede",
+   "metadata": {},
+   "source": [
+    "Let's study first which are the most common wrong tasks. I.e. How can we increase accuracy with a single next step (e.g. implement a new tool, modify sys message, etc.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0613fc4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df = historical_xp_results[historical_xp_results[\"is_correct\"]==0]\n",
+    "wrong_ans_df[\"fp_extension\"] = wrong_ans_df[\"file_path\"].map(lambda path: path.split(\".\")[-1])\n",
+    "wrong_ans_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43925ec4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Summary\n",
+    "wrong_ans_df.groupby(\"fp_extension\")[\"is_correct\"].count()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f77dabb",
+   "metadata": {},
+   "source": [
+    "The vast majority of remaining tasks do not include files to read. So we sould study them at first"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1379da15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension = wrong_ans_df[wrong_ans_df[\"file_path\"].map(lambda fp: len(fp)==0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e40e190",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wrong_ans_df_no_extension"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "14a8ebcf",
+   "metadata": {},
+   "source": [
+    "### Method\n",
+    "\n",
+    "We identified three main necessities to cover through web search\n",
+    "\n",
+    "1. Analize youtube videos (transcription and computer vision)\n",
+    "2. Scrap web pages (e.g. journals, pdfs, wikipedia articles)\n",
+    "\n",
+    "\n",
+    "So we decided to modularize the web_search_tool into three main modules\n",
+    "\n",
+    "1. High-level Web results\n",
+    "2. Youtube Video Class: To gather both audio and video\n",
+    "3. Web page scrapper: Given a web url, retrieve the text as markdown\n",
+    "4. Object detection tool: Ideally coded into handle_image tool\n",
+    "\n",
+    "So we plan to generate three main tools: `youtube_tool` (Class + Langchain runnable), `web_search` (Expanded) and `handle_images` with an initial module `object_detection` using YoloV4-tiny"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b63c5f35",
+   "metadata": {},
+   "source": [
+    "### Youtube Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2141e4ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "892f468e",
+   "metadata": {},
+   "source": [
+    "#### 1. FIlter tasks to solve with the tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e6b3b0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pattern = r\".*youtube.*\"\n",
+    "yt_tasks = gaia_df.copy()\n",
+    "yt_tasks = yt_tasks[yt_tasks[\"Question\"].map(lambda question: bool(re.search(pattern, question, re.IGNORECASE)))]\n",
+    "yt_tasks"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d41b2f29",
+   "metadata": {},
+   "source": [
+    "#### 2. Tool PoC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64c2891e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_yt_url = \"https://www.youtube.com/watch?v=L1vXCYZAYYM\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9b82ace7",
+   "metadata": {},
+   "source": [
+    "Lets import that video"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f5ad954c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5c12ac9f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "output_dir = \"../../data/temp/\"\n",
+    "\n",
+    "yt_audio_filename = \"yt_audio.mp3\"\n",
+    "yt_audio_path = os.path.join(output_dir, yt_audio_filename)\n",
+    "\n",
+    "raw_yt_video_filename = \"raw_yt_video.mp4\"\n",
+    "processed_yt_video_filename = \"processed_yt_video.mp4\"\n",
+    "\n",
+    "raw_yt_video_path = os.path.join(output_dir, raw_yt_video_filename)\n",
+    "processed_yt_video_path = os.path.join(output_dir, processed_yt_video_filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f1e8a09",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pytubefix import YouTube"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67f68f09",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yt = YouTube(sample_yt_url)\n",
+    "#yt_audio = yt.streams.filter(only_audio=True).first()\n",
+    "#yt_video = yt.streams.filter(only_video=True).first()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f04e971",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yt_audio = yt.streams.filter(only_audio=True).first()\n",
+    "yt_audio_filename = \"yt_audio.mp3\"\n",
+    "yt_audio_path = os.path.join(output_dir, yt_audio_filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4c19f5a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yt_video = yt.streams.filter(only_video=True, fps=25, res=\"144p\").order_by(\"fps\").asc().first()\n",
+    "yt_video_filename = \"yt_video.mp4\"\n",
+    "yt_video_path = os.path.join(output_dir, raw_yt_video_filename)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3e2ee8ec",
+   "metadata": {},
+   "source": [
+    "We imported a binary representation of both YT video and audio. The idea now is to try to pass these objects to a CV model (e.g. YoloV4) and to a Transcriber model (e.g. Whisper)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ff5864bc",
+   "metadata": {},
+   "source": [
+    "YoLo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5fef61e0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yt_video.download(output_path=output_dir, filename=raw_yt_video_filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd04bfdf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "\n",
+    "new_fps = 1\n",
+    "reduce_fps_cmd = f'ffmpeg -y -i {raw_yt_video_path} -filter:v \"fps={new_fps}\" -an {processed_yt_video_path}'\n",
+    "subprocess.run(\n",
+    "    reduce_fps_cmd, shell=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0383e2f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ultralytics import YOLO\n",
+    "\n",
+    "cv_model = YOLO(model=\"yolov8s.pt\", task=\"detect\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8c6851e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cv_results = cv_model.predict(source=processed_yt_video_path, stream=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "04e3ecf3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.sys.getsizeof(cv_results)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ddf175f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def summarize(results):# -> Generator[dict[str, Any], Any, None]:# -> Generator[dict[str, Any], Any, None]:\n",
+    "    for i, result in enumerate(results):\n",
+    "        yield {\n",
+    "            \"frame\": i + 1,\n",
+    "            \"result\": [result.names[int(cls)] for cls in result.boxes.cls]\n",
+    "           }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93b55a97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = list(summarize(cv_results))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd51f5b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: make this a typed dict to let the agent process this data\n",
+    "processed_data = {\n",
+    "    \"video_url\": sample_yt_url,\n",
+    "    \"detected_objects\": data\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2595c49b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "processed_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "293a9894",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# delete unnecesary stuff\n",
+    "import subprocess\n",
+    "remove_videos_cmd = f\"rm {raw_yt_video_path} {processed_yt_video_path}\"\n",
+    "subprocess.run(remove_videos_cmd, shell=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6e928c09",
+   "metadata": {},
+   "source": [
+    "Whisper custom tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06d786cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "yt_audio.download(output_path=output_dir, filename=yt_audio_filename)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "408eae4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tools_dir = \"../../src/tools\"\n",
+    "os.sys.path.append(tools_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fb62c338",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import transcriber"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10ab97b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "transcript = transcriber.transcriber.invoke(input={\"audio_path\": yt_audio_path})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4729fe7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "transcript"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d3271ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "subprocess.run(cmd=f\"rm {yt_audio_path}\", shell=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "715d48ab",
+   "metadata": {},
+   "source": [
+    "#### 3. Test Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "009aba7c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.sys.path.append(\"../../src/\")\n",
+    "os.sys.path.append(\"../../src/agents/\")\n",
+    "os.sys.path.append(\"../../src/utils/\")\n",
+    "\n",
+    "import react"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9194249b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "yt_tasks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fcc960f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_yt_task = yt_tasks.iloc[0]\n",
+    "sample_yt_question = sample_yt_task[\"Question\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "952a70a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_yt_question"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e178e9f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "react.run_app(user_query=sample_yt_question + \". Just use the pull_youtube_video and the transcriber tools. Try to figure out the number of bird species from the transcript as those are mentioned\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9712cc9",
+   "metadata": {},
+   "source": [
+    "### Web search Tool (Web scrapping)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c06f6e50",
+   "metadata": {},
+   "source": [
+    "#### 1. FIlter tasks to solve with the tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "4765f8e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sample_task_ids = [\n",
+    "    \"5d0080cb-90d7-4712-bc33-848150e917d3\",\n",
+    "    \"46719c30-f4c3-4cad-be07-d5cb21eee6bb\",\n",
+    "    \"b816bfce-3d80-4913-a07d-69b752ce6377\",\n",
+    "    \"b415aba4-4b68-4fc6-9b89-2c812e55a3e1\",\n",
+    "    \"935e2cff-ae78-4218-b3f5-115589b19dae\",\n",
+    "    \"5188369a-3bbe-43d8-8b94-11558f909a08\",\n",
+    "    \"7673d772-ef80-4f0f-a602-1bf4485c9b43\",\n",
+    "    \"c365c1c7-a3db-4d5e-a9a1-66f56eae7865\",\n",
+    "    \"7d4a7d1d-cac6-44a8-96e8-ea9584a70825\",\n",
+    "    \"3f57289b-8c60-48be-bd80-01f8099ca449\",\n",
+    "    \"23dd907f-1261-4488-b21c-e9185af91d5e\",\n",
+    "    \"840bfca7-4f7b-481a-8794-c560c340185d\",\n",
+    "    \"a0068077-79f4-461a-adfe-75c1a4148545\",\n",
+    "    \"a0c07678-e491-4bbc-8f0b-07405144218f\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "ef998b32",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "l = [\"a\", \"b\", \"c\"]\n",
+    "b = \"a\"\n",
+    "\n",
+    "b in l"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d318749c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
+       "columns": [
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "index",
+         "rawType": "int64",
+         "type": "integer"
+        },
+        {
+         "name": "task_id",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Question",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Level",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Final answer",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_name",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "file_path",
+         "rawType": "object",
+         "type": "string"
+        },
+        {
+         "name": "Annotator Metadata",
+         "rawType": "object",
+         "type": "unknown"
+        }
+       ],
+       "ref": "c253276a-1799-43b2-bc36-438a090ebf7d",
+       "rows": [
+        [
+         "0",
+         "3",
+         "5d0080cb-90d7-4712-bc33-848150e917d3",
+         "What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"",
+         "1",
+         "0.1777",
+         "",
+         "",
+         "{'Steps': '1. Searched \\'\"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"\\' on Google.\\n2. Opened \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\" at https://journals.le.ac.uk/ojs1/index.php/jist/article/view/733.\\n3. Clicked \"PDF\".\\n4. Found the calculations for the volume of the fish bag and noted them.', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. PDF access', 'Number of tools': '3'}"
+        ],
+        [
+         "1",
+         "5",
+         "46719c30-f4c3-4cad-be07-d5cb21eee6bb",
+         "Of the authors (First M. Last) that worked on the paper \"Pie Menus or Linear Menus, Which Is Better?\" in 2015, what was the title of the first paper authored by the one that had authored prior papers?",
+         "1",
+         "Mapping Human Oriented Information to Software Agents for Online Systems Usage",
+         "",
+         "",
+         "{'Steps': '1. Searched \"Pie Menus or Linear Menus, Which Is Better?\" on Google.\\n2. Opened \"Pie Menus or Linear Menus, Which Is Better?\" on https://oda.oslomet.no/oda-xmlui/handle/10642/3162.\\n3. Clicked each author\\'s name.\\n4. Noted the name that had no other papers listed.\\n5. Searched \"Murano, Pietro\" on Google.\\n6. Opened http://www.pietromurano.org/.\\n7. Clicked \"Publications\".\\n8. Found the earliest paper he contributed to.', 'Number of steps': '8', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "2",
+         "12",
+         "b816bfce-3d80-4913-a07d-69b752ce6377",
+         "In Emily Midkiff's June 2014 article in a journal named for the one of Hreidmar's sons that guarded his house, what word was quoted from two different authors in distaste for the nature of dragon depictions?",
+         "1",
+         "fluffy",
+         "",
+         "",
+         "{'Steps': '1. Searched \"Hreidmar\\'s sons\" on Google.\\n2. Opened https://en.wikipedia.org/wiki/Hrei%C3%B0marr.\\n3. Noted Fafnir guarded his house.\\n4. Searched \"Emily Midkiff June 2014 Fafnir\" on Google.\\n5. Opened \"Fafnir 2/2014 |\" at http://journal.finfar.org/journal/archive/fafnir-22014/.\\n6. Clicked the title \\'“Dragons are Tricksy”: The Uncanny Dragons of Children’s Literature\\'.\\n7. Found the word in quotation marks from two different authors (Ruth Stein and Margaret Blount) in the text.', 'Number of steps': '7', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "3",
+         "15",
+         "b415aba4-4b68-4fc6-9b89-2c812e55a3e1",
+         "In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.",
+         "1",
+         "diamond",
+         "",
+         "",
+         "{'Steps': '1. Searched \"nature scientific reports\" on Google.\\n2. Opened https://www.nature.com/srep/.\\n3. Selected Explore Content > Research Articles.\\n4. Filtered for Conference Proceedings from 2012.\\n5. Opened each article link.\\n6. Checked for \"plasmon\" or \"plasmonic\".\\n7. Noted the nano-compound in the article that did not include either.', 'Number of steps': '7', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "4",
+         "17",
+         "935e2cff-ae78-4218-b3f5-115589b19dae",
+         "In the year 2022, and before December, what does \"R\" stand for in the three core policies of the type of content that was violated in the public logs on the Legume Wikipedia page?",
+         "1",
+         "research",
+         "",
+         "",
+         "{'Steps': '1. Searched \"legume wikipedia\" on Google.\\n2. Opened \"Legume\" on Wikipedia.\\n3. Clicked \"View history\".\\n4. Clicked \"View logs for this page\".\\n5. Checked all types of logs.\\n6. Set the date to November 2022.\\n7. Followed the BLP link of the violation.\\n8. Noted the meaning of \"R\".', 'Number of steps': '8', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "5",
+         "19",
+         "5188369a-3bbe-43d8-8b94-11558f909a08",
+         "What writer is quoted by Merriam-Webster for the Word of the Day from June 27, 2022?",
+         "1",
+         "Annie Levin",
+         "",
+         "",
+         "{'Steps': '1. Search \"merriam-webster word of the day\" on Google search.\\n2. Opened the top \"Word of the Day\" result from the Merriam-Webster dictionary online.\\n3. Clicked \"SEE ALL WORDS OF THE DAY\" at the bottom.\\n4. Scrolled down to June 27, 2022.\\n5. Opened the Word of the Day (\"jingoism\").\\n6. Scrolled down and identified context quote for \"jingoism\".\\n7. Noted the name attributed to the quote. ', 'Number of steps': '7', 'How long did this take?': '8 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. Audio capability', 'Number of tools': '3'}"
+        ],
+        [
+         "6",
+         "38",
+         "7673d772-ef80-4f0f-a602-1bf4485c9b43",
+         "On Cornell Law School website's legal information institute, under the fifth section of federal rules alphabetically, what word was deleted in the last amendment to the first rule in the article that has \"witnesses\" in the most titles as of 2021?",
+         "1",
+         "inference",
+         "",
+         "",
+         "{'Steps': '1. Searched \"Cornell Law School legal information institute\" on Google.\\n2. Opened https://www.law.cornell.edu/.\\n3. Clicked Get The Law > Federal Rules > Federal Rules of Evidence (fourth section down).\\n4. Found the article that has \"witnesses\" in the most titles (VII).\\n5. Opened the first rule (701).\\n6. Scrolled to the last amendment as of 2021 (2011 amendment).\\n7. Found the word that was deleted (inference).', 'Number of steps': '7', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "7",
+         "39",
+         "c365c1c7-a3db-4d5e-a9a1-66f56eae7865",
+         "Of the cities within the United States where U.S. presidents were born, which two are the farthest apart from the westernmost to the easternmost going east, giving the city names only? Give them to me in alphabetical order, in a comma-separated list",
+         "1",
+         "Braintree, Honolulu",
+         "",
+         "",
+         "{'Steps': '1. Searched \"cities where us presidents are born\" on Google.\\n2. Opened \"List of presidents of the United States by home state\" on Wikipedia.\\n3. Searched the eastern cities to find the easternmost one (Braintree, MA).\\n4. Checked the westernmost city (Honolulu, HI).', 'Number of steps': '4', 'How long did this take?': '8 minutes', 'Tools': '1. Search engine\\n2. Web browser', 'Number of tools': '3'}"
+        ],
+        [
+         "8",
+         "40",
+         "7d4a7d1d-cac6-44a8-96e8-ea9584a70825",
+         "According to Girls Who Code, how long did it take in years for the percentage of computer scientists that were women to change by 13% from a starting point of 37%?",
+         "1",
+         "22",
+         "",
+         "",
+         "{'Steps': '1. Searched \"Girls Who Code\" on Google.\\n2. Opened https://girlswhocode.com/.\\n3. Clicked \"About Us\".\\n4. Noted that the chart started at 37% and declined to 24%.\\n5. Subtracted the marked years to find the number of years (2017 - 1995 = 22).', 'Number of steps': '5', 'How long did this take?': '10 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. Calculator', 'Number of tools': '3'}"
+        ],
+        [
+         "9",
+         "42",
+         "3f57289b-8c60-48be-bd80-01f8099ca449",
+         "How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?",
+         "1",
+         "519",
+         "",
+         "",
+         "{'Steps': '1. Search \"yankee stats\" to find their MLB stats page.\\n2. Set the data to the 1977 regular season.\\n3. Sort to find the most walks.\\n4. See how many at bats the player had.', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. web browser\\n2. search engine', 'Number of tools': '2'}"
+        ],
+        [
+         "10",
+         "43",
+         "23dd907f-1261-4488-b21c-e9185af91d5e",
+         "In Audre Lorde’s poem “Father Son and Holy Ghost”, what is the number of the stanza in which some lines are indented?",
+         "1",
+         "2",
+         "",
+         "",
+         "{'Steps': '1. Search the web for “Audre Lorde Father Son and Holy Ghost”.\\n2. Click on Poetry Foundation result.\\n3. Note the stanza that appears to have lines indented, the second one.\\n4. Return to search results to confirm.\\n5. Click on second result.\\n6. Confirm that the indentation appears in the second stanza here as well.', 'Number of steps': '6', 'How long did this take?': '5 minutes', 'Tools': '1. Search engine\\n2. Web browser', 'Number of tools': '2'}"
+        ],
+        [
+         "11",
+         "45",
+         "840bfca7-4f7b-481a-8794-c560c340185d",
+         "On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",
+         "1",
+         "80GSFC21M0002",
+         "",
+         "",
+         "{'Steps': '1. Google \"June 6, 2023 Carolyn Collins Petersen Universe Today\"\\n2. Find the relevant link to the scientific paper and follow that link\\n3. Open the PDF. \\n4. Search for NASA award number', 'Number of steps': '4', 'How long did this take?': '5 minutes', 'Tools': '1. Web browser\\n2. Search engine\\n3. Access to academic journal websites', 'Number of tools': '2'}"
+        ],
+        [
+         "12",
+         "46",
+         "a0068077-79f4-461a-adfe-75c1a4148545",
+         "What was the actual enrollment count of the clinical trial on H. pylori in acne vulgaris patients from Jan-May 2018 as listed on the NIH website?",
+         "1",
+         "90",
+         "",
+         "",
+         "{'Steps': '1. Searched \"nih\" on Google search.\\n2. Clicked the top link to nih.gov.\\n3. Searched \"h pylori acne\" in the search box.\\n4. Clicked \"More\" and selected \"Clinical Trials\".\\n5. Clicked the result about H. Pylori and acne.\\n6. Checked the date to confirm it was January to May 2018.\\n7. Opened \"Tabular View\".\\n8. Scrolled down to Actual Enrollment and recorded the number.', 'Number of steps': '8', 'How long did this take?': '8 minutes', 'Tools': '1. Search engine\\n2. Web browser', 'Number of tools': '2'}"
+        ],
+        [
+         "13",
+         "50",
+         "a0c07678-e491-4bbc-8f0b-07405144218f",
+         "Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",
+         "1",
+         "Yoshida, Uehara",
+         "",
+         "",
+         "{'Steps': '1. Look up Taishō Tamai on Wikipedia\\n2. See the pitcher with the number 18 (before) is Kōsei Yoshida and number 20 (after) is Kenta Uehara', 'Number of steps': '2', 'How long did this take?': '5 minutes', 'Tools': '1. Wikipedia', 'Number of tools': '1'}"
+        ]
+       ],
+       "shape": {
+        "columns": 8,
+        "rows": 14
+       }
+      },
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>task_id</th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Level</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>file_name</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Annotator Metadata</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>3</td>\n",
+       "      <td>5d0080cb-90d7-4712-bc33-848150e917d3</td>\n",
+       "      <td>What was the volume in m^3 of the fish bag tha...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.1777</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched '\"Can Hiccup Supply Eno...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>5</td>\n",
+       "      <td>46719c30-f4c3-4cad-be07-d5cb21eee6bb</td>\n",
+       "      <td>Of the authors (First M. Last) that worked on ...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Mapping Human Oriented Information to Software...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"Pie Menus or Linear Me...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>12</td>\n",
+       "      <td>b816bfce-3d80-4913-a07d-69b752ce6377</td>\n",
+       "      <td>In Emily Midkiff's June 2014 article in a jour...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>fluffy</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"Hreidmar's sons\" on Go...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>15</td>\n",
+       "      <td>b415aba4-4b68-4fc6-9b89-2c812e55a3e1</td>\n",
+       "      <td>In Nature journal's Scientific Reports confere...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>diamond</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"nature scientific repo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>17</td>\n",
+       "      <td>935e2cff-ae78-4218-b3f5-115589b19dae</td>\n",
+       "      <td>In the year 2022, and before December, what do...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>research</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"legume wikipedia\" on G...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>19</td>\n",
+       "      <td>5188369a-3bbe-43d8-8b94-11558f909a08</td>\n",
+       "      <td>What writer is quoted by Merriam-Webster for t...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Annie Levin</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Search \"merriam-webster word of ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>38</td>\n",
+       "      <td>7673d772-ef80-4f0f-a602-1bf4485c9b43</td>\n",
+       "      <td>On Cornell Law School website's legal informat...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>inference</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"Cornell Law School leg...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>39</td>\n",
+       "      <td>c365c1c7-a3db-4d5e-a9a1-66f56eae7865</td>\n",
+       "      <td>Of the cities within the United States where U...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Braintree, Honolulu</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"cities where us presid...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>40</td>\n",
+       "      <td>7d4a7d1d-cac6-44a8-96e8-ea9584a70825</td>\n",
+       "      <td>According to Girls Who Code, how long did it t...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>22</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"Girls Who Code\" on Goo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>42</td>\n",
+       "      <td>3f57289b-8c60-48be-bd80-01f8099ca449</td>\n",
+       "      <td>How many at bats did the Yankee with the most ...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>519</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Search \"yankee stats\" to find th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>43</td>\n",
+       "      <td>23dd907f-1261-4488-b21c-e9185af91d5e</td>\n",
+       "      <td>In Audre Lorde’s poem “Father Son and Holy Gho...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Search the web for “Audre Lorde ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>45</td>\n",
+       "      <td>840bfca7-4f7b-481a-8794-c560c340185d</td>\n",
+       "      <td>On June 6, 2023, an article by Carolyn Collins...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>80GSFC21M0002</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Google \"June 6, 2023 Carolyn Col...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>46</td>\n",
+       "      <td>a0068077-79f4-461a-adfe-75c1a4148545</td>\n",
+       "      <td>What was the actual enrollment count of the cl...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>90</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Searched \"nih\" on Google search....</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>50</td>\n",
+       "      <td>a0c07678-e491-4bbc-8f0b-07405144218f</td>\n",
+       "      <td>Who are the pitchers with the number before an...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Yoshida, Uehara</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. Look up Taishō Tamai on Wikipedi...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    index                               task_id  \\\n",
+       "0       3  5d0080cb-90d7-4712-bc33-848150e917d3   \n",
+       "1       5  46719c30-f4c3-4cad-be07-d5cb21eee6bb   \n",
+       "2      12  b816bfce-3d80-4913-a07d-69b752ce6377   \n",
+       "3      15  b415aba4-4b68-4fc6-9b89-2c812e55a3e1   \n",
+       "4      17  935e2cff-ae78-4218-b3f5-115589b19dae   \n",
+       "5      19  5188369a-3bbe-43d8-8b94-11558f909a08   \n",
+       "6      38  7673d772-ef80-4f0f-a602-1bf4485c9b43   \n",
+       "7      39  c365c1c7-a3db-4d5e-a9a1-66f56eae7865   \n",
+       "8      40  7d4a7d1d-cac6-44a8-96e8-ea9584a70825   \n",
+       "9      42  3f57289b-8c60-48be-bd80-01f8099ca449   \n",
+       "10     43  23dd907f-1261-4488-b21c-e9185af91d5e   \n",
+       "11     45  840bfca7-4f7b-481a-8794-c560c340185d   \n",
+       "12     46  a0068077-79f4-461a-adfe-75c1a4148545   \n",
+       "13     50  a0c07678-e491-4bbc-8f0b-07405144218f   \n",
+       "\n",
+       "                                             Question Level  \\\n",
+       "0   What was the volume in m^3 of the fish bag tha...     1   \n",
+       "1   Of the authors (First M. Last) that worked on ...     1   \n",
+       "2   In Emily Midkiff's June 2014 article in a jour...     1   \n",
+       "3   In Nature journal's Scientific Reports confere...     1   \n",
+       "4   In the year 2022, and before December, what do...     1   \n",
+       "5   What writer is quoted by Merriam-Webster for t...     1   \n",
+       "6   On Cornell Law School website's legal informat...     1   \n",
+       "7   Of the cities within the United States where U...     1   \n",
+       "8   According to Girls Who Code, how long did it t...     1   \n",
+       "9   How many at bats did the Yankee with the most ...     1   \n",
+       "10  In Audre Lorde’s poem “Father Son and Holy Gho...     1   \n",
+       "11  On June 6, 2023, an article by Carolyn Collins...     1   \n",
+       "12  What was the actual enrollment count of the cl...     1   \n",
+       "13  Who are the pitchers with the number before an...     1   \n",
+       "\n",
+       "                                         Final answer file_name file_path  \\\n",
+       "0                                              0.1777                       \n",
+       "1   Mapping Human Oriented Information to Software...                       \n",
+       "2                                              fluffy                       \n",
+       "3                                             diamond                       \n",
+       "4                                            research                       \n",
+       "5                                         Annie Levin                       \n",
+       "6                                           inference                       \n",
+       "7                                 Braintree, Honolulu                       \n",
+       "8                                                  22                       \n",
+       "9                                                 519                       \n",
+       "10                                                  2                       \n",
+       "11                                      80GSFC21M0002                       \n",
+       "12                                                 90                       \n",
+       "13                                    Yoshida, Uehara                       \n",
+       "\n",
+       "                                   Annotator Metadata  \n",
+       "0   {'Steps': '1. Searched '\"Can Hiccup Supply Eno...  \n",
+       "1   {'Steps': '1. Searched \"Pie Menus or Linear Me...  \n",
+       "2   {'Steps': '1. Searched \"Hreidmar's sons\" on Go...  \n",
+       "3   {'Steps': '1. Searched \"nature scientific repo...  \n",
+       "4   {'Steps': '1. Searched \"legume wikipedia\" on G...  \n",
+       "5   {'Steps': '1. Search \"merriam-webster word of ...  \n",
+       "6   {'Steps': '1. Searched \"Cornell Law School leg...  \n",
+       "7   {'Steps': '1. Searched \"cities where us presid...  \n",
+       "8   {'Steps': '1. Searched \"Girls Who Code\" on Goo...  \n",
+       "9   {'Steps': '1. Search \"yankee stats\" to find th...  \n",
+       "10  {'Steps': '1. Search the web for “Audre Lorde ...  \n",
+       "11  {'Steps': '1. Google \"June 6, 2023 Carolyn Col...  \n",
+       "12  {'Steps': '1. Searched \"nih\" on Google search....  \n",
+       "13  {'Steps': '1. Look up Taishō Tamai on Wikipedi...  "
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sample_tasks = gaia_df[gaia_df[\"task_id\"].map(lambda row: row in sample_task_ids)]\n",
+    "sample_tasks.reset_index(inplace=True)\n",
+    "sample_tasksb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c3a2fc9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1b8f4daf",
+   "metadata": {},
+   "source": [
+    "#### 2. Tool PoC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "66478316",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.tools.playwright.utils import create_async_playwright_browser\n",
+    "from langchain_community.agent_toolkits import PlayWrightBrowserToolkit\n",
+    "\n",
+    "PlayWrightBrowserToolkit.model_rebuild()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "597fcf82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio # Just for jupyter notebooks\n",
+    "\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "# Create async browser context to launch browser \n",
+    "async_browser = create_async_playwright_browser()\n",
+    "toolkit = PlayWrightBrowserToolkit.from_browser(async_browser=async_browser)\n",
+    "tools = toolkit.get_tools()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "35333b1e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.prebuilt import create_react_agent\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(model=\"gpt-4o\", temperature=0.0)\n",
+    "agent = create_react_agent(model=llm, tools=tools)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0a9abe8b",
+   "metadata": {},
+   "source": [
+    "#### 3. Test Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "8974a8ff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sys_message = \"\"\n",
+    "\n",
+    "with open(\"../../prompts/agent/gaia_system_message.md\", \"r\") as f:\n",
+    "    for line in f:\n",
+    "        sys_message += line"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "35d2e16b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = [\n",
+    "    {\n",
+    "        \"task_id\": task_id,\n",
+    "        \"answer\": None\n",
+    "    }\n",
+    "    for task_id in sample_task_ids\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "id": "08932eb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[ClickTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " NavigateTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " NavigateBackTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " ExtractTextTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " ExtractHyperlinksTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " GetElementsTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>),\n",
+       " CurrentWebPageTool(async_browser=<Browser type=<BrowserType name=chromium executable_path=/home/santiagoal/.cache/ms-playwright/chromium-1179/chrome-linux/chrome> version=138.0.7204.23>)]"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tools"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "id": "b1ddc9a5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "for task_item in results:\n",
+    "    task_id = task_item[\"task_id\"]\n",
+    "    answer = task_item[\"answer\"]\n",
+    "\n",
+    "    # get question\n",
+    "    question = sample_tasks[sample_tasks[\"task_id\"]==task_id] # Filter record\n",
+    "    #print(question)\n",
+    "    question_processed = question[\"Question\"].iloc[0] # filter field\n",
+    "    \n",
+    "    # fill question with sys message \n",
+    "    #question_processed = sys_message + \"\\n\" + question\n",
+    "    #print(question_processed)\n",
+    "\n",
+    "    # pass processed question to react agent\n",
+    "\n",
+    "    agent_response = agent.invoke(input={\"messages\": question_processed})\n",
+    "    task_item[\"answer\"] = agent_response[\"messages\"][-1].content\n",
+    "    time.sleep(4)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "95e8c66e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "content='It seems there is an issue with accessing the webpage directly. You can try visiting the [University of Leicester\\'s journal page](https://journals.le.ac.uk/ojs1/index.php/jist/article/view/2985) directly in your browser to find the information about the fish bag volume in the paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\". If you have any other questions or need further assistance, feel free to ask!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 93, 'prompt_tokens': 453, 'total_tokens': 546, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZL932hfEp4hA2xH97IFjkieQe5r', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--a49dce90-e0e0-49cb-9e4f-3666cab83ad0-0' usage_metadata={'input_tokens': 453, 'output_tokens': 93, 'total_tokens': 546, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems there\\'s an issue with navigating to external websites directly. However, you can manually search for the paper \"Pie Menus or Linear Menus, Which Is Better?\" on Google Scholar to find the authors. Once you have the list of authors, you can search for their publication history to identify the first paper authored by the one with prior publications. If you need further assistance with this process, feel free to ask!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 85, 'prompt_tokens': 382, 'total_tokens': 467, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZLH0iQmPE8EmQEJB4kDWOyLX0b5', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--c04f3176-6860-4449-bc9d-6abe56c8b776-0' usage_metadata={'input_tokens': 382, 'output_tokens': 85, 'total_tokens': 467, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems I can\\'t access external websites directly to retrieve the article. However, I can guide you on how to find the information.\\n\\n1. **Journal Name**: The journal you\\'re referring to is likely \"Saga-Book,\" named after Fafnir, one of Hreidmar\\'s sons who guarded his house in Norse mythology.\\n\\n2. **Finding the Article**:\\n   - Visit a database like JSTOR or your institution\\'s library portal.\\n   - Search for Emily Midkiff\\'s article from June 2014 in the \"Saga-Book\" journal.\\n\\n3. **Identifying the Quoted Word**:\\n   - Once you have access to the article, look for sections discussing dragon depictions.\\n   - Pay attention to any quotes from other authors expressing distaste for these depictions.\\n\\nIf you have access to the article, you can follow these steps to find the specific word quoted in distaste. If you need further assistance, feel free to ask!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 197, 'prompt_tokens': 400, 'total_tokens': 597, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZLQLqyUJ9gQ7hpaOyWXimDsurFM', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--527dd0d5-c1b9-4187-9eba-277f3ec9c483-0' usage_metadata={'input_tokens': 400, 'output_tokens': 197, 'total_tokens': 597, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems there is an issue with accessing the webpage directly. You can visit the [Nature Scientific Reports conference proceedings for 2012](https://www.nature.com/srep/conference-proceedings/2012) and search for the article that does not mention plasmons or plasmonics to find the nano-compound studied. If you have any other questions or need further assistance, feel free to ask!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 84, 'prompt_tokens': 399, 'total_tokens': 483, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZLcnLiuIxBJnmbzCBIoLxXJ6MuH', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--1e784dc3-7a50-4766-a08b-be3d1c5bcf00-0' usage_metadata={'input_tokens': 399, 'output_tokens': 84, 'total_tokens': 483, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems I am unable to access the webpage directly at the moment. However, I can guide you on how to find this information:\\n\\n1. Go to the [Legume Wikipedia page](https://en.wikipedia.org/wiki/Legume).\\n2. On the left-hand side, click on \"View history\" to access the page\\'s edit history.\\n3. Look for any public logs or discussions related to content violations.\\n4. The three core policies often referred to in Wikipedia are \"Neutral point of view,\" \"Verifiability,\" and \"No original research.\" The \"R\" in these policies typically stands for \"Research\" in \"No original research.\"\\n\\nIf you need further assistance, you can try accessing the page directly through your browser.' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 150, 'prompt_tokens': 380, 'total_tokens': 530, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZLkx0v7VlxdKIKROu9Utpvrhooh', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--aad2df13-a5e9-4b39-93fa-b2bc73a46839-0' usage_metadata={'input_tokens': 380, 'output_tokens': 150, 'total_tokens': 530, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content=\"It seems there is an issue with accessing the Merriam-Webster website directly. You can visit the [Merriam-Webster Word of the Day Archive](https://www.merriam-webster.com/word-of-the-day/calendar) and navigate to June 27, 2022, to find the writer quoted for that day's word.\" additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 70, 'prompt_tokens': 372, 'total_tokens': 442, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZLtOIpU3OPCd5tkbwTt5YLchm7X', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--34ea79ab-6408-495f-bf96-24571d945e22-0' usage_metadata={'input_tokens': 372, 'output_tokens': 70, 'total_tokens': 442, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems there is an issue with navigating to the website directly. However, I can guide you on how to find the information you need:\\n\\n1. **Visit the Cornell Law School\\'s Legal Information Institute website**: Go to [https://www.law.cornell.edu/](https://www.law.cornell.edu/).\\n\\n2. **Access the Federal Rules**: Look for a section or link related to \"Federal Rules\" on the homepage. This might be under a menu or a direct link.\\n\\n3. **Find the Fifth Section Alphabetically**: Once you are in the Federal Rules section, identify the fifth section alphabetically. This could be something like \"Federal Rules of Evidence\" or similar.\\n\\n4. **Search for Articles with \"Witnesses\" in the Title**: Within the identified section, look for articles or rules that have \"witnesses\" in their titles.\\n\\n5. **Check the Amendments**: For the first rule in the article with the most titles containing \"witnesses,\" look at the amendment history. The website should provide details on what changes were made, including any words that were deleted.\\n\\nIf you follow these steps, you should be able to find the specific word that was deleted in the last amendment. If you need further assistance, please let me know!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 267, 'prompt_tokens': 392, 'total_tokens': 659, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZM19gRtBkrW7j9zs384oBAetICC', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--64bb8616-03b7-45bd-a560-20e30d813c1e-0' usage_metadata={'input_tokens': 392, 'output_tokens': 267, 'total_tokens': 659, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='Honolulu, Norfolk' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 5, 'prompt_tokens': 292, 'total_tokens': 297, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMBOrJ52lxUEKqPl9643vEG3Uur', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--5b4d6367-af83-4664-8a8a-97013f00bb34-0' usage_metadata={'input_tokens': 292, 'output_tokens': 5, 'total_tokens': 297, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='I am unable to access the Girls Who Code website directly. However, I can guide you on how to find this information:\\n\\n1. Visit the [Girls Who Code website](https://girlswhocode.com).\\n2. Look for sections related to statistics or reports on women in computer science.\\n3. Check for any historical data or reports that mention changes in the percentage of women in computer science over the years.\\n\\nIf you have any specific questions or need further assistance, feel free to ask!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 99, 'prompt_tokens': 369, 'total_tokens': 468, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMJSFKAE3PbT5yaQjwOStt5kH5I', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--efa3cfa7-ed0e-4cf7-8b9b-89fb435df512-0' usage_metadata={'input_tokens': 369, 'output_tokens': 99, 'total_tokens': 468, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='In the 1977 regular season, the New York Yankees player with the most walks was Reggie Jackson, who had 86 walks. During that same season, Reggie Jackson had 582 at-bats.' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 44, 'prompt_tokens': 263, 'total_tokens': 307, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMPTirhNjOLLpQ7aY97f3kiwgKc', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--2bdd593f-25c6-4447-b61f-b4b89a341835-0' usage_metadata={'input_tokens': 263, 'output_tokens': 44, 'total_tokens': 307, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems there is an issue with accessing the webpage directly. However, you can find the poem \"Father Son and Holy Ghost\" by Audre Lorde on the Poetry Foundation website or other poetry resources online. Look for the stanza where some lines are indented to identify its number. If you have access to the text, you can also share it here, and I can help analyze it for you.' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 82, 'prompt_tokens': 452, 'total_tokens': 534, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMYHeM8Ya1yxiL0hu5TKzmzHIrq', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--0617cb01-94a9-4b32-897d-27c0b5e4d401-0' usage_metadata={'input_tokens': 452, 'output_tokens': 82, 'total_tokens': 534, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content=\"It seems there is an issue with navigating to the Universe Today website. Unfortunately, I can't directly access the web to find the article and the linked paper. However, you can visit the [Universe Today website](https://www.universetoday.com/) and search for the article by Carolyn Collins Petersen published on June 6, 2023. Once you find the article, you can look for the link to the paper at the bottom of the article to find the NASA award number associated with R. G. Arendt's work.\" additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 109, 'prompt_tokens': 452, 'total_tokens': 561, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMi1y6B1LLAZANMww4hNlhi8IIF', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--7f8ca547-269d-4543-a918-f7a0698a203a-0' usage_metadata={'input_tokens': 452, 'output_tokens': 109, 'total_tokens': 561, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content='It seems there\\'s an issue with accessing the website directly. You can visit [ClinicalTrials.gov](https://clinicaltrials.gov) and search for the clinical trial on H. pylori in acne vulgaris patients to find the actual enrollment count. Use keywords like \"H. pylori,\" \"acne vulgaris,\" and the date range \"Jan-May 2018\" to narrow down the search results.' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 83, 'prompt_tokens': 365, 'total_tokens': 448, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZMrw062kbVH3s0i4Vx8s3sjopAH', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--f2f168c9-9531-42ed-b0ec-9b7ee9ed5234-0' usage_metadata={'input_tokens': 365, 'output_tokens': 83, 'total_tokens': 448, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n",
+      "content=\"It seems there's an issue with accessing the website directly. You can visit the [official NPB website](https://npb.jp/bis/eng/teams/index2023.html) to find the team rosters and identify the pitchers with numbers before and after Taishō Tamai's number. If you have any other questions or need further assistance, feel free to ask!\" additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 77, 'prompt_tokens': 454, 'total_tokens': 531, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_name': 'gpt-4o-2024-08-06', 'system_fingerprint': 'fp_07871e2ad8', 'id': 'chatcmpl-BmZN15zPT7foUv3zhjFLjSqzUzZCt', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None} id='run--4ae3e703-097f-4b65-b682-f1fada191dcd-0' usage_metadata={'input_tokens': 454, 'output_tokens': 77, 'total_tokens': 531, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in range(len(sample_task_ids)):\n",
+    "    print(results[i][\"answer\"][\"messages\"][-1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "id": "12119c36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pd.DataFrame(results).to_csv(\"../../data/temp/results_web_search.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b0e4521f",
+   "metadata": {},
+   "source": [
+    "### Handle Images Tool (Object detection)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4cd203e5",
+   "metadata": {},
+   "source": [
+    "#### 1. FIlter tasks to solve with the tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "109858cf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f7d03390",
+   "metadata": {},
+   "source": [
+    "#### 2. Tool PoC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca297e23",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4b926609",
+   "metadata": {},
+   "source": [
+    "#### 3. Test Tool"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "adc49004",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "efb50438",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "chappie-CHLGiFC_-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}