agent dev checkpoint

local.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "660cfadd",
    "metadata": {},
    "outputs": [
@@ -39,6 +39,10 @@
        "      <th>question</th>\n",
        "      <th>Level</th>\n",
        "      <th>file_name</th>\n",
+       "      <th>Question</th>\n",
+       "      <th>Final answer</th>\n",
+       "      <th>file_path</th>\n",
+       "      <th>Annotator Metadata</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -48,6 +52,10 @@
        "      <td>20</td>\n",
        "      <td>20</td>\n",
        "      <td>20</td>\n",
+       "      <td>20</td>\n",
+       "      <td>20</td>\n",
+       "      <td>20</td>\n",
+       "      <td>20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>unique</th>\n",
@@ -55,6 +63,10 @@
        "      <td>20</td>\n",
        "      <td>1</td>\n",
        "      <td>6</td>\n",
+       "      <td>20</td>\n",
+       "      <td>19</td>\n",
+       "      <td>6</td>\n",
+       "      <td>20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>top</th>\n",
@@ -62,6 +74,10 @@
        "      <td>How many studio albums were published by Merce...</td>\n",
        "      <td>1</td>\n",
        "      <td></td>\n",
+       "      <td>How many studio albums were published by Merce...</td>\n",
+       "      <td>3</td>\n",
+       "      <td></td>\n",
+       "      <td>{'Steps': '1. I did a search for Mercedes Sosa...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>freq</th>\n",
@@ -69,6 +85,10 @@
        "      <td>1</td>\n",
        "      <td>20</td>\n",
        "      <td>15</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>15</td>\n",
+       "      <td>1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -81,230 +101,55 @@
        "top     8e867cd7-cff9-4e6c-867a-ff5ddc2550be   \n",
        "freq                                       1   \n",
        "\n",
-       "                                                 question Level file_name  \n",
-       "count                                                  20    20        20  \n",
-       "unique                                                 20     1         6  \n",
-       "top     How many studio albums were published by Merce...     1            \n",
-       "freq                                                    1    20        15  "
+       "                                                 question Level file_name  \\\n",
+       "count                                                  20    20        20   \n",
+       "unique                                                 20     1         6   \n",
+       "top     How many studio albums were published by Merce...     1             \n",
+       "freq                                                    1    20        15   \n",
+       "\n",
+       "                                                 Question Final answer  \\\n",
+       "count                                                  20           20   \n",
+       "unique                                                 20           19   \n",
+       "top     How many studio albums were published by Merce...            3   \n",
+       "freq                                                    1            2   \n",
+       "\n",
+       "       file_path                                 Annotator Metadata  \n",
+       "count         20                                                 20  \n",
+       "unique         6                                                 20  \n",
+       "top               {'Steps': '1. I did a search for Mercedes Sosa...  \n",
+       "freq          15                                                  1  "
       ]
      },
-     "execution_count": 1,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "import os\n",
     "from connectors import fetch_questions\n",
+    "from datasets import load_dataset\n",
     "\n",
     "DEFAULT_API_URL = \"https://agents-course-unit4-scoring.hf.space\"\n",
     "QUESTIONS_URL = f\"{DEFAULT_API_URL}/questions\"\n",
     "SUBMIT_URL = f\"{DEFAULT_API_URL}/submit\"\n",
-    "HFTOKEN=os.getenv(\"HF_TOKEN\")\n",
     "\n",
-    "questions = fetch_questions(QUESTIONS_URL)\n",
-    "questions.describe()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "41b1c193",
-   "metadata": {},
-   "source": [
-    "# Agent"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "02fe107c",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\Users\\dgl17\\Desktop\\Lavoro\\Personal\\Final_Assignment_Template\\hfagent\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
-   "source": [
-    "from langgraph.graph import StateGraph, START, END\n",
-    "from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline\n",
-    "from typing import TypedDict, Any\n",
-    "from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer\n",
-    "import torch\n",
-    "import random as rng"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4cb1b488",
-   "metadata": {},
-   "source": [
-    "## Schemas"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "01b8a368",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class ActionsSchema(TypedDict):\n",
-    "  name: str\n",
-    "  arguments: dict[str, Any] # name - value\n",
-    "  observation: Any | None\n",
+    "MODELID = \"nemotron-3-nano:4b\"\n",
     "\n",
-    "class AgentSchema(TypedDict):\n",
-    "  query: str\n",
-    "  answer: str\n",
-    "  actions: list[ActionsSchema]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c8687b82",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Fetching 3 files:   0%|          | 0/3 [00:00<?, ?it/s]"
-     ]
-    }
-   ],
-   "source": [
-    "class RootAgent():\n",
-    "  __MODELID = \"mistralai/Mistral-7B-Instruct-v0.2\"\n",
-    "  \n",
-    "  def __init__(self):\n",
-    "    sg = StateGraph(AgentSchema)\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(self.__MODELID)\n",
-    "    model = AutoModelForCausalLM.from_pretrained(self.__MODELID, dtype=torch.bfloat16, device_map=\"auto\")\n",
-    "    pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, temperature=0.3, do_sample=True, max_new_tokens=512)\n",
-    "    self.brain = ChatHuggingFace(llm=HuggingFacePipeline(pipeline=pipe))\n",
-    "    sg.add_node(\"plan\", self.__plan)\n",
-    "    sg.add_edge(START, \"plan\")\n",
-    "    sg.add_edge(\"plan\", END)\n",
-    "    self.agentflow = sg.compile()\n",
-    "    return\n",
-    "  \n",
-    "  def __call__(self, query: str):\n",
-    "    return self.agentflow.invoke({\"actions\": [], \"query\": query, \"answer\": \"\"})\n",
-    "  \n",
-    "  ## nodes ##\n",
-    "  def __plan(self, state: AgentSchema) -> AgentSchema:\n",
-    "    stageprompt = f\"\"\"\n",
-    "      The user asked the following question: {state.get(\"query\", \"\")}.\n",
-    "      In past iterations you observed the following results: {state.get(\"actions\", [])}\n",
-    "\n",
-    "      You are the planning manager entitled to develop an efficient plan to obtain the answer, for this reason you must follow these rules:\n",
-    "      1. IF you can answer directly without using any tools, given the information you have, generate the [Answer Structured JSON](#Answer-Structured-JSON) format\n",
-    "      2. OTHERWISE generate a plan consisting of tool calls for which you have all the data to pass as arguments following the [Plan Structured JSON](#Plan-Structured-JSON) format\n",
-    "\n",
-    "      !IMPORTANT: ALWAYS use one of the two JSON format for your answer!\n",
-    "      \n",
-    "      #Answer Structured JSON\n",
-    "      ```json\n",
-    "      {\"{'Answer': '<answer1>,<answer2>,...'}\"}\n",
-    "      ```\n",
-    "      !IMPORTANT: Only provide the comma separated answers that are directly asked by the user, without any formatting or measurement unit!\n",
-    "      ## Examples\n",
-    "      1. user: 'what is the capital of Italy?' --> {\"{'Answer': 'Rome'}\"}\n",
-    "      2. user: 'What is the nearest city to Firenze? How many kilometers? --> past iterations: [{\"{'name': 'web_search', 'arguments': {'query': 'Distance between Firenze and Pisa in Kilometers'}, 'observation': '150 KM'}\"}] --> {\"{'Answer': 'Pisa,150'}\"}\n",
-    "\n",
-    "      #Plan Structured JSON\n",
-    "      ```json\n",
-    "      {\"\"\"{'Plan': [\n",
-    "        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1, '<argument_name_2>': argument_value_2, ...}},\n",
-    "        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1}},\n",
-    "        ...\n",
-    "      ]}\"\"\"}\n",
-    "      ```\n",
-    "      !IMPORTANT: Those will be used as function calls so be extremely precise on argument<-->value mappings, moreover to better understand\n",
-    "      the task, always place a Reasoning string before the plan JSON\n",
-    "      ## Examples\n",
-    "      1. user: 'What is the distance between Rome and Paris?' --> Reasoning: To get the distance I need to search the web, since it is a very\n",
-    "      specific question and I can choose between web_search and wiki_search, I think it is better to search in the web with query='distance Rome to Paris in Kilometers' --> {\"\"\"{\n",
-    "        'Plan': [\n",
-    "          {'name': '<web_search>', 'arguments': {'query': 'distance Rome to Paris'}},\n",
-    "          ...\n",
-    "        ]\n",
-    "      }\"\"\"}\n",
-    "    \"\"\"\n",
-    "    response = self.brain.invoke(stageprompt)\n",
-    "    return {\"query\": state.get(\"query\"), \"answer\": response, \"actions\": []}\n",
+    "questions = fetch_questions(QUESTIONS_URL)\n",
+    "dataset = load_dataset(\"gaia-benchmark/GAIA\", \"2023_level1\")[\"validation\"].to_pandas()\n",
     "\n",
-    "agent = RootAgent()"
+    "questiondf = questions.merge(right=dataset, how=\"left\", on=\"task_id\").drop(columns=[\"Level_y\", \"file_name_y\"]).rename(columns={\n",
+    "    \"Level_x\": \"Level\", \"file_name_x\": \"file_name\"\n",
+    "})\n",
+    "questiondf.describe()\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "64fb34d8",
+   "cell_type": "markdown",
+   "id": "99632633",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The user asked the following question: Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters..\n",
-      "      In past iterations you observed the following results: []\n",
-      "\n",
-      "      You are the planning manager entitled to develop an efficient plan to obtain the answer, for this reason you must follow these rules:\n",
-      "      1. IF you can answer directly without using any tools, given the information you have, generate the [Answer Structured JSON](#Answer-Structured-JSON) format\n",
-      "      2. OTHERWISE generate a plan consisting of tool calls for which you have all the data to pass as arguments following the [Plan Structured JSON](#Plan-Structured-JSON) format\n",
-      "\n",
-      "      !IMPORTANT: ALWAYS use one of the two JSON format for your answer!\n",
-      "\n",
-      "      #Answer Structured JSON\n",
-      "      ```json\n",
-      "      {'Answer': '<answer1>,<answer2>,...'}\n",
-      "      ```\n",
-      "      !IMPORTANT: Only provide the comma separated answers that are directly asked by the user, without any formatting or measurement unit!\n",
-      "      ## Examples\n",
-      "      1. user: 'what is the capital of Italy?' --> {'Answer': 'Rome'}\n",
-      "      2. user: 'What is the nearest city to Firenze? How many kilometers? --> past iterations: [{'name': 'web_search', 'arguments': {'query': 'Distance between Firenze and Pisa in Kilometers'}, 'observation': '150 KM'}] --> {'Answer': 'Pisa,150'}\n",
-      "\n",
-      "      #Plan Structured JSON\n",
-      "      ```json\n",
-      "      {'Plan': [\n",
-      "        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1, '<argument_name_2>': argument_value_2, ...}},\n",
-      "        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1}},\n",
-      "        ...\n",
-      "      ]}\n",
-      "      ```\n",
-      "      !IMPORTANT: Those will be used as function calls so be extremely precise on argument<-->value mappings, moreover to better understand\n",
-      "      the task, always place a Reasoning string before the plan JSON\n",
-      "      ## Examples\n",
-      "      1. user: 'What is the distance between Rome and Paris?' --> Reasoning: To get the distance I need to search the web, since it is a very\n",
-      "      specific question and I can choose between web_search and wiki_search, I think it is better to search in the web with query='distance Rome to Paris in Kilometers' --> {\n",
-      "        'Plan': [\n",
-      "          {'name': '<web_search>', 'arguments': {'query': 'distance Rome to Paris'}},\n",
-      "          ...\n",
-      "        ]\n",
-      "      }\n",
-      "{'query': \"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.\", 'answer': \"The user asked the following question: Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters..\\n      In past iterations you observed the following results: []\\n\\n      You are the planning manager entitled to develop an efficient plan to obtain the answer, for this reason you must follow these rules:\\n      1. IF you can answer directly without using any tools, given the information you have, generate the [Answer Structured JSON](#Answer-Structured-JSON) format\\n      2. OTHERWISE generate a plan consisting of tool calls for which you have all the data to pass as arguments following the [Plan Structured JSON](#Plan-Structured-JSON) format\\n\\n      !IMPORTANT: ALWAYS use one of the two JSON format for your answer!\\n\\n      #Answer Structured JSON\\n      ```json\\n      {'Answer': '<answer1>,<answer2>,...'}\\n      ```\\n      !IMPORTANT: Only provide the comma separated answers that are directly asked by the user, without any formatting or measurement unit!\\n      ## Examples\\n      1. user: 'what is the capital of Italy?' --> {'Answer': 'Rome'}\\n      2. user: 'What is the nearest city to Firenze? How many kilometers? --> past iterations: [{'name': 'web_search', 'arguments': {'query': 'Distance between Firenze and Pisa in Kilometers'}, 'observation': '150 KM'}] --> {'Answer': 'Pisa,150'}\\n\\n      #Plan Structured JSON\\n      ```json\\n      {'Plan': [\\n        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1, '<argument_name_2>': argument_value_2, ...}},\\n        {'name': '<tool_name>', 'arguments': {'<argument_name_1>': argument_value_1}},\\n        ...\\n      ]}\\n      ```\\n      !IMPORTANT: Those will be used as function calls so be extremely precise on argument<-->value mappings, moreover to better understand\\n      the task, always place a Reasoning string before the plan JSON\\n      ## Examples\\n      1. user: 'What is the distance between Rome and Paris?' --> Reasoning: To get the distance I need to search the web, since it is a very\\n      specific question and I can choose between web_search and wiki_search, I think it is better to search in the web with query='distance Rome to Paris in Kilometers' --> {\\n        'Plan': [\\n          {'name': '<web_search>', 'arguments': {'query': 'distance Rome to Paris'}},\\n          ...\\n        ]\\n      }\", 'actions': []}\n"
-     ]
-    },
-    {
-     "ename": "",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31mnotebook controller is DISPOSED. \n",
-      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
-     ]
-    }
-   ],
    "source": [
-    "query = questions[\"question\"].iloc[rng.randint(0, len(questions[\"question\"]))]\n",
-    "print(agent(query))"
+    "# Execution"
    ]
   }
  ],

src/__init__.py

@@ -0,0 +1,3 @@
+from . import agent
+
+__all__ = ["agent"]

src/agent.py

@@ -0,0 +1,57 @@
+from langchain_ollama import ChatOllama
+from langgraph.graph import StateGraph, START, END
+from typing import TypedDict, Literal, Any
+from tools import TOOL_LIST
+from prompts import PLANNER, ANSWER
+
+class ObservationSchema(TypedDict):
+  tool_name: str
+  arguments: dict[str, Any]
+  observation: Any
+  
+class AgentSchema(TypedDict):
+  query: str
+  pastit: list[ObservationSchema]
+  answer: str
+
+class ReActAgent():
+  def __init__(self, modelid: str, verbose: bool = False):
+    self.brain = ChatOllama(model=modelid, validate_model_on_init=True)
+    workflow = StateGraph(AgentSchema)
+    self.verbose = verbose
+
+    # nodes #
+
+    # edges #
+    workflow.add_conditional_edges(START, self.plan)
+
+    # compile
+    self.workflow = workflow.compile(); return
+
+  def __call__(self, query: str):
+    return self.workflow.invoke({"query": query, "pastit": [], "answer": ""})
+  
+  # nodes #
+  def answer(self, state: AgentSchema) -> AgentSchema:
+    self.brain.reasoning = False; self.brain.temperature = 0
+    messages = [
+      {'role': 'system', 'content': ANSWER.format(past_iterations=state.get("pastit", []))},
+      {'role': 'user', 'content': state.get("query", "")}
+    ]
+    response = self.brain.invoke(messages)
+    if self.verbose: print(response.content)
+    return {"query": state.get("query", ""), "pastit": state.get("pastit", []), "answer": response.content}
+  
+  def reason(self, state: AgentSchema) -> AgentSchema:
+    self.brain.reasoning = True; self.brain.temperature = 0.3
+  
+  # edges #
+  def plan(self, state: AgentSchema) -> Literal["answer", "reason", "tool_call"]:
+    self.brain.reasoning = True; self.brain.temperature = 0
+    messages = [
+      {'role': 'system', 'content': PLANNER.format(past_iterations=state.get("pastit", []))},
+      {'role': 'user', 'content': state.get("query", "")}
+    ]
+    response = self.brain.invoke(messages)
+    if self.verbose: print(f"#------ Reasoning ------#\n{response.additional_kwargs.get("reasoning_content", "")}\n#---------------------#\n\n#------ Response -------#\n{response.content}\n#-------------------------#\n")
+    return ["answer", "reason", "tool_call"][int(response.content)-1]

src/prompts.py

@@ -0,0 +1,25 @@
+PLANNER = (
+"""
+You are planning the best way to answer the user questions, you have the following info from past iterations:
+{past_iterations}
+
+Now you need to carefully plan the next step, you can decide one of the following options:
+1. You have all the necessary information to provide the answers to the user questions
+2. You have all the necessary information but you need to reason on the questions before providing correct answers
+3. You don't have all the necessary information to proceed with an answer, it will be necessary to call some tools before proceeding
+
+After reasoning on what to do, provide the number corresponding to your choice for the next step.
+**IMPORTANT:** Only provide a single digit number corresponding to the option you want to follow
+"""
+)[0].strip()
+
+ANSWER = (
+"""
+You got the following information from past iterations: {past_iterations}
+
+Answer the user's questions by providing a single unformatted word/digit answer for each question he asked.
+
+**Format Examples:**
+1. User: "What is the first name of the singer of Thriller and at what age did he die?" -> Answer: "Michael, 51"
+"""
+)[0].strip()

src/tools.py

@@ -0,0 +1,16 @@
+from langchain_core.tools import tool
+
+@tool
+def web_search(query: str) -> str:
+  """
+  This tool can be used to search the web for information
+
+  Arguments:
+    * query -> search string
+  
+  Returns:
+    * string containing the research results
+  """
+  return "Hello World!"
+
+TOOL_LIST = [web_search]