updated

agent.py

@@ -139,11 +139,34 @@ def get_graph(llm):
             query: The search query."""
         print("\n-------------------- Tool (Wikipedia) has been called --------------------\n")
         search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
-        formatted_search_docs = "\n\n---\n\n".join(
-            [
-                f'<Document source="{doc.metadata["source"]}" page="{doc.metadata["title"]} page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-                for doc in search_docs
-            ])
+
+        parts: list[str] = []
+
+        for doc in search_docs:
+            parts.append(
+            f'<Document source="{doc.metadata["source"]}" '
+            f'title="{doc.metadata["title"]}" '
+            f'page="{doc.metadata.get("page", "")}">\n'
+            f'{doc.page_content}\n</Document>'
+            )
+
+            try:
+                print("---------------------------------")
+                print("Loading tables from: ", doc.metadata["source"])
+                print("---------------------------------")
+                tables = WikipediaTableLoader(url=doc.metadata["source"], title=doc.metadata["title"]).load()
+                for i, table in enumerate(tables):
+                    parts.append(
+                        f'<Document source="{table.metadata["source"]}" '
+                        f'title="{table.metadata["title"]}" '
+                        f'table_index="{i}">\n'
+                        f'{table.page_content}\n</Document>'
+                    )
+            except Exception:
+                pass
+
+        formatted_search_docs = "\n\n---\n\n".join(parts)
+
         return formatted_search_docs
 
     @tool

langchain_custom.py

@@ -1,3 +1,5 @@
+from io import StringIO
+
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_core.documents import Document
 import pandas as pd
@@ -12,7 +14,7 @@ class WikipediaTableLoader(BaseLoader):
     def load(self):
 
         html  = requests.get(self.url, timeout=10).text
-        dfs   = pd.read_html(html)          # grab the tables
+        dfs   = pd.read_html(StringIO(html))          # grab the tables
         docs  = []
 
         for i, df in enumerate(dfs):

playground_api.ipynb

@@ -226,8 +226,8 @@
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-04-30T13:04:59.351968Z",
-     "start_time": "2025-04-30T13:04:59.338937Z"
+     "end_time": "2025-05-03T17:49:49.209881Z",
+     "start_time": "2025-05-03T17:49:48.876295Z"
     }
    },
    "cell_type": "code",
@@ -242,13 +242,13 @@
    ],
    "id": "799bcaab50ebff55",
    "outputs": [],
-   "execution_count": 16
+   "execution_count": 1
   },
   {
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-04-30T13:04:59.536418Z",
-     "start_time": "2025-04-30T13:04:59.526232Z"
+     "end_time": "2025-05-03T17:49:49.230082Z",
+     "start_time": "2025-05-03T17:49:49.221606Z"
     }
    },
    "cell_type": "code",
@@ -495,12 +495,12 @@
        "</div>"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
-   "execution_count": 17
+   "execution_count": 2
   },
   {
    "metadata": {

requirements.txt

@@ -17,4 +17,6 @@ matplotlib ~=3.10.1
 arxiv ~=2.2.0
 pymupdf ~=1.25.5
 youtube-transcript-api ~=1.0.3
-assemblyai ~=0.40.2
+assemblyai ~=0.40.2
+lxml ~=5.4.0
+tabulate ~=0.9.0

test_playground.ipynb

@@ -0,0 +1,190 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-05-03T18:40:33.108625Z",
+     "start_time": "2025-05-03T18:40:29.698376Z"
+    }
+   },
+   "source": [
+    "from io import StringIO\n",
+    "\n",
+    "from dotenv import load_dotenv\n",
+    "from gradio.server_messages import BaseMessage\n",
+    "from jinja2 import BaseLoader\n",
+    "\n",
+    "from agent import *\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "llm = get_llm()\n",
+    "\n",
+    "graph = get_graph(llm)\n",
+    "\n",
+    "# graph.invoke({\"messages\": [HumanMessage(content=question),]})\n"
+   ],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/dennis/PycharmProjects/Final_Assignment_Template/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-05-03T18:41:12.685273Z",
+     "start_time": "2025-05-03T18:40:33.126333Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "result = graph.invoke(\n",
+    "    {\n",
+    "        \"messages\": [\n",
+    "            HumanMessage(\n",
+    "                content=\"How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.\"\n",
+    "            ),\n",
+    "        ]\n",
+    "    }\n",
+    ")"
+   ],
+   "id": "96cdb32d9145e6b3",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "-------------------- Starting to create a plan --------------------\n",
+      "\n",
+      "Waiting for 5 seconds...\n",
+      "The plan is:  1. Access the English Wikipedia (2022 version if available).\n",
+      "2. Search for \"Mercedes Sosa discography\".\n",
+      "3. Identify the section listing studio albums.\n",
+      "4. Filter the list to include only albums released between 2000 and 2009 (inclusive).\n",
+      "5. Count the number of albums that meet the criteria.\n",
+      "6. Provide the count as the answer.\n",
+      "\n",
+      "-------------------- Agent has been called -----------------------------------\n",
+      "\n",
+      "Waiting for 5 seconds...\n",
+      "Agent has made a decision:\n",
+      "  [{'name': 'wiki_search', 'args': {'query': 'Mercedes Sosa discography'}, 'id': '416cac28-6fa8-4f45-a5db-f6bc6d8e1d1a', 'type': 'tool_call'}]\n",
+      "\n",
+      "-------------------- Decision of forwarding has been made --------------------\n",
+      "\n",
+      "Waiting for 2 seconds...\n",
+      "This is round:  2\n",
+      "The last message is:  content='' additional_kwargs={'function_call': {'name': 'wiki_search', 'arguments': '{\"query\": \"Mercedes Sosa discography\"}'}} response_metadata={'prompt_feedback': {'block_reason': 0, 'safety_ratings': []}, 'finish_reason': 'STOP', 'model_name': 'gemini-2.0-flash', 'safety_ratings': []} id='run-80c57808-f76d-495c-bc1a-e22b89556475-0' tool_calls=[{'name': 'wiki_search', 'args': {'query': 'Mercedes Sosa discography'}, 'id': '416cac28-6fa8-4f45-a5db-f6bc6d8e1d1a', 'type': 'tool_call'}] usage_metadata={'input_tokens': 622, 'output_tokens': 7, 'total_tokens': 629, 'input_token_details': {'cache_read': 0}}\n",
+      "\n",
+      "-------------------- Tool (Wikipedia) has been called --------------------\n",
+      "\n",
+      "---------------------------------\n",
+      "Loading tables from:  https://en.wikipedia.org/wiki/Mercedes_Sosa\n",
+      "---------------------------------\n",
+      "---------------------------------\n",
+      "Loading tables from:  https://en.wikipedia.org/wiki/Joan_Baez_discography\n",
+      "---------------------------------\n",
+      "\n",
+      "-------------------- Agent has been called -----------------------------------\n",
+      "\n",
+      "Waiting for 5 seconds...\n",
+      "Agent has made a decision:\n",
+      "  [{'name': 'wiki_search', 'args': {'query': 'Mercedes Sosa discography'}, 'id': '77eb4086-528b-48ea-b139-70382e84a80c', 'type': 'tool_call'}]\n",
+      "\n",
+      "-------------------- Decision of forwarding has been made --------------------\n",
+      "\n",
+      "Waiting for 2 seconds...\n",
+      "This is round:  3\n",
+      "The last message is:  content='' additional_kwargs={'function_call': {'name': 'wiki_search', 'arguments': '{\"query\": \"Mercedes Sosa discography\"}'}} response_metadata={'prompt_feedback': {'block_reason': 0, 'safety_ratings': []}, 'finish_reason': 'STOP', 'model_name': 'gemini-2.0-flash', 'safety_ratings': []} id='run-3240832b-3a9d-48b5-9c58-0f9aa5dd8658-0' tool_calls=[{'name': 'wiki_search', 'args': {'query': 'Mercedes Sosa discography'}, 'id': '77eb4086-528b-48ea-b139-70382e84a80c', 'type': 'tool_call'}] usage_metadata={'input_tokens': 17925, 'output_tokens': 7, 'total_tokens': 17932, 'input_token_details': {'cache_read': 0}}\n",
+      "\n",
+      "-------------------- Tool (Wikipedia) has been called --------------------\n",
+      "\n",
+      "---------------------------------\n",
+      "Loading tables from:  https://en.wikipedia.org/wiki/Mercedes_Sosa\n",
+      "---------------------------------\n",
+      "---------------------------------\n",
+      "Loading tables from:  https://en.wikipedia.org/wiki/Joan_Baez_discography\n",
+      "---------------------------------\n",
+      "\n",
+      "-------------------- Agent has been called -----------------------------------\n",
+      "\n",
+      "Waiting for 5 seconds...\n",
+      "Agent has made a decision:\n",
+      " Based on the Wikipedia search results for \"Mercedes Sosa discography\", the studio albums published by Mercedes Sosa between 2000 and 2009 (inclusive) are:\n",
+      "\n",
+      "*   2005 - Corazón Libre\n",
+      "*   2009 - Cantora 1\n",
+      "*   2009 - Cantora 2\n",
+      "\n",
+      "Therefore, the answer is 3. []\n",
+      "\n",
+      "-------------------- Decision of forwarding has been made --------------------\n",
+      "\n",
+      "Waiting for 2 seconds...\n",
+      "This is round:  4\n",
+      "The last message is:  content='Based on the Wikipedia search results for \"Mercedes Sosa discography\", the studio albums published by Mercedes Sosa between 2000 and 2009 (inclusive) are:\\n\\n*   2005 - Corazón Libre\\n*   2009 - Cantora 1\\n*   2009 - Cantora 2\\n\\nTherefore, the answer is 3.' additional_kwargs={} response_metadata={'prompt_feedback': {'block_reason': 0, 'safety_ratings': []}, 'finish_reason': 'STOP', 'model_name': 'gemini-2.0-flash', 'safety_ratings': []} id='run-59b32104-a935-46ed-a8fc-c530bbe2816e-0' usage_metadata={'input_tokens': 35228, 'output_tokens': 79, 'total_tokens': 35307, 'input_token_details': {'cache_read': 0}}\n",
+      "\n",
+      "-------------------- Generating Answer -----------------------------------\n",
+      "\n",
+      "Waiting for 5 seconds...\n",
+      "The final answer is:  3\n"
+     ]
+    }
+   ],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-05-03T18:30:59.650996Z",
+     "start_time": "2025-05-03T18:30:59.647709Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "",
+   "id": "ce8f566e12feb242",
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "27e7946b3191e98f"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}