commit

app.py

@@ -12,11 +12,11 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 # testing --------------------------------------------
 testing_mode = True
 questions_to_run = [
-    #"8e867cd7-cff9-4e6c-867a-ff5ddc2550be",     # good answer, wrong albums
+    "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",     # good answer, wrong albums
     #"a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
     #"2d83110e-a098-4ebb-9987-066c06fa42d0",     # OK
     # "cca530fc-4052-43b2-b130-b30968d8aa44",     # OK, need to remove FINAL_ANSWER string
-    "4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",     # <--------
+    #"4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",     # <--------
     #"6f37996b-2ac7-44b0-8e68-6d28256631b4",
     #"9d191bce-651d-4746-be2d-7ef8ecadb9c2",
     #"cabe07ed-9eca-40ea-8ead-410ef5e83f91",

my_agent.py

@@ -9,6 +9,8 @@ from my_tool_chess_board import ChessBoard
 from my_tool_fen import FENTool
 from my_tool_chess_analysis import ChessAnalysisTool
 from my_prompt_config import MyPromptConfig
+from my_tool_wiki_page_section import MyWikiPageSectionTool
+from my_tool_wiki_filter_tables import MyWikiTableFilterTool
 from dotenv import load_dotenv
 
 # https://huggingface.co/docs/transformers/model_doc/mistral?usage=Pipeline
@@ -55,6 +57,8 @@ class MyAgent:
                 ChessBoard(self.chess_board_model_name, self.chess_board_model_dir),
                 FENTool(),
                 ChessAnalysisTool(),
+                MyWikiPageSectionTool(),
+                MyWikiTableFilterTool(),
                 WebSearchTool(),
                 FinalAnswerTool(),
             ],
@@ -69,13 +73,18 @@ class MyAgent:
                 "matplotlib.pyplot",
                 "stockfish",
                 "requests",
-                "wikipediaapi",
                 "bs4",
+                "json",
+                "lxml",
+                "wikitextparser",
+                "mwparserfromhell"
                 "my_tool_chess_analysis",
                 "my_tool_chess_board",
                 "my_tool_fen",
                 "my_tool_image_load",
                 "my_tool_reverse_string",
+                "my_tool_wiki_page_section",
+                "my_tool_wiki_filter_tables"
             ],
         )
         #web_search_agent = CodeAgent(

my_base_wiki_api.py

@@ -0,0 +1,81 @@
+import requests
+import json
+import wikitextparser as wtp
+
+# https://gist.github.com/scionoftech/0f35d5e231be2cf46823d774023268b6
+# https://www.mediawiki.org/wiki/API:Main_page
+
+class MyWikiAPI:
+    WIKI_BASE_URL = "https://en.wikipedia.org/w/api.php"
+    user_agent_headers = {"user-Agent":
+        "AgentsCourseAssignment/1.0 (https://huggingface.co/spaces/krzsam/Agents-Course-Assignment)"}
+
+    def __init__(self):
+        print(f"***KS*** Initializing Wiki API")
+
+    def __find_section_on_page__(self, page_title, section_title):
+        response = requests.get(
+            self.WIKI_BASE_URL,
+            headers=self.user_agent_headers,
+            params={
+                'action': 'parse',
+                'format': 'json',
+                'page': page_title,
+                'prop': 'sections',
+            }).json()
+        sections = response['parse']['sections']
+        section_ix_found = None
+        for sec in sections:
+            if sec["line"] == section_title:
+                section_ix_found = sec["index"]
+        return section_ix_found
+
+    def __get_page_section_content__(self, page_title, section_id, format="wikitext"):
+        response = requests.get(
+            self.WIKI_BASE_URL,
+            headers=self.user_agent_headers,
+            params={
+                'action': 'parse',
+                'format': 'json',
+                'page': page_title,
+                'prop': format,
+                'section': section_id
+            }).json()
+        return response["parse"][format]["*"]
+
+    def get_page_section(self, page_title, section_title):
+        section_id = self.__find_section_on_page__(page_title, section_title)
+        _ret = ""
+        if section_id is not None:
+            _ret = self.__get_page_section_content__(page_title, section_id)
+        # print(f"***KS*** Got page section for {page_title} / {section_title} \n{_ret}")
+        return _ret
+
+    def filter_section_and_table(self, section_content, sub_section_name, year_start, year_end):
+        parsed = wtp.parse(section_content)
+        #print(f"***KS*** Parsed section:\n{parsed}")
+
+        sections = parsed.sections
+        print(f"Sections {len(sections)}")
+        section_found = None
+        for sec in sections:
+            print("---------------------------------------------------------------------")
+            print(f"{sec.title if sec.title is not None else ''}")
+            if sec.title is not None and sec.title.find(sub_section_name) >= 0:
+                section_found = sec
+
+        print(f"Found matching subsection: {section_found}")
+
+        print(f"Section has tables: {section_found.tables}")
+
+        table_data = section_found.tables[0].data()
+        print(f"Table has data: {table_data}")
+        rows_collected = []
+        for row in table_data:
+            print("---------------------------------------")
+            print(f"Table has data: {row}")
+
+            # TODO do filtering of rows and return the filtered table
+
+
+        return ["aaa"]

my_prompt_config.py

@@ -1,16 +1,8 @@
 from smolagents import PromptTemplates, PlanningPromptTemplate, FinalAnswerPromptTemplate, ManagedAgentPromptTemplate
 
-#             Additional tools provided :
-#             * _my_tool_chess_analysis : analyze chess position provided in FEN notation and provide best move,
-#                 available arguments: fen, player_color
-#             * _my_tool_chess_board : process an image and extract list of chess pieces, available arguments: img ,
-#             * _my_tool_fen : convert list of chess pieces into FEN notation, available arguments: chest_pieces ,
-#             * _my_tool_image_load : load an image for given task_id, available arguments: task_id ,
-#
 #             Use <code></code> examples as here
 #             https://github.com/huggingface/smolagents/blob/main/src/smolagents/vision_web_browser.py
-#
-#             * _my_tool_ reverse_string : reverse a string, available arguments: input_str ,
+
 
 class MyPromptConfig:
     PROMPT_TEMPLATES = PromptTemplates(
@@ -27,14 +19,18 @@ class MyPromptConfig:
             If you are going to use a tool, describe in detail how you are going
             to invoke the tool and explain parameters used to invoke the tool.
             
-            The list of tools available is provided below. The allow to solve a variety of various questions asked.
-            Decide and Pick and use only tools which are necessary to answer the question asked.
+            The list of tools available is provided below. They allow to solve a variety of various questions asked.
+            Decide and pick and use only tools which are necessary to answer the question asked.
+            If the tool looks proper for the task, use it in the first place, instead of generating the code
+            yourself to achieve the same task.
             
             * _my_tool_chess_analysis : analyze chess position provided as FEN notation
             * _my_tool_chess_board : extract list of chess pieces from chess board image
             * _my_tool_fen : convert list of chess pieces into FEN notation
             * _my_tool_image_load : load image of the chess board for given task_id
             * _my_tool_reverse_string : reverse provided string
+            * _my_tool_wiki_page_section : get contents of a section of the Wikipedia page
+            * _my_tool_wiki_table_filter : filter the subsection and tables on the Wikipedia page for provided years
             
             If the question mentions image or other file, use one of provided tools to load it
             using task_id associated with the question.
@@ -47,18 +43,6 @@ class MyPromptConfig:
             plain text unless specified otherwise.
             If you are asked for a comma separated list, apply the above rules depending of whether the element to be put
             in the list is a number or a string.
-            
-            Report your answer using final_answer tool
-            
-            YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of
-            numbers and/or strings.
-            If you are asked for a number, don’t use comma to write your number neither use units such as $ or 
-            percent sign unless specified otherwise.
-            If you are asked for a string, don’t use articles, neither abbreviations (e.g. for cities), 
-            and write the digits in plain text unless specified otherwise.
-            If you are asked for a comma separated list, apply the above rules depending of whether the element 
-            to be put in the list is a number or a string
-            
         """,
         planning=PlanningPromptTemplate(
             initial_plan="""
@@ -75,23 +59,19 @@ class MyPromptConfig:
         final_answer=FinalAnswerPromptTemplate(
             pre_messages="",
             post_messages="""
-                
+            Report your answer using final_answer tool
+            
+            YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of
+            numbers and/or strings.
+            If you are asked for a number, don’t use comma to write your number neither use units such as $ or 
+            percent sign unless specified otherwise.
+            If you are asked for a string, don’t use articles, neither abbreviations (e.g. for cities), 
+            and write the digits in plain text unless specified otherwise.
+            If you are asked for a comma separated list, apply the above rules depending of whether the element 
+            to be put in the list is a number or a string
             """
         ),
     )
 
     def __init__(self):
         print("Prompt Templates initialized")
-
-
-
-#EMPTY_PROMPT_TEMPLATES = PromptTemplates(
-#    system_prompt="",
-#    planning=PlanningPromptTemplate(
-#        initial_plan="",
-#        update_plan_pre_messages="",
-#        update_plan_post_messages="",
-#    ),
-#    managed_agent=ManagedAgentPromptTemplate(task="", report=""),
-#    final_answer=FinalAnswerPromptTemplate(pre_messages="", post_messages=""),
-#)

my_tool_wiki_filter_tables.py

@@ -0,0 +1,59 @@
+from smolagents import Tool
+from my_base_wiki_api import MyWikiAPI
+
+
+#AUTHORIZED_TYPES = [
+#    "string",
+#    "boolean",
+#    "integer",
+#    "number",
+#    "image",
+#    "audio",
+#    "array",
+#    "object",
+#    "any",
+#    "null",
+#]
+
+# https://wikipedia-api.readthedocs.io/en/latest/API.html
+# https://github.com/martin-majlis/Wikipedia-API/tree/master
+class MyWikiTableFilterTool(Tool):
+    name = "_my_tool_wiki_table_filter"
+    description = """
+        Extract contents of the page section for the provided a Wikipedia page
+        To invoke the tool use code as below
+        <code>
+        section_contents = _my_tool_wiki_table_filter(section_content='aaa', sub_section_name='bbb', 
+        year_start='1998', year_end='2005')
+        </code>
+    """
+
+    inputs = {
+        "section_content": {
+            "type": "string",
+            "description": "Wikipedia page title",
+        },
+        "sub_section_name": {
+            "type": "string",
+            "description": "Wikipedia page title",
+        },
+        "year_start": {
+            "type": "string",
+            "description": "Name of the section on the Wikipedia page",
+        },
+        "year_end": {
+            "type": "string",
+            "description": "Name of the section on the Wikipedia page",
+        },
+    }
+
+    output_type = "string"
+
+    is_initialized = True
+
+    def __init__(self):
+        print(f"***KS*** Wiki table filter tool initializing ...")
+        self.wiki = MyWikiAPI()
+
+    def forward(self, section_content, sub_section_name, year_start, year_end) -> str:
+        return self.wiki.filter_section_and_table(section_content,sub_section_name, year_start, year_end)

my_tool_wiki_page_section.py

@@ -0,0 +1,75 @@
+from smolagents import Tool
+from my_base_wiki_api import MyWikiAPI
+
+
+#AUTHORIZED_TYPES = [
+#    "string",
+#    "boolean",
+#    "integer",
+#    "number",
+#    "image",
+#    "audio",
+#    "array",
+#    "object",
+#    "any",
+#    "null",
+#]
+
+# https://wikipedia-api.readthedocs.io/en/latest/API.html
+# https://github.com/martin-majlis/Wikipedia-API/tree/master
+class MyWikiPageSectionTool(Tool):
+    name = "_my_tool_wiki_page_section"
+    description = """
+        Extract contents of the page section for the provided a Wikipedia page
+        To invoke the tool use code as below
+        <code>
+        section_contents = _my_tool_wiki_page_section(page_title='aaa', section_title='bbb', version='2022')
+        </code>
+    """
+
+    inputs = {
+        "page": {
+            "type": "string",
+            "description": "Wikipedia page title",
+            "nullable": True
+        },
+        "page_title": {
+            "type": "string",
+            "description": "Wikipedia page title",
+            "nullable": True
+        },
+        "section": {
+            "type": "string",
+            "description": "Name of the section on the Wikipedia page",
+            "nullable": True
+        },
+        "section_title": {
+            "type": "string",
+            "description": "Name of the section on the Wikipedia page",
+            "nullable": True
+        },
+        "version": {
+            "type": "string",
+            "description": "Year version of the Wikipedia page",
+            "nullable": True
+        },
+        "wiki_version": {
+            "type": "string",
+            "description": "Year version of the Wikipedia page",
+            "nullable": True
+        },
+    }
+
+    output_type = "string"
+
+    is_initialized = True
+
+    def __init__(self):
+        print(f"***KS*** Wiki page section tool initializing ...")
+        self.wiki = MyWikiAPI()
+
+    def forward(self, page=None, page_title=None, section=None, section_title=None, version="2025",
+                wiki_version="2025") -> str:
+        return self.wiki.get_page_section(
+            page if page is not None else page_title,
+            section if section is not None else section_title)

requirements.txt

@@ -8,7 +8,6 @@ PyQt6
 opencv-python
 torch
 stockfish
-wikipedia-api
 bs4
-mistral-inference
-mistralai
+wikitextparser
+mwparserfromhell

test_tools.py

@@ -3,19 +3,33 @@ from my_tool_image_load import ImageLoadTool
 from my_tool_chess_board import ChessBoard
 from my_tool_fen import FENTool
 from my_tool_chess_analysis import ChessAnalysisTool
+from my_tool_wiki_page_section import MyWikiPageSectionTool
+from my_tool_wiki_filter_tables import MyWikiTableFilterTool
 import pytest
-import matplotlib.pyplot as plt
 import matplotlib as mp
 
 #pytest --capture=no
 
+
+@pytest.mark.parametrize("_page,_section,_sub_section,_year_start,_year_end,_exp",
+    [("Mercedes Sosa", "Discography", "Studio albums", 2000, 2009, 3)])
+def test_tool_wiki_page_section(_page, _section, _sub_section, _year_start, _year_end, _exp):
+    w = MyWikiPageSectionTool()
+    f = MyWikiTableFilterTool()
+    result = w(page=_page, section=_section)
+    # print(f"Got page section content: \n{result}")
+    filtered = f(result, _sub_section, _year_start, _year_end)
+    print(f"Table filtered: \n{filtered}")
+    assert len(filtered) == _exp
+
+
 @pytest.mark.skip(reason="disabled")
-@pytest.mark.parametrize("_inp,_exp",[("abc", "cba"),("ihg fed cba", "abc def ghi")])
+@pytest.mark.parametrize("_inp,_exp", [("abc", "cba"), ("ihg fed cba", "abc def ghi")])
 def test_tool_reverse_string(_inp,_exp):
     assert ReverseStringTool().forward(_inp) == _exp
 
 @pytest.mark.skip(reason="disabled")
-@pytest.mark.parametrize("_task_id,_exp",[("cca530fc-4052-43b2-b130-b30968d8aa44", "")])
+@pytest.mark.parametrize("_task_id,_exp", [("cca530fc-4052-43b2-b130-b30968d8aa44", "")])
 def test_tool_image_load(_task_id,_exp):
     print(f"\nLoading image for task id: {_task_id}")
     t = ImageLoadTool()
@@ -26,7 +40,8 @@ def test_tool_image_load(_task_id,_exp):
     #plt.show()
 
 @pytest.mark.skip(reason="disabled")
-@pytest.mark.parametrize("_task_id,_exp",[("cca530fc-4052-43b2-b130-b30968d8aa44", "1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111")])
+@pytest.mark.parametrize("_task_id,_exp", [("cca530fc-4052-43b2-b130-b30968d8aa44",
+                                            "1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111")])
 def test_tool_chess_board(_task_id,_exp):
     print(f"\nLoading image for task id: {_task_id}")
     t = ImageLoadTool()
@@ -39,7 +54,7 @@ def test_tool_chess_board(_task_id,_exp):
     assert pieces == _exp
 
 
-#@pytest.mark.skip(reason="disabled")
+@pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_pieces_list,_exp",
     [
         ("1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111",
@@ -52,10 +67,10 @@ def test_tool_fen(_pieces_list,_exp):
     print(f"Got result: {fen}")
     assert fen == _exp
 
-
+@pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_fen,_exp",
                          [
-                             ("3r2k1/pp3pp1/4b2p/7Q/3n4/PqBBR2P/5PP1/6K1 b - - 0 1","d8d5")
+                             ("3r2k1/pp3pp1/4b2p/7Q/3n4/PqBBR2P/5PP1/6K1 b - - 0 1", "d8d5")
                          ])
 def test_tool_chess_analysis(_fen,_exp):
     print(f"\nAnalysing FEN: {_fen}")