commit

app.py

@@ -12,11 +12,11 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 # testing --------------------------------------------
 testing_mode = True
 questions_to_run = [
-    "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",     # good answer, wrong albums
-    #"a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
+    #"8e867cd7-cff9-4e6c-867a-ff5ddc2550be",     # not using tools
+    #"a1e91b78-d3d8-4675-bb8d-62741b4b68a6",     # (!)
     #"2d83110e-a098-4ebb-9987-066c06fa42d0",     # OK
-    # "cca530fc-4052-43b2-b130-b30968d8aa44",     # OK, need to remove FINAL_ANSWER string
-    #"4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",     # <--------
+    #"cca530fc-4052-43b2-b130-b30968d8aa44",     # OK, need to remove FINAL_ANSWER string
+    "4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",     # <--------
     #"6f37996b-2ac7-44b0-8e68-6d28256631b4",
     #"9d191bce-651d-4746-be2d-7ef8ecadb9c2",
     #"cabe07ed-9eca-40ea-8ead-410ef5e83f91",

my_agent.py

@@ -84,7 +84,7 @@ class MyAgent:
                 "my_tool_image_load",
                 "my_tool_reverse_string",
                 "my_tool_wiki_page_section",
-                "my_tool_wiki_filter_tables"
+                "_my_tool_wiki_table_filter"
             ],
         )
         #web_search_agent = CodeAgent(

my_base_wiki_api.py

@@ -43,6 +43,108 @@ class MyWikiAPI:
             }).json()
         return response["parse"][format]["*"]
 
+    def __get_category_pages__(self, category, year):
+        # https://en.wikipedia.org/w/api.php?action=help&modules=main#main/datatype/timestamp
+        # https://www.mediawiki.org/wiki/API:Query#Generators
+        # https://www.mediawiki.org/wiki/API:Continue
+        # https://stackoverflow.com/questions/35826469/how-to-combine-two-wikipedia-api-calls-into-one
+        # 2001-01-15T14:56:00Z
+        # api.php?action=query&generator=categorymembers&gcmtitle=Category:Physics&prop=info [open in sandbox]
+
+        cmstart = f"{year}-01-01T00:00:00Z"
+        cmend =   f"{year}-12-31T23:59:59Z"
+
+        print(f"Listing pages in category {category} from {cmstart} to {cmend}")
+
+        response = requests.get(
+            self.WIKI_BASE_URL,
+            headers=self.user_agent_headers,
+            params={
+                'action': 'query',
+                'format': 'json',
+                #'prop': 'extracts',
+                'list': 'categorymembers',
+                #'generator': 'categorymembers',
+                'cmtitle': f'Category:{category}',
+                'cmsort': 'timestamp',
+                'cmprop': 'ids|title|timestamp',
+                'cmstart': cmstart,
+                'cmend': cmend,
+                'cmlimit': '500',
+                #'cmlimit': '20',
+            }).json()
+        print(f"Response: \n{json.dumps(response,indent=4)}")
+
+        print(f"Got artickles: {len(response['query']['categorymembers'])}")
+
+        #cmcontinue = ""
+        #while(True):
+        #    if "continue" in response and "cmcontinue" in response["continue"]:
+        #        cmcontinue = response["continue"]["cmcontinue"]
+        #    else:
+        #        break
+
+        #    print(f"cmcontinue: {cmcontinue}")
+
+        #    response = requests.get(
+        #        self.WIKI_BASE_URL,
+        #        headers=self.user_agent_headers,
+        #        params={
+        #            'action': 'query',
+        #            'format': 'json',
+        #            #'prop': 'extracts',
+        #            'list': 'categorymembers',
+        #            #'generator': 'categorymembers',
+        #            'cmcontinue': cmcontinue,
+        #            'cmtitle': f'Category:{category}',
+        #            'cmsort': 'timestamp',
+        #            'cmprop': 'ids|title|timestamp',
+        #            'cmstart': cmstart,
+        #            #'cmend': cmend,
+        #            #'cmlimit': '500',
+        #            'cmlimit': '20',
+        #        }).json()
+        #    print(f"Response: \n{json.dumps(response,indent=4)}")
+
+
+        #response = requests.get(
+        #    self.WIKI_BASE_URL,
+        #    headers=self.user_agent_headers,
+        #    params={
+        #        'action': 'query',
+        #        'format': 'json',
+        #        'prop': 'extracts',
+        #        #'list': 'categorymembers',
+        #        'generator': 'categorymembers',
+        #        'gcmtitle': f'Category:{category}',
+        #        'gcmsort': 'timestamp',
+        #        'gcmprop': 'ids|title|timestamp',
+        #        'gcmstart': cmstart,
+        #        #'cmend': cmend,
+        #        #'cmlimit': '500',
+        #        'gcmlimit': '20',
+        #    }).json()
+
+        # ['query']['categorymembers']
+        # list -->
+        #   title
+
+
+
+        return response
+
+    def __is_int__(self, s):
+        try:
+            int(s)
+        except ValueError:
+            return False
+        else:
+            return True
+
+    def get_category(self, category, year):
+        ret = self.__get_category_pages__(category, year)
+        print(f"Got category: {category}\n{ret}")
+
     def get_page_section(self, page_title, section_title):
         section_id = self.__find_section_on_page__(page_title, section_title)
         _ret = ""
@@ -53,29 +155,36 @@ class MyWikiAPI:
 
     def filter_section_and_table(self, section_content, sub_section_name, year_start, year_end):
         parsed = wtp.parse(section_content)
-        #print(f"***KS*** Parsed section:\n{parsed}")
-
         sections = parsed.sections
-        print(f"Sections {len(sections)}")
         section_found = None
         for sec in sections:
-            print("---------------------------------------------------------------------")
-            print(f"{sec.title if sec.title is not None else ''}")
             if sec.title is not None and sec.title.find(sub_section_name) >= 0:
                 section_found = sec
 
-        print(f"Found matching subsection: {section_found}")
-
-        print(f"Section has tables: {section_found.tables}")
+        print(f"Found matching subsection: {section_found.title}")
 
-        table_data = section_found.tables[0].data()
-        print(f"Table has data: {table_data}")
         rows_collected = []
-        for row in table_data:
-            print("---------------------------------------")
-            print(f"Table has data: {row}")
+        if section_found is not None and section_found.tables is not None and len(section_found.tables) > 0:
+            table_data = section_found.tables[0].data()
+            for row in table_data:
+                if self.__is_int__(row[0]) and year_start <= int(row[0]) <= year_end:
+                    rows_collected.append(row)
+
+        return rows_collected
+
+    def get_featured_articles(self, year):
+        featured_url = f"https://api.wikimedia.org/feed/v1/wikipedia/en/featured/{year}/01/01"
+
+        response = requests.get(
+            featured_url,
+            headers=self.user_agent_headers,
+        ).json()
 
-            # TODO do filtering of rows and return the filtered table
+        #print(f"Got featured list: {json.dumps(response,indent=4)}")
 
+        tfa = response['tfa']
+        print(f"tfa: {json.dumps(tfa,indent=4)}")
+        title = tfa['titles']['canonical']
+        extract = tfa['extract']
 
-        return ["aaa"]
+        print(f"tfa:\ntitle: {title}\nextract: {extract}")

my_tool_wiki_filter_tables.py

@@ -56,4 +56,6 @@ class MyWikiTableFilterTool(Tool):
         self.wiki = MyWikiAPI()
 
     def forward(self, section_content, sub_section_name, year_start, year_end) -> str:
-        return self.wiki.filter_section_and_table(section_content,sub_section_name, year_start, year_end)
+        result = self.wiki.filter_section_and_table(section_content,sub_section_name, year_start, year_end)
+        print(f"***KS*** Wiki table filter tool, result: {result}")
+        return result

test_tools.py

@@ -5,12 +5,27 @@ from my_tool_fen import FENTool
 from my_tool_chess_analysis import ChessAnalysisTool
 from my_tool_wiki_page_section import MyWikiPageSectionTool
 from my_tool_wiki_filter_tables import MyWikiTableFilterTool
+from my_base_wiki_api import MyWikiAPI
 import pytest
 import matplotlib as mp
 
 #pytest --capture=no
 
+@pytest.mark.parametrize("_year,_exp", [(2023, 3)])
+def test_tool_wiki_contributions(_year, _exp):
+    api = MyWikiAPI()
+    #api.get_featured_articles(_year)
+    # https://en.wikipedia.org/wiki/Category:Featured_articles
+    category = 'Featured articles'
+    api.get_category(category, _year)
 
+    category = 'Former featured articles'
+    api.get_category(category, _year)
+
+    assert _exp == _exp
+
+
+@pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_page,_section,_sub_section,_year_start,_year_end,_exp",
     [("Mercedes Sosa", "Discography", "Studio albums", 2000, 2009, 3)])
 def test_tool_wiki_page_section(_page, _section, _sub_section, _year_start, _year_end, _exp):
@@ -28,6 +43,7 @@ def test_tool_wiki_page_section(_page, _section, _sub_section, _year_start, _yea
 def test_tool_reverse_string(_inp,_exp):
     assert ReverseStringTool().forward(_inp) == _exp
 
+
 @pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_task_id,_exp", [("cca530fc-4052-43b2-b130-b30968d8aa44", "")])
 def test_tool_image_load(_task_id,_exp):
@@ -39,6 +55,7 @@ def test_tool_image_load(_task_id,_exp):
     #plt.imshow(result)
     #plt.show()
 
+
 @pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_task_id,_exp", [("cca530fc-4052-43b2-b130-b30968d8aa44",
                                             "1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111")])
@@ -67,6 +84,7 @@ def test_tool_fen(_pieces_list,_exp):
     print(f"Got result: {fen}")
     assert fen == _exp
 
+
 @pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_fen,_exp",
                          [