commit

my_base_libretexts_api.py

@@ -0,0 +1,97 @@
+import requests
+import json
+from lxml import etree
+from lxml import html
+
+# https://gist.github.com/scionoftech/0f35d5e231be2cf46823d774023268b6
+# https://www.mediawiki.org/wiki/API:Main_page
+
+class MyLibreTextsAPI:
+    LIBRETEXTS_BASE_URL = "https://chem.libretexts.org"
+    user_agent_headers = {"user-Agent":
+        "AgentsCourseAssignment/1.0 (https://huggingface.co/spaces/krzsam/Agents-Course-Assignment)"}
+
+    def __init__(self):
+        print(f"***KS*** Initializing LibreTexts API")
+
+    def get_bookshelves(self):
+        html_content = requests.get(
+            f"{self.LIBRETEXTS_BASE_URL}/Bookshelves",
+            headers=self.user_agent_headers,
+            ).text
+        tree = html.fromstring(html_content)
+        link_class = "mt-sortable-listing-link mt-edit-section internal"
+        elements = tree.xpath(f"//a[@class='{link_class}']")
+        #for element in elements:
+        #    print("-----------------------------------")
+        #    print(f"Tag {element.tag} : title: {element.attrib['title']} href: {element.attrib['href']}")
+
+        bookshelf = [(element.attrib['title'], element.attrib['href']) for element in elements]
+        print(f"Bookshelf:\n{bookshelf}\n\n")
+
+        return bookshelf
+
+    def get_books(self, bookshelf_url):
+        html_content = requests.get(
+            bookshelf_url,
+            headers=self.user_agent_headers,
+        ).text
+        tree = html.fromstring(html_content)
+        link_class = "mt-sortable-listing-link mt-edit-section internal"
+        elements = tree.xpath(f"//a[@class='{link_class}']")
+        #for element in elements:
+        #    print("-----------------------------------")
+        #    print(f"Tag {element.tag} : title: {element.attrib['title']} href: {element.attrib['href']}")
+
+        books = [(element.attrib['title'], element.attrib['href']) for element in elements]
+        print(f"Books:\n{books}\n\n")
+
+        return books
+
+    def get_book_sections(self, book_url):
+        html_content = requests.get(
+            book_url,
+            headers=self.user_agent_headers,
+        ).text
+        tree = html.fromstring(html_content)
+        link_class = "mt-sortable-listing-link mt-edit-section internal"
+        elements = tree.xpath(f"//a[@class='{link_class}']")
+        #for element in elements:
+        #    print("-----------------------------------")
+        #    print(f"Tag {element.tag} : title: {element.attrib['title']} href: {element.attrib['href']}")
+
+        sections = [(element.attrib['title'], element.attrib['href']) for element in elements]
+        print(f"Sections:\n{sections}\n\n")
+
+        return sections
+
+    def get_book_section_paragraphs(self, section_url):
+        html_content = requests.get(
+            section_url,
+            headers=self.user_agent_headers,
+        ).text
+        tree = html.fromstring(html_content)
+        link_class = "internal"
+        elements = tree.xpath(f"//a[@class='{link_class}']")
+        #for element in elements:
+        #    print("-----------------------------------")
+        #    print(f"Tag {element.tag} : title: {element.attrib['title']} href: {element.attrib['href']}")
+
+        paragraphs = [(element.attrib['title'], element.attrib['href']) for element in elements]
+        print(f"Paragraphs:\n{paragraphs}\n\n")
+
+        return paragraphs
+
+    def get_section_contents(self, paragraph_url):
+        html_content = requests.get(
+            paragraph_url,
+            headers=self.user_agent_headers,
+        ).text
+        tree = html.fromstring(html_content)
+        link_class = "internal"
+        elements = tree.xpath(f"//a[@class='{link_class}']")
+
+        # TODO get contents and convert to MD format
+
+        return contents
+

my_base_wiki_api.py

@@ -61,9 +61,6 @@ class MyWikiAPI:
         elements = tree.xpath(f"//div[@class='{article_heading}']")
         element_texts = []
         for element in elements:
-            #print("-----------------------------------")
-            #print(f"Tag {element.tag}")
-
             n1 = element.getnext()
             n1_text = " ".join(n1.itertext())
 
@@ -71,12 +68,10 @@ class MyWikiAPI:
             n2_text = " ".join(n2.itertext())
             element_text = f"{n1_text} {n2_text}"
 
-            #print(f"Element text: {element_text}")
             element_texts.append(element_text)
 
         return element_texts
 
-
     def __is_int__(self, s):
         try:
             int(s)

my_tool_libretext_book_shelves.py

@@ -0,0 +1,49 @@
+from smolagents import Tool
+from my_base_wiki_api import MyWikiAPI
+
+
+#AUTHORIZED_TYPES = [
+#    "string",
+#    "boolean",
+#    "integer",
+#    "number",
+#    "image",
+#    "audio",
+#    "array",
+#    "object",
+#    "any",
+#    "null",
+#]
+
+class MyWikiFeaturedArticles(Tool):
+    name = "_my_tool_wiki_featured_articles"
+    description = """
+        Extracts a list of nominators and summaries of featured Wikipedia articles for the provided month and year
+        To invoke the tool use code as below
+        <code>
+        featured = _my_tool_wiki_featured_articles(month='January', year='2005')
+        </code>
+    """
+
+    inputs = {
+        "month": {
+            "type": "string",
+            "description": "name of the month for example January",
+        },
+        "year": {
+            "type": "integer",
+            "description": "Year expressed in 4 digit notation, for example 2009",
+        },
+    }
+
+    output_type = "string"
+
+    is_initialized = True
+
+    def __init__(self):
+        print(f"***KS*** Wiki featured articles tool initializing ...")
+        self.wiki = MyWikiAPI()
+
+    def forward(self, month, year):
+        result = self.wiki.get_featured_articles(month, year)
+        return result

test_tools.py

@@ -7,21 +7,53 @@ from my_tool_wiki_page_section import MyWikiPageSectionTool
 from my_tool_wiki_filter_tables import MyWikiTableFilterTool
 from my_base_wiki_api import MyWikiAPI
 from my_tool_wiki_featured_articles import MyWikiFeaturedArticles
+from my_base_libretexts_api import MyLibreTextsAPI
 import pytest
 import matplotlib as mp
 
 #pytest --capture=no
 
-@pytest.mark.parametrize("_month,_year,_exp", [('November',2016, "Giganotosaurus")])
-def test_tool_wiki_contributions(_month,_year, _exp):
-    #api = MyWikiAPI()
+@pytest.mark.parametrize("_exp", [("Introductory")])
+def test_tool_libretextx_bookshelves(_exp):
+    api = MyLibreTextsAPI()
+    bookshelves = api.get_bookshelves()
+
+    shelf_1 = bookshelves[0]
+    shelf_1_url = shelf_1[1]
+
+    books = api.get_books(shelf_1_url)
+
+    book_2 = books[1]
+    book_2_url = book_2[1]
+
+    sections = api.get_book_sections(book_2_url)
+
+    section_2 = sections[1]
+    section_2_url = section_2[1]
+
+    paragraphs = api.get_book_section_paragraphs(section_2_url)
+
+    paragraph_8 = paragraphs[7]
+    paragraph_8_url = paragraph_8[1]
+
     #content = api.get_featured_articles(_month,_year)
+    #tool = MyWikiFeaturedArticles()
+    #results = tool(month=_month, year=_year)
+    #results_str = "\n".join(results)
+    #print(f"Results: {len(results)}\n{results_str}")
+
+    assert 0 == 0
+
+
+@pytest.mark.skip(reason="disabled")
+@pytest.mark.parametrize("_month,_year,_exp", [('November',2016, "FunkMonk")])
+def test_tool_wiki_contributions(_month,_year, _exp):
     tool = MyWikiFeaturedArticles()
     results = tool(month=_month, year=_year)
     results_str = "\n".join(results)
     print(f"Results: {len(results)}\n{results_str}")
 
-    assert 0 == 0
+    assert _exp in results_str
 
 
 @pytest.mark.skip(reason="disabled")
@@ -31,7 +63,6 @@ def test_tool_wiki_page_section(_page, _section, _sub_section, _year_start, _yea
     w = MyWikiPageSectionTool()
     f = MyWikiTableFilterTool()
     result = w(page=_page, section=_section)
-    # print(f"Got page section content: \n{result}")
     filtered = f(result, _sub_section, _year_start, _year_end)
     print(f"Table filtered: \n{filtered}")
     assert len(filtered) == _exp
@@ -51,8 +82,6 @@ def test_tool_image_load(_task_id,_exp):
     result = t.forward(_task_id)
     print(f"Got result: {result}")
     mp.use('QtAgg')
-    #plt.imshow(result)
-    #plt.show()
 
 
 @pytest.mark.skip(reason="disabled")