Add bs4 dependency and update Wikipedia tool for HTML parsing

pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
     "arize-phoenix-otel>=0.12.0",
+    "bs4>=0.0.2",
     "gradio[oauth]>=5.42.0",
     "grandalf>=0.8",
     "ipykernel>=6.30.1",

tools/wikipedia.py

@@ -1,5 +1,7 @@
 from langchain_core.tools import tool
 import wikipediaapi
+import requests
+from bs4 import BeautifulSoup
 
 @tool
 def wiki_search(query: str) -> str:
@@ -9,14 +11,14 @@ def wiki_search(query: str) -> str:
     Args:
         query (str): The search query to find relevant Wikipedia articles.
     """
-    # Initialize Wikipedia API with a proper user agent
-    wiki_wiki = wikipediaapi.Wikipedia(
+    # Initialize Wikipedia API with additional parameters for more info
+    wiki = wikipediaapi.Wikipedia(
         user_agent='OracleBot/0.1.0 (https://github.com/abtsousa/oraclebot)',
         language='en',
     )
     
     # Get the page
-    page = wiki_wiki.page(query)
+    page = wiki.page(query)
     
     # Check if page exists
     if not page.exists():
@@ -24,3 +26,75 @@ def wiki_search(query: str) -> str:
     
     # Return the full text content (summary + all sections)
     return f"Title: {page.title}\n\nURL: {page.fullurl}\n\n{page.text}"
+
+@tool
+def wiki_parse_html(page_title: str, section_id: int | None = None) -> str:
+    """
+    Get Wikipedia page HTML content using the parse API.
+    Use only if the standard wiki_search tool returns insufficient text for a section.
+
+    Args:
+        page_title (str): The exact title of the Wikipedia page.
+        section_id (int, optional): The section ID number to parse (e.g., "1" for first section). 
+                                   If None, returns the entire page.
+    """
+    url = "https://en.wikipedia.org/w/api.php"
+    params = {
+        'action': 'parse',
+        'page': page_title,
+        'format': 'json',
+        'prop': 'text'
+    }
+    
+    # Add section parameter if provided
+    if section_id is not None:
+        params['section'] = str(section_id)
+    
+    headers = {
+        'User-Agent': 'OracleBot/0.1.0 (https://github.com/abtsousa/oraclebot)'
+    }
+    
+    try:
+        response = requests.get(url, params=params, headers=headers)
+        response.raise_for_status()
+        data = response.json()
+        
+        if 'error' in data:
+            return f"Error: {data['error']['info']}"
+        
+        if 'parse' not in data or 'text' not in data['parse']:
+            return f"No content found for page '{page_title}'"
+        
+        # Raw HTML content from Wikipedia
+        raw_html = data['parse']['text']['*']
+
+        # Sanitize HTML: remove style/script tags and strip all attributes while keeping tag structure
+        try:
+            soup = BeautifulSoup(raw_html, 'html.parser')
+
+            # Remove unwanted tags entirely
+            for tag in soup(['style', 'script']):
+                tag.decompose()
+
+            # Strip attributes from all remaining tags (e.g., <div class=".." id=".."> -> <div>)
+            from bs4.element import Tag as _Tag
+            for tag in soup.find_all(True):
+                if isinstance(tag, _Tag):
+                    tag.attrs.clear()
+
+            # Optional: collapse excessive whitespace
+            text = str(soup)
+            return text
+        except Exception as e:
+            # Fallback to raw HTML if sanitization fails
+            return raw_html
+        
+    except requests.RequestException as e:
+        return f"Error fetching page: {str(e)}"
+    except Exception as e:
+        return f"Error parsing response: {str(e)}"
+
+if __name__ == "__main__":
+    query = "Malko Competition"
+    result = wiki_parse_html(query)
+    print(result)

uv.lock

@@ -235,6 +235,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517, upload-time = "2024-10-18T12:32:54.066Z" },
 ]
 
+[[package]]
+name = "bs4"
+version = "0.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "beautifulsoup4" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/aa/4acaf814ff901145da37332e05bb510452ebed97bc9602695059dd46ef39/bs4-0.0.2.tar.gz", hash = "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925", size = 698, upload-time = "2024-01-17T18:15:47.371Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/51/bb/bf7aab772a159614954d84aa832c129624ba6c32faa559dfb200a534e50b/bs4-0.0.2-py2.py3-none-any.whl", hash = "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc", size = 1189, upload-time = "2024-01-17T18:15:48.613Z" },
+]
+
 [[package]]
 name = "cachetools"
 version = "5.5.2"
@@ -1802,6 +1814,7 @@ version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
     { name = "arize-phoenix-otel" },
+    { name = "bs4" },
     { name = "gradio", extra = ["oauth"] },
     { name = "grandalf" },
     { name = "ipykernel" },
@@ -1827,6 +1840,7 @@ dependencies = [
 [package.metadata]
 requires-dist = [
     { name = "arize-phoenix-otel", specifier = ">=0.12.0" },
+    { name = "bs4", specifier = ">=0.0.2" },
     { name = "gradio", extras = ["oauth"], specifier = ">=5.42.0" },
     { name = "grandalf", specifier = ">=0.8" },
     { name = "ipykernel", specifier = ">=6.30.1" },