Add more readable version of wikipedia pages when visiting web page (live or archived)

tools/text_web_browser.py

@@ -13,6 +13,7 @@ from urllib.parse import unquote, urljoin, urlparse
 
 import pathvalidate
 import requests
+import trafilatura
 # from serpapi import GoogleSearch
 
 from markdownify import markdownify
@@ -471,10 +472,19 @@ class VisitTool(Tool):
         self.browser = browser
 
     def forward(self, url: str) -> str:
-        self.browser.visit_page(url)
-        header, content = self.browser._state()
-        # return header.strip() + "\n=======================\n" + content
-        return markdownify(content)
+        if 'wikipedia.org/wiki/' in url:
+            downloaded = trafilatura.fetch_url(url)
+            if downloaded is None:
+                return "Failed to fetch the Wikipedia page content."
+            extracted = trafilatura.extract(downloaded)
+            if not extracted:
+                return "Failed to extract main content from the Wikipedia page."
+            return extracted
+        else:
+            self.browser.visit_page(url)
+            header, content = self.browser._state()
+            # return header.strip() + "\n=======================\n" + content
+            return markdownify(content)
 
 
 class DownloadTool(Tool):
@@ -545,15 +555,25 @@ class ArchiveSearchTool(Tool):
         else:
             raise Exception(f"Your {url=} was not archived on Wayback Machine, try a different url.")
         target_url = closest["url"]
-        self.browser.visit_page(target_url)
-        header, content = self.browser._state()
-        # return (
-        #     f"Web archive for url {url}, snapshot taken at date {closest['timestamp'][:8]}:\n"
-        #     + header.strip()
-        #     + "\n=======================\n"
-        #     + content
-        # )
-        return markdownify(content)
+
+        if 'wikipedia.org/wiki/' in target_url:
+            downloaded = trafilatura.fetch_url(target_url)
+            if downloaded is None:
+                return "Failed to fetch the Wikipedia page content."
+            extracted = trafilatura.extract(downloaded)
+            if not extracted:
+                return "Failed to extract main content from the Wikipedia page."
+            return extracted
+        else:
+            self.browser.visit_page(target_url)
+            header, content = self.browser._state()
+            # return (
+            #     f"Web archive for url {url}, snapshot taken at date {closest['timestamp'][:8]}:\n"
+            #     + header.strip()
+            #     + "\n=======================\n"
+            #     + content
+            # )
+            return markdownify(content)
 
 
 class PageUpTool(Tool):