Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -1,45 +1,116 @@
-from typing import Any, Optional
-from smolagents.tools import Tool
+import re
 import requests
-import markdownify
-import smolagents
+from markdownify import markdownify
+
+from requests.exceptions import RequestException
+from smolagents.tools import Tool
+from smolagents.utils import truncate_content
+
 
 class VisitWebpageTool(Tool):
+    """
+    Visits a webpage and converts its contents into Markdown.
+    """
+
     name = "visit_webpage"
-    description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
-    inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
+
+    description = (
+        "Visits a webpage given its URL and returns the readable "
+        "markdown version of the webpage. Use this after performing "
+        "a web search to extract detailed information."
+    )
+
+    inputs = {
+        "url": {
+            "type": "string",
+            "description": "URL of the webpage to visit."
+        }
+    }
+
     output_type = "string"
 
+    def __init__(self):
+
+        super().__init__()
+
     def forward(self, url: str) -> str:
+
+        headers = {
+            "User-Agent": (
+                "Mozilla/5.0 "
+                "(Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 "
+                "(KHTML, like Gecko) "
+                "Chrome/138.0 Safari/537.36"
+            )
+        }
+
         try:
-            import requests
-            from markdownify import markdownify
-            from requests.exceptions import RequestException
-
-            from smolagents.utils import truncate_content
-        except ImportError as e:
-            raise ImportError(
-                "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
-            ) from e
-        try:
-            # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20)
-            response.raise_for_status()  # Raise an exception for bad status codes
 
-            # Convert the HTML content to Markdown
-            markdown_content = markdownify(response.text).strip()
+            response = requests.get(
+
+                url,
+
+                headers=headers,
+
+                timeout=20,
+
+                allow_redirects=True,
+
+            )
+
+            response.raise_for_status()
 
-            # Remove multiple line breaks
-            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
+            markdown = markdownify(
 
-            return truncate_content(markdown_content, 10000)
+                response.text,
+
+                heading_style="ATX"
+
+            )
+
+            markdown = re.sub(
+
+                r"\n{3,}",
+
+                "\n\n",
+
+                markdown
+
+            )
+
+            markdown = markdown.strip()
+
+            markdown = truncate_content(
+
+                markdown,
+
+                20000
+
+            )
+
+            return markdown
 
         except requests.exceptions.Timeout:
-            return "The request timed out. Please try again later or check the URL."
+
+            return (
+                "The webpage request timed out."
+            )
+
+        except requests.exceptions.HTTPError as e:
+
+            return (
+                f"HTTP Error: {e}"
+            )
+
         except RequestException as e:
-            return f"Error fetching the webpage: {str(e)}"
+
+            return (
+                f"Request failed: {e}"
+            )
+
         except Exception as e:
-            return f"An unexpected error occurred: {str(e)}"
 
-    def __init__(self, *args, **kwargs):
-        self.is_initialized = False
+            return (
+                f"Unexpected error: {e}"
+            )