Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -3,7 +3,7 @@ from smolagents.tools import Tool
 import requests
 import markdownify
 import smolagents
-import re  # Import the re module for regular expressions
+import re
 
 class VisitWebpageTool(Tool):
     name = "visit_webpage"
@@ -13,7 +13,7 @@ class VisitWebpageTool(Tool):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.is_initialized = True  # Example initialization logic
+        self.is_initialized = True
 
     def forward(self, url: str) -> str:
         try:
@@ -27,17 +27,19 @@ class VisitWebpageTool(Tool):
                 "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
             ) from e
         try:
-            # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20)
-            response.raise_for_status()  # Raise an exception for bad status codes
+            # Add User-Agent header to mimic a browser
+            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
+            response = requests.get(url, headers=headers, timeout=20)
+            response.raise_for_status()
 
-            # Convert the HTML content to Markdown
             markdown_content = markdownify(response.text).strip()
-
-            # Remove multiple line breaks
             markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
 
             return truncate_content(markdown_content, 10000)
 
         except requests.exceptions.Timeout:
-            return
+            return "The request timed out. Please try again later or check the URL."
+        except RequestException as e:
+            return f"Error fetching the webpage: {str(e)}"
+        except Exception as e:
+            return f"An unexpected error occurred: {str(e)}"