import re from typing import Any, Optional from smolagents.tools import Tool import requests import markdownify import smolagents class VisitWebpageTool(Tool): name = "visit_webpage" description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages." inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}} output_type = "string" def forward(self, url: str) -> str: try: # Add headers to mimic a browser headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } # Send a GET request to the URL with a 20-second timeout response = requests.get(url, timeout=20, headers=headers) response.raise_for_status() # Convert the HTML content to Markdown markdown_content = markdownify.markdownify(response.text).strip() # Remove multiple line breaks markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content) return markdown_content[:10000] # Limit content length except requests.exceptions.Timeout: return "The request timed out. Please try again later or check the URL." except requests.exceptions.RequestException as e: return f"Error fetching the webpage: {str(e)}" except Exception as e: return f"An unexpected error occurred: {str(e)}" def __init__(self, *args, **kwargs): self.is_initialized = False