import re import requests from markdownify import markdownify from requests.exceptions import RequestException from smolagents.tools import Tool from smolagents.utils import truncate_content class VisitWebpageTool(Tool): """ Visits a webpage and converts its contents into Markdown. """ name = "visit_webpage" description = ( "Visits a webpage given its URL and returns the readable " "markdown version of the webpage. Use this after performing " "a web search to extract detailed information." ) inputs = { "url": { "type": "string", "description": "URL of the webpage to visit." } } output_type = "string" def __init__(self): super().__init__() def forward(self, url: str) -> str: headers = { "User-Agent": ( "Mozilla/5.0 " "(Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 " "(KHTML, like Gecko) " "Chrome/138.0 Safari/537.36" ) } try: response = requests.get( url, headers=headers, timeout=20, allow_redirects=True, ) response.raise_for_status() markdown = markdownify( response.text, heading_style="ATX" ) markdown = re.sub( r"\n{3,}", "\n\n", markdown ) markdown = markdown.strip() markdown = truncate_content( markdown, 20000 ) return markdown except requests.exceptions.Timeout: return ( "The webpage request timed out." ) except requests.exceptions.HTTPError as e: return ( f"HTTP Error: {e}" ) except RequestException as e: return ( f"Request failed: {e}" ) except Exception as e: return ( f"Unexpected error: {e}" )