tools/visit_webpage.py

from typing import Any, Optional
from smolagents.tools import Tool
import requests
import markdownify
import re
from smolagents.utils import truncate_content

class VisitWebpageTool(Tool):
    name = "visit_webpage"
    description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
    inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
    output_type = "string"

    def forward(self, url: str) -> str:
        try:
            # Send a GET request to the URL with a 20-second timeout
            response = requests.get(url, timeout=20)
            response.raise_for_status()  # Raise an exception for bad status codes
            html = response.text
        except Exception as e:
            return f"Error fetching the webpage: {str(e)}"
        try:
            # Convert the HTML content to Markdown
            markdown_content = markdownify.markdownify(html).strip()
            # Remove multiple line breaks
            markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
            return truncate_content(markdown_content, 10000)
        except Exception as e:
            return f"Error processing content: {str(e)}"

    def __init__(self, *args, **kwargs):
        self.is_initialized = False