tools/visit_webpage.py

from smolagents.tools import Tool
import requests
from markdownify import markdownify
import re
from urllib.parse import urlparse

class VisitWebpageTool(Tool):
    name = "visit_webpage"
    description = "Visits a webpage and returns its content as markdown."
    inputs = {'url': {'type': 'string', 'description': 'The URL to visit'}}
    output_type = "string"

    def forward(self, url: str) -> str:
        try:
            if not re.match(r'^https?://', url):
                return "Error: Invalid URL protocol"
                
            parsed = urlparse(url)
            if not parsed.netloc:
                return "Error: Invalid URL format"

            response = requests.get(url, timeout=20)
            response.raise_for_status()
            
            content = markdownify(response.text)
            content = re.sub(r'\n{3,}', '\n\n', content.strip())
            
            return content[:10000]  # Limit output length
            
        except requests.exceptions.RequestException as e:
            return f"Error fetching page: {str(e)}"
        except Exception as e:
            return f"Unexpected error: {str(e)}"