Spaces:
Sleeping
Sleeping
| import re | |
| import requests | |
| from markdownify import markdownify | |
| from requests.exceptions import RequestException | |
| from smolagents.tools import Tool | |
| from smolagents.utils import truncate_content | |
| class VisitWebpageTool(Tool): | |
| """ | |
| Visits a webpage and converts its contents into Markdown. | |
| """ | |
| name = "visit_webpage" | |
| description = ( | |
| "Visits a webpage given its URL and returns the readable " | |
| "markdown version of the webpage. Use this after performing " | |
| "a web search to extract detailed information." | |
| ) | |
| inputs = { | |
| "url": { | |
| "type": "string", | |
| "description": "URL of the webpage to visit." | |
| } | |
| } | |
| output_type = "string" | |
| def __init__(self): | |
| super().__init__() | |
| def forward(self, url: str) -> str: | |
| headers = { | |
| "User-Agent": ( | |
| "Mozilla/5.0 " | |
| "(Windows NT 10.0; Win64; x64) " | |
| "AppleWebKit/537.36 " | |
| "(KHTML, like Gecko) " | |
| "Chrome/138.0 Safari/537.36" | |
| ) | |
| } | |
| try: | |
| response = requests.get( | |
| url, | |
| headers=headers, | |
| timeout=20, | |
| allow_redirects=True, | |
| ) | |
| response.raise_for_status() | |
| markdown = markdownify( | |
| response.text, | |
| heading_style="ATX" | |
| ) | |
| markdown = re.sub( | |
| r"\n{3,}", | |
| "\n\n", | |
| markdown | |
| ) | |
| markdown = markdown.strip() | |
| markdown = truncate_content( | |
| markdown, | |
| 20000 | |
| ) | |
| return markdown | |
| except requests.exceptions.Timeout: | |
| return ( | |
| "The webpage request timed out." | |
| ) | |
| except requests.exceptions.HTTPError as e: | |
| return ( | |
| f"HTTP Error: {e}" | |
| ) | |
| except RequestException as e: | |
| return ( | |
| f"Request failed: {e}" | |
| ) | |
| except Exception as e: | |
| return ( | |
| f"Unexpected error: {e}" | |
| ) |