Spaces:
Sleeping
Sleeping
File size: 2,215 Bytes
9c2a00b 487a7fe 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b 8fe992b 9c2a00b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | import re
import requests
from markdownify import markdownify
from requests.exceptions import RequestException
from smolagents.tools import Tool
from smolagents.utils import truncate_content
class VisitWebpageTool(Tool):
"""
Visits a webpage and converts its contents into Markdown.
"""
name = "visit_webpage"
description = (
"Visits a webpage given its URL and returns the readable "
"markdown version of the webpage. Use this after performing "
"a web search to extract detailed information."
)
inputs = {
"url": {
"type": "string",
"description": "URL of the webpage to visit."
}
}
output_type = "string"
def __init__(self):
super().__init__()
def forward(self, url: str) -> str:
headers = {
"User-Agent": (
"Mozilla/5.0 "
"(Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 "
"(KHTML, like Gecko) "
"Chrome/138.0 Safari/537.36"
)
}
try:
response = requests.get(
url,
headers=headers,
timeout=20,
allow_redirects=True,
)
response.raise_for_status()
markdown = markdownify(
response.text,
heading_style="ATX"
)
markdown = re.sub(
r"\n{3,}",
"\n\n",
markdown
)
markdown = markdown.strip()
markdown = truncate_content(
markdown,
20000
)
return markdown
except requests.exceptions.Timeout:
return (
"The webpage request timed out."
)
except requests.exceptions.HTTPError as e:
return (
f"HTTP Error: {e}"
)
except RequestException as e:
return (
f"Request failed: {e}"
)
except Exception as e:
return (
f"Unexpected error: {e}"
) |