Spaces:
Sleeping
Sleeping
File size: 2,278 Bytes
0451b7d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import os
from firecrawl import FirecrawlApp
from pydantic import BaseModel, Field
from typing import List
from smolagents import Tool
import urllib.parse
class FireCrawlTool(Tool):
name = "firecrawl_website_qa"
description = """This tool scrapes websites using an API call with URL validation"""
inputs = {
"website": {
"type": "string",
"description": "A singular website address"
}
}
output_type = "string"
def validate_url(self, url: str) -> str:
"""
Validate the input URL and return an appropriate message.
Args:
url (str): The URL to validate
Returns:
str: Validated URL or error message
"""
# Check if URL is blank or None
if not url or url.strip() == '':
return 'No URL was entered by user'
try:
# Attempt to parse the URL
result = urllib.parse.urlparse(url)
# Check if the URL has a valid scheme (http/https) and netloc (domain)
if not all([result.scheme in ['http', 'https'], result.netloc]):
return 'URL entered by user is invalid'
return url # Return the original URL if it passes validation
except Exception:
return 'URL entered by user is invalid'
def forward(self, website: str):
# Validate the URL first
url_validation = self.validate_url(website)
# If validation returns an error message, return it
if url_validation in ['No URL was entered by user', 'URL entered by user is invalid']:
return url_validation
# Initialize the FirecrawlApp with the API key
api_key = os.getenv("api_key")
app = FirecrawlApp(api_key=api_key)
# Scrape a website
try:
scrape_result = app.scrape_url(website, params={
'location': {
'country': 'AU'
}
})
# Truncate the result to 7000 characters
return scrape_result['markdown'][:7000]
except Exception as e:
return f"Error scraping website: {str(e)}" |