firecrawler.py

import os
from firecrawl import FirecrawlApp
from pydantic import BaseModel, Field
from typing import List
from smolagents import Tool
import urllib.parse

class FireCrawlTool(Tool):
    name = "firecrawl_website_qa"
    description = """This tool scrapes websites using an API call with URL validation"""
    inputs = {
        "website": {
            "type": "string", 
            "description": "A singular website address"
        }
    }
    output_type = "string"

    def validate_url(self, url: str) -> str:
        """
        Validate the input URL and return an appropriate message.
        
        Args:
            url (str): The URL to validate
        
        Returns:
            str: Validated URL or error message
        """
        # Check if URL is blank or None
        if not url or url.strip() == '':
            return 'No URL was entered by user'
        
        try:
            # Attempt to parse the URL
            result = urllib.parse.urlparse(url)
            
            # Check if the URL has a valid scheme (http/https) and netloc (domain)
            if not all([result.scheme in ['http', 'https'], result.netloc]):
                return 'URL entered by user is invalid'
            
            return url  # Return the original URL if it passes validation
        
        except Exception:
            return 'URL entered by user is invalid'

    def forward(self, website: str):
        # Validate the URL first
        url_validation = self.validate_url(website)
        
        # If validation returns an error message, return it
        if url_validation in ['No URL was entered by user', 'URL entered by user is invalid']:
            return url_validation
        
        # Initialize the FirecrawlApp with the API key
        api_key = os.getenv("api_key")
        app = FirecrawlApp(api_key=api_key)
        
        # Scrape a website
        try:
            scrape_result = app.scrape_url(website, params={
                'location': {
                    'country': 'AU'
                }
            })
            
            # Truncate the result to 7000 characters
            return scrape_result['markdown'][:7000]
        
        except Exception as e:
            return f"Error scraping website: {str(e)}"