tools/scrape_website.py

from crewai_tools import ScrapeWebsiteTool
import requests
from bs4 import BeautifulSoup

scrape_tool = ScrapeWebsiteTool()

def CustomScrapeWebsiteTool(url):
    response = requests.get(url)
    parsed = BeautifulSoup(response.content, "html.parser")
    text = parsed.get_text()
    text = '\n'.join([i for i in text.split('\n') if i.strip() != ''])
    text = ' '.join([i for i in text.split(' ') if i.strip() != ''])

    return text