File size: 1,240 Bytes
3241f25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | from crewai import Agent, Crew, Process, Task
from crewai.project import CrewBase, agent, crew, task
#from src.ats.crews.web_scraper_crew.tools.webscraper_tool import WebscraperTool
from crewai_tools import ScrapeWebsiteTool
import os
@CrewBase
class WebScraperCrew:
"""Lead Filter Crew"""
agents_config = "config/agents.yaml"
tasks_config = "config/tasks.yaml"
@agent
def web_scraper_agent(self) -> Agent:
return Agent(
config=self.agents_config["web_scraper_agent"],
tools=[ScrapeWebsiteTool(
args={
'pageOptions': {
'onlyMainContent': True,
'includeHtml': False
},
'timeout': 60000 # 60 seconds
}
)],
verbose=True,
)
@task
def web_scraper_task(self) -> Task:
return Task(
config=self.tasks_config["web_scraper_task"],
)
@crew
def crew(self) -> Crew:
"""Creates the Web Scraper Crew"""
return Crew(
agents=self.agents,
tasks=self.tasks,
process=Process.sequential,
verbose=True,
)
|