SuccessfulCrab commited on
Commit
d89aafb
·
verified ·
1 Parent(s): 475d3c6

Update firecrawler.py

Browse files
Files changed (1) hide show
  1. firecrawler.py +17 -23
firecrawler.py CHANGED
@@ -12,45 +12,39 @@ api_key = os.getenv("api_key")
12
  class FireCrawlTool(Tool):
13
  name = "firecrawl_website_qa"
14
  description = """
15
- This tool returns informatuion about websites using an API call"""
16
 
17
  inputs = {
18
- "websites": {
19
  "type": "string",
20
- "description": "A singlular or multiple website addresses",
21
  }
22
  }
23
 
24
  output_type = "string"
25
 
26
 
27
- def forward(self, websites: str):
28
 
29
  # Initialize the FirecrawlApp with the API key
30
  app = FirecrawlApp(api_key = api_key)
31
 
32
  # Convert the websites string to a list
33
  # Split by commas, newlines, or spaces and strip whitespace
34
- websites_list = [
35
- website.strip()
36
- for website in websites.replace(',', ' ').replace('\n', ' ').split()
37
- if website.strip()
38
- ]
39
-
40
- # Define the extraction schema
41
- class ExtractSchema(BaseModel):
42
- company_name: str
43
- website_purpose: str
44
 
45
- prompt_string = 'Extract the website purpose and company name.'
46
-
47
- # Execute the API call
48
- data = app.extract(
49
- websites_list,
50
- {
51
- 'prompt': prompt_string,
52
- 'schema': ExtractSchema.model_json_schema(),
53
  }
54
  )
55
 
56
- return data
 
 
12
  class FireCrawlTool(Tool):
13
  name = "firecrawl_website_qa"
14
  description = """
15
+ This tool scrapes websites using an API call"""
16
 
17
  inputs = {
18
+ "website": {
19
  "type": "string",
20
+ "description": "A singlular website address",
21
  }
22
  }
23
 
24
  output_type = "string"
25
 
26
 
27
+ def forward(self, website: str):
28
 
29
  # Initialize the FirecrawlApp with the API key
30
  app = FirecrawlApp(api_key = api_key)
31
 
32
  # Convert the websites string to a list
33
  # Split by commas, newlines, or spaces and strip whitespace
34
+ #websites_list = [
35
+ #website.strip()
36
+ #for website in websites.replace(',', ' ').replace('\n', ' ').split()
37
+ #if website.strip()
38
+ #]
 
 
 
 
 
39
 
40
+ # Scrape a website:
41
+ scrape_result = app.scrape_url(website,
42
+ params={
43
+ 'location': {
44
+ 'country': 'AU'
45
+ }
 
 
46
  }
47
  )
48
 
49
+
50
+ return scrape_result