Spaces:
Sleeping
Sleeping
Update firecrawler.py
Browse files- firecrawler.py +17 -23
firecrawler.py
CHANGED
|
@@ -12,45 +12,39 @@ api_key = os.getenv("api_key")
|
|
| 12 |
class FireCrawlTool(Tool):
|
| 13 |
name = "firecrawl_website_qa"
|
| 14 |
description = """
|
| 15 |
-
This tool
|
| 16 |
|
| 17 |
inputs = {
|
| 18 |
-
"
|
| 19 |
"type": "string",
|
| 20 |
-
"description": "A singlular
|
| 21 |
}
|
| 22 |
}
|
| 23 |
|
| 24 |
output_type = "string"
|
| 25 |
|
| 26 |
|
| 27 |
-
def forward(self,
|
| 28 |
|
| 29 |
# Initialize the FirecrawlApp with the API key
|
| 30 |
app = FirecrawlApp(api_key = api_key)
|
| 31 |
|
| 32 |
# Convert the websites string to a list
|
| 33 |
# Split by commas, newlines, or spaces and strip whitespace
|
| 34 |
-
websites_list = [
|
| 35 |
-
website.strip()
|
| 36 |
-
for website in websites.replace(',', ' ').replace('\n', ' ').split()
|
| 37 |
-
if website.strip()
|
| 38 |
-
]
|
| 39 |
-
|
| 40 |
-
# Define the extraction schema
|
| 41 |
-
class ExtractSchema(BaseModel):
|
| 42 |
-
company_name: str
|
| 43 |
-
website_purpose: str
|
| 44 |
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
'prompt': prompt_string,
|
| 52 |
-
'schema': ExtractSchema.model_json_schema(),
|
| 53 |
}
|
| 54 |
)
|
| 55 |
|
| 56 |
-
|
|
|
|
|
|
| 12 |
class FireCrawlTool(Tool):
|
| 13 |
name = "firecrawl_website_qa"
|
| 14 |
description = """
|
| 15 |
+
This tool scrapes websites using an API call"""
|
| 16 |
|
| 17 |
inputs = {
|
| 18 |
+
"website": {
|
| 19 |
"type": "string",
|
| 20 |
+
"description": "A singlular website address",
|
| 21 |
}
|
| 22 |
}
|
| 23 |
|
| 24 |
output_type = "string"
|
| 25 |
|
| 26 |
|
| 27 |
+
def forward(self, website: str):
|
| 28 |
|
| 29 |
# Initialize the FirecrawlApp with the API key
|
| 30 |
app = FirecrawlApp(api_key = api_key)
|
| 31 |
|
| 32 |
# Convert the websites string to a list
|
| 33 |
# Split by commas, newlines, or spaces and strip whitespace
|
| 34 |
+
#websites_list = [
|
| 35 |
+
#website.strip()
|
| 36 |
+
#for website in websites.replace(',', ' ').replace('\n', ' ').split()
|
| 37 |
+
#if website.strip()
|
| 38 |
+
#]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
+
# Scrape a website:
|
| 41 |
+
scrape_result = app.scrape_url(website,
|
| 42 |
+
params={
|
| 43 |
+
'location': {
|
| 44 |
+
'country': 'AU'
|
| 45 |
+
}
|
|
|
|
|
|
|
| 46 |
}
|
| 47 |
)
|
| 48 |
|
| 49 |
+
|
| 50 |
+
return scrape_result
|