Spaces:
Paused
Paused
Update main.py
Browse files
main.py
CHANGED
|
@@ -33,11 +33,11 @@ class CrawlerInput(BaseModel):
|
|
| 33 |
class CrawlerOutput(BaseModel):
|
| 34 |
data: List[dict]
|
| 35 |
|
| 36 |
-
async def simple_crawl():
|
| 37 |
async with AsyncWebCrawler(verbose=True) as crawler:
|
| 38 |
-
result = await crawler.arun(url=
|
| 39 |
-
|
| 40 |
-
|
| 41 |
|
| 42 |
@app.post("/crawl", response_model=CrawlerOutput)
|
| 43 |
async def crawl(input: CrawlerInput, api_key: str = Depends(verify_api_key)):
|
|
@@ -78,7 +78,7 @@ async def test_url(api_key: str = Depends(verify_api_key), url: str = Query(...,
|
|
| 78 |
|
| 79 |
@app.get("/test")
|
| 80 |
async def test(api_key: str = Depends(verify_api_key)):
|
| 81 |
-
result = await simple_crawl()
|
| 82 |
return {"markdown": result.markdown}
|
| 83 |
|
| 84 |
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
| 33 |
class CrawlerOutput(BaseModel):
|
| 34 |
data: List[dict]
|
| 35 |
|
| 36 |
+
async def simple_crawl(url: str):
|
| 37 |
async with AsyncWebCrawler(verbose=True) as crawler:
|
| 38 |
+
result = await crawler.arun(url=url)
|
| 39 |
+
print(len(result.markdown))
|
| 40 |
+
return result
|
| 41 |
|
| 42 |
@app.post("/crawl", response_model=CrawlerOutput)
|
| 43 |
async def crawl(input: CrawlerInput, api_key: str = Depends(verify_api_key)):
|
|
|
|
| 78 |
|
| 79 |
@app.get("/test")
|
| 80 |
async def test(api_key: str = Depends(verify_api_key)):
|
| 81 |
+
result = await simple_crawl("https://www.nbcnews.com/business")
|
| 82 |
return {"markdown": result.markdown}
|
| 83 |
|
| 84 |
from fastapi.middleware.cors import CORSMiddleware
|