Rewatiramans commited on
Commit
ed2b12e
·
verified ·
1 Parent(s): 9f68d25

Upload scraping_utils.py

Browse files
Files changed (1) hide show
  1. src/scraping_utils.py +11 -0
src/scraping_utils.py ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ from crawl4ai import *
3
+
4
+ async def scrape_url(url):
5
+ async with AsyncWebCrawler() as crawler:
6
+ result = await crawler.arun(
7
+ url,
8
+ )
9
+ code_code = ""
10
+ return result.markdown,status_code
11
+