File size: 2,585 Bytes
325b94c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from urllib.parse import urljoin
async def crawl_parse_url(url: str):
"""Scrape a single URL using Crawlee + ParselCrawler and return one record"""
result_container = {"data": None}
crawler = ParselCrawler(max_requests_per_crawl=1)
@crawler.router.default_handler
async def handle_page(ctx: ParselCrawlingContext):
selector = ctx.selector
req_url = ctx.request.url
# ---------- Title ----------
title = selector.xpath("//title/text()").get() or "No title"
# ---------- Extract page text ----------
text = selector.xpath("string(//body)").get()
text = text.strip() if text else ""
# ---------- Multimedia ----------
img_urls = [
urljoin(req_url, src) for src in selector.xpath("//img/@src").getall()
]
video_urls = [
urljoin(req_url, src) for src in selector.xpath("//video/@src").getall()
] + [
urljoin(req_url, src)
for src in selector.xpath("//source[@type='video/mp4']/@src").getall()
]
audio_urls = [
urljoin(req_url, src) for src in selector.xpath("//audio/@src").getall()
] + [
urljoin(req_url, src)
for src in selector.xpath("//source[@type='audio/mpeg']/@src").getall()
]
pdf_urls = [
urljoin(req_url, href)
for href in selector.xpath("//a[contains(@href, '.pdf')]/@href").getall()
]
# ---------- Final structured result ----------
record = {
"page_url": req_url,
"title": title,
"content": text,
"img_url": img_urls,
"video_url": video_urls,
"audio_url": audio_urls,
"pdf_url": pdf_urls,
"agent_recommendation_rank": 4.2,
"agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.",
"header": "Web Scraping Test",
"sub_header": "Crawlee Version",
}
result_container["data"] = record
await ctx.push_data(record)
await crawler.run([url])
# fallback
return result_container["data"] or {
"page_url": url,
"title": "No title",
"content": "",
"img_url": [],
"video_url": [],
"audio_url": [],
"pdf_url": [],
"agent_recommendation_rank": 4.2,
"agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.",
}
|