tools/scraper/no_agent/crawlee_parse_scraper.py

import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from urllib.parse import urljoin


async def crawl_parse_url(url: str):
    """Scrape a single URL using Crawlee + ParselCrawler and return one record"""

    result_container = {"data": None}

    crawler = ParselCrawler(max_requests_per_crawl=1)



    @crawler.router.default_handler
    async def handle_page(ctx: ParselCrawlingContext):
        selector = ctx.selector
        req_url = ctx.request.url

        # ---------- Title ----------
        title = selector.xpath("//title/text()").get() or "No title"

        # ---------- Extract page text ----------
        text = selector.xpath("string(//body)").get()
        text = text.strip() if text else ""

        # ---------- Multimedia ----------
        img_urls = [
            urljoin(req_url, src) for src in selector.xpath("//img/@src").getall()
        ]

        video_urls = [
            urljoin(req_url, src) for src in selector.xpath("//video/@src").getall()
        ] + [
            urljoin(req_url, src)
            for src in selector.xpath("//source[@type='video/mp4']/@src").getall()
        ]

        audio_urls = [
            urljoin(req_url, src) for src in selector.xpath("//audio/@src").getall()
        ] + [
            urljoin(req_url, src)
            for src in selector.xpath("//source[@type='audio/mpeg']/@src").getall()
        ]

        pdf_urls = [
            urljoin(req_url, href)
            for href in selector.xpath("//a[contains(@href, '.pdf')]/@href").getall()
        ]

        # ---------- Final structured result ----------
        record = {
            "page_url": req_url,
            "title": title,
            "content": text,
            "img_url": img_urls,
            "video_url": video_urls,
            "audio_url": audio_urls,
            "pdf_url": pdf_urls,
            "agent_recommendation_rank": 4.2,
            "agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.",
            "header": "Web Scraping Test",
            "sub_header": "Crawlee Version",
        }

        result_container["data"] = record
        await ctx.push_data(record)

    await crawler.run([url])

    # fallback
    return result_container["data"] or {
        "page_url": url,
        "title": "No title",
        "content": "",
        "img_url": [],
        "video_url": [],
        "audio_url": [],
        "pdf_url": [],
        "agent_recommendation_rank": 4.2,
        "agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.",
    }