backend/test_crawl4ai.py

import asyncio
import os
import sys
from pathlib import Path

sys.path.append(str(Path(__file__).parent))

from core.crawl4ai_client import scrape_url_to_markdown

PARP_GRANTS_URL = "https://www.parp.gov.pl/component/grants/?task=grants.grant_list&type=0"


async def test_crawl4ai():
    try:
        markdown = await scrape_url_to_markdown(PARP_GRANTS_URL)
        if not markdown:
            print("EMPTY")
            return
        print("SUCCESS")
        print(markdown[:1500])
        out_path = Path(__file__).parent / "parp_crawl4ai.md"
        out_path.write_text(markdown, encoding="utf-8")
        print(f"Saved to {out_path}")
    except Exception as e:
        print("ERROR:", e)


asyncio.run(test_crawl4ai())