fix: replace arun_many with asyncio.gather+semaphore for true parallel crawling

src/crawler.py

@@ -7,7 +7,7 @@ from urllib.parse import urljoin, urlparse
 import httpx
 import polars as pl
 from bs4 import BeautifulSoup
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig, SemaphoreDispatcher, RateLimiter
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
 
 from src.config import Config
 from src.utils import content_hash, normalize_url
@@ -149,14 +149,7 @@ class Crawler:
             remove_overlay_elements=True,
             remove_consent_popups=True,
         )
-        dispatcher = SemaphoreDispatcher(
-            semaphore_count=4,
-            rate_limiter=RateLimiter(
-                base_delay=(1.0, 2.5),
-                max_delay=30.0,
-                max_retries=2,
-            ),
-        )
+        html_semaphore = asyncio.Semaphore(4)
 
         async with httpx.AsyncClient(
             timeout=30.0,
@@ -204,12 +197,18 @@ class Crawler:
                                             next_layer.append(link)
 
                     if html_urls:
-                        html_results = await crawler.arun_many(
-                            urls=html_urls,
-                            config=crawl_config,
-                            dispatcher=dispatcher,
-                        )
+                        async def _fetch_one(c, url, cfg, sem):
+                            async with sem:
+                                return await c.arun(url=url, config=cfg)
+
+                        html_coros = [
+                            _fetch_one(crawler, u, crawl_config, html_semaphore)
+                            for u in html_urls
+                        ]
+                        html_results = await asyncio.gather(*html_coros, return_exceptions=True)
                         for result in html_results:
+                            if isinstance(result, Exception):
+                                continue
                             page = self._parse_arun_result(result, current_depth)
                             if page:
                                 links = page.pop("_links", [])