Increase max_workers to 50 for faster parallel crawling

src/crawler.py

@@ -342,8 +342,8 @@ class Crawler:
         logger.info(f"Starting crawl for domain: {base_domain}")
 
         # Use ThreadPoolExecutor for parallel crawling with high concurrency
-        # Adjusted to 20 for HuggingFace Spaces (2 vCPU usually) to avoid context switching overhead
-        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+        # Adjusted to 50 for much faster I/O bound concurrency
+        with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
             # Map of future -> url
             future_to_url = {}