Increase max_workers to 200 for extreme concurrency

src/crawler.py

@@ -342,8 +342,8 @@ class Crawler:
         logger.info(f"Starting crawl for domain: {base_domain}")
 
         # Use ThreadPoolExecutor for parallel crawling with high concurrency
-        # Adjusted to 50 for much faster I/O bound concurrency
-        with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
+        # Pushed to 200 for extreme concurrency (WARNING: High risk of DDoS/Blocking from target)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=200) as executor:
             # Map of future -> url
             future_to_url = {}