Optimize connection pool and concurrency to prevent target server crashes

src/crawler.py

@@ -21,10 +21,14 @@ class Crawler:
         # impersonate="chrome120" is key. verify=False to speed up SSL handshakes.
         self.session = requests.Session(impersonate="chrome120", verify=False)
         self.ua = UserAgent()
-        # ssl_context=None might not be exposed, but we can try to optimize
+        # Increase connection pool sizes to handle concurrency without warnings
+        import requests as std_requests
+        adapter = std_requests.adapters.HTTPAdapter(pool_connections=50, pool_maxsize=50)
         self.scraper = cloudscraper.create_scraper(
             browser={'browser': 'chrome', 'platform': 'windows', 'desktop': True}
         )
+        self.scraper.mount('http://', adapter)
+        self.scraper.mount('https://', adapter)
         
         # Concurrency control for Playwright (heavy resource)
         self.playwright_semaphore = threading.Semaphore(3) 
@@ -342,8 +346,8 @@ class Crawler:
         logger.info(f"Starting crawl for domain: {base_domain}")
 
         # Use ThreadPoolExecutor for parallel crawling with high concurrency
-        # Pushed to 200 for extreme concurrency (WARNING: High risk of DDoS/Blocking from target)
-        with concurrent.futures.ThreadPoolExecutor(max_workers=200) as executor:
+        # Sweet spot is 30. Any higher causes the target server (bestcheck.in) to drop SSL connections (SSLEOFError)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
             # Map of future -> url
             future_to_url = {}