Fix crawl evasion: Add WAF block detection to Tier 1 fetch

src/crawler.py

@@ -53,14 +53,21 @@ class Crawler:
         # Tier 1: curl_cffi
         content = self._fetch_http(url)
         if content:
-             return content
+             # Validate content isn't a block page
+             if not self._is_blocked(content):
+                 return content
+             else:
+                 logger.warning(f"Tier 1 fetched blocked content for {url}. Escalating...")
              
         # Tier 2: Cloudscraper (Intermediate, handles JS challenges)
         logger.info(f"Tier 1 failed. Trying Cloudscraper info for {url}...")
         try:
              resp = self.scraper.get(url, timeout=10)
              if 200 <= resp.status_code < 300:
-                  return resp.text
+                  if not self._is_blocked(resp.text):
+                      return resp.text
+                  else:
+                      logger.warning(f"Cloudscraper also blocked for {url}.")
         except Exception as e:
              logger.warning(f"Cloudscraper failed: {e}")
 
@@ -71,6 +78,34 @@ class Crawler:
             
         return None
 
+    def _is_blocked(self, content):
+        """
+        Detects if the content is likely a WAF block page, CAPTCHA, or 'Just a moment'.
+        """
+        if not content or len(content) < 500:
+            return True # Too small, suspicious
+            
+        lower_content = content.lower()
+        block_keywords = [
+            "just a moment...",
+            "enable javascript",
+            "verify you are human",
+            "access denied",
+            "cloudflare",
+            "captcha",
+            "security check",
+            "turn on javascript",
+            "challenge.js"
+        ]
+        
+        if any(k in lower_content for k in block_keywords):
+            # Double check: sometimes legitimate pages mention these words.
+            # But usually, if it's < 5KB and has these words, it's a block.
+            if len(content) < 5000:
+                return True
+                
+        return False
+
     def _fetch_http(self, url):
         """
         Fast HTTP fetch with aggressive timeouts and minimal retries.