Fix crawling for eminentcoders.com

src/crawler.py

@@ -102,7 +102,23 @@ class Crawler:
                 return None
 
             except Exception as e:
-                logger.error(f"Exception fetching {url}: {e}. Retrying...")
+                logger.error(f"Exception fetching {url} with curl_cffi: {e}. Attempting standard requests fallback...")
+                try:
+                    # Fallback to standard requests (sometimes works better for simpler/older server configs)
+                    import requests as standard_requests
+                    # random user agent
+                    headers = {
+                        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
+                    }
+                    resp = standard_requests.get(url, headers=headers, timeout=30, verify=False)
+                    if 200 <= resp.status_code < 300:
+                        return resp.text
+                    else:
+                         logger.error(f"Standard requests failed with status {resp.status_code}")
+                except Exception as std_e:
+                    logger.error(f"Standard requests fallback failed: {std_e}")
+
                 time.sleep(2 * (attempt + 1))
         
         return None