Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -5,6 +5,7 @@ import asyncio
 from typing import Optional, List
 import time
 from dotenv import load_dotenv
+from aiohttp_socks import ProxyConnector  # Better proxy connector
 
 # Load environment variables
 load_dotenv()
@@ -26,7 +27,6 @@ WEBSHARE_PROXIES = [
 ]
 
 # Track proxy usage and failures
-proxy_usage_count = {}
 proxy_failure_count = {}
 last_proxy_index = -1
 
@@ -53,13 +53,11 @@ def get_next_proxy() -> str:
         if proxy_failure_count.get(proxy_str, 0) >= 3:
             continue
             
-        # Track usage
-        proxy_usage_count[proxy_str] = proxy_usage_count.get(proxy_str, 0) + 1
-        
         return format_proxy_url(proxy_str)
     
     # If all proxies have failures, reset failure counts and try again
     proxy_failure_count.clear()
+    logger.warning("All proxies have failure records, resetting counts")
     return get_next_proxy()
 
 def mark_proxy_failure(proxy_url: str):
@@ -70,10 +68,10 @@ def mark_proxy_failure(proxy_url: str):
             proxy_failure_count[proxy_str] = proxy_failure_count.get(proxy_str, 0) + 1
             logger.warning(f"Marked proxy as failed: {proxy_url} (failure count: {proxy_failure_count[proxy_str]})")
             
-            # Reset failure count after 10 minutes to give proxy a second chance
+            # Reset failure count after 5 minutes
             if proxy_failure_count[proxy_str] >= 3:
                 logger.warning(f"Proxy {proxy_url} has failed multiple times, cooling down")
-                asyncio.create_task(reset_proxy_failure(proxy_str, 600))  # 10 minutes cooldown
+                asyncio.create_task(reset_proxy_failure(proxy_str, 300))  # 5 minutes cooldown
             break
 
 async def reset_proxy_failure(proxy_str: str, delay: int):
@@ -83,6 +81,34 @@ async def reset_proxy_failure(proxy_str: str, delay: int):
         proxy_failure_count[proxy_str] = 0
         logger.info(f"Reset failure count for proxy: {proxy_str}")
 
+async def fetch_with_session(session: aiohttp.ClientSession, url: str, proxy_url: str, headers: dict) -> Optional[str]:
+    """Make a request with the given session and proxy"""
+    try:
+        async with session.get(
+            url,
+            headers=headers,
+            proxy=proxy_url,
+            timeout=30,
+            allow_redirects=True
+        ) as response:
+            # Accept both 200 and 202 status codes
+            if response.status in (200, 202):
+                content = await response.text()
+                
+                # Verify we got actual content
+                if len(content) > 1000 and ("<html" in content or "<!DOCTYPE" in content):
+                    logger.info(f"Successfully retrieved content ({len(content)} bytes)")
+                    return content
+                else:
+                    logger.warning(f"Response too small or not HTML: {len(content)} bytes")
+                    return None
+            else:
+                logger.warning(f"Response status code: {response.status}")
+                return None
+    except Exception as e:
+        logger.error(f"Request failed: {str(e)}")
+        return None
+
 async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
     """Fetch a page using WebShare proxies with retry logic"""
     logger.info(f"Requesting URL: {url}")
@@ -92,20 +118,21 @@ async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) ->
         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.9",
-        "Accept-Encoding": "gzip, deflate, br",
+        # Don't specify Accept-Encoding to let aiohttp handle it with brotli installed
         "Connection": "keep-alive",
         "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
         "Cache-Control": "max-age=0",
-        "TE": "Trailers",
         "Referer": "https://www.google.com/"
     }
     # Update with any custom headers provided
     enhanced_headers.update(headers)
     
     # Try up to 3 different proxies
-    max_proxy_attempts = 3
-    
-    for attempt in range(max_proxy_attempts):
+    for attempt in range(3):
         proxy_url = get_next_proxy()
         if not proxy_url:
             logger.error("Failed to get a valid proxy")
@@ -114,37 +141,20 @@ async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) ->
         logger.info(f"Using proxy {proxy_url} (attempt {attempt+1})")
         
         try:
-            # Try with this proxy
-            async with session.get(
-                url,
-                headers=enhanced_headers,
-                proxy=proxy_url,
-                timeout=30,
-                allow_redirects=True
-            ) as response:
-                if response.status == 200:
-                    content = await response.text()
-                    
-                    # Verify we got actual content (common anti-bot techniques return empty pages)
-                    if len(content) > 1000 and ("<html" in content or "<!DOCTYPE" in content):
-                        logger.info(f"Successfully retrieved content ({len(content)} bytes)")
-                        return content
-                    else:
-                        logger.warning(f"Response too small or not HTML: {len(content)} bytes")
-                        mark_proxy_failure(proxy_url)
+            # Create a new session for each proxy to avoid connection issues
+            connector = ProxyConnector.from_url(proxy_url)
+            async with aiohttp.ClientSession(connector=connector) as proxy_session:
+                content = await fetch_with_session(proxy_session, url, None, enhanced_headers)
+                if content:
+                    return content
                 else:
-                    logger.warning(f"Response status code: {response.status}")
                     mark_proxy_failure(proxy_url)
-                    
-        except (aiohttp.ClientError, asyncio.TimeoutError) as e:
-            logger.error(f"Request failed with proxy {proxy_url}: {str(e)}")
-            mark_proxy_failure(proxy_url)
         except Exception as e:
-            logger.error(f"Unexpected error: {str(e)}")
+            logger.error(f"Request failed with proxy {proxy_url}: {str(e)}")
             mark_proxy_failure(proxy_url)
         
         # Wait before trying next proxy
-        await asyncio.sleep(1)
+        await asyncio.sleep(2)
     
     logger.error("All proxy attempts failed")
     return None