Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -1,151 +1,150 @@
 import aiohttp
 import logging
-from typing import Optional, List
-import asyncio
-from fp.fp import FreeProxy
 import random
+import asyncio
+from typing import Optional, List
 import time
-import contextlib
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
 
 logger = logging.getLogger(__name__)
 
-# Cache for working proxies
-WORKING_PROXIES = []
-PROXY_REFRESH_TIME = 0
-PROXY_REFRESH_INTERVAL = 60 * 10  # 10 minutes
+# WebShare proxies list (format: IP:PORT:USERNAME:PASSWORD)
+WEBSHARE_PROXIES = [
+    "198.23.239.134:6540:zvubytfw:ak6yit5k2tvj",
+    "207.244.217.165:6712:zvubytfw:ak6yit5k2tvj",
+    "107.172.163.27:6543:zvubytfw:ak6yit5k2tvj",
+    "161.123.152.115:6360:zvubytfw:ak6yit5k2tvj",
+    "23.94.138.75:6349:zvubytfw:ak6yit5k2tvj",
+    "216.10.27.159:6837:zvubytfw:ak6yit5k2tvj",
+    "136.0.207.84:6661:zvubytfw:ak6yit5k2tvj",
+    "64.64.118.149:6732:zvubytfw:ak6yit5k2tvj",
+    "142.147.128.93:6593:zvubytfw:ak6yit5k2tvj",
+    "154.36.110.199:6853:zvubytfw:ak6yit5k2tvj"
+]
 
-def get_working_proxies() -> List[str]:
-    """Get a list of working proxies"""
-    global WORKING_PROXIES, PROXY_REFRESH_TIME
-    
-    current_time = time.time()
+# Track proxy usage and failures
+proxy_usage_count = {}
+proxy_failure_count = {}
+last_proxy_index = -1
+
+def format_proxy_url(proxy_str: str) -> str:
+    """Convert proxy string to proxy URL format"""
+    parts = proxy_str.split(':')
+    if len(parts) != 4:
+        logger.error(f"Invalid proxy format: {proxy_str}")
+        return None
     
-    # If we have proxies and they're not expired, use them
-    if WORKING_PROXIES and (current_time - PROXY_REFRESH_TIME) < PROXY_REFRESH_INTERVAL:
-        return WORKING_PROXIES
+    ip, port, username, password = parts
+    return f"http://{username}:{password}@{ip}:{port}"
+
+def get_next_proxy() -> str:
+    """Get the next proxy using a round-robin approach with failure consideration"""
+    global last_proxy_index
     
-    # Get new proxies
-    try:
-        proxies = []
-        # Try to get 5 working proxies
-        for _ in range(5):
-            try:
-                proxy = FreeProxy(https=True, rand=True, timeout=1).get()
-                if proxy and proxy not in proxies:
-                    proxies.append(proxy)
-            except Exception:
-                pass
-                
-        if proxies:
-            WORKING_PROXIES = proxies
-            PROXY_REFRESH_TIME = current_time
-            logger.info(f"Refreshed proxy list, found {len(proxies)} working proxies")
-            return WORKING_PROXIES
-    except Exception as e:
-        logger.error(f"Error refreshing proxy list: {e}")
+    # Simple round-robin selection with failure skipping
+    for _ in range(len(WEBSHARE_PROXIES)):
+        last_proxy_index = (last_proxy_index + 1) % len(WEBSHARE_PROXIES)
+        proxy_str = WEBSHARE_PROXIES[last_proxy_index]
+        
+        # Skip proxies with too many recent failures
+        if proxy_failure_count.get(proxy_str, 0) >= 3:
+            continue
+            
+        # Track usage
+        proxy_usage_count[proxy_str] = proxy_usage_count.get(proxy_str, 0) + 1
+        
+        return format_proxy_url(proxy_str)
     
-    return WORKING_PROXIES  # Return whatever we have, even if it's empty
+    # If all proxies have failures, reset failure counts and try again
+    proxy_failure_count.clear()
+    return get_next_proxy()
+
+def mark_proxy_failure(proxy_url: str):
+    """Mark a proxy as having a failure"""
+    # Extract the original proxy string from the URL
+    for proxy_str in WEBSHARE_PROXIES:
+        if proxy_str.split(':')[0] in proxy_url and proxy_str.split(':')[2] in proxy_url:
+            proxy_failure_count[proxy_str] = proxy_failure_count.get(proxy_str, 0) + 1
+            logger.warning(f"Marked proxy as failed: {proxy_url} (failure count: {proxy_failure_count[proxy_str]})")
+            
+            # Reset failure count after 10 minutes to give proxy a second chance
+            if proxy_failure_count[proxy_str] >= 3:
+                logger.warning(f"Proxy {proxy_url} has failed multiple times, cooling down")
+                asyncio.create_task(reset_proxy_failure(proxy_str, 600))  # 10 minutes cooldown
+            break
+
+async def reset_proxy_failure(proxy_str: str, delay: int):
+    """Reset the failure count for a proxy after a delay"""
+    await asyncio.sleep(delay)
+    if proxy_str in proxy_failure_count:
+        proxy_failure_count[proxy_str] = 0
+        logger.info(f"Reset failure count for proxy: {proxy_str}")
 
 async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
-    """Fetch a page using aiohttp with free proxies and retry logic"""
+    """Fetch a page using WebShare proxies with retry logic"""
     logger.info(f"Requesting URL: {url}")
     
-    # Get list of working proxies
-    proxies = get_working_proxies()
-    
-    # Try with proxies if available
-    if proxies:
-        # Shuffle proxies for better distribution
-        random.shuffle(proxies)
-        
-        # Try each proxy until one works
-        for proxy in proxies:
-            try:
-                logger.info(f"Trying with proxy: {proxy}")
-                
-                # Use standard session with proxy instead of RetryClient
-                try:
-                    async with session.get(
-                        url, 
-                        headers=headers,
-                        proxy=proxy,
-                        timeout=20,
-                        ssl=False  # Some free proxies don't support SSL verification
-                    ) as response:
-                        # Handle 202 status code as potential success
-                        if response.status in (200, 202):
-                            content = await response.text()
-                            # Verify we got actual HTML content, not just a waiting page
-                            if len(content) > 5000 and ("<html" in content or "<!DOCTYPE" in content):
-                                logger.info(f"Successfully retrieved content via proxy")
-                                return content
-                            else:
-                                logger.warning(f"Proxy {proxy} returned too short content or non-HTML")
-                        else:
-                            logger.warning(f"Proxy {proxy} failed with status {response.status}")
-                except Exception as e:
-                    logger.warning(f"Error using proxy {proxy}: {str(e)}")
-                    continue
-            except Exception as e:
-                logger.warning(f"Outer error with proxy {proxy}: {str(e)}")
-                continue
-    
-    # If all proxies failed or no proxies available, try direct request with extensive disguise
-    logger.info("All proxies failed or no proxies available, trying direct request with disguised headers")
-    return await direct_request(session, url, headers)
-
-async def direct_request(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
-    """Attempt a direct request with enhanced browser-like headers"""
-    # Enhance headers to look more like a real browser
-    enhanced_headers = headers.copy()
-    enhanced_headers.update({
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+    # Enhanced headers that look more like a browser
+    enhanced_headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.9",
         "Accept-Encoding": "gzip, deflate, br",
         "Connection": "keep-alive",
-        "Cache-Control": "max-age=0",
-        "Sec-Ch-Ua": '"Google Chrome";v="123", "Not:A-Brand";v="8"',
-        "Sec-Ch-Ua-Mobile": "?0", 
-        "Sec-Ch-Ua-Platform": '"Windows"',
-        "Sec-Fetch-Dest": "document",
-        "Sec-Fetch-Mode": "navigate", 
-        "Sec-Fetch-Site": "none",
-        "Sec-Fetch-User": "?1",
         "Upgrade-Insecure-Requests": "1",
+        "Cache-Control": "max-age=0",
+        "TE": "Trailers",
         "Referer": "https://www.google.com/"
-    })
+    }
+    # Update with any custom headers provided
+    enhanced_headers.update(headers)
     
-    try:
-        # Use standard session.get instead of RetryClient
-        # Add retry logic manually to ensure proper cleanup
-        max_retries = 3
-        for attempt in range(max_retries):
-            try:
-                async with session.get(url, headers=enhanced_headers, timeout=20) as response:
-                    # Handle 202 status code as potential success
-                    if response.status in (200, 202):
-                        content = await response.text()
-                        # Verify we got actual HTML content, not just a waiting page
-                        if len(content) > 5000 and ("<html" in content or "<!DOCTYPE" in content):
-                            logger.info(f"Successfully retrieved content directly")
-                            return content
-                        else:
-                            logger.warning(f"Direct request returned too short content or non-HTML")
-                    else:
-                        logger.error(f"Direct request failed with status code {response.status}")
-                
-                # If we get here, the attempt failed, so wait before retry
-                if attempt < max_retries - 1:
-                    delay = (attempt + 1) * 2  # Exponential backoff
-                    await asyncio.sleep(delay)
+    # Try up to 3 different proxies
+    max_proxy_attempts = 3
+    
+    for attempt in range(max_proxy_attempts):
+        proxy_url = get_next_proxy()
+        if not proxy_url:
+            logger.error("Failed to get a valid proxy")
+            return None
+            
+        logger.info(f"Using proxy {proxy_url} (attempt {attempt+1})")
+        
+        try:
+            # Try with this proxy
+            async with session.get(
+                url,
+                headers=enhanced_headers,
+                proxy=proxy_url,
+                timeout=30,
+                allow_redirects=True
+            ) as response:
+                if response.status == 200:
+                    content = await response.text()
                     
-            except Exception as e:
-                logger.error(f"Direct request attempt {attempt+1} failed: {e}")
-                if attempt < max_retries - 1:
-                    delay = (attempt + 1) * 2
-                    await asyncio.sleep(delay)
+                    # Verify we got actual content (common anti-bot techniques return empty pages)
+                    if len(content) > 1000 and ("<html" in content or "<!DOCTYPE" in content):
+                        logger.info(f"Successfully retrieved content ({len(content)} bytes)")
+                        return content
+                    else:
+                        logger.warning(f"Response too small or not HTML: {len(content)} bytes")
+                        mark_proxy_failure(proxy_url)
+                else:
+                    logger.warning(f"Response status code: {response.status}")
+                    mark_proxy_failure(proxy_url)
                     
-        return None  # All attempts failed
-    except Exception as e:
-        logger.error(f"All direct request attempts failed: {e}")
-        return None
+        except (aiohttp.ClientError, asyncio.TimeoutError) as e:
+            logger.error(f"Request failed with proxy {proxy_url}: {str(e)}")
+            mark_proxy_failure(proxy_url)
+        except Exception as e:
+            logger.error(f"Unexpected error: {str(e)}")
+            mark_proxy_failure(proxy_url)
+        
+        # Wait before trying next proxy
+        await asyncio.sleep(1)
+    
+    logger.error("All proxy attempts failed")
+    return None