Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -5,7 +5,6 @@ import asyncio
 from typing import Optional, List
 import time
 from dotenv import load_dotenv
-from aiohttp_socks import ProxyConnector  # Better proxy connector
 
 # Load environment variables
 load_dotenv()
@@ -81,58 +80,14 @@ async def reset_proxy_failure(proxy_str: str, delay: int):
         proxy_failure_count[proxy_str] = 0
         logger.info(f"Reset failure count for proxy: {proxy_str}")
 
-async def fetch_with_session(session: aiohttp.ClientSession, url: str, proxy_url: str, headers: dict) -> Optional[str]:
-    """Make a request with the given session and proxy"""
-    try:
-        async with session.get(
-            url,
-            headers=headers,
-            proxy=proxy_url,
-            timeout=30,
-            allow_redirects=True
-        ) as response:
-            # Accept both 200 and 202 status codes
-            if response.status in (200, 202):
-                content = await response.text()
-                
-                # Verify we got actual content
-                if len(content) > 1000 and ("<html" in content or "<!DOCTYPE" in content):
-                    logger.info(f"Successfully retrieved content ({len(content)} bytes)")
-                    return content
-                else:
-                    logger.warning(f"Response too small or not HTML: {len(content)} bytes")
-                    return None
-            else:
-                logger.warning(f"Response status code: {response.status}")
-                return None
-    except Exception as e:
-        logger.error(f"Request failed: {str(e)}")
-        return None
-
 async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
     """Fetch a page using WebShare proxies with retry logic"""
     logger.info(f"Requesting URL: {url}")
     
-    # Enhanced headers that look more like a browser
-    enhanced_headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.9",
-        # Don't specify Accept-Encoding to let aiohttp handle it with brotli installed
-        "Connection": "keep-alive",
-        "Upgrade-Insecure-Requests": "1",
-        "Sec-Fetch-Dest": "document",
-        "Sec-Fetch-Mode": "navigate",
-        "Sec-Fetch-Site": "none",
-        "Sec-Fetch-User": "?1",
-        "Cache-Control": "max-age=0",
-        "Referer": "https://www.google.com/"
-    }
-    # Update with any custom headers provided
-    enhanced_headers.update(headers)
-    
     # Try up to 3 different proxies
-    for attempt in range(3):
+    max_proxy_attempts = 3
+    
+    for attempt in range(max_proxy_attempts):
         proxy_url = get_next_proxy()
         if not proxy_url:
             logger.error("Failed to get a valid proxy")
@@ -141,20 +96,37 @@ async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) ->
         logger.info(f"Using proxy {proxy_url} (attempt {attempt+1})")
         
         try:
-            # Create a new session for each proxy to avoid connection issues
-            connector = ProxyConnector.from_url(proxy_url)
-            async with aiohttp.ClientSession(connector=connector) as proxy_session:
-                content = await fetch_with_session(proxy_session, url, None, enhanced_headers)
-                if content:
-                    return content
+            # Try with this proxy
+            async with session.get(
+                url,
+                headers=headers,
+                proxy=proxy_url,
+                timeout=30,
+                ssl=False
+            ) as response:
+                if response.status in [200, 202]:
+                    content = await response.text()
+                    
+                    # Verify we got actual content (common anti-bot techniques return empty pages)
+                    if len(content) > 1000 and ("<html" in content or "<!DOCTYPE" in content):
+                        logger.info(f"Successfully retrieved content ({len(content)} bytes)")
+                        return content
+                    else:
+                        logger.warning(f"Response too small or not HTML: {len(content)} bytes")
+                        mark_proxy_failure(proxy_url)
                 else:
+                    logger.warning(f"Response status code: {response.status}")
                     mark_proxy_failure(proxy_url)
-        except Exception as e:
+                    
+        except (aiohttp.ClientError, asyncio.TimeoutError) as e:
             logger.error(f"Request failed with proxy {proxy_url}: {str(e)}")
             mark_proxy_failure(proxy_url)
+        except Exception as e:
+            logger.error(f"Unexpected error: {str(e)}")
+            mark_proxy_failure(proxy_url)
         
         # Wait before trying next proxy
-        await asyncio.sleep(2)
+        await asyncio.sleep(1)
     
     logger.error("All proxy attempts failed")
     return None