Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -1,141 +1,16 @@
 import aiohttp
 import logging
 import random
-import asyncio
 from typing import Optional
 
-# WebShare proxies
-WEBSHARE_PROXIES = [
-    "198.23.239.134:6540:zvubytfw:ak6yit5k2tvj",
-    "207.244.217.165:6712:zvubytfw:ak6yit5k2tvj",
-    "107.172.163.27:6543:zvubytfw:ak6yit5k2tvj",
-    "161.123.152.115:6360:zvubytfw:ak6yit5k2tvj",
-    "23.94.138.75:6349:zvubytfw:ak6yit5k2tvj",
-    "216.10.27.159:6837:zvubytfw:ak6yit5k2tvj",
-    "136.0.207.84:6661:zvubytfw:ak6yit5k2tvj",
-    "64.64.118.149:6732:zvubytfw:ak6yit5k2tvj",
-    "142.147.128.93:6593:zvubytfw:ak6yit5k2tvj",
-    "154.36.110.199:6853:zvubytfw:ak6yit5k2tvj"
-]
-
-# Track proxy performance
-proxy_failures = {}
-current_proxy_idx = -1
+from .browser_utils import fetch_page_with_browser
 
 logger = logging.getLogger(__name__)
 
-def get_proxy_url() -> str:
-    """Get a proxy URL using round-robin with failure detection"""
-    global current_proxy_idx
-    
-    # Try all proxies if necessary
-    for _ in range(len(WEBSHARE_PROXIES)):
-        current_proxy_idx = (current_proxy_idx + 1) % len(WEBSHARE_PROXIES)
-        proxy_str = WEBSHARE_PROXIES[current_proxy_idx]
-        
-        # Skip frequently failing proxies
-        if proxy_failures.get(proxy_str, 0) >= 3:
-            continue
-            
-        # Format proxy for aiohttp
-        parts = proxy_str.split(':')
-        if len(parts) == 4:
-            ip, port, username, password = parts
-            return f"http://{username}:{password}@{ip}:{port}"
-    
-    # If all proxies have failures, reset and try again
-    proxy_failures.clear()
-    return get_proxy_url()
-
-def mark_proxy_failure(proxy_url: str) -> None:
-    """Mark a proxy as failing"""
-    for proxy_str in WEBSHARE_PROXIES:
-        if proxy_str.split(':')[0] in proxy_url:
-            proxy_failures[proxy_str] = proxy_failures.get(proxy_str, 0) + 1
-            if proxy_failures[proxy_str] >= 3:
-                asyncio.create_task(reset_proxy_after_delay(proxy_str, 300))
-            break
-
-async def reset_proxy_after_delay(proxy_str: str, delay: int) -> None:
-    """Reset a proxy's failure count after a delay"""
-    await asyncio.sleep(delay)
-    if proxy_str in proxy_failures:
-        proxy_failures[proxy_str] = 0
-
 async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
-    """Fetch a page using proxies with retry logic"""
-    logger.info(f"Requesting URL: {url}")
-    
-    # Enhanced headers
-    enhanced_headers = headers.copy()
-    enhanced_headers.update({
-        "Accept-Encoding": "gzip, deflate, br",  # Explicitly specify brotli support
-        "Cache-Control": "no-cache",
-        "Pragma": "no-cache",
-        "Sec-Fetch-Dest": "document",
-        "Sec-Fetch-Mode": "navigate",
-        "Sec-Fetch-Site": "none",
-        "Sec-Fetch-User": "?1",
-        "Referer": "https://www.google.com/",
-    })
-    
-    # Try up to 3 proxies
-    for attempt in range(3):
-        proxy_url = get_proxy_url()
-        logger.info(f"Using proxy {proxy_url} (attempt {attempt+1})")
-        
-        try:
-            # Use a longer timeout for proxies
-            async with session.get(
-                url,
-                headers=enhanced_headers,
-                proxy=proxy_url,
-                timeout=25,
-                ssl=False,
-                allow_redirects=True
-            ) as response:
-                if response.status in [200, 202]:
-                    content = await response.text()
-                    
-                    # Check if we got actual content, not a bot detection page
-                    if len(content) > 5000 and ("<html" in content or "<!DOCTYPE" in content):
-                        # Check for property content on search pages
-                        if "searchresults" in url or "search" in url:
-                            if "property-card" in content or "sr_property_block" in content or "sr_item" in content:
-                                logger.info(f"Successfully retrieved search results ({len(content)} bytes)")
-                                return content
-                            else:
-                                logger.warning("No property cards found in search results")
-                                mark_proxy_failure(proxy_url)
-                        else:
-                            logger.info(f"Successfully retrieved content ({len(content)} bytes)")
-                            return content
-                    else:
-                        logger.warning(f"Response too short or not HTML: {len(content)} bytes")
-                        mark_proxy_failure(proxy_url)
-                else:
-                    logger.warning(f"Response status {response.status} from proxy {proxy_url}")
-                    mark_proxy_failure(proxy_url)
-        except Exception as e:
-            logger.error(f"Proxy request failed: {str(e)}")
-            mark_proxy_failure(proxy_url)
-        
-        # Wait before next attempt
-        await asyncio.sleep(2)
-    
-    # If proxies failed, try direct connection as last resort
-    logger.warning("All proxies failed, trying direct connection")
-    try:
-        async with session.get(
-            url,
-            headers=enhanced_headers,
-            timeout=15
-        ) as response:
-            if response.status == 200:
-                content = await response.text()
-                if len(content) > 5000:
-                    return content
-    except Exception as e:
-        logger.error(f"Direct request also failed: {str(e)}")
+    """Fetch a page using browser automation with proxies"""
+    # Get user agent from headers
+    user_agent = headers.get('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
     
-    return None
+    # Use the browser-based approach
+    return await fetch_page_with_browser(url, user_agent)