Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -1,54 +1,20 @@
 import aiohttp # type: ignore
 import logging
-import asyncio
-import random
-import os
-from typing import Optional, Dict
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
-async def fetch_page(session: aiohttp.ClientSession, url: str, headers: Dict[str, str]) -> Optional[str]:
-    """Fetch a page using aiohttp with retry logic for 202 status codes"""
-    # Add random cookies to mimic browser behavior
-    cookies = {
-        "bkng": f"11UmFuZG9tSVYkc2RlIyh9YSk7IZWFkeH97ry3Z{random.randint(10000, 99999)}=",
-        "bkng_sso_session": "e30",
-        "bkng_sso_ses": str(random.randint(1000000, 9999999))
-    }
-    
-    # Get proxy from environment variable if available
-    proxy = os.environ.get("PROXY_URL")
-    
-    for attempt in range(3):  # Try up to 3 times
-        try:
-            logger.info(f"Requesting URL: {url} (attempt {attempt+1})")
-            
-            request_kwargs = {
-                "headers": headers,
-                "cookies": cookies,
-                "timeout": 30,
-                "allow_redirects": True
-            }
-            
-            # Add proxy if configured
-            if proxy:
-                request_kwargs["proxy"] = proxy
-                
-            async with session.get(url, **request_kwargs) as response:
-                if response.status == 200:
-                    logger.debug(f"Successfully retrieved content from {url}")
-                    return await response.text()
-                elif response.status == 202:
-                    # For 202 status codes, wait and retry
-                    wait_time = random.uniform(5, 10) * (attempt + 1)
-                    logger.warning(f"Got 202 status code, waiting {wait_time:.1f}s before retry")
-                    await asyncio.sleep(wait_time)
-                else:
-                    logger.error(f"Error retrieving URL {url}: Status code {response.status}")
-                    return None
-                    
-        except Exception as e:
-            logger.error(f"Request failed for {url}: {e}")
-            await asyncio.sleep(random.uniform(2, 5))
-    
-    return None
+async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
+    """Fetch a page using aiohttp"""
+    try:
+        logger.info(f"Requesting URL: {url}")
+        async with session.get(url, headers=headers, timeout=15) as response:
+            if response.status == 200:
+                logger.debug(f"Successfully retrieved content from {url}")
+                return await response.text()
+            else:
+                logger.error(f"Error retrieving URL {url}: Status code {response.status}")
+                return None
+    except Exception as e:
+        logger.error(f"Request failed for {url}: {e}")
+        return None