Update services/utils/http_utils.py

services/utils/http_utils.py

@@ -1,20 +1,54 @@
-import aiohttp # type: ignore
-import logging
-from typing import Optional
-
-logger = logging.getLogger(__name__)
-
-async def fetch_page(session: aiohttp.ClientSession, url: str, headers: dict) -> Optional[str]:
-    """Fetch a page using aiohttp"""
-    try:
-        logger.info(f"Requesting URL: {url}")
-        async with session.get(url, headers=headers, timeout=15) as response:
-            if response.status == 200:
-                logger.debug(f"Successfully retrieved content from {url}")
-                return await response.text()
-            else:
-                logger.error(f"Error retrieving URL {url}: Status code {response.status}")
-                return None
-    except Exception as e:
-        logger.error(f"Request failed for {url}: {e}")
-        return None
+import aiohttp # type: ignore
+import logging
+import asyncio
+import random
+import os
+from typing import Optional, Dict
+
+logger = logging.getLogger(__name__)
+
+async def fetch_page(session: aiohttp.ClientSession, url: str, headers: Dict[str, str]) -> Optional[str]:
+    """Fetch a page using aiohttp with retry logic for 202 status codes"""
+    # Add random cookies to mimic browser behavior
+    cookies = {
+        "bkng": f"11UmFuZG9tSVYkc2RlIyh9YSk7IZWFkeH97ry3Z{random.randint(10000, 99999)}=",
+        "bkng_sso_session": "e30",
+        "bkng_sso_ses": str(random.randint(1000000, 9999999))
+    }
+    
+    # Get proxy from environment variable if available
+    proxy = os.environ.get("PROXY_URL")
+    
+    for attempt in range(3):  # Try up to 3 times
+        try:
+            logger.info(f"Requesting URL: {url} (attempt {attempt+1})")
+            
+            request_kwargs = {
+                "headers": headers,
+                "cookies": cookies,
+                "timeout": 30,
+                "allow_redirects": True
+            }
+            
+            # Add proxy if configured
+            if proxy:
+                request_kwargs["proxy"] = proxy
+                
+            async with session.get(url, **request_kwargs) as response:
+                if response.status == 200:
+                    logger.debug(f"Successfully retrieved content from {url}")
+                    return await response.text()
+                elif response.status == 202:
+                    # For 202 status codes, wait and retry
+                    wait_time = random.uniform(5, 10) * (attempt + 1)
+                    logger.warning(f"Got 202 status code, waiting {wait_time:.1f}s before retry")
+                    await asyncio.sleep(wait_time)
+                else:
+                    logger.error(f"Error retrieving URL {url}: Status code {response.status}")
+                    return None
+                    
+        except Exception as e:
+            logger.error(f"Request failed for {url}: {e}")
+            await asyncio.sleep(random.uniform(2, 5))
+    
+    return None