Update core/scraper.py

core/scraper.py

@@ -19,11 +19,10 @@ class HotelScraper:
         
         # Use ClientSession with TCP connector options for better reliability
         connector = aiohttp.TCPConnector(
-            limit=5,  # Limit number of simultaneous connections
+            limit=5,            # Limit number of simultaneous connections
             ttl_dns_cache=300,  # Cache DNS results for 5 minutes
-            enable_cleanup_closed=True,  # Clean up closed connections
-            force_close=True,  # Force close connections
-            ssl=False  # Some proxies don't support SSL
+            force_close=True,   # Force close connections after each use
+            ssl=False           # Some proxies don't support SSL
         )
         
         timeout = aiohttp.ClientTimeout(total=60)  # Longer timeout for proxies
@@ -31,26 +30,27 @@ class HotelScraper:
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
             tasks = []
             for query in hotel_queries:
-                task = self.booking_service.search_hotel(
+                task = asyncio.create_task(self.booking_service.search_hotel(
                     session=session,
                     destination=query.destination,
                     hotel_name=query.hotel_name
-                )
+                ))
                 tasks.append(task)
             
-            # Run all tasks concurrently with a limit to avoid overloading proxies
+            # Process hotels in small batches to avoid overloading proxies
             processed_results = []
-            chunk_size = 3  # Process 3 hotels at a time
+            chunk_size = 2  # Process 2 hotels at a time
             
             for i in range(0, len(tasks), chunk_size):
                 chunk = tasks[i:i+chunk_size]
-                results = await asyncio.gather(*chunk, return_exceptions=True)
+                chunk_results = await asyncio.gather(*chunk, return_exceptions=True)
                 
-                for j, result in enumerate(results):
+                # Process results
+                for j, result in enumerate(chunk_results):
                     query_index = i + j
-                    if query_index < len(hotel_queries):  # Safety check
+                    if query_index < len(hotel_queries):
                         if isinstance(result, Exception):
-                            logger.error(f"Error scraping hotel {hotel_queries[query_index].hotel_name}: {result}")
+                            logger.error(f"Error scraping {hotel_queries[query_index].hotel_name}: {result}")
                             processed_results.append({
                                 "destination": hotel_queries[query_index].destination,
                                 "hotel_name": hotel_queries[query_index].hotel_name,
@@ -59,8 +59,8 @@ class HotelScraper:
                         else:
                             processed_results.append(result)
                 
-                # Small delay between chunks to be gentle on proxies
+                # Add delay between chunks
                 if i + chunk_size < len(tasks):
-                    await asyncio.sleep(1)
+                    await asyncio.sleep(2)  # 2 second delay between batches
             
             return processed_results