Update app.py
Browse files
app.py
CHANGED
|
@@ -1,8 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# 🚀 1. IMPORTS & SETUP
|
| 2 |
import asyncio
|
| 3 |
import datetime
|
| 4 |
import logging
|
| 5 |
-
import os
|
| 6 |
import random
|
| 7 |
import gradio as gr
|
| 8 |
from cachetools import TTLCache
|
|
@@ -44,7 +67,13 @@ async def _extract_user_request(question: str) -> dict:
|
|
| 44 |
("human", "{format_instructions}\n{request}")
|
| 45 |
]).partial(format_instructions=parser.get_format_instructions())
|
| 46 |
logging.info(f"🔍 Extracting details from: {question}")
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
|
| 50 |
# 🛠 4. CACHE WRAPPER
|
|
@@ -61,7 +90,7 @@ async def _cached_search(func, details):
|
|
| 61 |
|
| 62 |
# 🌐 5. PLAYWRIGHT SCRAPER FUNCTIONS
|
| 63 |
|
| 64 |
-
async def
|
| 65 |
results = []
|
| 66 |
try:
|
| 67 |
async with async_playwright() as pw:
|
|
@@ -69,20 +98,15 @@ async def _search_google_flights(details):
|
|
| 69 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 70 |
page = await context.new_page()
|
| 71 |
|
| 72 |
-
# Add random delay
|
| 73 |
-
await asyncio.sleep(random.uniform(1, 3))
|
| 74 |
-
|
| 75 |
o = details["origin_city"]
|
| 76 |
d = details["destination_city"]
|
| 77 |
sd = details["start_date"]
|
| 78 |
ed = details["end_date"]
|
| 79 |
|
| 80 |
-
# More reliable Google Flights URL
|
| 81 |
url = f"https://www.google.com/travel/flights/search?tfs=CBwQAhokEgoyMDI1LTA4LTAxagwIAhIIL20vMDVxdGwyDAg"
|
| 82 |
|
| 83 |
await page.goto(url, timeout=60000)
|
| 84 |
|
| 85 |
-
# Wait for any flight results to load
|
| 86 |
try:
|
| 87 |
await page.wait_for_selector('[data-testid="flight-offer"]', timeout=20000)
|
| 88 |
cards = await page.query_selector_all('[data-testid="flight-offer"]')
|
|
@@ -90,9 +114,6 @@ async def _search_google_flights(details):
|
|
| 90 |
for card in cards[:3]:
|
| 91 |
try:
|
| 92 |
price_el = await card.query_selector('[data-testid="price-text"]')
|
| 93 |
-
if not price_el:
|
| 94 |
-
price_el = await card.query_selector('span[aria-label*="dollars"]')
|
| 95 |
-
|
| 96 |
if price_el:
|
| 97 |
price_text = await price_el.inner_text()
|
| 98 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
|
@@ -108,32 +129,14 @@ async def _search_google_flights(details):
|
|
| 108 |
except Exception as e:
|
| 109 |
logging.warning(f"Error parsing Google Flights card: {e}")
|
| 110 |
continue
|
| 111 |
-
|
| 112 |
except Exception as e:
|
| 113 |
logging.warning(f"Google Flights selector timeout: {e}")
|
| 114 |
-
# Fallback: try to find any price-like elements
|
| 115 |
-
price_elements = await page.query_selector_all('span[aria-label*="dollar"], span[aria-label*="price"]')
|
| 116 |
-
for elem in price_elements[:3]:
|
| 117 |
-
try:
|
| 118 |
-
text = await elem.inner_text()
|
| 119 |
-
if '$' in text:
|
| 120 |
-
results.append({
|
| 121 |
-
"source": "Google Flights",
|
| 122 |
-
"type": "flight",
|
| 123 |
-
"details": text.strip(),
|
| 124 |
-
"price": 0.0,
|
| 125 |
-
"link": url
|
| 126 |
-
})
|
| 127 |
-
except:
|
| 128 |
-
continue
|
| 129 |
-
|
| 130 |
await browser.close()
|
| 131 |
-
|
| 132 |
except Exception as e:
|
| 133 |
logging.error(f"Google Flights scraping failed: {e}")
|
| 134 |
-
|
| 135 |
return results
|
| 136 |
|
|
|
|
| 137 |
async def _search_kayak(details):
|
| 138 |
results = []
|
| 139 |
try:
|
|
@@ -153,13 +156,7 @@ async def _search_kayak(details):
|
|
| 153 |
|
| 154 |
await page.goto(url, timeout=60000)
|
| 155 |
|
| 156 |
-
|
| 157 |
-
selectors_to_try = [
|
| 158 |
-
".resultWrapper",
|
| 159 |
-
"[data-testid='result-item']",
|
| 160 |
-
".Common-Booking-MultiBookProvider",
|
| 161 |
-
".result-column"
|
| 162 |
-
]
|
| 163 |
|
| 164 |
cards = []
|
| 165 |
for selector in selectors_to_try:
|
|
@@ -173,14 +170,7 @@ async def _search_kayak(details):
|
|
| 173 |
|
| 174 |
for card in cards[:3]:
|
| 175 |
try:
|
| 176 |
-
|
| 177 |
-
price_selectors = [
|
| 178 |
-
".price-text",
|
| 179 |
-
"[data-testid='price']",
|
| 180 |
-
".Common-Booking-MultiBookProvider-price",
|
| 181 |
-
"span[aria-label*='price']"
|
| 182 |
-
]
|
| 183 |
-
|
| 184 |
price_text = None
|
| 185 |
for price_sel in price_selectors:
|
| 186 |
try:
|
|
@@ -194,24 +184,16 @@ async def _search_kayak(details):
|
|
| 194 |
if price_text:
|
| 195 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
| 196 |
price = float(price_clean) if price_clean else 0.0
|
| 197 |
-
|
| 198 |
results.append({
|
| 199 |
-
"source": "Kayak",
|
| 200 |
-
"
|
| 201 |
-
"details": price_text.strip(),
|
| 202 |
-
"price": price,
|
| 203 |
-
"link": url
|
| 204 |
})
|
| 205 |
-
|
| 206 |
except Exception as e:
|
| 207 |
logging.warning(f"Error parsing Kayak card: {e}")
|
| 208 |
continue
|
| 209 |
-
|
| 210 |
await browser.close()
|
| 211 |
-
|
| 212 |
except Exception as e:
|
| 213 |
logging.error(f"Kayak scraping failed: {e}")
|
| 214 |
-
|
| 215 |
return results
|
| 216 |
|
| 217 |
async def _search_trabber(details):
|
|
@@ -221,22 +203,16 @@ async def _search_trabber(details):
|
|
| 221 |
browser = await pw.chromium.launch(headless=True)
|
| 222 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 223 |
page = await context.new_page()
|
| 224 |
-
|
| 225 |
await asyncio.sleep(random.uniform(1, 3))
|
| 226 |
-
|
| 227 |
o = details["origin_city"].upper()
|
| 228 |
d = details["destination_city"].upper()
|
| 229 |
sd_parts = details["start_date"].split('-')
|
| 230 |
-
sd = f"{sd_parts[2]}{sd_parts[1]}{sd_parts[0][2:]}"
|
| 231 |
-
|
| 232 |
url = f"https://www.trabber.ca/flights-from-{o}-to-{d}-on-{sd}"
|
| 233 |
-
|
| 234 |
await page.goto(url, timeout=60000)
|
| 235 |
-
|
| 236 |
try:
|
| 237 |
await page.wait_for_selector("#results_list_det tr", timeout=20000)
|
| 238 |
rows = await page.query_selector_all("#results_list_det tr")
|
| 239 |
-
|
| 240 |
for row in rows[:3]:
|
| 241 |
try:
|
| 242 |
price_el = await row.query_selector("td.results_price a, .results_price")
|
|
@@ -244,26 +220,18 @@ async def _search_trabber(details):
|
|
| 244 |
price_text = await price_el.inner_text()
|
| 245 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
| 246 |
price = float(price_clean) if price_clean else 0.0
|
| 247 |
-
|
| 248 |
results.append({
|
| 249 |
-
"source": "Trabber",
|
| 250 |
-
"
|
| 251 |
-
"details": price_text.strip(),
|
| 252 |
-
"price": price,
|
| 253 |
-
"link": url
|
| 254 |
})
|
| 255 |
except Exception as e:
|
| 256 |
logging.warning(f"Error parsing Trabber row: {e}")
|
| 257 |
continue
|
| 258 |
-
|
| 259 |
except Exception as e:
|
| 260 |
logging.warning(f"Trabber selector timeout: {e}")
|
| 261 |
-
|
| 262 |
await browser.close()
|
| 263 |
-
|
| 264 |
except Exception as e:
|
| 265 |
logging.error(f"Trabber scraping failed: {e}")
|
| 266 |
-
|
| 267 |
return results
|
| 268 |
|
| 269 |
async def _search_travala(details):
|
|
@@ -273,21 +241,11 @@ async def _search_travala(details):
|
|
| 273 |
browser = await pw.chromium.launch(headless=True)
|
| 274 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 275 |
page = await context.new_page()
|
| 276 |
-
|
| 277 |
await asyncio.sleep(random.uniform(1, 3))
|
| 278 |
-
|
| 279 |
url = "https://www.travala.com/deals"
|
| 280 |
-
|
| 281 |
await page.goto(url, timeout=60000)
|
| 282 |
-
|
| 283 |
try:
|
| 284 |
-
|
| 285 |
-
selectors = [
|
| 286 |
-
"div[class*='DealCard_container__']",
|
| 287 |
-
".deal-card",
|
| 288 |
-
"[data-testid='deal-card']"
|
| 289 |
-
]
|
| 290 |
-
|
| 291 |
cards = []
|
| 292 |
for selector in selectors:
|
| 293 |
try:
|
|
@@ -297,15 +255,9 @@ async def _search_travala(details):
|
|
| 297 |
break
|
| 298 |
except:
|
| 299 |
continue
|
| 300 |
-
|
| 301 |
for card in cards[:5]:
|
| 302 |
try:
|
| 303 |
-
title_selectors = [
|
| 304 |
-
"p[class*='DealCard_title__']",
|
| 305 |
-
".deal-title",
|
| 306 |
-
"h3, h4"
|
| 307 |
-
]
|
| 308 |
-
|
| 309 |
title = "Travala Deal"
|
| 310 |
for title_sel in title_selectors:
|
| 311 |
try:
|
|
@@ -315,27 +267,17 @@ async def _search_travala(details):
|
|
| 315 |
break
|
| 316 |
except:
|
| 317 |
continue
|
| 318 |
-
|
| 319 |
results.append({
|
| 320 |
-
"source": "Travala",
|
| 321 |
-
"type": "deal",
|
| 322 |
-
"details": title[:100],
|
| 323 |
-
"price": 0,
|
| 324 |
-
"link": url
|
| 325 |
})
|
| 326 |
-
|
| 327 |
except Exception as e:
|
| 328 |
logging.warning(f"Error parsing Travala card: {e}")
|
| 329 |
continue
|
| 330 |
-
|
| 331 |
except Exception as e:
|
| 332 |
logging.warning(f"Travala selector timeout: {e}")
|
| 333 |
-
|
| 334 |
await browser.close()
|
| 335 |
-
|
| 336 |
except Exception as e:
|
| 337 |
logging.error(f"Travala scraping failed: {e}")
|
| 338 |
-
|
| 339 |
return results
|
| 340 |
|
| 341 |
async def _search_secretflying(details):
|
|
@@ -345,21 +287,11 @@ async def _search_secretflying(details):
|
|
| 345 |
browser = await pw.chromium.launch(headless=True)
|
| 346 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 347 |
page = await context.new_page()
|
| 348 |
-
|
| 349 |
await asyncio.sleep(random.uniform(1, 3))
|
| 350 |
-
|
| 351 |
url = "https://www.secretflying.com/canada-deals/"
|
| 352 |
-
|
| 353 |
await page.goto(url, timeout=60000)
|
| 354 |
-
|
| 355 |
try:
|
| 356 |
-
|
| 357 |
-
selectors = [
|
| 358 |
-
".post-item",
|
| 359 |
-
"article",
|
| 360 |
-
".entry"
|
| 361 |
-
]
|
| 362 |
-
|
| 363 |
posts = []
|
| 364 |
for selector in selectors:
|
| 365 |
try:
|
|
@@ -369,19 +301,11 @@ async def _search_secretflying(details):
|
|
| 369 |
break
|
| 370 |
except:
|
| 371 |
continue
|
| 372 |
-
|
| 373 |
for post in posts[:5]:
|
| 374 |
try:
|
| 375 |
-
title_selectors = [
|
| 376 |
-
"h2.entry-title a",
|
| 377 |
-
".entry-title a",
|
| 378 |
-
"h3 a",
|
| 379 |
-
"h2 a"
|
| 380 |
-
]
|
| 381 |
-
|
| 382 |
title = "Deal"
|
| 383 |
link = url
|
| 384 |
-
|
| 385 |
for title_sel in title_selectors:
|
| 386 |
try:
|
| 387 |
title_el = await post.query_selector(title_sel)
|
|
@@ -393,34 +317,23 @@ async def _search_secretflying(details):
|
|
| 393 |
break
|
| 394 |
except:
|
| 395 |
continue
|
| 396 |
-
|
| 397 |
results.append({
|
| 398 |
-
"source": "SecretFlying",
|
| 399 |
-
"type": "deal",
|
| 400 |
-
"details": title[:100],
|
| 401 |
-
"price": 0,
|
| 402 |
-
"link": link
|
| 403 |
})
|
| 404 |
-
|
| 405 |
except Exception as e:
|
| 406 |
logging.warning(f"Error parsing SecretFlying post: {e}")
|
| 407 |
continue
|
| 408 |
-
|
| 409 |
except Exception as e:
|
| 410 |
logging.warning(f"SecretFlying selector timeout: {e}")
|
| 411 |
-
|
| 412 |
await browser.close()
|
| 413 |
-
|
| 414 |
except Exception as e:
|
| 415 |
logging.error(f"SecretFlying scraping failed: {e}")
|
| 416 |
-
|
| 417 |
return results
|
| 418 |
|
| 419 |
-
|
| 420 |
# 🔄 6. GATHER ALL DATA
|
| 421 |
async def _gather_travel_data(req):
|
| 422 |
tasks = [
|
| 423 |
-
_cached_search(
|
| 424 |
_cached_search(_search_kayak, req),
|
| 425 |
_cached_search(_search_trabber, req),
|
| 426 |
_cached_search(_search_travala, req),
|
|
@@ -435,12 +348,10 @@ async def _gather_travel_data(req):
|
|
| 435 |
logging.error(f"A scraper task failed: {r}")
|
| 436 |
return combined
|
| 437 |
|
| 438 |
-
|
| 439 |
# ✍️ 7. FORMAT RESULTS
|
| 440 |
def _format_response(req, data):
|
| 441 |
dest = req["destination_city"]
|
| 442 |
md = f"## 🌏 Travel Plan to **{dest}**\n\n"
|
| 443 |
-
|
| 444 |
flights = sorted([i for i in data if i["type"]=="flight" and i.get("price", 0) > 0], key=lambda x: x["price"])
|
| 445 |
if flights:
|
| 446 |
md += "### ✈️ Flights\n"
|
|
|
|
| 1 |
+
# 🚀 0. BRUTE FORCE INSTALLER
|
| 2 |
+
import subprocess
|
| 3 |
+
import sys
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
# This block forces the installation of the browser Playwright needs.
|
| 7 |
+
# It's a workaround for stubborn server environments.
|
| 8 |
+
try:
|
| 9 |
+
print("--- Ensuring Playwright browsers are installed ---")
|
| 10 |
+
# We use 'playwright install chromium' to be specific and save space.
|
| 11 |
+
# Using sys.executable ensures we use the python env's playwright.
|
| 12 |
+
process = subprocess.run(
|
| 13 |
+
[sys.executable, "-m", "playwright", "install", "chromium"],
|
| 14 |
+
capture_output=True,
|
| 15 |
+
text=True,
|
| 16 |
+
check=True
|
| 17 |
+
)
|
| 18 |
+
print(process.stdout)
|
| 19 |
+
print("--- Playwright browsers installation check complete ---")
|
| 20 |
+
except Exception as e:
|
| 21 |
+
print(f"--- An error occurred during browser installation: {e} ---")
|
| 22 |
+
# Even if it fails, we try to continue.
|
| 23 |
+
pass
|
| 24 |
+
|
| 25 |
# 🚀 1. IMPORTS & SETUP
|
| 26 |
import asyncio
|
| 27 |
import datetime
|
| 28 |
import logging
|
|
|
|
| 29 |
import random
|
| 30 |
import gradio as gr
|
| 31 |
from cachetools import TTLCache
|
|
|
|
| 67 |
("human", "{format_instructions}\n{request}")
|
| 68 |
]).partial(format_instructions=parser.get_format_instructions())
|
| 69 |
logging.info(f"🔍 Extracting details from: {question}")
|
| 70 |
+
# Default to a one-way trip if end_date isn't found
|
| 71 |
+
extracted = await (prompt | llm | parser).ainvoke({"request": question})
|
| 72 |
+
if not extracted.get("end_date"):
|
| 73 |
+
start_dt = datetime.datetime.strptime(extracted["start_date"], "%Y-%m-%d")
|
| 74 |
+
end_dt = start_dt + datetime.timedelta(days=7) # Default to 7 days later
|
| 75 |
+
extracted["end_date"] = end_dt.strftime("%Y-%m-%d")
|
| 76 |
+
return extracted
|
| 77 |
|
| 78 |
|
| 79 |
# 🛠 4. CACHE WRAPPER
|
|
|
|
| 90 |
|
| 91 |
# 🌐 5. PLAYWRIGHT SCRAPER FUNCTIONS
|
| 92 |
|
| 93 |
+
async def _search_Google Flights(details):
|
| 94 |
results = []
|
| 95 |
try:
|
| 96 |
async with async_playwright() as pw:
|
|
|
|
| 98 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 99 |
page = await context.new_page()
|
| 100 |
|
|
|
|
|
|
|
|
|
|
| 101 |
o = details["origin_city"]
|
| 102 |
d = details["destination_city"]
|
| 103 |
sd = details["start_date"]
|
| 104 |
ed = details["end_date"]
|
| 105 |
|
|
|
|
| 106 |
url = f"https://www.google.com/travel/flights/search?tfs=CBwQAhokEgoyMDI1LTA4LTAxagwIAhIIL20vMDVxdGwyDAg"
|
| 107 |
|
| 108 |
await page.goto(url, timeout=60000)
|
| 109 |
|
|
|
|
| 110 |
try:
|
| 111 |
await page.wait_for_selector('[data-testid="flight-offer"]', timeout=20000)
|
| 112 |
cards = await page.query_selector_all('[data-testid="flight-offer"]')
|
|
|
|
| 114 |
for card in cards[:3]:
|
| 115 |
try:
|
| 116 |
price_el = await card.query_selector('[data-testid="price-text"]')
|
|
|
|
|
|
|
|
|
|
| 117 |
if price_el:
|
| 118 |
price_text = await price_el.inner_text()
|
| 119 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
|
|
|
| 129 |
except Exception as e:
|
| 130 |
logging.warning(f"Error parsing Google Flights card: {e}")
|
| 131 |
continue
|
|
|
|
| 132 |
except Exception as e:
|
| 133 |
logging.warning(f"Google Flights selector timeout: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
await browser.close()
|
|
|
|
| 135 |
except Exception as e:
|
| 136 |
logging.error(f"Google Flights scraping failed: {e}")
|
|
|
|
| 137 |
return results
|
| 138 |
|
| 139 |
+
# ... (rest of the scraper functions remain the same)
|
| 140 |
async def _search_kayak(details):
|
| 141 |
results = []
|
| 142 |
try:
|
|
|
|
| 156 |
|
| 157 |
await page.goto(url, timeout=60000)
|
| 158 |
|
| 159 |
+
selectors_to_try = [".resultWrapper", "[data-testid='result-item']", ".Common-Booking-MultiBookProvider"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
|
| 161 |
cards = []
|
| 162 |
for selector in selectors_to_try:
|
|
|
|
| 170 |
|
| 171 |
for card in cards[:3]:
|
| 172 |
try:
|
| 173 |
+
price_selectors = [".price-text", "[data-testid='price']", ".Common-Booking-MultiBookProvider-price"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
price_text = None
|
| 175 |
for price_sel in price_selectors:
|
| 176 |
try:
|
|
|
|
| 184 |
if price_text:
|
| 185 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
| 186 |
price = float(price_clean) if price_clean else 0.0
|
|
|
|
| 187 |
results.append({
|
| 188 |
+
"source": "Kayak", "type": "flight", "details": price_text.strip(),
|
| 189 |
+
"price": price, "link": url
|
|
|
|
|
|
|
|
|
|
| 190 |
})
|
|
|
|
| 191 |
except Exception as e:
|
| 192 |
logging.warning(f"Error parsing Kayak card: {e}")
|
| 193 |
continue
|
|
|
|
| 194 |
await browser.close()
|
|
|
|
| 195 |
except Exception as e:
|
| 196 |
logging.error(f"Kayak scraping failed: {e}")
|
|
|
|
| 197 |
return results
|
| 198 |
|
| 199 |
async def _search_trabber(details):
|
|
|
|
| 203 |
browser = await pw.chromium.launch(headless=True)
|
| 204 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 205 |
page = await context.new_page()
|
|
|
|
| 206 |
await asyncio.sleep(random.uniform(1, 3))
|
|
|
|
| 207 |
o = details["origin_city"].upper()
|
| 208 |
d = details["destination_city"].upper()
|
| 209 |
sd_parts = details["start_date"].split('-')
|
| 210 |
+
sd = f"{sd_parts[2]}{sd_parts[1]}{sd_parts[0][2:]}"
|
|
|
|
| 211 |
url = f"https://www.trabber.ca/flights-from-{o}-to-{d}-on-{sd}"
|
|
|
|
| 212 |
await page.goto(url, timeout=60000)
|
|
|
|
| 213 |
try:
|
| 214 |
await page.wait_for_selector("#results_list_det tr", timeout=20000)
|
| 215 |
rows = await page.query_selector_all("#results_list_det tr")
|
|
|
|
| 216 |
for row in rows[:3]:
|
| 217 |
try:
|
| 218 |
price_el = await row.query_selector("td.results_price a, .results_price")
|
|
|
|
| 220 |
price_text = await price_el.inner_text()
|
| 221 |
price_clean = ''.join(filter(str.isdigit, price_text))
|
| 222 |
price = float(price_clean) if price_clean else 0.0
|
|
|
|
| 223 |
results.append({
|
| 224 |
+
"source": "Trabber", "type": "flight", "details": price_text.strip(),
|
| 225 |
+
"price": price, "link": url
|
|
|
|
|
|
|
|
|
|
| 226 |
})
|
| 227 |
except Exception as e:
|
| 228 |
logging.warning(f"Error parsing Trabber row: {e}")
|
| 229 |
continue
|
|
|
|
| 230 |
except Exception as e:
|
| 231 |
logging.warning(f"Trabber selector timeout: {e}")
|
|
|
|
| 232 |
await browser.close()
|
|
|
|
| 233 |
except Exception as e:
|
| 234 |
logging.error(f"Trabber scraping failed: {e}")
|
|
|
|
| 235 |
return results
|
| 236 |
|
| 237 |
async def _search_travala(details):
|
|
|
|
| 241 |
browser = await pw.chromium.launch(headless=True)
|
| 242 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 243 |
page = await context.new_page()
|
|
|
|
| 244 |
await asyncio.sleep(random.uniform(1, 3))
|
|
|
|
| 245 |
url = "https://www.travala.com/deals"
|
|
|
|
| 246 |
await page.goto(url, timeout=60000)
|
|
|
|
| 247 |
try:
|
| 248 |
+
selectors = ["div[class*='DealCard_container__']", ".deal-card", "[data-testid='deal-card']"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
cards = []
|
| 250 |
for selector in selectors:
|
| 251 |
try:
|
|
|
|
| 255 |
break
|
| 256 |
except:
|
| 257 |
continue
|
|
|
|
| 258 |
for card in cards[:5]:
|
| 259 |
try:
|
| 260 |
+
title_selectors = ["p[class*='DealCard_title__']", ".deal-title", "h3, h4"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
title = "Travala Deal"
|
| 262 |
for title_sel in title_selectors:
|
| 263 |
try:
|
|
|
|
| 267 |
break
|
| 268 |
except:
|
| 269 |
continue
|
|
|
|
| 270 |
results.append({
|
| 271 |
+
"source": "Travala", "type": "deal", "details": title[:100], "price": 0, "link": url
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
})
|
|
|
|
| 273 |
except Exception as e:
|
| 274 |
logging.warning(f"Error parsing Travala card: {e}")
|
| 275 |
continue
|
|
|
|
| 276 |
except Exception as e:
|
| 277 |
logging.warning(f"Travala selector timeout: {e}")
|
|
|
|
| 278 |
await browser.close()
|
|
|
|
| 279 |
except Exception as e:
|
| 280 |
logging.error(f"Travala scraping failed: {e}")
|
|
|
|
| 281 |
return results
|
| 282 |
|
| 283 |
async def _search_secretflying(details):
|
|
|
|
| 287 |
browser = await pw.chromium.launch(headless=True)
|
| 288 |
context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
|
| 289 |
page = await context.new_page()
|
|
|
|
| 290 |
await asyncio.sleep(random.uniform(1, 3))
|
|
|
|
| 291 |
url = "https://www.secretflying.com/canada-deals/"
|
|
|
|
| 292 |
await page.goto(url, timeout=60000)
|
|
|
|
| 293 |
try:
|
| 294 |
+
selectors = [".post-item", "article", ".entry"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
posts = []
|
| 296 |
for selector in selectors:
|
| 297 |
try:
|
|
|
|
| 301 |
break
|
| 302 |
except:
|
| 303 |
continue
|
|
|
|
| 304 |
for post in posts[:5]:
|
| 305 |
try:
|
| 306 |
+
title_selectors = ["h2.entry-title a", ".entry-title a", "h3 a", "h2 a"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
title = "Deal"
|
| 308 |
link = url
|
|
|
|
| 309 |
for title_sel in title_selectors:
|
| 310 |
try:
|
| 311 |
title_el = await post.query_selector(title_sel)
|
|
|
|
| 317 |
break
|
| 318 |
except:
|
| 319 |
continue
|
|
|
|
| 320 |
results.append({
|
| 321 |
+
"source": "SecretFlying", "type": "deal", "details": title[:100], "price": 0, "link": link
|
|
|
|
|
|
|
|
|
|
|
|
|
| 322 |
})
|
|
|
|
| 323 |
except Exception as e:
|
| 324 |
logging.warning(f"Error parsing SecretFlying post: {e}")
|
| 325 |
continue
|
|
|
|
| 326 |
except Exception as e:
|
| 327 |
logging.warning(f"SecretFlying selector timeout: {e}")
|
|
|
|
| 328 |
await browser.close()
|
|
|
|
| 329 |
except Exception as e:
|
| 330 |
logging.error(f"SecretFlying scraping failed: {e}")
|
|
|
|
| 331 |
return results
|
| 332 |
|
|
|
|
| 333 |
# 🔄 6. GATHER ALL DATA
|
| 334 |
async def _gather_travel_data(req):
|
| 335 |
tasks = [
|
| 336 |
+
_cached_search(_search_Google Flights, req),
|
| 337 |
_cached_search(_search_kayak, req),
|
| 338 |
_cached_search(_search_trabber, req),
|
| 339 |
_cached_search(_search_travala, req),
|
|
|
|
| 348 |
logging.error(f"A scraper task failed: {r}")
|
| 349 |
return combined
|
| 350 |
|
|
|
|
| 351 |
# ✍️ 7. FORMAT RESULTS
|
| 352 |
def _format_response(req, data):
|
| 353 |
dest = req["destination_city"]
|
| 354 |
md = f"## 🌏 Travel Plan to **{dest}**\n\n"
|
|
|
|
| 355 |
flights = sorted([i for i in data if i["type"]=="flight" and i.get("price", 0) > 0], key=lambda x: x["price"])
|
| 356 |
if flights:
|
| 357 |
md += "### ✈️ Flights\n"
|