AlexanderStaniel commited on
Commit
d07ff34
·
verified ·
1 Parent(s): a2c4f97

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +47 -136
app.py CHANGED
@@ -1,8 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # 🚀 1. IMPORTS & SETUP
2
  import asyncio
3
  import datetime
4
  import logging
5
- import os
6
  import random
7
  import gradio as gr
8
  from cachetools import TTLCache
@@ -44,7 +67,13 @@ async def _extract_user_request(question: str) -> dict:
44
  ("human", "{format_instructions}\n{request}")
45
  ]).partial(format_instructions=parser.get_format_instructions())
46
  logging.info(f"🔍 Extracting details from: {question}")
47
- return await (prompt | llm | parser).ainvoke({"request": question})
 
 
 
 
 
 
48
 
49
 
50
  # 🛠 4. CACHE WRAPPER
@@ -61,7 +90,7 @@ async def _cached_search(func, details):
61
 
62
  # 🌐 5. PLAYWRIGHT SCRAPER FUNCTIONS
63
 
64
- async def _search_google_flights(details):
65
  results = []
66
  try:
67
  async with async_playwright() as pw:
@@ -69,20 +98,15 @@ async def _search_google_flights(details):
69
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
70
  page = await context.new_page()
71
 
72
- # Add random delay
73
- await asyncio.sleep(random.uniform(1, 3))
74
-
75
  o = details["origin_city"]
76
  d = details["destination_city"]
77
  sd = details["start_date"]
78
  ed = details["end_date"]
79
 
80
- # More reliable Google Flights URL
81
  url = f"https://www.google.com/travel/flights/search?tfs=CBwQAhokEgoyMDI1LTA4LTAxagwIAhIIL20vMDVxdGwyDAg"
82
 
83
  await page.goto(url, timeout=60000)
84
 
85
- # Wait for any flight results to load
86
  try:
87
  await page.wait_for_selector('[data-testid="flight-offer"]', timeout=20000)
88
  cards = await page.query_selector_all('[data-testid="flight-offer"]')
@@ -90,9 +114,6 @@ async def _search_google_flights(details):
90
  for card in cards[:3]:
91
  try:
92
  price_el = await card.query_selector('[data-testid="price-text"]')
93
- if not price_el:
94
- price_el = await card.query_selector('span[aria-label*="dollars"]')
95
-
96
  if price_el:
97
  price_text = await price_el.inner_text()
98
  price_clean = ''.join(filter(str.isdigit, price_text))
@@ -108,32 +129,14 @@ async def _search_google_flights(details):
108
  except Exception as e:
109
  logging.warning(f"Error parsing Google Flights card: {e}")
110
  continue
111
-
112
  except Exception as e:
113
  logging.warning(f"Google Flights selector timeout: {e}")
114
- # Fallback: try to find any price-like elements
115
- price_elements = await page.query_selector_all('span[aria-label*="dollar"], span[aria-label*="price"]')
116
- for elem in price_elements[:3]:
117
- try:
118
- text = await elem.inner_text()
119
- if '$' in text:
120
- results.append({
121
- "source": "Google Flights",
122
- "type": "flight",
123
- "details": text.strip(),
124
- "price": 0.0,
125
- "link": url
126
- })
127
- except:
128
- continue
129
-
130
  await browser.close()
131
-
132
  except Exception as e:
133
  logging.error(f"Google Flights scraping failed: {e}")
134
-
135
  return results
136
 
 
137
  async def _search_kayak(details):
138
  results = []
139
  try:
@@ -153,13 +156,7 @@ async def _search_kayak(details):
153
 
154
  await page.goto(url, timeout=60000)
155
 
156
- # Multiple selector attempts
157
- selectors_to_try = [
158
- ".resultWrapper",
159
- "[data-testid='result-item']",
160
- ".Common-Booking-MultiBookProvider",
161
- ".result-column"
162
- ]
163
 
164
  cards = []
165
  for selector in selectors_to_try:
@@ -173,14 +170,7 @@ async def _search_kayak(details):
173
 
174
  for card in cards[:3]:
175
  try:
176
- # Try multiple price selectors
177
- price_selectors = [
178
- ".price-text",
179
- "[data-testid='price']",
180
- ".Common-Booking-MultiBookProvider-price",
181
- "span[aria-label*='price']"
182
- ]
183
-
184
  price_text = None
185
  for price_sel in price_selectors:
186
  try:
@@ -194,24 +184,16 @@ async def _search_kayak(details):
194
  if price_text:
195
  price_clean = ''.join(filter(str.isdigit, price_text))
196
  price = float(price_clean) if price_clean else 0.0
197
-
198
  results.append({
199
- "source": "Kayak",
200
- "type": "flight",
201
- "details": price_text.strip(),
202
- "price": price,
203
- "link": url
204
  })
205
-
206
  except Exception as e:
207
  logging.warning(f"Error parsing Kayak card: {e}")
208
  continue
209
-
210
  await browser.close()
211
-
212
  except Exception as e:
213
  logging.error(f"Kayak scraping failed: {e}")
214
-
215
  return results
216
 
217
  async def _search_trabber(details):
@@ -221,22 +203,16 @@ async def _search_trabber(details):
221
  browser = await pw.chromium.launch(headless=True)
222
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
223
  page = await context.new_page()
224
-
225
  await asyncio.sleep(random.uniform(1, 3))
226
-
227
  o = details["origin_city"].upper()
228
  d = details["destination_city"].upper()
229
  sd_parts = details["start_date"].split('-')
230
- sd = f"{sd_parts[2]}{sd_parts[1]}{sd_parts[0][2:]}" # DDMMYY format
231
-
232
  url = f"https://www.trabber.ca/flights-from-{o}-to-{d}-on-{sd}"
233
-
234
  await page.goto(url, timeout=60000)
235
-
236
  try:
237
  await page.wait_for_selector("#results_list_det tr", timeout=20000)
238
  rows = await page.query_selector_all("#results_list_det tr")
239
-
240
  for row in rows[:3]:
241
  try:
242
  price_el = await row.query_selector("td.results_price a, .results_price")
@@ -244,26 +220,18 @@ async def _search_trabber(details):
244
  price_text = await price_el.inner_text()
245
  price_clean = ''.join(filter(str.isdigit, price_text))
246
  price = float(price_clean) if price_clean else 0.0
247
-
248
  results.append({
249
- "source": "Trabber",
250
- "type": "flight",
251
- "details": price_text.strip(),
252
- "price": price,
253
- "link": url
254
  })
255
  except Exception as e:
256
  logging.warning(f"Error parsing Trabber row: {e}")
257
  continue
258
-
259
  except Exception as e:
260
  logging.warning(f"Trabber selector timeout: {e}")
261
-
262
  await browser.close()
263
-
264
  except Exception as e:
265
  logging.error(f"Trabber scraping failed: {e}")
266
-
267
  return results
268
 
269
  async def _search_travala(details):
@@ -273,21 +241,11 @@ async def _search_travala(details):
273
  browser = await pw.chromium.launch(headless=True)
274
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
275
  page = await context.new_page()
276
-
277
  await asyncio.sleep(random.uniform(1, 3))
278
-
279
  url = "https://www.travala.com/deals"
280
-
281
  await page.goto(url, timeout=60000)
282
-
283
  try:
284
- # Try multiple selectors for deals
285
- selectors = [
286
- "div[class*='DealCard_container__']",
287
- ".deal-card",
288
- "[data-testid='deal-card']"
289
- ]
290
-
291
  cards = []
292
  for selector in selectors:
293
  try:
@@ -297,15 +255,9 @@ async def _search_travala(details):
297
  break
298
  except:
299
  continue
300
-
301
  for card in cards[:5]:
302
  try:
303
- title_selectors = [
304
- "p[class*='DealCard_title__']",
305
- ".deal-title",
306
- "h3, h4"
307
- ]
308
-
309
  title = "Travala Deal"
310
  for title_sel in title_selectors:
311
  try:
@@ -315,27 +267,17 @@ async def _search_travala(details):
315
  break
316
  except:
317
  continue
318
-
319
  results.append({
320
- "source": "Travala",
321
- "type": "deal",
322
- "details": title[:100],
323
- "price": 0,
324
- "link": url
325
  })
326
-
327
  except Exception as e:
328
  logging.warning(f"Error parsing Travala card: {e}")
329
  continue
330
-
331
  except Exception as e:
332
  logging.warning(f"Travala selector timeout: {e}")
333
-
334
  await browser.close()
335
-
336
  except Exception as e:
337
  logging.error(f"Travala scraping failed: {e}")
338
-
339
  return results
340
 
341
  async def _search_secretflying(details):
@@ -345,21 +287,11 @@ async def _search_secretflying(details):
345
  browser = await pw.chromium.launch(headless=True)
346
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
347
  page = await context.new_page()
348
-
349
  await asyncio.sleep(random.uniform(1, 3))
350
-
351
  url = "https://www.secretflying.com/canada-deals/"
352
-
353
  await page.goto(url, timeout=60000)
354
-
355
  try:
356
- # Try multiple selectors for posts
357
- selectors = [
358
- ".post-item",
359
- "article",
360
- ".entry"
361
- ]
362
-
363
  posts = []
364
  for selector in selectors:
365
  try:
@@ -369,19 +301,11 @@ async def _search_secretflying(details):
369
  break
370
  except:
371
  continue
372
-
373
  for post in posts[:5]:
374
  try:
375
- title_selectors = [
376
- "h2.entry-title a",
377
- ".entry-title a",
378
- "h3 a",
379
- "h2 a"
380
- ]
381
-
382
  title = "Deal"
383
  link = url
384
-
385
  for title_sel in title_selectors:
386
  try:
387
  title_el = await post.query_selector(title_sel)
@@ -393,34 +317,23 @@ async def _search_secretflying(details):
393
  break
394
  except:
395
  continue
396
-
397
  results.append({
398
- "source": "SecretFlying",
399
- "type": "deal",
400
- "details": title[:100],
401
- "price": 0,
402
- "link": link
403
  })
404
-
405
  except Exception as e:
406
  logging.warning(f"Error parsing SecretFlying post: {e}")
407
  continue
408
-
409
  except Exception as e:
410
  logging.warning(f"SecretFlying selector timeout: {e}")
411
-
412
  await browser.close()
413
-
414
  except Exception as e:
415
  logging.error(f"SecretFlying scraping failed: {e}")
416
-
417
  return results
418
 
419
-
420
  # 🔄 6. GATHER ALL DATA
421
  async def _gather_travel_data(req):
422
  tasks = [
423
- _cached_search(_search_google_flights, req),
424
  _cached_search(_search_kayak, req),
425
  _cached_search(_search_trabber, req),
426
  _cached_search(_search_travala, req),
@@ -435,12 +348,10 @@ async def _gather_travel_data(req):
435
  logging.error(f"A scraper task failed: {r}")
436
  return combined
437
 
438
-
439
  # ✍️ 7. FORMAT RESULTS
440
  def _format_response(req, data):
441
  dest = req["destination_city"]
442
  md = f"## 🌏 Travel Plan to **{dest}**\n\n"
443
-
444
  flights = sorted([i for i in data if i["type"]=="flight" and i.get("price", 0) > 0], key=lambda x: x["price"])
445
  if flights:
446
  md += "### ✈️ Flights\n"
 
1
+ # 🚀 0. BRUTE FORCE INSTALLER
2
+ import subprocess
3
+ import sys
4
+ import os
5
+
6
+ # This block forces the installation of the browser Playwright needs.
7
+ # It's a workaround for stubborn server environments.
8
+ try:
9
+ print("--- Ensuring Playwright browsers are installed ---")
10
+ # We use 'playwright install chromium' to be specific and save space.
11
+ # Using sys.executable ensures we use the python env's playwright.
12
+ process = subprocess.run(
13
+ [sys.executable, "-m", "playwright", "install", "chromium"],
14
+ capture_output=True,
15
+ text=True,
16
+ check=True
17
+ )
18
+ print(process.stdout)
19
+ print("--- Playwright browsers installation check complete ---")
20
+ except Exception as e:
21
+ print(f"--- An error occurred during browser installation: {e} ---")
22
+ # Even if it fails, we try to continue.
23
+ pass
24
+
25
  # 🚀 1. IMPORTS & SETUP
26
  import asyncio
27
  import datetime
28
  import logging
 
29
  import random
30
  import gradio as gr
31
  from cachetools import TTLCache
 
67
  ("human", "{format_instructions}\n{request}")
68
  ]).partial(format_instructions=parser.get_format_instructions())
69
  logging.info(f"🔍 Extracting details from: {question}")
70
+ # Default to a one-way trip if end_date isn't found
71
+ extracted = await (prompt | llm | parser).ainvoke({"request": question})
72
+ if not extracted.get("end_date"):
73
+ start_dt = datetime.datetime.strptime(extracted["start_date"], "%Y-%m-%d")
74
+ end_dt = start_dt + datetime.timedelta(days=7) # Default to 7 days later
75
+ extracted["end_date"] = end_dt.strftime("%Y-%m-%d")
76
+ return extracted
77
 
78
 
79
  # 🛠 4. CACHE WRAPPER
 
90
 
91
  # 🌐 5. PLAYWRIGHT SCRAPER FUNCTIONS
92
 
93
+ async def _search_Google Flights(details):
94
  results = []
95
  try:
96
  async with async_playwright() as pw:
 
98
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
99
  page = await context.new_page()
100
 
 
 
 
101
  o = details["origin_city"]
102
  d = details["destination_city"]
103
  sd = details["start_date"]
104
  ed = details["end_date"]
105
 
 
106
  url = f"https://www.google.com/travel/flights/search?tfs=CBwQAhokEgoyMDI1LTA4LTAxagwIAhIIL20vMDVxdGwyDAg"
107
 
108
  await page.goto(url, timeout=60000)
109
 
 
110
  try:
111
  await page.wait_for_selector('[data-testid="flight-offer"]', timeout=20000)
112
  cards = await page.query_selector_all('[data-testid="flight-offer"]')
 
114
  for card in cards[:3]:
115
  try:
116
  price_el = await card.query_selector('[data-testid="price-text"]')
 
 
 
117
  if price_el:
118
  price_text = await price_el.inner_text()
119
  price_clean = ''.join(filter(str.isdigit, price_text))
 
129
  except Exception as e:
130
  logging.warning(f"Error parsing Google Flights card: {e}")
131
  continue
 
132
  except Exception as e:
133
  logging.warning(f"Google Flights selector timeout: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  await browser.close()
 
135
  except Exception as e:
136
  logging.error(f"Google Flights scraping failed: {e}")
 
137
  return results
138
 
139
+ # ... (rest of the scraper functions remain the same)
140
  async def _search_kayak(details):
141
  results = []
142
  try:
 
156
 
157
  await page.goto(url, timeout=60000)
158
 
159
+ selectors_to_try = [".resultWrapper", "[data-testid='result-item']", ".Common-Booking-MultiBookProvider"]
 
 
 
 
 
 
160
 
161
  cards = []
162
  for selector in selectors_to_try:
 
170
 
171
  for card in cards[:3]:
172
  try:
173
+ price_selectors = [".price-text", "[data-testid='price']", ".Common-Booking-MultiBookProvider-price"]
 
 
 
 
 
 
 
174
  price_text = None
175
  for price_sel in price_selectors:
176
  try:
 
184
  if price_text:
185
  price_clean = ''.join(filter(str.isdigit, price_text))
186
  price = float(price_clean) if price_clean else 0.0
 
187
  results.append({
188
+ "source": "Kayak", "type": "flight", "details": price_text.strip(),
189
+ "price": price, "link": url
 
 
 
190
  })
 
191
  except Exception as e:
192
  logging.warning(f"Error parsing Kayak card: {e}")
193
  continue
 
194
  await browser.close()
 
195
  except Exception as e:
196
  logging.error(f"Kayak scraping failed: {e}")
 
197
  return results
198
 
199
  async def _search_trabber(details):
 
203
  browser = await pw.chromium.launch(headless=True)
204
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
205
  page = await context.new_page()
 
206
  await asyncio.sleep(random.uniform(1, 3))
 
207
  o = details["origin_city"].upper()
208
  d = details["destination_city"].upper()
209
  sd_parts = details["start_date"].split('-')
210
+ sd = f"{sd_parts[2]}{sd_parts[1]}{sd_parts[0][2:]}"
 
211
  url = f"https://www.trabber.ca/flights-from-{o}-to-{d}-on-{sd}"
 
212
  await page.goto(url, timeout=60000)
 
213
  try:
214
  await page.wait_for_selector("#results_list_det tr", timeout=20000)
215
  rows = await page.query_selector_all("#results_list_det tr")
 
216
  for row in rows[:3]:
217
  try:
218
  price_el = await row.query_selector("td.results_price a, .results_price")
 
220
  price_text = await price_el.inner_text()
221
  price_clean = ''.join(filter(str.isdigit, price_text))
222
  price = float(price_clean) if price_clean else 0.0
 
223
  results.append({
224
+ "source": "Trabber", "type": "flight", "details": price_text.strip(),
225
+ "price": price, "link": url
 
 
 
226
  })
227
  except Exception as e:
228
  logging.warning(f"Error parsing Trabber row: {e}")
229
  continue
 
230
  except Exception as e:
231
  logging.warning(f"Trabber selector timeout: {e}")
 
232
  await browser.close()
 
233
  except Exception as e:
234
  logging.error(f"Trabber scraping failed: {e}")
 
235
  return results
236
 
237
  async def _search_travala(details):
 
241
  browser = await pw.chromium.launch(headless=True)
242
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
243
  page = await context.new_page()
 
244
  await asyncio.sleep(random.uniform(1, 3))
 
245
  url = "https://www.travala.com/deals"
 
246
  await page.goto(url, timeout=60000)
 
247
  try:
248
+ selectors = ["div[class*='DealCard_container__']", ".deal-card", "[data-testid='deal-card']"]
 
 
 
 
 
 
249
  cards = []
250
  for selector in selectors:
251
  try:
 
255
  break
256
  except:
257
  continue
 
258
  for card in cards[:5]:
259
  try:
260
+ title_selectors = ["p[class*='DealCard_title__']", ".deal-title", "h3, h4"]
 
 
 
 
 
261
  title = "Travala Deal"
262
  for title_sel in title_selectors:
263
  try:
 
267
  break
268
  except:
269
  continue
 
270
  results.append({
271
+ "source": "Travala", "type": "deal", "details": title[:100], "price": 0, "link": url
 
 
 
 
272
  })
 
273
  except Exception as e:
274
  logging.warning(f"Error parsing Travala card: {e}")
275
  continue
 
276
  except Exception as e:
277
  logging.warning(f"Travala selector timeout: {e}")
 
278
  await browser.close()
 
279
  except Exception as e:
280
  logging.error(f"Travala scraping failed: {e}")
 
281
  return results
282
 
283
  async def _search_secretflying(details):
 
287
  browser = await pw.chromium.launch(headless=True)
288
  context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
289
  page = await context.new_page()
 
290
  await asyncio.sleep(random.uniform(1, 3))
 
291
  url = "https://www.secretflying.com/canada-deals/"
 
292
  await page.goto(url, timeout=60000)
 
293
  try:
294
+ selectors = [".post-item", "article", ".entry"]
 
 
 
 
 
 
295
  posts = []
296
  for selector in selectors:
297
  try:
 
301
  break
302
  except:
303
  continue
 
304
  for post in posts[:5]:
305
  try:
306
+ title_selectors = ["h2.entry-title a", ".entry-title a", "h3 a", "h2 a"]
 
 
 
 
 
 
307
  title = "Deal"
308
  link = url
 
309
  for title_sel in title_selectors:
310
  try:
311
  title_el = await post.query_selector(title_sel)
 
317
  break
318
  except:
319
  continue
 
320
  results.append({
321
+ "source": "SecretFlying", "type": "deal", "details": title[:100], "price": 0, "link": link
 
 
 
 
322
  })
 
323
  except Exception as e:
324
  logging.warning(f"Error parsing SecretFlying post: {e}")
325
  continue
 
326
  except Exception as e:
327
  logging.warning(f"SecretFlying selector timeout: {e}")
 
328
  await browser.close()
 
329
  except Exception as e:
330
  logging.error(f"SecretFlying scraping failed: {e}")
 
331
  return results
332
 
 
333
  # 🔄 6. GATHER ALL DATA
334
  async def _gather_travel_data(req):
335
  tasks = [
336
+ _cached_search(_search_Google Flights, req),
337
  _cached_search(_search_kayak, req),
338
  _cached_search(_search_trabber, req),
339
  _cached_search(_search_travala, req),
 
348
  logging.error(f"A scraper task failed: {r}")
349
  return combined
350
 
 
351
  # ✍️ 7. FORMAT RESULTS
352
  def _format_response(req, data):
353
  dest = req["destination_city"]
354
  md = f"## 🌏 Travel Plan to **{dest}**\n\n"
 
355
  flights = sorted([i for i in data if i["type"]=="flight" and i.get("price", 0) > 0], key=lambda x: x["price"])
356
  if flights:
357
  md += "### ✈️ Flights\n"