Update app.py

app.py

@@ -1,22 +1,52 @@
-# 🚀 0. BRUTE FORCE INSTALLER
+# 🚀 0. IMPROVED BRUTE FORCE INSTALLER
 import subprocess
 import sys
 import os
 
-# This block forces the installation of the browser Playwright needs.
-try:
-    print("--- Ensuring Playwright browsers are installed ---")
-    process = subprocess.run(
+def install_playwright_browsers():
+    """More robust Playwright browser installation"""
+    commands_to_try = [
+        # Try installing all browsers first
+        [sys.executable, "-m", "playwright", "install"],
+        # Then try just chromium
         [sys.executable, "-m", "playwright", "install", "chromium"],
-        capture_output=True,
-        text=True,
-        check=True
-    )
-    print(process.stdout)
-    print("--- Playwright browsers installation check complete ---")
-except Exception as e:
-    print(f"--- An error occurred during browser installation: {e} ---")
-    pass
+        # Try with --with-deps flag
+        [sys.executable, "-m", "playwright", "install", "--with-deps", "chromium"],
+        # Try installing webkit as backup
+        [sys.executable, "-m", "playwright", "install", "webkit"],
+    ]
+    
+    for i, cmd in enumerate(commands_to_try):
+        try:
+            print(f"--- Attempt {i+1}: {' '.join(cmd)} ---")
+            process = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout
+                check=True
+            )
+            print(process.stdout)
+            if process.stderr:
+                print("STDERR:", process.stderr)
+            print(f"--- Attempt {i+1} succeeded ---")
+            return True
+        except subprocess.TimeoutExpired:
+            print(f"--- Attempt {i+1} timed out ---")
+            continue
+        except Exception as e:
+            print(f"--- Attempt {i+1} failed: {e} ---")
+            continue
+    
+    print("--- All installation attempts failed ---")
+    return False
+
+# Run the installation
+print("--- Starting Playwright browser installation ---")
+success = install_playwright_browsers()
+if not success:
+    print("--- WARNING: Browser installation failed, scrapers may not work ---")
+print("--- Installation process complete ---")
 
 # 🚀 1. IMPORTS & SETUP
 import asyncio
@@ -27,7 +57,7 @@ import gradio as gr
 from cachetools import TTLCache
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field  # Fixed: Using pydantic directly instead of pydantic_v1
 from langchain_openai import ChatOpenAI
 from playwright.async_api import async_playwright
 
@@ -41,6 +71,7 @@ ttl_cache = TTLCache(maxsize=100, ttl=3600)
 USER_AGENTS = [
     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
 ]
 
 # 📦 2. DATA MODEL
@@ -52,24 +83,23 @@ class TravelRequest(BaseModel):
     budget: int = Field(description="Budget in USD (informational only)")
     interests: list[str] = Field(description="List of travel interests")
 
-
 # ✨ 3. EXTRACT USER REQUEST
 async def _extract_user_request(question: str) -> dict:
     llm = ChatOpenAI(model="gpt-4o", temperature=0)
     parser = JsonOutputParser(pydantic_object=TravelRequest)
     prompt = ChatPromptTemplate.from_messages([
-        ("system", f"You are a travel assistant. Extract flight details from the user's request. Assume the current year is {datetime.datetime.now().year} unless specified otherwise. Today's date is {datetime.date.today().isoformat()}."),
+        ("system", "You are a travel assistant. Extract flight details from the user's request. Assume the current year is 2025 unless specified otherwise. Today's date is July 8, 2025."),
         ("human", "{format_instructions}\n{request}")
     ]).partial(format_instructions=parser.get_format_instructions())
     logging.info(f"🔍 Extracting details from: {question}")
+    # Default to a one-way trip if end_date isn't found
     extracted = await (prompt | llm | parser).ainvoke({"request": question})
-    if not extracted.get("end_date") and extracted.get("start_date"):
+    if not extracted.get("end_date"):
         start_dt = datetime.datetime.strptime(extracted["start_date"], "%Y-%m-%d")
-        end_dt = start_dt + datetime.timedelta(days=7) # Default to a 7-day trip
+        end_dt = start_dt + datetime.timedelta(days=7) # Default to 7 days later
         extracted["end_date"] = end_dt.strftime("%Y-%m-%d")
     return extracted
 
-
 # 🛠 4. CACHE WRAPPER
 async def _cached_search(func, details):
     key = (func.__name__, details['origin_city'], details['destination_city'], details['start_date'])
@@ -81,36 +111,106 @@ async def _cached_search(func, details):
     ttl_cache[key] = res
     return res
 
-
 # 🌐 5. PLAYWRIGHT SCRAPER FUNCTIONS
 
-async def _search_Google Flights(details):
+async def _get_browser_and_page(pw):
+    """Helper function to get browser and page with fallback options"""
+    browser_types = [
+        ('webkit', pw.webkit),
+        ('chromium', pw.chromium),
+        ('firefox', pw.firefox)
+    ]
+    
+    for browser_name, browser_type in browser_types:
+        try:
+            browser = await browser_type.launch(
+                headless=True,
+                args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
+            )
+            context = await browser.new_context(
+                user_agent=random.choice(USER_AGENTS),
+                viewport={'width': 1920, 'height': 1080}
+            )
+            page = await context.new_page()
+            logging.info(f"✅ Successfully launched {browser_name} browser")
+            return browser, page
+        except Exception as e:
+            logging.warning(f"❌ Failed to launch {browser_name}: {e}")
+            continue
+    
+    raise Exception("❌ All browser types failed to launch")
+
+async def _search_google_flights(details):
     results = []
     try:
         async with async_playwright() as pw:
-            browser = await pw.chromium.launch(headless=True)
-            context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
-            page = await context.new_page()
+            browser, page = await _get_browser_and_page(pw)
             
             o = details["origin_city"]
             d = details["destination_city"]
             sd = details["start_date"]
             ed = details["end_date"]
             
-            url = f"https://www.google.com/travel/flights/search?tfs=CBwQAhokEgoyMDI1LTA4LTAxagwIAhIIL20vMDVxdGwyDAg"
+            # More robust Google Flights URL construction
+            base_url = "https://www.google.com/travel/flights"
+            url = f"{base_url}?q=flights+from+{o}+to+{d}+on+{sd}"
+            
             await page.goto(url, timeout=60000)
+            await asyncio.sleep(3)  # Wait for page load
+            
+            # Multiple selectors to try
+            selectors_to_try = [
+                '[data-testid="flight-offer"]',
+                '.pIav2d',
+                '.yR1fYc',
+                '.JMc5Xc'
+            ]
+            
+            cards = []
+            for selector in selectors_to_try:
+                try:
+                    await page.wait_for_selector(selector, timeout=10000)
+                    cards = await page.query_selector_all(selector)
+                    if cards:
+                        logging.info(f"✅ Found {len(cards)} Google Flights results with selector: {selector}")
+                        break
+                except:
+                    continue
+            
+            for card in cards[:3]:
+                try:
+                    price_selectors = [
+                        '[data-testid="price-text"]',
+                        '.YMlIz',
+                        '.U3gSDe',
+                        'span[aria-label*="dollars"]'
+                    ]
+                    
+                    price_text = None
+                    for price_sel in price_selectors:
+                        try:
+                            price_el = await card.query_selector(price_sel)
+                            if price_el:
+                                price_text = await price_el.inner_text()
+                                break
+                        except:
+                            continue
+                    
+                    if price_text:
+                        price_clean = ''.join(filter(str.isdigit, price_text))
+                        price = float(price_clean) if price_clean else 0.0
+                        
+                        results.append({
+                            "source": "Google Flights", 
+                            "type": "flight", 
+                            "details": price_text.strip(),
+                            "price": price,
+                            "link": url
+                        })
+                except Exception as e:
+                    logging.warning(f"Error parsing Google Flights card: {e}")
+                    continue
             
-            await page.wait_for_selector('div[role="main"]', timeout=20000)
-            price_elements = await page.query_selector_all('span[aria-label*="dollars"]')
-            
-            for elem in price_elements[:3]:
-                price_text = await elem.inner_text()
-                price_clean = ''.join(filter(str.isdigit, price_text))
-                if price_clean:
-                    results.append({
-                        "source": "Google Flights", "type": "flight", "details": price_text.strip(),
-                        "price": float(price_clean) / 100, "link": url
-                    })
             await browser.close()
     except Exception as e:
         logging.error(f"Google Flights scraping failed: {e}")
@@ -120,9 +220,9 @@ async def _search_kayak(details):
     results = []
     try:
         async with async_playwright() as pw:
-            browser = await pw.chromium.launch(headless=True)
-            context = await browser.new_context(user_agent=random.choice(USER_AGENTS))
-            page = await context.new_page()
+            browser, page = await _get_browser_and_page(pw)
+            
+            await asyncio.sleep(random.uniform(1, 3))
             
             o = details["origin_city"].upper()
             d = details["destination_city"].upper()
@@ -130,34 +230,296 @@ async def _search_kayak(details):
             ed = details["end_date"]
             
             url = f"https://www.kayak.com/flights/{o}-{d}/{sd}/{ed}"
+            
             await page.goto(url, timeout=60000)
+            await asyncio.sleep(5)  # Wait for results to load
+            
+            selectors_to_try = [
+                ".resultWrapper", 
+                "[data-testid='result-item']", 
+                ".Common-Booking-MultiBookProvider",
+                ".item.flights",
+                ".resultInner"
+            ]
             
-            await page.wait_for_selector(".resultWrapper", timeout=20000)
-            cards = await page.query_selector_all(".resultWrapper")
+            cards = []
+            for selector in selectors_to_try:
+                try:
+                    await page.wait_for_selector(selector, timeout=15000)
+                    cards = await page.query_selector_all(selector)
+                    if cards:
+                        logging.info(f"✅ Found {len(cards)} Kayak results with selector: {selector}")
+                        break
+                except:
+                    continue
             
             for card in cards[:3]:
-                price_el = await card.query_selector(".price-text")
-                if price_el:
-                    price_text = await price_el.inner_text()
-                    price_clean = ''.join(filter(str.isdigit, price_text))
-                    if price_clean:
+                try:
+                    price_selectors = [
+                        ".price-text", 
+                        "[data-testid='price']", 
+                        ".Common-Booking-MultiBookProvider-price",
+                        ".price",
+                        ".f8F1-price-text"
+                    ]
+                    
+                    price_text = None
+                    for price_sel in price_selectors:
+                        try:
+                            price_el = await card.query_selector(price_sel)
+                            if price_el:
+                                price_text = await price_el.inner_text()
+                                break
+                        except:
+                            continue
+                    
+                    if price_text:
+                        price_clean = ''.join(filter(str.isdigit, price_text))
+                        price = float(price_clean) if price_clean else 0.0
                         results.append({
-                            "source": "Kayak", "type": "flight", "details": price_text.strip(),
-                            "price": float(price_clean), "link": url
+                            "source": "Kayak", 
+                            "type": "flight", 
+                            "details": price_text.strip(),
+                            "price": price, 
+                            "link": url
                         })
+                except Exception as e:
+                    logging.warning(f"Error parsing Kayak card: {e}")
+                    continue
+            
             await browser.close()
     except Exception as e:
         logging.error(f"Kayak scraping failed: {e}")
     return results
 
-# ... other scrapers ...
+async def _search_trabber(details):
+    results = []
+    try:
+        async with async_playwright() as pw:
+            browser, page = await _get_browser_and_page(pw)
+            
+            await asyncio.sleep(random.uniform(1, 3))
+            
+            o = details["origin_city"].upper()
+            d = details["destination_city"].upper()
+            sd_parts = details["start_date"].split('-')
+            sd = f"{sd_parts[2]}{sd_parts[1]}{sd_parts[0][2:]}"
+            
+            url = f"https://www.trabber.ca/flights-from-{o}-to-{d}-on-{sd}"
+            
+            await page.goto(url, timeout=60000)
+            await asyncio.sleep(3)
+            
+            try:
+                selectors_to_try = [
+                    "#results_list_det tr",
+                    ".results_row",
+                    "tr[class*='result']"
+                ]
+                
+                rows = []
+                for selector in selectors_to_try:
+                    try:
+                        await page.wait_for_selector(selector, timeout=15000)
+                        rows = await page.query_selector_all(selector)
+                        if rows:
+                            logging.info(f"✅ Found {len(rows)} Trabber results")
+                            break
+                    except:
+                        continue
+                
+                for row in rows[:3]:
+                    try:
+                        price_selectors = [
+                            "td.results_price a", 
+                            ".results_price",
+                            "td[class*='price']"
+                        ]
+                        
+                        price_text = None
+                        for price_sel in price_selectors:
+                            try:
+                                price_el = await row.query_selector(price_sel)
+                                if price_el:
+                                    price_text = await price_el.inner_text()
+                                    break
+                            except:
+                                continue
+                        
+                        if price_text:
+                            price_clean = ''.join(filter(str.isdigit, price_text))
+                            price = float(price_clean) if price_clean else 0.0
+                            results.append({
+                                "source": "Trabber", 
+                                "type": "flight", 
+                                "details": price_text.strip(),
+                                "price": price, 
+                                "link": url
+                            })
+                    except Exception as e:
+                        logging.warning(f"Error parsing Trabber row: {e}")
+                        continue
+            except Exception as e:
+                logging.warning(f"Trabber selector timeout: {e}")
+            
+            await browser.close()
+    except Exception as e:
+        logging.error(f"Trabber scraping failed: {e}")
+    return results
+
+async def _search_travala(details):
+    results = []
+    try:
+        async with async_playwright() as pw:
+            browser, page = await _get_browser_and_page(pw)
+            
+            await asyncio.sleep(random.uniform(1, 3))
+            
+            url = "https://www.travala.com/deals"
+            
+            await page.goto(url, timeout=60000)
+            await asyncio.sleep(3)
+            
+            try:
+                selectors = [
+                    "div[class*='DealCard_container__']", 
+                    ".deal-card", 
+                    "[data-testid='deal-card']",
+                    ".deal-item",
+                    "[class*='deal']"
+                ]
+                
+                cards = []
+                for selector in selectors:
+                    try:
+                        await page.wait_for_selector(selector, timeout=10000)
+                        cards = await page.query_selector_all(selector)
+                        if cards:
+                            logging.info(f"✅ Found {len(cards)} Travala deals")
+                            break
+                    except:
+                        continue
+                
+                for card in cards[:5]:
+                    try:
+                        title_selectors = [
+                            "p[class*='DealCard_title__']", 
+                            ".deal-title", 
+                            "h3, h4",
+                            "[class*='title']"
+                        ]
+                        
+                        title = "Travala Deal"
+                        for title_sel in title_selectors:
+                            try:
+                                title_el = await card.query_selector(title_sel)
+                                if title_el:
+                                    title = await title_el.inner_text()
+                                    break
+                            except:
+                                continue
+                        
+                        results.append({
+                            "source": "Travala", 
+                            "type": "deal", 
+                            "details": title[:100], 
+                            "price": 0, 
+                            "link": url
+                        })
+                    except Exception as e:
+                        logging.warning(f"Error parsing Travala card: {e}")
+                        continue
+            except Exception as e:
+                logging.warning(f"Travala selector timeout: {e}")
+            
+            await browser.close()
+    except Exception as e:
+        logging.error(f"Travala scraping failed: {e}")
+    return results
+
+async def _search_secretflying(details):
+    results = []
+    try:
+        async with async_playwright() as pw:
+            browser, page = await _get_browser_and_page(pw)
+            
+            await asyncio.sleep(random.uniform(1, 3))
+            
+            url = "https://www.secretflying.com/canada-deals/"
+            
+            await page.goto(url, timeout=60000)
+            await asyncio.sleep(3)
+            
+            try:
+                selectors = [
+                    ".post-item", 
+                    "article", 
+                    ".entry",
+                    ".post",
+                    "[class*='post']"
+                ]
+                
+                posts = []
+                for selector in selectors:
+                    try:
+                        await page.wait_for_selector(selector, timeout=10000)
+                        posts = await page.query_selector_all(selector)
+                        if posts:
+                            logging.info(f"✅ Found {len(posts)} SecretFlying deals")
+                            break
+                    except:
+                        continue
+                
+                for post in posts[:5]:
+                    try:
+                        title_selectors = [
+                            "h2.entry-title a", 
+                            ".entry-title a", 
+                            "h3 a", 
+                            "h2 a",
+                            ".title a"
+                        ]
+                        
+                        title = "Deal"
+                        link = url
+                        for title_sel in title_selectors:
+                            try:
+                                title_el = await post.query_selector(title_sel)
+                                if title_el:
+                                    title = await title_el.inner_text()
+                                    href = await title_el.get_attribute("href")
+                                    if href:
+                                        link = href if href.startswith('http') else f"https://www.secretflying.com{href}"
+                                    break
+                            except:
+                                continue
+                        
+                        results.append({
+                            "source": "SecretFlying", 
+                            "type": "deal", 
+                            "details": title[:100], 
+                            "price": 0, 
+                            "link": link
+                        })
+                    except Exception as e:
+                        logging.warning(f"Error parsing SecretFlying post: {e}")
+                        continue
+            except Exception as e:
+                logging.warning(f"SecretFlying selector timeout: {e}")
+            
+            await browser.close()
+    except Exception as e:
+        logging.error(f"SecretFlying scraping failed: {e}")
+    return results
 
 # 🔄 6. GATHER ALL DATA
 async def _gather_travel_data(req):
     tasks = [
-        _cached_search(_search_Google Flights, req),
+        _cached_search(_search_google_flights, req),
         _cached_search(_search_kayak, req),
-        # Add other scrapers here as we build them
+        _cached_search(_search_trabber, req),
+        _cached_search(_search_travala, req),
+        _cached_search(_search_secretflying, req),
     ]
     results = await asyncio.gather(*tasks, return_exceptions=True)
     combined = []
@@ -168,46 +530,69 @@ async def _gather_travel_data(req):
             logging.error(f"A scraper task failed: {r}")
     return combined
 
-
 # ✍️ 7. FORMAT RESULTS
 def _format_response(req, data):
     dest = req["destination_city"]
     md = f"## 🌏 Travel Plan to **{dest}**\n\n"
+    
     flights = sorted([i for i in data if i["type"]=="flight" and i.get("price", 0) > 0], key=lambda x: x["price"])
     if flights:
         md += "### ✈️ Flights\n"
         for item in flights[:8]:
             md += f"- **{item['source']}**: `{item['details']}` — **${item['price']:.2f}** ([Book]({item['link']}))\n"
-    else:
+    
+    deals = [i for i in data if i["type"]=="deal"]
+    if deals:
+        md += "\n### 💸 Hot Deals\n"
+        for item in deals[:8]:
+            md += f"- **{item['source']}**: {item['details']} ([Link]({item['link']}))\n"
+    
+    if not flights and not deals:
         md += "No results found. The websites may be blocking our scrapers or there are no flights available for those dates.\n\n"
         md += "**Debug Info:**\n"
         md += f"- Searched for: {req['origin_city']} → {req['destination_city']}\n"
         md += f"- Dates: {req['start_date']} to {req['end_date']}\n"
+        md += f"- Total scrapers attempted: 5\n"
+        md += f"- Browser installation status: {'✅ Success' if success else '❌ Failed'}\n"
 
     return md
 
-
 # 💬 8. MAIN BOT & UI
 async def ask_bot(question):
     if not question:
         return "❓ Tell me where you want to go!"
     try:
         req = await _extract_user_request(question)
-        if not req.get('origin_city') or not req.get('destination_city') or not req.get('start_date'):
-            return "🤔 I couldn't understand all the details. Please provide at least a starting city, a destination, and a date."
         all_data = await _gather_travel_data(req)
         return _format_response(req, all_data)
     except Exception as e:
         logging.error(f"An error occurred in the main process: {e}")
-        return f"😵 Oh no! Something went wrong: {str(e)}\n\nPlease try asking in a different way."
+        return f"😵 Oh no! Something went wrong: {str(e)}\n\nPlease try asking in a different way or check if your OpenAI API key is set correctly."
 
+# 🎨 9. GRADIO INTERFACE
 iface = gr.Interface(
     fn=ask_bot,
-    inputs=gr.Textbox(lines=4, label="🌍 Your dream trip?", placeholder="e.g., I want to fly from Halifax to Toronto for a week starting August 1st"),
+    inputs=gr.Textbox(
+        lines=4, 
+        label="🌍 Your dream trip?", 
+        placeholder="e.g., I want to fly from Halifax to Toronto for a week starting August 1st",
+        info="Tell me where you want to go, when, and from where!"
+    ),
     outputs=gr.Markdown(label="🚀 Your Travel Plan"),
     title="🛫 High Flyer AI Bot",
-    description="Your personal AI travel agent. Uses Playwright to scrape multiple sources in real-time."
+    description="Your personal AI travel agent. Uses multiple browsers to scrape flight prices and deals in real-time.",
+    theme=gr.themes.Soft(),
+    examples=[
+        ["I want to fly from Halifax to Tokyo tomorrow after 3 pm"],
+        ["Find me cheap flights from YHZ to London UK next month"],
+        ["I need a flight from Toronto to New York on December 15th, budget $500"]
+    ]
 )
 
 if __name__ == "__main__":
-    iface.launch(server_name="0.0.0.0", server_port=7860)
+    iface.launch(
+        server_name="0.0.0.0", 
+        server_port=7860,
+        share=False,
+        show_error=True
+    )