utils/fetch_articles.py

# This must be run locally only. 
#!/usr/bin/env python3
"""
VRBO Help Center Scraper - With Customer Type Filtering for RAG
Separates articles by user type: Traveler, Owner, Property Manager
Optimized for LangChain cosine similarity retrieval.
"""

import json
import time
import re
from collections import defaultdict
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager


class VRBOScraper:
    def __init__(self):
        self.base_url = "https://help.vrbo.com"
        self.articles = []          # All articles with metadata
        self.driver = None

    def start_browser(self):
        """Start headless Chrome."""
        print("🚀 Starting Chrome...")
        options = Options()
        options.add_argument("--headless")
        options.add_argument("--disable-gpu")
        options.add_argument("--window-size=1920,1080")
        options.add_argument("--disable-blink-features=AutomationControlled")
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)

        service = Service(ChromeDriverManager().install())
        self.driver = webdriver.Chrome(service=service, options=options)
        self.driver.execute_script(
            "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
        )
        print("✅ Browser ready")

    def load_page(self, url, wait_seconds=2):
        """Load page and wait for JS to render."""
        self.driver.get(url)
        time.sleep(wait_seconds)
        return self.driver.page_source

    def get_categories(self):
        """Extract category tree with customer types from portalConfigJSON."""
        html = self.load_page(self.base_url)
        match = re.search(r'_globals\.portalConfigJSON\s*=\s*(\{.*?\});', html, re.DOTALL)
        if not match:
            print("❌ Could not find portalConfigJSON")
            return []

        config = json.loads(match.group(1))
        categories = config.get('categories', [])
        print(f"✅ Found {len(categories)} categories")
        return categories

    def get_articles_from_page(self, url, category_name, customer_types, subcategory_name=None):
        """
        Get article links from a loaded category/section page.
        customer_types: list like ['Traveler'], ['Owner'], ['Owner', 'Property Manager'], or None
        """
        self.load_page(url)

        articles = []
        links = self.driver.find_elements(By.CSS_SELECTOR, "a[href*='/articles/']")

        for link in links:
            href = link.get_attribute('href')
            title = link.text.strip()
            if href and title and '/articles/' in href:
                articles.append({
                    'url': href,
                    'title': title,
                    'category': category_name,
                    'subcategory': subcategory_name,
                    'customer_types': customer_types  # <-- KEY: store user types
                })

        # Remove duplicates
        seen = set()
        unique = []
        for a in articles:
            if a['url'] not in seen:
                seen.add(a['url'])
                unique.append(a)

        return unique

    def fetch_article(self, article):
        """Fetch full content of a single article."""
        print(f"      📝 {article['title'][:60]}...")
        self.load_page(article['url'])

        # Get title
        try:
            title = self.driver.find_element(By.TAG_NAME, 'h1').text.strip()
        except:
            title = article['title']

        # Get content
        content = ""
        for selector in ['.article-body', '.article-content', 'article', 'main', '[class*="article-body"]']:
            try:
                elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                text = elem.text.strip()
                if len(text) > 100:
                    content = text
                    break
            except:
                continue

        # Extract ID from URL
        id_match = re.search(r'/articles/(\d+)', article['url'])
        article_id = id_match.group(1) if id_match else str(hash(article['url']) % 100000000)

        return {
            'id': article_id,
            'url': article['url'],
            'title': title,
            'content': content,
            'category': article['category'],
            'subcategory': article.get('subcategory'),
            'customer_types': article.get('customer_types')  # <-- preserved
        } if len(content) > 100 else None

    def run(self):
        """Main workflow."""
        print("=" * 60)
        print("VRBO Help Center Scraper - With User Type Filtering")
        print("=" * 60)

        self.start_browser()

        try:
            # Step 1: Get categories with customer types
            categories = self.get_categories()

            # Step 2: Collect all article links WITH customer type info
            all_links = []

            for cat in categories:
                cat_name = cat['name']
                cat_value = cat['value']
                cat_types = cat.get('customerTypes')  # e.g., ['Traveler'] or None

                cat_url = f"{self.base_url}/category/{cat_value}"
                print(f"📂 {cat_name} (types: {cat_types})")
                links = self.get_articles_from_page(cat_url, cat_name, cat_types)
                print(f"   {len(links)} articles")
                all_links.extend(links)

                # Subcategories
                for child in cat.get('children', []):
                    child_name = child['name']
                    child_value = child['value']
                    child_types = child.get('customerTypes')  # may override parent types

                    child_url = f"{self.base_url}/category/{child_value}"
                    print(f"   📁 {child_name} (types: {child_types})")
                    links = self.get_articles_from_page(child_url, cat_name, child_types, child_name)
                    print(f"      {len(links)} articles")
                    all_links.extend(links)

            # Remove duplicates (same article might appear in multiple categories)
            seen = set()
            unique_links = []
            for a in all_links:
                if a['url'] not in seen:
                    seen.add(a['url'])
                    unique_links.append(a)

            print(f"{'=' * 60}")
            print(f"📊 Total unique articles: {len(unique_links)}")
            print(f"{'=' * 60}")

            # Step 3: Fetch each article's content
            for i, link in enumerate(unique_links, 1):
                print(f"[{i}/{len(unique_links)}]", end="")
                result = self.fetch_article(link)
                if result:
                    self.articles.append(result)
                    print(f" ✅ ({len(result['content'])} chars)")
                else:
                    print(f" ❌ Empty/failed")

            # Step 4: Save outputs
            print(f"{'=' * 60}")
            print(f"📈 Results: {len(self.articles)} successful")
            print(f"{'=' * 60}")

            self.save_outputs()

        finally:
            self.driver.quit()
            print("🛑 Browser closed")

    def save_outputs(self):
        """
        Save articles in multiple formats for RAG flexibility:
        1. vrbo_articles_all.json - Everything in one file
        2. vrbo_articles_by_type.json - Grouped by customer type
        3. vrbo_articles_traveler.json - Traveler only
        4. vrbo_articles_owner.json - Owner only
        5. vrbo_articles_pm.json - Property Manager only
        """

        # --- 1. All articles in one file ---
        output_all = {
            'source': 'help.vrbo.com',
            'total_articles': len(self.articles),
            'categories': list(set(a['category'] for a in self.articles)),
            'articles': self.articles
        }
        with open('vrbo_articles_all.json', 'w', encoding='utf-8') as f:
            json.dump(output_all, f, indent=2, ensure_ascii=False)
        print("💾 Saved: vrbo_articles_all.json")

        # --- 2. Grouped by customer type ---
        # An article with types=['Traveler','Owner'] goes into BOTH groups
        by_type = defaultdict(list)

        for article in self.articles:
            types = article.get('customer_types')
            if not types:
                # If no type specified, article applies to ALL user types
                for t in ['Traveler', 'Owner', 'Property Manager']:
                    by_type[t].append(article)
            else:
                for t in types:
                    by_type[t].append(article)

        output_by_type = {
            'source': 'help.vrbo.com',
            'grouped_by_customer_type': {
                user_type: {
                    'count': len(articles),
                    'articles': articles
                }
                for user_type, articles in by_type.items()
            }
        }
        with open('data/vrbo_articles_by_type.json', 'w', encoding='utf-8') as f:
            json.dump(output_by_type, f, indent=2, ensure_ascii=False)
        print("💾 Saved: vrbo_articles_by_type.json")

        # --- 3. Individual files per type (cleanest for RAG) ---
        type_files = {
            'Traveler': 'vrbo_articles_traveler.json',
            'Owner': 'vrbo_articles_owner.json',
            'Property Manager': 'vrbo_articles_pm.json',
        }

        for user_type, filename in type_files.items():
            articles_of_type = by_type.get(user_type, [])
            output = {
                'source': 'help.vrbo.com',
                'customer_type': user_type,
                'total_articles': len(articles_of_type),
                'articles': articles_of_type
            }
            with open(filename, 'w', encoding='utf-8') as f:
                json.dump(output, f, indent=2, ensure_ascii=False)
            print(f"💾 Saved: {filename} ({len(articles_of_type)} articles)")

        # --- Print summary ---
        print(f"{'=' * 60}")
        print("📊 SUMMARY")
        print(f"{'=' * 60}")
        print(f"Total unique articles: {len(self.articles)}")
        for user_type in ['Traveler', 'Owner', 'Property Manager']:
            count = len(by_type.get(user_type, []))
            print(f"  {user_type}: {count} articles")


# ========================
# LANGCHAIN RAG INTEGRATION
# ========================
"""
How to use with LangChain for cosine similarity + user type filtering:

Option A: Pre-filter by user type BEFORE embedding
    - Load vrbo_articles_traveler.json for traveler queries
    - Load vrbo_articles_owner.json for owner queries
    - This gives the cleanest, most relevant results

Option B: Add user type to the document text for embedding
    - Modify the 'content' to include: "[Traveler] How to book..."
    - Then filter results by checking customer_types after retrieval

Option C: Metadata filtering with vector store
    - Most vector stores (Chroma, Pinecone, Weaviate) support metadata filtering
    - Example with Chroma:

    from langchain_community.vectorstores import Chroma
    from langchain_openai import OpenAIEmbeddings

    # Load all articles
    with open('vrbo_articles_all.json') as f:
        data = json.load(f)

    texts = [a['content'] for a in data['articles']]
    metadatas = [{
        'id': a['id'],
        'title': a['title'],
        'category': a['category'],
        'subcategory': a['subcategory'],
        'customer_types': a.get('customer_types', []),
        'url': a['url']
    } for a in data['articles']]

    embeddings = OpenAIEmbeddings()
    vectorstore = Chroma.from_texts(
        texts=texts,
        embedding=embeddings,
        metadatas=metadatas
    )

    # Query with metadata filter
    results = vectorstore.similarity_search(
        "how do I cancel my reservation?",
        k=5,
        filter={"customer_types": {"$contains": "Traveler"}}
    )
"""


if __name__ == "__main__":
    scraper = VRBOScraper()
    scraper.run()