core/scraper.py

"""
Web Scraper - AI-friendly web content extraction using Playwright.

Handles:
- Rendering JavaScript-heavy pages (Wellfound, React apps)
- Extracting page text content cleanly
- Following redirects
- Screenshot capture for AI analysis (fallback)
- Rate limiting and retry logic
"""

import asyncio
import re
from typing import Optional, Dict, Any, Tuple
from urllib.parse import urlparse

from bs4 import BeautifulSoup
from tenacity import retry, stop_after_attempt, wait_exponential


class WebScraper:
    """Intelligent web scraper with JS rendering support."""

    def __init__(self, headless: bool = True, timeout: int = 30000):
        self.headless = headless
        self.timeout = timeout
        self._browser = None
        self._context = None
        self._playwright = None

    async def start(self):
        """Initialize Playwright browser."""
        try:
            from playwright.async_api import async_playwright
        except ImportError:
            raise ImportError(
                "Playwright not installed. Run: pip install playwright && playwright install chromium"
            )

        self._playwright = await async_playwright().start()
        self._browser = await self._playwright.chromium.launch(
            headless=self.headless,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--no-sandbox",
                "--disable-dev-shm-usage",
                "--disable-web-security",
                "--disable-features=IsolateOrigins,site-per-process",
            ],
        )

    async def stop(self):
        """Clean up Playwright resources."""
        if self._browser:
            await self._browser.close()
        if self._playwright:
            await self._playwright.stop()

    async def _get_page(self):
        """Get or create a browser page."""
        if not self._context:
            self._context = await self._browser.new_context(
                user_agent=(
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/125.0.0.0 Safari/537.36"
                ),
                viewport={"width": 1920, "height": 1080},
                java_script_enabled=True,
            )
        return await self._context.new_page()

    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=15))
    async def fetch_page(self, url: str, wait_for_selector: Optional[str] = None) -> Dict[str, Any]:
        """Fetch and render a web page, return content and metadata."""
        page = await self._get_page()
        result = {
            "url": url,
            "final_url": url,
            "title": "",
            "text": "",
            "html": "",
            "status": 200,
            "error": None,
            "emails": [],
            "links": [],
        }

        try:
            response = await page.goto(url, wait_until="domcontentloaded", timeout=self.timeout)

            if response:
                result["status"] = response.status
                result["final_url"] = response.url

            if result["status"] >= 400:
                result["error"] = f"HTTP {result['status']}"
                return result

            # Wait for key content to load
            if wait_for_selector:
                try:
                    await page.wait_for_selector(wait_for_selector, timeout=10000)
                except Exception:
                    pass

            # Let JS finish rendering
            await asyncio.sleep(1.5)

            # Scroll to load lazy content
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            await asyncio.sleep(0.8)
            await page.evaluate("window.scrollTo(0, 0)")
            await asyncio.sleep(0.5)

            # Extract content
            result["title"] = await page.title()
            result["html"] = await page.content()

            # Clean text extraction (keep footer — addresses/emails often there)
            text = await page.evaluate("""
                () => {
                    // Remove script, style, nav, iframe, noscript elements
                    // Keep footer — it often contains contact info and addresses
                    const clone = document.body.cloneNode(true);
                    const removes = clone.querySelectorAll('script, style, iframe, noscript, [aria-hidden="true"]');
                    removes.forEach(el => el.remove());
                    return clone.innerText || clone.textContent || '';
                }
            """)
            result["text"] = self._clean_text(text)

            # Extract emails
            result["emails"] = self._extract_emails(result["text"])

            # Extract important links (broader set for deep scanning)
            result["links"] = await page.evaluate("""
                () => {
                    const links = [];
                    const anchors = document.querySelectorAll('a[href]');
                    const keywords = ['contact', 'about', 'career', 'job', 'team', 'press',
                                      'investor', 'blog', 'support', 'help', 'location',
                                      'office', 'join', 'work', 'talent', 'opening',
                                      'apply', 'who-we-are', 'our-story', 'people',
                                      'headquarter', 'map', 'address', 'reach'];
                    anchors.forEach(a => {
                        const href = a.href || '';
                        const text = (a.innerText || '').trim().toLowerCase();
                        const hrefLower = href.toLowerCase();
                        if (keywords.some(k => hrefLower.includes(k) || text.includes(k))) {
                            links.push({ url: href, text: a.innerText?.trim() || '' });
                        }
                    });
                    // Also capture footer links (often contain contact info)
                    const footerLinks = document.querySelectorAll('footer a[href]');
                    footerLinks.forEach(a => {
                        const href = a.href || '';
                        const text = (a.innerText || '').trim().toLowerCase();
                        const hrefLower = href.toLowerCase();
                        if (href && !href.startsWith('javascript:')) {
                            links.push({ url: href, text: a.innerText?.trim() || '' });
                        }
                    });
                    // Deduplicate by URL
                    const seen = new Set();
                    return links.filter(l => {
                        if (seen.has(l.url)) return false;
                        seen.add(l.url);
                        return true;
                    }).slice(0, 80);
                }
            """)

        except Exception as e:
            result["error"] = str(e)[:500]
        finally:
            await page.close()

        return result

    async def fetch_wellfound_page(self, url: str) -> Dict[str, Any]:
        """Specialized fetch for Wellfound company pages to extract funding data."""
        result = await self.fetch_page(url, wait_for_selector="[class*='company'], [class*='profile'], h1")

        if not result["error"]:
            # Wellfound-specific extraction: look for funding/valuation patterns in text
            funding_data = self._extract_wellfound_funding(result["text"])
            result["funding_data"] = funding_data

            # Additional: try to extract structured data from meta tags
            soup = BeautifulSoup(result["html"], "lxml")
            meta_tags = {}
            for meta in soup.find_all("meta"):
                name = meta.get("name") or meta.get("property", "")
                content = meta.get("content", "")
                if name and content:
                    meta_tags[name] = content
            result["meta"] = meta_tags

        return result

    async def fetch_company_website(self, url: str) -> Dict[str, Any]:
        """Fetch company website to extract contact info and address."""
        result = await self.fetch_page(url)

        if not result["error"]:
            # Extract additional contact patterns
            soup = BeautifulSoup(result["html"], "lxml")

            # Find contact page links
            contact_links = []
            for a in soup.find_all("a", href=True):
                href = a.get("href", "").lower()
                text = (a.get_text() or "").strip().lower()
                if any(kw in href or kw in text for kw in
                       ["contact", "support", "help", "about", "career", "job", "team"]):
                    contact_links.append({
                        "url": a["href"],
                        "text": a.get_text(strip=True),
                        "type": self._classify_link(href, text),
                    })

            result["contact_links"] = contact_links[:30]

            # Extract phone numbers
            phones = re.findall(
                r'(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}',
                result["text"]
            )
            result["phones"] = list(set(phones))[:10]

            # Extract physical addresses (US-focused)
            address_patterns = re.findall(
                r'\d{1,5}\s+[\w\s.,]+(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Drive|Dr|Lane|Ln|Way|Court|Ct|Plaza|Plz|Suite|Ste)[\w\s.,]*(?:[A-Z]{2}\s+\d{5})?',
                result["text"]
            )
            result["addresses"] = list(set(address_patterns))[:5]

        return result

    def _clean_text(self, text: str) -> str:
        """Clean extracted text."""
        # Remove excessive whitespace
        text = re.sub(r'[ \t]+', ' ', text)
        text = re.sub(r'\n\s*\n+', '\n\n', text)
        # Remove very short lines (usually noise)
        lines = [l.strip() for l in text.split('\n') if len(l.strip()) > 1]
        return '\n'.join(lines)

    def _extract_emails(self, text: str) -> list:
        """Extract email addresses from text."""
        email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
        emails = re.findall(email_pattern, text)

        # Filter out common false positives
        excluded = {'example.com', 'test.com', 'domain.com', 'email.com', 'yourcompany.com'}
        filtered = []
        for email in emails:
            domain = email.split('@')[1].lower() if '@' in email else ''
            if domain not in excluded and not domain.startswith('example'):
                filtered.append(email.lower())

        return list(set(filtered))

    def _extract_wellfound_funding(self, text: str) -> Dict[str, Any]:
        """Extract funding-related information from Wellfound page text."""
        data = {
            "valuation": None,
            "total_raised": None,
            "rounds": None,
            "series": None,
            "investors": [],
        }

        # Look for funding amounts
        amount_patterns = [
            (r'\$(\d+(?:\.\d+)?)\s*(?:million|M|Billion|B)', lambda m: f"${m.group(1)}{'M' if 'million' in m.group(0).lower() or m.group(0).strip().endswith('M') else 'B'}"),
            (r'\$(\d{1,3}(?:,\d{3})*(?:\.\d+)?)\s*(?:million|M)', lambda m: f"${m.group(1)}M"),
            (r'\$(\d{1,3}(?:,\d{3})*(?:\.\d+)?)\s*(?:billion|B)', lambda m: f"${m.group(1)}B"),
            (r'valuation[:\s]*\$(\d+(?:\.\d+)?)\s*(million|M|billion|B)', lambda m: f"${m.group(1)}{m.group(2)[0].upper()}"),
            (r'total funding[:\s]*\$(\d+(?:\.\d+)?)\s*(million|M|billion|B)', lambda m: f"${m.group(1)}{m.group(2)[0].upper()}"),
        ]

        for line in text.split('\n'):
            line_lower = line.lower().strip()

            # Valuation
            if any(kw in line_lower for kw in ['valuation', 'valued at']):
                for pattern, fmt in amount_patterns:
                    match = re.search(pattern, line, re.IGNORECASE)
                    if match:
                        data["valuation"] = fmt(match)
                        break

            # Total raised
            if any(kw in line_lower for kw in ['total funding', 'total raised', 'raised']):
                for pattern, fmt in amount_patterns:
                    match = re.search(pattern, line, re.IGNORECASE)
                    if match:
                        data["total_raised"] = fmt(match)
                        break

            # Series/Rounds
            series_match = re.search(
                r'(?:Series|Seed|Pre[-\s]?seed)\s*([A-F])?\b',
                line, re.IGNORECASE
            )
            if series_match:
                data["series"] = series_match.group(0).strip()

            # Round count
            rounds_match = re.search(r'(\d+)\s*(?:funding\s*)?rounds?', line, re.IGNORECASE)
            if rounds_match:
                data["rounds"] = int(rounds_match.group(1))

            # Investors
            investor_keywords = ['investor', 'backed by', 'funded by']
            if any(kw in line_lower for kw in investor_keywords):
                # Extract what comes after
                for kw in investor_keywords:
                    if kw in line_lower:
                        idx = line_lower.index(kw) + len(kw)
                        investors_text = line[idx:].strip(' :,;')
                        data["investors"].append(investors_text)
                        break

        return data

    def _classify_link(self, href: str, text: str) -> str:
        """Classify a link by its type."""
        combined = f"{href} {text}".lower()
        if any(kw in combined for kw in ['contact', 'reach', 'get in touch']):
            return 'contact'
        elif any(kw in combined for kw in ['career', 'job', 'work with', 'join']):
            return 'career'
        elif any(kw in combined for kw in ['about', 'company', 'our story']):
            return 'about'
        elif any(kw in combined for kw in ['team', 'people', 'leadership']):
            return 'team'
        elif any(kw in combined for kw in ['support', 'help', 'faq']):
            return 'support'
        else:
            return 'other'