app.py

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
import asyncio
import re
import sys
from urllib.parse import urlparse
from typing import List
from datetime import datetime

if sys.platform == "win32":
    # Playwright launches a driver subprocess; Proactor loop supports subprocess APIs on Windows.
    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())

app = FastAPI(title="HackTrack Scraper", version="3.0.0")

# Global Playwright runtime objects reused across requests.
playwright = None
browser = None

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["*"],
    allow_headers=["*"],
)


class ScrapeRequest(BaseModel):
    url: str


class ScrapeResponse(BaseModel):
    name: str = ""
    platform: str = ""
    banner_url: str = ""
    description: str = ""
    registration_deadline: str = ""
    submission_deadline: str = ""
    result_date: str = ""
    start_date: str = ""
    end_date: str = ""
    prize_pool: str = ""
    team_size: dict = Field(default_factory=lambda: {"min": 1, "max": 4})
    problem_statements: List[dict] = Field(default_factory=list)
    resource_links: List[dict] = Field(default_factory=list)
    scrape_success: bool = False
    url: str = ""


def detect_platform(url: str) -> str:
    domain = urlparse(url).netloc.lower()
    if "devfolio" in domain:
        return "Devfolio"
    elif "unstop" in domain:
        return "Unstop"
    elif "devpost" in domain:
        return "Devpost"
    elif "dorahacks" in domain:
        return "DoraHacks"
    return "Other"


# ============================================================
# DATE PARSING — robust multi-format
# ============================================================
MONTH_MAP = {
    "jan": 1, "january": 1, "feb": 2, "february": 2, "mar": 3, "march": 3,
    "apr": 4, "april": 4, "may": 5, "jun": 6, "june": 6,
    "jul": 7, "july": 7, "aug": 8, "august": 8, "sep": 9, "sept": 9, "september": 9,
    "oct": 10, "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12,
}

DATE_FORMATS = [
    "%Y-%m-%d", "%Y/%m/%d",
    "%d %B %Y", "%d %b %Y", "%d %B, %Y", "%d %b, %Y",
    "%B %d, %Y", "%b %d, %Y", "%B %d %Y", "%b %d %Y",
    "%m/%d/%Y", "%d/%m/%Y",
    "%B %d", "%b %d",
]


def parse_any_date(text: str, fallback_year: int = None) -> str:
    """Parse many date formats to YYYY-MM-DD. Handles partial dates."""
    if not text:
        return ""
    text = text.strip()
    text = re.sub(r"(\d+)(st|nd|rd|th)", r"\1", text)
    text = re.sub(r"\s+", " ", text)

    if not fallback_year:
        fallback_year = datetime.now().year

    for fmt in DATE_FORMATS:
        try:
            dt = datetime.strptime(text, fmt)
            if dt.year == 1900:  # no year in format
                dt = dt.replace(year=fallback_year)
                if dt < datetime.now():
                    dt = dt.replace(year=fallback_year + 1)
            return dt.strftime("%Y-%m-%d")
        except ValueError:
            continue
    return ""


def find_dates_near(text: str, keywords: List[str], window: int = 400) -> str:
    """Find dates within `window` chars after any keyword."""
    lower = text.lower()
    all_date_patterns = [
        r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})",
        r"(\d{1,2}\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)[,]?\s+\d{4})",
        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2}(?:st|nd|rd|th)?[,]?\s*\d{0,4})",
        r"(\d{1,2}/\d{1,2}/\d{4})",
    ]
    for kw in keywords:
        idx = lower.find(kw.lower())
        if idx == -1:
            continue
        chunk = text[idx:idx + window]
        for pat in all_date_patterns:
            match = re.search(pat, chunk, re.IGNORECASE)
            if match:
                parsed = parse_any_date(match.group(1))
                if parsed:
                    return parsed
    return ""


# ============================================================
# EXTRACT from full page innerText (the reliable approach)
# ============================================================

def extract_all_from_text(body_text: str, platform: str) -> dict:
    """Extract hackathon details from page innerText using text patterns."""
    result = {
        "registration_deadline": "",
        "submission_deadline": "",
        "result_date": "",
        "start_date": "",
        "end_date": "",
        "prize_pool": "",
        "team_size": {"min": 1, "max": 4},
        "problem_statements": [],
    }

    # ---- DATES ----
    # Registration deadline
    result["registration_deadline"] = find_dates_near(body_text, [
        "registration close", "registrations close", "register by",
        "last date to register", "registration deadline", "applications close",
        "apply by", "registration ends", "sign up deadline",
    ])

    # Submission deadline
    result["submission_deadline"] = find_dates_near(body_text, [
        "submission deadline", "submission closes", "submissions close",
        "submit by", "last date to submit", "submission end",
        "final submission", "project submission",
        "deadline",  # generic fallback last
    ])

    # Start date — Devfolio uses "Runs from Mar 25 - 26, 2026"
    runs_from = re.search(
        r"(?:runs?\s+from|starts?\s+(?:on|from)?|begins?\s+(?:on)?|commences?\s+(?:on)?)\s*[:\-]?\s*"
        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2})"
        r"(?:\s*[-–]\s*(\d{1,2}))?"
        r"(?:[,\s]+(\d{4}))?",
        body_text, re.IGNORECASE
    )
    if runs_from:
        start_text = runs_from.group(1)
        year = runs_from.group(3) or str(datetime.now().year)
        result["start_date"] = parse_any_date(f"{start_text} {year}")
        if runs_from.group(2) and runs_from.group(1):
            month = runs_from.group(1).split()[0]
            result["end_date"] = parse_any_date(f"{month} {runs_from.group(2)} {year}")

    if not result["start_date"]:
        result["start_date"] = find_dates_near(body_text, [
            "start date", "starts on", "begins on", "hackathon starts",
            "event starts", "event date", "dates:",
        ])

    if not result["end_date"]:
        result["end_date"] = find_dates_near(body_text, [
            "end date", "ends on", "hackathon ends", "event ends",
        ])

    # Result date
    result["result_date"] = find_dates_near(body_text, [
        "result", "winners announced", "announcement", "winner announcement",
        "results declared", "shortlist",
    ])

    # ---- PRIZE POOL ----
    prize_patterns = [
        r"(₹\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
        r"(\$\s*[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M|million|thousand))?)",
        r"(€\s*[\d,]+(?:\.\d+)?)",
        r"(£\s*[\d,]+(?:\.\d+)?)",
        r"(INR\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
        r"(Rs\.?\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
    ]

    # Find prize amounts near keywords like "prize", "reward", "worth", "win"
    prize_lower = body_text.lower()
    for kw in ["prize", "reward", "worth", "winning", "bounty", "in cash", "in prizes"]:
        idx = prize_lower.find(kw)
        if idx == -1:
            continue
        # Search ±200 chars around keyword
        start = max(0, idx - 200)
        chunk = body_text[start:idx + 200]
        for pat in prize_patterns:
            match = re.search(pat, chunk, re.IGNORECASE)
            if match:
                result["prize_pool"] = match.group(1).strip()
                break
        if result["prize_pool"]:
            break

    # Fallback: any large currency amount
    if not result["prize_pool"]:
        for pat in prize_patterns:
            match = re.search(pat, body_text)
            if match:
                result["prize_pool"] = match.group(1).strip()
                break

    # ---- TEAM SIZE ----
    team_patterns = [
        r"team\s*size[:\s]*(\d+)\s*[-–to]+\s*(\d+)",
        r"(\d+)\s*[-–to]+\s*(\d+)\s*(?:members?|people|participants?|per team)",
        r"teams?\s+of\s+(?:up\s+to\s+)?(\d+)",
        r"max(?:imum)?\s*(?:team)?\s*(?:size)?\s*[:\s]*(\d+)",
        r"(\d+)\s*[-–]\s*(\d+)\s*$",  # in FAQ: "2 - 4"
    ]
    for pat in team_patterns:
        match = re.search(pat, body_text, re.IGNORECASE)
        if match:
            groups = [g for g in match.groups() if g]
            if len(groups) == 2:
                result["team_size"] = {"min": int(groups[0]), "max": int(groups[1])}
            elif len(groups) == 1:
                result["team_size"] = {"min": 1, "max": int(groups[0])}
            break

    # ---- PROBLEM STATEMENTS / TRACKS / DOMAINS ----
    ps = []
    seen_ps = set()

    # Pattern 1: "Domains: AI, ML, Web App" (Devfolio style)
    domain_match = re.search(
        r"(?:domains?|themes?|tracks?|categories|verticals|areas?)[:\s]+([^\n💡🏆🎁🎟️📍📅⏳📞🌮]+)",
        body_text, re.IGNORECASE
    )
    if domain_match:
        items = re.split(r"[,•|/]", domain_match.group(1))
        for item in items:
            item = item.strip().rstrip(".")
            if 3 < len(item) < 150 and item.lower() not in seen_ps:
                seen_ps.add(item.lower())
                ps.append({"track": "", "title": item})

    # Pattern 2: Numbered problem statements: "PS1: ...", "Problem Statement 1 - ..."
    for match in re.finditer(
        r"(?:PS|Problem\s*Statement|Theme|Track|Challenge)\s*#?(\d+)\s*[:\-–]\s*(.{5,200})",
        body_text, re.IGNORECASE
    ):
        num = match.group(1)
        title = match.group(2).strip().split("\n")[0]
        if title.lower() not in seen_ps and len(title) > 4:
            seen_ps.add(title.lower())
            ps.append({"track": f"Track {num}", "title": title})

    # Pattern 3: Devpost-style theme tags (already in themes list from JS)
    # Pattern 4: Bulleted lists after "Themes" or "Tracks" heading
    for match in re.finditer(
        r"(?:themes?|tracks?|problem\s*statements?|challenges?|domains?)\s*[:\n]"
        r"((?:\s*[-•●▸]\s*.{5,200}\n?)+)",
        body_text, re.IGNORECASE
    ):
        items = re.findall(r"[-•●▸]\s*(.{5,200})", match.group(1))
        for item in items:
            item = item.strip().split("\n")[0]
            if item.lower() not in seen_ps and 4 < len(item) < 200:
                seen_ps.add(item.lower())
                ps.append({"track": "", "title": item})

    result["problem_statements"] = ps[:20]
    return result


# ============================================================
# PLAYWRIGHT SCRAPER — gets innerText + meta from rendered page
# ============================================================

EXTRACT_SCRIPT = """() => {
    const getMeta = (name) => {
        const el = document.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
        return el ? el.getAttribute('content') || '' : '';
    };

    // Name: try multiple selectors
    const nameSelectors = [
        'h1',
        '.hackathon-name', '.event-name', '.challenge-title',
        '#challenge-title', '.opp-title',
    ];
    let name = '';
    for (const sel of nameSelectors) {
        const el = document.querySelector(sel);
        if (el && el.textContent.trim().length > 2) {
            name = el.textContent.trim();
            break;
        }
    }
    name = name || getMeta('og:title') || document.title.split('|')[0].trim();

    // Banner
    const banner = getMeta('og:image') || '';

    // Description
    let description = getMeta('og:description') || getMeta('description') || '';

    // Full page text for parsing
    const bodyText = document.body.innerText;

    // For Devpost: extract themes from tag links
    const themes = [];
    document.querySelectorAll('a[href*="themes"]').forEach(a => {
        const t = a.textContent.trim();
        if (t && t.length > 2 && t.length < 100) themes.push(t);
    });

    // Devpost sidebar prize text
    let sidebarPrize = '';
    document.querySelectorAll('a[href*="prizes"], .prize, [class*="prize"]').forEach(el => {
        const t = el.textContent.trim();
        if (t && t.length > 2) sidebarPrize += t + ' ';
    });

    // Resource links: PDFs, Google Drive, problem statements, rules, guidelines
    const resourceLinks = [];
    const seenHrefs = new Set();
    const linkKeywords = ['problem', 'statement', 'pdf', 'rule', 'guideline', 'brochure', 'document', 'brief', 'challenge', 'track', 'theme', 'schedule', 'timeline'];
    document.querySelectorAll('a[href]').forEach(a => {
        const href = a.href || '';
        const text = a.textContent.trim();
        const hrefLower = href.toLowerCase();
        const textLower = text.toLowerCase();
        if (seenHrefs.has(href) || !href || href === '#') return;

        const isPdf = hrefLower.endsWith('.pdf') || hrefLower.includes('/pdf');
        const isDrive = hrefLower.includes('drive.google.com') || hrefLower.includes('docs.google.com');
        const isDropbox = hrefLower.includes('dropbox.com');
        const isRelevant = linkKeywords.some(kw => textLower.includes(kw) || hrefLower.includes(kw));

        if (isPdf || isDrive || isDropbox || isRelevant) {
            seenHrefs.add(href);
            resourceLinks.push({
                text: text.substring(0, 150) || 'Document',
                url: href,
                type: isPdf ? 'pdf' : isDrive ? 'google_drive' : isDropbox ? 'dropbox' : 'link',
            });
        }
    });

    return {
        name: name.substring(0, 200),
        description: description.substring(0, 2000),
        banner_url: banner,
        bodyText: bodyText.substring(0, 30000),
        themes: themes,
        sidebarPrize: sidebarPrize.trim(),
        resourceLinks: resourceLinks.slice(0, 30),
    };
}"""


@app.on_event("startup")
async def startup() -> None:
    global playwright, browser
    from playwright.async_api import async_playwright

    playwright = await async_playwright().start()
    browser = await playwright.chromium.launch(
        headless=True,
        args=["--no-sandbox", "--disable-setuid-sandbox"],
    )
    print("[Scraper] Playwright browser initialized")


@app.on_event("shutdown")
async def shutdown() -> None:
    global playwright, browser

    try:
        if browser is not None:
            await browser.close()
            print("[Scraper] Browser closed")
    finally:
        browser = None

    try:
        if playwright is not None:
            await playwright.stop()
            print("[Scraper] Playwright stopped")
    finally:
        playwright = None

async def scrape_with_playwright(url: str, platform: str) -> dict:
    """Scrape using Playwright — renders JS, grabs full innerText for parsing."""
    global browser
    try:
        if browser is None:
            return {
                "scrape_success": False,
                "error": "Browser is not initialized. Service startup failed.",
            }

        context = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
            viewport={"width": 1920, "height": 1080},
        )

        try:
            page = await context.new_page()

            print(f"[Scraper] Navigating to {url} (platform: {platform})")
            await page.goto(url, wait_until="domcontentloaded", timeout=20000)

            # Wait for JS rendering — longer for SPAs
            wait_time = 8 if platform in ("Unstop",) else 5
            print(f"[Scraper] Waiting {wait_time}s for JS rendering...")
            await page.wait_for_timeout(wait_time * 1000)

            # Scroll to trigger lazy content
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)")
            await asyncio.sleep(1)
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight * 2 / 3)")
            await asyncio.sleep(1)
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            await asyncio.sleep(1)
            await page.evaluate("window.scrollTo(0, 0)")
            await asyncio.sleep(0.5)

            # Extract structured + raw text data
            data = await page.evaluate(EXTRACT_SCRIPT)

            body_text = data.get("bodyText", "")
            name = data.get("name", "")
            description = data.get("description", "")

            print(f"[Scraper] Extracted name: '{name}', bodyText length: {len(body_text)}")

            # Parse all fields from full innerText
            extracted = extract_all_from_text(body_text, platform)

            # Devpost themes from sidebar tags
            themes = data.get("themes", [])
            if themes and not extracted["problem_statements"]:
                seen = set()
                for t in themes:
                    if t.lower() not in seen:
                        seen.add(t.lower())
                        extracted["problem_statements"].append({"track": "Theme", "title": t})

            # Sidebar prize fallback (Devpost)
            if not extracted["prize_pool"] and data.get("sidebarPrize"):
                prize_text = data["sidebarPrize"]
                for pat in [r"(\$[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M))?)", r"(₹[\d,]+)"]:
                    m = re.search(pat, prize_text)
                    if m:
                        extracted["prize_pool"] = m.group(1)
                        break
                if not extracted["prize_pool"]:
                    extracted["prize_pool"] = prize_text[:100]

            return {
                "name": name,
                "description": description,
                "banner_url": data.get("banner_url", ""),
                "scrape_success": bool(name and len(name) > 2),
                "resource_links": data.get("resourceLinks", []),
                **extracted,
            }
        finally:
            await context.close()

    except Exception as e:
        print(f"[Scraper] Error: {e}")
        import traceback
        traceback.print_exc()
        return {"scrape_success": False, "error": str(e)}


# ============================================================
# API ROUTES
# ============================================================

@app.get("/")
async def root():
    return {"status": "ok", "service": "HackTrack Scraper v3"}


@app.post("/scrape", response_model=ScrapeResponse)
async def scrape(request: ScrapeRequest):
    url = request.url.strip()
    platform = detect_platform(url)
    print(f"\n[Scraper] === New scrape request: {url} (platform={platform}) ===")

    try:
        data = await scrape_with_playwright(url, platform)

        response = ScrapeResponse(
            name=data.get("name", ""),
            platform=platform,
            banner_url=data.get("banner_url", ""),
            description=data.get("description", ""),
            registration_deadline=data.get("registration_deadline", ""),
            submission_deadline=data.get("submission_deadline", ""),
            result_date=data.get("result_date", ""),
            start_date=data.get("start_date", ""),
            end_date=data.get("end_date", ""),
            prize_pool=data.get("prize_pool", ""),
            team_size=data.get("team_size", {"min": 1, "max": 4}),
            problem_statements=data.get("problem_statements", []),
            resource_links=data.get("resource_links", []),
            scrape_success=data.get("scrape_success", False),
            url=url,
        )

        print(f"[Scraper] Result: name='{response.name}', dates=({response.start_date}, {response.end_date}, reg={response.registration_deadline}, sub={response.submission_deadline}), prize='{response.prize_pool}', team={response.team_size}, ps={len(response.problem_statements)}")
        return response

    except Exception as e:
        print(f"[Scraper] Endpoint error: {e}")
        return ScrapeResponse(platform=platform, url=url, scrape_success=False)

# ============================================================
# LISTING PAGE CRAWLERS — for discovery / public_hackathons
# ============================================================

class CrawledHackathon(BaseModel):
    name: str = ""
    platform: str = ""
    banner_url: str = ""
    description: str = ""
    start_date: str = ""
    end_date: str = ""
    registration_deadline: str = ""
    prize_pool: str = ""
    tags: List[str] = Field(default_factory=list)
    source_url: str = ""
    status: str = "open"


class CrawlResponse(BaseModel):
    platform: str
    count: int = 0
    hackathons: List[CrawledHackathon] = Field(default_factory=list)
    error: str = ""


DEVFOLIO_EXTRACT = """() => {
    // Devfolio uses subdomain links like https://code-recet-3.devfolio.co/
    const allLinks = document.querySelectorAll('a[href*=".devfolio.co"]');
    const results = [];
    const seen = new Set();

    // Also grab any links that contain h3 tags (hackathon card pattern)
    const h3Links = document.querySelectorAll('a:has(h3)');
    const combined = new Set([...allLinks, ...h3Links]);

    combined.forEach(card => {
        try {
            const href = card.href || '';
            if (!href || seen.has(href)) return;

            // Skip non-hackathon links
            const hostname = new URL(href).hostname;
            if (hostname === 'devfolio.co' || hostname === 'www.devfolio.co') return;
            if (!hostname.endsWith('.devfolio.co')) return;
            // Skip common non-hackathon subdomains
            if (['api', 'docs', 'blog', 'app'].some(s => hostname.startsWith(s + '.'))) return;

            seen.add(href);

            const nameEl = card.querySelector('h3, h2, [class*="name"], [class*="title"]');
            const name = nameEl ? nameEl.textContent.trim() : '';
            if (!name || name.length < 3) return;

            // Walk up to the card container to find banner and other data
            const container = card.closest('div') || card.parentElement?.closest('div') || card;

            const imgEl = container.querySelector('img') || card.querySelector('img');
            const banner = imgEl ? (imgEl.src || imgEl.getAttribute('data-src') || '') : '';

            const descEl = container.querySelector('p') || card.querySelector('p');
            const description = descEl ? descEl.textContent.trim().substring(0, 500) : '';

            const allText = (container.textContent || card.textContent || '');

            // Extract prize
            let prize = '';
            const prizeMatch = allText.match(/[\u20B9$\u20AC\u00A3]\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|K|k|L|M))?/);
            if (prizeMatch) prize = prizeMatch[0].trim();

            // Extract dates like "Mar 25 - 27, 2026" or "Runs from ..."
            let startDate = '';
            let endDate = '';
            const dateMatch = allText.match(/((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2})(?:\s*[-\u2013]\s*(\d{1,2}))?(?:[,\s]+(\d{4}))?/i);
            if (dateMatch) {
                const year = dateMatch[3] || new Date().getFullYear().toString();
                startDate = dateMatch[1] + ' ' + year;
                if (dateMatch[2]) {
                    const month = dateMatch[1].split(/\s+/)[0];
                    endDate = month + ' ' + dateMatch[2] + ' ' + year;
                }
            }

            // Extract tags from spans/badges
            const tags = [];
            const tagEls = container.querySelectorAll('span, [class*="tag"], [class*="badge"], [class*="chip"], [class*="Pill"]');
            tagEls.forEach(el => {
                const t = el.textContent.trim();
                if (t && t.length > 1 && t.length < 50 && !t.includes('\u20B9') && !t.includes('$') && t !== name) {
                    tags.push(t);
                }
            });

            results.push({
                name,
                source_url: href,
                banner_url: banner,
                description,
                prize_pool: prize,
                start_date: startDate,
                end_date: endDate,
                tags: [...new Set(tags)].slice(0, 10),
            });
        } catch(e) {}
    });

    return results;
}"""


DEVPOST_EXTRACT = """() => {
    const cards = document.querySelectorAll('.hackathon-tile, a[data-hackathon-slug], [class*="hackathon"]');
    const results = [];
    const seen = new Set();

    // Fallback: also try generic link approach
    const allLinks = document.querySelectorAll('a[href*="devpost.com/hackathons/"]');
    const combined = [...cards, ...allLinks];

    combined.forEach(card => {
        try {
            let href = card.href || card.querySelector('a')?.href || '';
            if (!href.startsWith('http')) {
                const aEl = card.closest('a') || card.querySelector('a');
                if (aEl) href = aEl.href;
            }
            if (!href || seen.has(href)) return;
            if (href.endsWith('/hackathons') || href.endsWith('/hackathons/')) return;
            seen.add(href);

            const nameEl = card.querySelector('h2, h3, .title, [class*="title"], [class*="name"]');
            const name = nameEl ? nameEl.textContent.trim() : (card.textContent || '').split('\\n')[0].trim().substring(0, 100);
            if (!name || name.length < 3) return;

            const imgEl = card.querySelector('img');
            const banner = imgEl ? (imgEl.src || '') : '';

            const descEl = card.querySelector('.tagline, .description, p');
            const description = descEl ? descEl.textContent.trim().substring(0, 500) : '';

            const allText = card.textContent || '';
            let prize = '';
            const prizeMatch = allText.match(/\\$\\s*[\\d,]+(?:\\.\\d+)?(?:\\s*(?:K|k|M|million))?/);
            if (prizeMatch) prize = prizeMatch[0].trim();

            // Dates
            let deadline = '';
            const dateMatch = allText.match(/(?:Submission|Deadline|Ends?)[:\\s]+([A-Za-z]+ \\d{1,2},?\\s*\\d{4})/i);
            if (dateMatch) deadline = dateMatch[1];

            const tags = [];
            card.querySelectorAll('.themes a, [class*="tag"], [class*="theme"]').forEach(el => {
                const t = el.textContent.trim();
                if (t && t.length > 1 && t.length < 50) tags.push(t);
            });

            results.push({
                name,
                source_url: href,
                banner_url: banner,
                description,
                prize_pool: prize,
                registration_deadline: deadline,
                tags: tags.slice(0, 10),
            });
        } catch(e) {}
    });

    return results;
}"""


UNSTOP_EXTRACT = """() => {
    const cards = document.querySelectorAll('[class*="card"], [class*="listing"], a[href*="/hackathons/"], a[href*="/competition/"]');
    const results = [];
    const seen = new Set();

    cards.forEach(card => {
        try {
            let href = card.href || '';
            if (!href.startsWith('http')) {
                const aEl = card.querySelector('a[href*="hackathon"], a[href*="competition"]');
                if (aEl) href = aEl.href;
            }
            if (!href || seen.has(href)) return;
            if (!href.includes('hackathon') && !href.includes('competition')) return;
            seen.add(href);

            const nameEl = card.querySelector('h3, h2, .title, [class*="title"], [class*="name"], p.semi-bold');
            const name = nameEl ? nameEl.textContent.trim() : '';
            if (!name || name.length < 3) return;

            const imgEl = card.querySelector('img');
            const banner = imgEl ? (imgEl.src || '') : '';

            const allText = card.textContent || '';

            let prize = '';
            const prizeMatch = allText.match(/(?:₹|INR|Rs\\.?)\\s*[\\d,]+(?:\\.\\d+)?(?:\\s*(?:Lakhs?|Lacs?|Crores?|K|k|L))?/i);
            if (prizeMatch) prize = prizeMatch[0].trim();

            const tags = [];
            card.querySelectorAll('[class*="chip"], [class*="tag"], [class*="badge"]').forEach(el => {
                const t = el.textContent.trim();
                if (t && t.length > 1 && t.length < 50 && !t.includes('₹')) tags.push(t);
            });

            const descEl = card.querySelector('p:not(.semi-bold)');
            const description = descEl ? descEl.textContent.trim().substring(0, 500) : '';

            results.push({
                name,
                source_url: href,
                banner_url: banner,
                description,
                prize_pool: prize,
                tags: tags.slice(0, 10),
            });
        } catch(e) {}
    });

    return results;
}"""


async def crawl_listing_page(url: str, platform: str, extract_script: str, scroll_count: int = 5, wait_secs: int = 5) -> List[dict]:
    """Generic listing page crawler: navigate, scroll to load lazy cards, extract."""
    global browser
    if browser is None:
        return []

    context = await browser.new_context(
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
        viewport={"width": 1920, "height": 1080},
    )

    try:
        page = await context.new_page()
        print(f"[Crawler] Navigating to {url} ({platform})")
        await page.goto(url, wait_until="domcontentloaded", timeout=30000)
        await page.wait_for_timeout(wait_secs * 1000)

        # Scroll multiple times to trigger lazy loading
        for i in range(scroll_count):
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            await asyncio.sleep(2)
            # Try clicking "Load More" / "Show More" buttons
            for selector in ['button:has-text("Load More")', 'button:has-text("Show More")', 'button:has-text("View More")', '[class*="load-more"]', '[class*="show-more"]']:
                try:
                    btn = page.locator(selector).first
                    if await btn.is_visible(timeout=500):
                        await btn.click()
                        await asyncio.sleep(2)
                except:
                    pass

        await page.evaluate("window.scrollTo(0, 0)")
        await asyncio.sleep(1)

        raw = await page.evaluate(extract_script)
        print(f"[Crawler] {platform}: extracted {len(raw)} entries")

        hackathons = []
        for item in raw:
            name = item.get("name", "").strip()
            source_url = item.get("source_url", "").strip()
            if not name or not source_url:
                continue

            # Parse dates if present
            reg_deadline = ""
            if item.get("registration_deadline"):
                reg_deadline = parse_any_date(item["registration_deadline"])

            hackathons.append({
                "name": name,
                "platform": platform,
                "banner_url": item.get("banner_url", ""),
                "description": item.get("description", ""),
                "start_date": parse_any_date(item.get("start_date", "")),
                "end_date": parse_any_date(item.get("end_date", "")),
                "registration_deadline": reg_deadline,
                "prize_pool": item.get("prize_pool", ""),
                "tags": item.get("tags", []),
                "source_url": source_url,
                "status": "open",
            })

        return hackathons
    except Exception as e:
        print(f"[Crawler] {platform} error: {e}")
        import traceback
        traceback.print_exc()
        return []
    finally:
        await context.close()


@app.post("/crawl/devfolio", response_model=CrawlResponse)
async def crawl_devfolio():
    results = await crawl_listing_page(
        url="https://devfolio.co/hackathons/open",
        platform="Devfolio",
        extract_script=DEVFOLIO_EXTRACT,
        scroll_count=5,
        wait_secs=6,
    )
    return CrawlResponse(platform="Devfolio", count=len(results), hackathons=[CrawledHackathon(**h) for h in results])


@app.post("/crawl/devpost", response_model=CrawlResponse)
async def crawl_devpost():
    results = await crawl_listing_page(
        url="https://devpost.com/hackathons?open_to[]=public&status[]=open",
        platform="DevPost",
        extract_script=DEVPOST_EXTRACT,
        scroll_count=4,
        wait_secs=5,
    )
    return CrawlResponse(platform="DevPost", count=len(results), hackathons=[CrawledHackathon(**h) for h in results])


@app.post("/crawl/unstop", response_model=CrawlResponse)
async def crawl_unstop():
    results = await crawl_listing_page(
        url="https://unstop.com/hackathons",
        platform="Unstop",
        extract_script=UNSTOP_EXTRACT,
        scroll_count=5,
        wait_secs=8,
    )
    return CrawlResponse(platform="Unstop", count=len(results), hackathons=[CrawledHackathon(**h) for h in results])


@app.post("/crawl/all")
async def crawl_all():
    """Crawl all platforms and return combined results."""
    print("\n[Crawler] === Starting full crawl ===")
    devfolio, devpost, unstop = await asyncio.gather(
        crawl_listing_page("https://devfolio.co/hackathons/open", "Devfolio", DEVFOLIO_EXTRACT, 5, 6),
        crawl_listing_page("https://devpost.com/hackathons?open_to[]=public&status[]=open", "DevPost", DEVPOST_EXTRACT, 4, 5),
        crawl_listing_page("https://unstop.com/hackathons", "Unstop", UNSTOP_EXTRACT, 5, 8),
    )
    all_results = devfolio + devpost + unstop
    print(f"[Crawler] === Full crawl complete: {len(all_results)} hackathons ===")
    return {
        "total": len(all_results),
        "by_platform": {
            "devfolio": len(devfolio),
            "devpost": len(devpost),
            "unstop": len(unstop),
        },
        "hackathons": all_results,
    }