Upload 3 files

Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    wget \
+    gnupg \
+    fonts-liberation \
+    fonts-noto-color-emoji \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libatspi2.0-0 \
+    libcairo2 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libglib2.0-0 \
+    libgtk-3-0 \
+    libnss3 \
+    libpango-1.0-0 \
+    libx11-6 \
+    libx11-xcb1 \
+    libxcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxkbcommon0 \
+    libxrandr2 \
+    xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# IMPORTANT FIX
+RUN playwright install --with-deps chromium
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,542 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+import asyncio
+import re
+import sys
+from urllib.parse import urlparse
+from typing import List
+from datetime import datetime
+
+if sys.platform == "win32":
+    # Playwright launches a driver subprocess; Proactor loop supports subprocess APIs on Windows.
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+app = FastAPI(title="HackTrack Scraper", version="3.0.0")
+
+# Global Playwright runtime objects reused across requests.
+playwright = None
+browser = None
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class ScrapeRequest(BaseModel):
+    url: str
+
+
+class ScrapeResponse(BaseModel):
+    name: str = ""
+    platform: str = ""
+    banner_url: str = ""
+    description: str = ""
+    registration_deadline: str = ""
+    submission_deadline: str = ""
+    result_date: str = ""
+    start_date: str = ""
+    end_date: str = ""
+    prize_pool: str = ""
+    team_size: dict = Field(default_factory=lambda: {"min": 1, "max": 4})
+    problem_statements: List[dict] = Field(default_factory=list)
+    resource_links: List[dict] = Field(default_factory=list)
+    scrape_success: bool = False
+    url: str = ""
+
+
+def detect_platform(url: str) -> str:
+    domain = urlparse(url).netloc.lower()
+    if "devfolio" in domain:
+        return "Devfolio"
+    elif "unstop" in domain:
+        return "Unstop"
+    elif "devpost" in domain:
+        return "Devpost"
+    elif "dorahacks" in domain:
+        return "DoraHacks"
+    return "Other"
+
+
+# ============================================================
+# DATE PARSING — robust multi-format
+# ============================================================
+MONTH_MAP = {
+    "jan": 1, "january": 1, "feb": 2, "february": 2, "mar": 3, "march": 3,
+    "apr": 4, "april": 4, "may": 5, "jun": 6, "june": 6,
+    "jul": 7, "july": 7, "aug": 8, "august": 8, "sep": 9, "sept": 9, "september": 9,
+    "oct": 10, "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12,
+}
+
+DATE_FORMATS = [
+    "%Y-%m-%d", "%Y/%m/%d",
+    "%d %B %Y", "%d %b %Y", "%d %B, %Y", "%d %b, %Y",
+    "%B %d, %Y", "%b %d, %Y", "%B %d %Y", "%b %d %Y",
+    "%m/%d/%Y", "%d/%m/%Y",
+    "%B %d", "%b %d",
+]
+
+
+def parse_any_date(text: str, fallback_year: int = None) -> str:
+    """Parse many date formats to YYYY-MM-DD. Handles partial dates."""
+    if not text:
+        return ""
+    text = text.strip()
+    text = re.sub(r"(\d+)(st|nd|rd|th)", r"\1", text)
+    text = re.sub(r"\s+", " ", text)
+
+    if not fallback_year:
+        fallback_year = datetime.now().year
+
+    for fmt in DATE_FORMATS:
+        try:
+            dt = datetime.strptime(text, fmt)
+            if dt.year == 1900:  # no year in format
+                dt = dt.replace(year=fallback_year)
+                if dt < datetime.now():
+                    dt = dt.replace(year=fallback_year + 1)
+            return dt.strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    return ""
+
+
+def find_dates_near(text: str, keywords: List[str], window: int = 400) -> str:
+    """Find dates within `window` chars after any keyword."""
+    lower = text.lower()
+    all_date_patterns = [
+        r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})",
+        r"(\d{1,2}\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)[,]?\s+\d{4})",
+        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2}(?:st|nd|rd|th)?[,]?\s*\d{0,4})",
+        r"(\d{1,2}/\d{1,2}/\d{4})",
+    ]
+    for kw in keywords:
+        idx = lower.find(kw.lower())
+        if idx == -1:
+            continue
+        chunk = text[idx:idx + window]
+        for pat in all_date_patterns:
+            match = re.search(pat, chunk, re.IGNORECASE)
+            if match:
+                parsed = parse_any_date(match.group(1))
+                if parsed:
+                    return parsed
+    return ""
+
+
+# ============================================================
+# EXTRACT from full page innerText (the reliable approach)
+# ============================================================
+
+def extract_all_from_text(body_text: str, platform: str) -> dict:
+    """Extract hackathon details from page innerText using text patterns."""
+    result = {
+        "registration_deadline": "",
+        "submission_deadline": "",
+        "result_date": "",
+        "start_date": "",
+        "end_date": "",
+        "prize_pool": "",
+        "team_size": {"min": 1, "max": 4},
+        "problem_statements": [],
+    }
+
+    # ---- DATES ----
+    # Registration deadline
+    result["registration_deadline"] = find_dates_near(body_text, [
+        "registration close", "registrations close", "register by",
+        "last date to register", "registration deadline", "applications close",
+        "apply by", "registration ends", "sign up deadline",
+    ])
+
+    # Submission deadline
+    result["submission_deadline"] = find_dates_near(body_text, [
+        "submission deadline", "submission closes", "submissions close",
+        "submit by", "last date to submit", "submission end",
+        "final submission", "project submission",
+        "deadline",  # generic fallback last
+    ])
+
+    # Start date — Devfolio uses "Runs from Mar 25 - 26, 2026"
+    runs_from = re.search(
+        r"(?:runs?\s+from|starts?\s+(?:on|from)?|begins?\s+(?:on)?|commences?\s+(?:on)?)\s*[:\-]?\s*"
+        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2})"
+        r"(?:\s*[-–]\s*(\d{1,2}))?"
+        r"(?:[,\s]+(\d{4}))?",
+        body_text, re.IGNORECASE
+    )
+    if runs_from:
+        start_text = runs_from.group(1)
+        year = runs_from.group(3) or str(datetime.now().year)
+        result["start_date"] = parse_any_date(f"{start_text} {year}")
+        if runs_from.group(2) and runs_from.group(1):
+            month = runs_from.group(1).split()[0]
+            result["end_date"] = parse_any_date(f"{month} {runs_from.group(2)} {year}")
+
+    if not result["start_date"]:
+        result["start_date"] = find_dates_near(body_text, [
+            "start date", "starts on", "begins on", "hackathon starts",
+            "event starts", "event date", "dates:",
+        ])
+
+    if not result["end_date"]:
+        result["end_date"] = find_dates_near(body_text, [
+            "end date", "ends on", "hackathon ends", "event ends",
+        ])
+
+    # Result date
+    result["result_date"] = find_dates_near(body_text, [
+        "result", "winners announced", "announcement", "winner announcement",
+        "results declared", "shortlist",
+    ])
+
+    # ---- PRIZE POOL ----
+    prize_patterns = [
+        r"(₹\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
+        r"(\$\s*[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M|million|thousand))?)",
+        r"(€\s*[\d,]+(?:\.\d+)?)",
+        r"(£\s*[\d,]+(?:\.\d+)?)",
+        r"(INR\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
+        r"(Rs\.?\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
+    ]
+
+    # Find prize amounts near keywords like "prize", "reward", "worth", "win"
+    prize_lower = body_text.lower()
+    for kw in ["prize", "reward", "worth", "winning", "bounty", "in cash", "in prizes"]:
+        idx = prize_lower.find(kw)
+        if idx == -1:
+            continue
+        # Search ±200 chars around keyword
+        start = max(0, idx - 200)
+        chunk = body_text[start:idx + 200]
+        for pat in prize_patterns:
+            match = re.search(pat, chunk, re.IGNORECASE)
+            if match:
+                result["prize_pool"] = match.group(1).strip()
+                break
+        if result["prize_pool"]:
+            break
+
+    # Fallback: any large currency amount
+    if not result["prize_pool"]:
+        for pat in prize_patterns:
+            match = re.search(pat, body_text)
+            if match:
+                result["prize_pool"] = match.group(1).strip()
+                break
+
+    # ---- TEAM SIZE ----
+    team_patterns = [
+        r"team\s*size[:\s]*(\d+)\s*[-–to]+\s*(\d+)",
+        r"(\d+)\s*[-–to]+\s*(\d+)\s*(?:members?|people|participants?|per team)",
+        r"teams?\s+of\s+(?:up\s+to\s+)?(\d+)",
+        r"max(?:imum)?\s*(?:team)?\s*(?:size)?\s*[:\s]*(\d+)",
+        r"(\d+)\s*[-–]\s*(\d+)\s*$",  # in FAQ: "2 - 4"
+    ]
+    for pat in team_patterns:
+        match = re.search(pat, body_text, re.IGNORECASE)
+        if match:
+            groups = [g for g in match.groups() if g]
+            if len(groups) == 2:
+                result["team_size"] = {"min": int(groups[0]), "max": int(groups[1])}
+            elif len(groups) == 1:
+                result["team_size"] = {"min": 1, "max": int(groups[0])}
+            break
+
+    # ---- PROBLEM STATEMENTS / TRACKS / DOMAINS ----
+    ps = []
+    seen_ps = set()
+
+    # Pattern 1: "Domains: AI, ML, Web App" (Devfolio style)
+    domain_match = re.search(
+        r"(?:domains?|themes?|tracks?|categories|verticals|areas?)[:\s]+([^\n💡🏆🎁🎟️📍📅⏳📞🌮]+)",
+        body_text, re.IGNORECASE
+    )
+    if domain_match:
+        items = re.split(r"[,•|/]", domain_match.group(1))
+        for item in items:
+            item = item.strip().rstrip(".")
+            if 3 < len(item) < 150 and item.lower() not in seen_ps:
+                seen_ps.add(item.lower())
+                ps.append({"track": "", "title": item})
+
+    # Pattern 2: Numbered problem statements: "PS1: ...", "Problem Statement 1 - ..."
+    for match in re.finditer(
+        r"(?:PS|Problem\s*Statement|Theme|Track|Challenge)\s*#?(\d+)\s*[:\-–]\s*(.{5,200})",
+        body_text, re.IGNORECASE
+    ):
+        num = match.group(1)
+        title = match.group(2).strip().split("\n")[0]
+        if title.lower() not in seen_ps and len(title) > 4:
+            seen_ps.add(title.lower())
+            ps.append({"track": f"Track {num}", "title": title})
+
+    # Pattern 3: Devpost-style theme tags (already in themes list from JS)
+    # Pattern 4: Bulleted lists after "Themes" or "Tracks" heading
+    for match in re.finditer(
+        r"(?:themes?|tracks?|problem\s*statements?|challenges?|domains?)\s*[:\n]"
+        r"((?:\s*[-•●▸]\s*.{5,200}\n?)+)",
+        body_text, re.IGNORECASE
+    ):
+        items = re.findall(r"[-•●▸]\s*(.{5,200})", match.group(1))
+        for item in items:
+            item = item.strip().split("\n")[0]
+            if item.lower() not in seen_ps and 4 < len(item) < 200:
+                seen_ps.add(item.lower())
+                ps.append({"track": "", "title": item})
+
+    result["problem_statements"] = ps[:20]
+    return result
+
+
+# ============================================================
+# PLAYWRIGHT SCRAPER — gets innerText + meta from rendered page
+# ============================================================
+
+EXTRACT_SCRIPT = """() => {
+    const getMeta = (name) => {
+        const el = document.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
+        return el ? el.getAttribute('content') || '' : '';
+    };
+
+    // Name: try multiple selectors
+    const nameSelectors = [
+        'h1',
+        '.hackathon-name', '.event-name', '.challenge-title',
+        '#challenge-title', '.opp-title',
+    ];
+    let name = '';
+    for (const sel of nameSelectors) {
+        const el = document.querySelector(sel);
+        if (el && el.textContent.trim().length > 2) {
+            name = el.textContent.trim();
+            break;
+        }
+    }
+    name = name || getMeta('og:title') || document.title.split('|')[0].trim();
+
+    // Banner
+    const banner = getMeta('og:image') || '';
+
+    // Description
+    let description = getMeta('og:description') || getMeta('description') || '';
+
+    // Full page text for parsing
+    const bodyText = document.body.innerText;
+
+    // For Devpost: extract themes from tag links
+    const themes = [];
+    document.querySelectorAll('a[href*="themes"]').forEach(a => {
+        const t = a.textContent.trim();
+        if (t && t.length > 2 && t.length < 100) themes.push(t);
+    });
+
+    // Devpost sidebar prize text
+    let sidebarPrize = '';
+    document.querySelectorAll('a[href*="prizes"], .prize, [class*="prize"]').forEach(el => {
+        const t = el.textContent.trim();
+        if (t && t.length > 2) sidebarPrize += t + ' ';
+    });
+
+    // Resource links: PDFs, Google Drive, problem statements, rules, guidelines
+    const resourceLinks = [];
+    const seenHrefs = new Set();
+    const linkKeywords = ['problem', 'statement', 'pdf', 'rule', 'guideline', 'brochure', 'document', 'brief', 'challenge', 'track', 'theme', 'schedule', 'timeline'];
+    document.querySelectorAll('a[href]').forEach(a => {
+        const href = a.href || '';
+        const text = a.textContent.trim();
+        const hrefLower = href.toLowerCase();
+        const textLower = text.toLowerCase();
+        if (seenHrefs.has(href) || !href || href === '#') return;
+
+        const isPdf = hrefLower.endsWith('.pdf') || hrefLower.includes('/pdf');
+        const isDrive = hrefLower.includes('drive.google.com') || hrefLower.includes('docs.google.com');
+        const isDropbox = hrefLower.includes('dropbox.com');
+        const isRelevant = linkKeywords.some(kw => textLower.includes(kw) || hrefLower.includes(kw));
+
+        if (isPdf || isDrive || isDropbox || isRelevant) {
+            seenHrefs.add(href);
+            resourceLinks.push({
+                text: text.substring(0, 150) || 'Document',
+                url: href,
+                type: isPdf ? 'pdf' : isDrive ? 'google_drive' : isDropbox ? 'dropbox' : 'link',
+            });
+        }
+    });
+
+    return {
+        name: name.substring(0, 200),
+        description: description.substring(0, 2000),
+        banner_url: banner,
+        bodyText: bodyText.substring(0, 30000),
+        themes: themes,
+        sidebarPrize: sidebarPrize.trim(),
+        resourceLinks: resourceLinks.slice(0, 30),
+    };
+}"""
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    global playwright, browser
+    from playwright.async_api import async_playwright
+
+    playwright = await async_playwright().start()
+    browser = await playwright.chromium.launch(
+        headless=True,
+        args=["--no-sandbox", "--disable-setuid-sandbox"],
+    )
+    print("[Scraper] Playwright browser initialized")
+
+
+@app.on_event("shutdown")
+async def shutdown() -> None:
+    global playwright, browser
+
+    try:
+        if browser is not None:
+            await browser.close()
+            print("[Scraper] Browser closed")
+    finally:
+        browser = None
+
+    try:
+        if playwright is not None:
+            await playwright.stop()
+            print("[Scraper] Playwright stopped")
+    finally:
+        playwright = None
+
+async def scrape_with_playwright(url: str, platform: str) -> dict:
+    """Scrape using Playwright — renders JS, grabs full innerText for parsing."""
+    global browser
+    try:
+        if browser is None:
+            return {
+                "scrape_success": False,
+                "error": "Browser is not initialized. Service startup failed.",
+            }
+
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+        )
+
+        try:
+            page = await context.new_page()
+
+            print(f"[Scraper] Navigating to {url} (platform: {platform})")
+            await page.goto(url, wait_until="domcontentloaded", timeout=20000)
+
+            # Wait for JS rendering — longer for SPAs
+            wait_time = 8 if platform in ("Unstop",) else 5
+            print(f"[Scraper] Waiting {wait_time}s for JS rendering...")
+            await page.wait_for_timeout(wait_time * 1000)
+
+            # Scroll to trigger lazy content
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight * 2 / 3)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, 0)")
+            await asyncio.sleep(0.5)
+
+            # Extract structured + raw text data
+            data = await page.evaluate(EXTRACT_SCRIPT)
+
+            body_text = data.get("bodyText", "")
+            name = data.get("name", "")
+            description = data.get("description", "")
+
+            print(f"[Scraper] Extracted name: '{name}', bodyText length: {len(body_text)}")
+
+            # Parse all fields from full innerText
+            extracted = extract_all_from_text(body_text, platform)
+
+            # Devpost themes from sidebar tags
+            themes = data.get("themes", [])
+            if themes and not extracted["problem_statements"]:
+                seen = set()
+                for t in themes:
+                    if t.lower() not in seen:
+                        seen.add(t.lower())
+                        extracted["problem_statements"].append({"track": "Theme", "title": t})
+
+            # Sidebar prize fallback (Devpost)
+            if not extracted["prize_pool"] and data.get("sidebarPrize"):
+                prize_text = data["sidebarPrize"]
+                for pat in [r"(\$[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M))?)", r"(₹[\d,]+)"]:
+                    m = re.search(pat, prize_text)
+                    if m:
+                        extracted["prize_pool"] = m.group(1)
+                        break
+                if not extracted["prize_pool"]:
+                    extracted["prize_pool"] = prize_text[:100]
+
+            return {
+                "name": name,
+                "description": description,
+                "banner_url": data.get("banner_url", ""),
+                "scrape_success": bool(name and len(name) > 2),
+                "resource_links": data.get("resourceLinks", []),
+                **extracted,
+            }
+        finally:
+            await context.close()
+
+    except Exception as e:
+        print(f"[Scraper] Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return {"scrape_success": False, "error": str(e)}
+
+
+# ============================================================
+# API ROUTES
+# ============================================================
+
+@app.get("/")
+async def root():
+    return {"status": "ok", "service": "HackTrack Scraper v3"}
+
+
+@app.post("/scrape", response_model=ScrapeResponse)
+async def scrape(request: ScrapeRequest):
+    url = request.url.strip()
+    platform = detect_platform(url)
+    print(f"\n[Scraper] === New scrape request: {url} (platform={platform}) ===")
+
+    try:
+        data = await scrape_with_playwright(url, platform)
+
+        response = ScrapeResponse(
+            name=data.get("name", ""),
+            platform=platform,
+            banner_url=data.get("banner_url", ""),
+            description=data.get("description", ""),
+            registration_deadline=data.get("registration_deadline", ""),
+            submission_deadline=data.get("submission_deadline", ""),
+            result_date=data.get("result_date", ""),
+            start_date=data.get("start_date", ""),
+            end_date=data.get("end_date", ""),
+            prize_pool=data.get("prize_pool", ""),
+            team_size=data.get("team_size", {"min": 1, "max": 4}),
+            problem_statements=data.get("problem_statements", []),
+            resource_links=data.get("resource_links", []),
+            scrape_success=data.get("scrape_success", False),
+            url=url,
+        )
+
+        print(f"[Scraper] Result: name='{response.name}', dates=({response.start_date}, {response.end_date}, reg={response.registration_deadline}, sub={response.submission_deadline}), prize='{response.prize_pool}', team={response.team_size}, ps={len(response.problem_statements)}")
+        return response
+
+    except Exception as e:
+        print(f"[Scraper] Endpoint error: {e}")
+        return ScrapeResponse(platform=platform, url=url, scrape_success=False)
+
+

requirements.txt

@@ -0,0 +1,4 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.0
+playwright==1.45.0
+pydantic==2.8.0