Update app.py

app.py

@@ -1,43 +1,20 @@
-"""
-HackTrack Scraper v4.0
-- Groq LLM (llama-3.3-70b-versatile) for intelligent extraction
-- Platforms: Devfolio, Unstop, Devpost, DoraHacks, MLH
-"""
-
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel, Field
 import asyncio
 import re
 import sys
-import os
-import json
 from urllib.parse import urlparse
 from typing import List
 from datetime import datetime
 
-# Groq client — free tier, llama-3.3-70b
-from groq import Groq
-
 if sys.platform == "win32":
+    # Playwright launches a driver subprocess; Proactor loop supports subprocess APIs on Windows.
     asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
 
-app = FastAPI(title="HackTrack Scraper", version="4.0.0")
-
-# ── Groq setup ────────────────────────────────────────────────────────────────
-_groq_client: Groq | None = None
-
-def get_groq() -> Groq | None:
-    global _groq_client
-    if _groq_client is None:
-        key = os.environ.get("GROQ_API_KEY", "")
-        if key:
-            _groq_client = Groq(api_key=key)
-    return _groq_client
-
-GROQ_MODEL = "llama-3.3-70b-versatile"
+app = FastAPI(title="HackTrack Scraper", version="3.0.0")
 
-# ── Global Playwright runtime ─────────────────────────────────────────────────
+# Global Playwright runtime objects reused across requests.
 playwright = None
 browser = None
 
@@ -49,10 +26,6 @@ app.add_middleware(
 )
 
 
-# ══════════════════════════════════════════════════════════════════════════════
-# MODELS
-# ══════════════════════════════════════════════════════════════════════════════
-
 class ScrapeRequest(BaseModel):
     url: str
 
@@ -73,50 +46,55 @@ class ScrapeResponse(BaseModel):
     resource_links: List[dict] = Field(default_factory=list)
     scrape_success: bool = False
     url: str = ""
-    llm_used: bool = False   # tells frontend whether Groq enriched this
-
 
-# ══════════════════════════════════════════════════════════════════════════════
-# PLATFORM DETECTION
-# ══════════════════════════════════════════════════════════════════════════════
 
 def detect_platform(url: str) -> str:
     domain = urlparse(url).netloc.lower()
-    if "devfolio" in domain:   return "Devfolio"
-    if "unstop"   in domain:   return "Unstop"
-    if "devpost"  in domain:   return "Devpost"
-    if "dorahacks" in domain:  return "DoraHacks"
-    if "mlh.io"   in domain:   return "MLH"
-    if "hackerearth" in domain: return "HackerEarth"
-    if "hackerrank"  in domain: return "HackerRank"
+    if "devfolio" in domain:
+        return "Devfolio"
+    elif "unstop" in domain:
+        return "Unstop"
+    elif "devpost" in domain:
+        return "Devpost"
+    elif "dorahacks" in domain:
+        return "DoraHacks"
     return "Other"
 
 
-# ══════════════════════════════════════════════════════════════════════════════
-# DATE PARSING
-# ══════════════════════════════════════════════════════════════════════════════
+# ============================================================
+# DATE PARSING — robust multi-format
+# ============================================================
+MONTH_MAP = {
+    "jan": 1, "january": 1, "feb": 2, "february": 2, "mar": 3, "march": 3,
+    "apr": 4, "april": 4, "may": 5, "jun": 6, "june": 6,
+    "jul": 7, "july": 7, "aug": 8, "august": 8, "sep": 9, "sept": 9, "september": 9,
+    "oct": 10, "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12,
+}
 
 DATE_FORMATS = [
     "%Y-%m-%d", "%Y/%m/%d",
     "%d %B %Y", "%d %b %Y", "%d %B, %Y", "%d %b, %Y",
     "%B %d, %Y", "%b %d, %Y", "%B %d %Y", "%b %d %Y",
-    "%m/%d/%Y",  "%d/%m/%Y",
-    "%B %d",     "%b %d",
+    "%m/%d/%Y", "%d/%m/%Y",
+    "%B %d", "%b %d",
 ]
 
 
 def parse_any_date(text: str, fallback_year: int = None) -> str:
+    """Parse many date formats to YYYY-MM-DD. Handles partial dates."""
     if not text:
         return ""
     text = text.strip()
     text = re.sub(r"(\d+)(st|nd|rd|th)", r"\1", text)
     text = re.sub(r"\s+", " ", text)
-    fallback_year = fallback_year or datetime.now().year
+
+    if not fallback_year:
+        fallback_year = datetime.now().year
 
     for fmt in DATE_FORMATS:
         try:
             dt = datetime.strptime(text, fmt)
-            if dt.year == 1900:
+            if dt.year == 1900:  # no year in format
                 dt = dt.replace(year=fallback_year)
                 if dt < datetime.now():
                     dt = dt.replace(year=fallback_year + 1)
@@ -127,8 +105,9 @@ def parse_any_date(text: str, fallback_year: int = None) -> str:
 
 
 def find_dates_near(text: str, keywords: List[str], window: int = 400) -> str:
+    """Find dates within `window` chars after any keyword."""
     lower = text.lower()
-    patterns = [
+    all_date_patterns = [
         r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})",
         r"(\d{1,2}\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)[,]?\s+\d{4})",
         r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2}(?:st|nd|rd|th)?[,]?\s*\d{0,4})",
@@ -138,87 +117,22 @@ def find_dates_near(text: str, keywords: List[str], window: int = 400) -> str:
         idx = lower.find(kw.lower())
         if idx == -1:
             continue
-        chunk = text[idx: idx + window]
-        for pat in patterns:
-            m = re.search(pat, chunk, re.IGNORECASE)
-            if m:
-                parsed = parse_any_date(m.group(1))
+        chunk = text[idx:idx + window]
+        for pat in all_date_patterns:
+            match = re.search(pat, chunk, re.IGNORECASE)
+            if match:
+                parsed = parse_any_date(match.group(1))
                 if parsed:
                     return parsed
     return ""
 
 
-# ══════════════════════════════════════════════════════════════════════════════
-# GROQ LLM EXTRACTION  (single call, returns full structured dict)
-# ══════════════════════════════════════════════════════════════════════════════
-
-def groq_extract(body_text: str, platform: str) -> dict | None:
-    """
-    One Groq call extracts ALL fields at once.
-    Returns None if Groq is unavailable or call fails.
-    """
-    client = get_groq()
-    if not client:
-        return None
-
-    # Trim to ~5000 chars to stay within token limits comfortably
-    excerpt = body_text[:5000]
-
-    prompt = f"""You are extracting structured data from a hackathon page ({platform}).
-
-Return ONLY valid JSON — no markdown, no explanation.
-
-Schema:
-{{
-  "registration_deadline": "YYYY-MM-DD or empty string",
-  "submission_deadline":   "YYYY-MM-DD or empty string",
-  "result_date":           "YYYY-MM-DD or empty string",
-  "start_date":            "YYYY-MM-DD or empty string",
-  "end_date":              "YYYY-MM-DD or empty string",
-  "prize_pool":            "raw string like ₹5,00,000 or $10,000 or empty string",
-  "team_size":             {{"min": 1, "max": 4}},
-  "problem_statements":    [
-    {{"track": "optional track label", "title": "PS or theme title"}}
-  ]
-}}
-
-Rules:
-- Dates: assume year {datetime.now().year} if missing; use YYYY-MM-DD format.
-- prize_pool: keep original currency symbol and denomination text (₹2 Lakh, $10K, etc.).
-- team_size: extract min/max members. Default {{"min":1,"max":4}} if not found.
-- problem_statements: list every unique track/theme/PS. Max 20 items.
-- If a field is not found, use "" or [] or the default value shown.
-
-Page text:
-{excerpt}"""
-
-    try:
-        resp = client.chat.completions.create(
-            model=GROQ_MODEL,
-            max_tokens=1200,
-            temperature=0.05,
-            messages=[
-                {
-                    "role": "system",
-                    "content": "You extract structured hackathon data. Respond with valid JSON only.",
-                },
-                {"role": "user", "content": prompt},
-            ],
-        )
-        raw = resp.choices[0].message.content.strip()
-        # Strip markdown fences if model wraps output
-        raw = re.sub(r"```(?:json)?", "", raw).strip().rstrip("`").strip()
-        return json.loads(raw)
-    except Exception as e:
-        print(f"[Groq] extraction failed: {e}")
-        return None
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# REGEX FALLBACK EXTRACTION  (same logic as v3, kept as safety net)
-# ════════════��═════════════════════════════════════════════════════════════════
+# ============================================================
+# EXTRACT from full page innerText (the reliable approach)
+# ============================================================
 
-def regex_extract(body_text: str, platform: str) -> dict:
+def extract_all_from_text(body_text: str, platform: str) -> dict:
+    """Extract hackathon details from page innerText using text patterns."""
     result = {
         "registration_deadline": "",
         "submission_deadline": "",
@@ -230,46 +144,56 @@ def regex_extract(body_text: str, platform: str) -> dict:
         "problem_statements": [],
     }
 
-    # Dates
+    # ---- DATES ----
+    # Registration deadline
     result["registration_deadline"] = find_dates_near(body_text, [
         "registration close", "registrations close", "register by",
         "last date to register", "registration deadline", "applications close",
         "apply by", "registration ends", "sign up deadline",
     ])
+
+    # Submission deadline
     result["submission_deadline"] = find_dates_near(body_text, [
         "submission deadline", "submission closes", "submissions close",
         "submit by", "last date to submit", "submission end",
-        "final submission", "project submission", "deadline",
+        "final submission", "project submission",
+        "deadline",  # generic fallback last
     ])
 
-    # "Runs from Mar 25 - 26, 2026"
+    # Start date — Devfolio uses "Runs from Mar 25 - 26, 2026"
     runs_from = re.search(
         r"(?:runs?\s+from|starts?\s+(?:on|from)?|begins?\s+(?:on)?|commences?\s+(?:on)?)\s*[:\-]?\s*"
-        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|"
-        r"Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2})"
-        r"(?:\s*[-–]\s*(\d{1,2}))?(?:[,\s]+(\d{4}))?",
-        body_text, re.IGNORECASE,
+        r"((?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2})"
+        r"(?:\s*[-–]\s*(\d{1,2}))?"
+        r"(?:[,\s]+(\d{4}))?",
+        body_text, re.IGNORECASE
     )
     if runs_from:
+        start_text = runs_from.group(1)
         year = runs_from.group(3) or str(datetime.now().year)
-        result["start_date"] = parse_any_date(f"{runs_from.group(1)} {year}")
-        if runs_from.group(2):
+        result["start_date"] = parse_any_date(f"{start_text} {year}")
+        if runs_from.group(2) and runs_from.group(1):
             month = runs_from.group(1).split()[0]
             result["end_date"] = parse_any_date(f"{month} {runs_from.group(2)} {year}")
 
     if not result["start_date"]:
         result["start_date"] = find_dates_near(body_text, [
-            "start date", "starts on", "begins on", "hackathon starts", "event starts",
+            "start date", "starts on", "begins on", "hackathon starts",
+            "event starts", "event date", "dates:",
         ])
+
     if not result["end_date"]:
         result["end_date"] = find_dates_near(body_text, [
             "end date", "ends on", "hackathon ends", "event ends",
         ])
+
+    # Result date
     result["result_date"] = find_dates_near(body_text, [
-        "result", "winners announced", "announcement", "results declared",
+        "result", "winners announced", "announcement", "winner announcement",
+        "results declared", "shortlist",
     ])
 
-    # Prize
+    # ---- PRIZE POOL ----
     prize_patterns = [
         r"(₹\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
         r"(\$\s*[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M|million|thousand))?)",
@@ -278,242 +202,161 @@ def regex_extract(body_text: str, platform: str) -> dict:
         r"(INR\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
         r"(Rs\.?\s*[\d,]+(?:\.\d+)?(?:\s*(?:Lakhs?|Lacs?|Crores?|Cr|K|k|L))?)",
     ]
+
+    # Find prize amounts near keywords like "prize", "reward", "worth", "win"
     prize_lower = body_text.lower()
     for kw in ["prize", "reward", "worth", "winning", "bounty", "in cash", "in prizes"]:
         idx = prize_lower.find(kw)
         if idx == -1:
             continue
-        chunk = body_text[max(0, idx - 200): idx + 200]
+        # Search ±200 chars around keyword
+        start = max(0, idx - 200)
+        chunk = body_text[start:idx + 200]
         for pat in prize_patterns:
-            m = re.search(pat, chunk, re.IGNORECASE)
-            if m:
-                result["prize_pool"] = m.group(1).strip()
+            match = re.search(pat, chunk, re.IGNORECASE)
+            if match:
+                result["prize_pool"] = match.group(1).strip()
                 break
         if result["prize_pool"]:
             break
+
+    # Fallback: any large currency amount
     if not result["prize_pool"]:
         for pat in prize_patterns:
-            m = re.search(pat, body_text)
-            if m:
-                result["prize_pool"] = m.group(1).strip()
+            match = re.search(pat, body_text)
+            if match:
+                result["prize_pool"] = match.group(1).strip()
                 break
 
-    # Team size
-    for pat in [
+    # ---- TEAM SIZE ----
+    team_patterns = [
         r"team\s*size[:\s]*(\d+)\s*[-–to]+\s*(\d+)",
         r"(\d+)\s*[-–to]+\s*(\d+)\s*(?:members?|people|participants?|per team)",
         r"teams?\s+of\s+(?:up\s+to\s+)?(\d+)",
         r"max(?:imum)?\s*(?:team)?\s*(?:size)?\s*[:\s]*(\d+)",
-    ]:
-        m = re.search(pat, body_text, re.IGNORECASE)
-        if m:
-            groups = [g for g in m.groups() if g]
-            result["team_size"] = (
-                {"min": int(groups[0]), "max": int(groups[1])} if len(groups) == 2
-                else {"min": 1, "max": int(groups[0])}
-            )
+        r"(\d+)\s*[-–]\s*(\d+)\s*$",  # in FAQ: "2 - 4"
+    ]
+    for pat in team_patterns:
+        match = re.search(pat, body_text, re.IGNORECASE)
+        if match:
+            groups = [g for g in match.groups() if g]
+            if len(groups) == 2:
+                result["team_size"] = {"min": int(groups[0]), "max": int(groups[1])}
+            elif len(groups) == 1:
+                result["team_size"] = {"min": 1, "max": int(groups[0])}
             break
 
-    # Problem statements
-    ps, seen = [], set()
+    # ---- PROBLEM STATEMENTS / TRACKS / DOMAINS ----
+    ps = []
+    seen_ps = set()
 
-    domain_m = re.search(
+    # Pattern 1: "Domains: AI, ML, Web App" (Devfolio style)
+    domain_match = re.search(
         r"(?:domains?|themes?|tracks?|categories|verticals|areas?)[:\s]+([^\n💡🏆🎁🎟️📍📅⏳📞🌮]+)",
-        body_text, re.IGNORECASE,
+        body_text, re.IGNORECASE
     )
-    if domain_m:
-        for item in re.split(r"[,•|/]", domain_m.group(1)):
+    if domain_match:
+        items = re.split(r"[,•|/]", domain_match.group(1))
+        for item in items:
             item = item.strip().rstrip(".")
-            if 3 < len(item) < 150 and item.lower() not in seen:
-                seen.add(item.lower())
+            if 3 < len(item) < 150 and item.lower() not in seen_ps:
+                seen_ps.add(item.lower())
                 ps.append({"track": "", "title": item})
 
-    for m in re.finditer(
+    # Pattern 2: Numbered problem statements: "PS1: ...", "Problem Statement 1 - ..."
+    for match in re.finditer(
         r"(?:PS|Problem\s*Statement|Theme|Track|Challenge)\s*#?(\d+)\s*[:\-–]\s*(.{5,200})",
-        body_text, re.IGNORECASE,
+        body_text, re.IGNORECASE
     ):
-        title = m.group(2).strip().split("\n")[0]
-        if title.lower() not in seen and len(title) > 4:
-            seen.add(title.lower())
-            ps.append({"track": f"Track {m.group(1)}", "title": title})
-
-    for m in re.finditer(
+        num = match.group(1)
+        title = match.group(2).strip().split("\n")[0]
+        if title.lower() not in seen_ps and len(title) > 4:
+            seen_ps.add(title.lower())
+            ps.append({"track": f"Track {num}", "title": title})
+
+    # Pattern 3: Devpost-style theme tags (already in themes list from JS)
+    # Pattern 4: Bulleted lists after "Themes" or "Tracks" heading
+    for match in re.finditer(
         r"(?:themes?|tracks?|problem\s*statements?|challenges?|domains?)\s*[:\n]"
         r"((?:\s*[-•●▸]\s*.{5,200}\n?)+)",
-        body_text, re.IGNORECASE,
+        body_text, re.IGNORECASE
     ):
-        for item in re.findall(r"[-•●▸]\s*(.{5,200})", m.group(1)):
+        items = re.findall(r"[-•●▸]\s*(.{5,200})", match.group(1))
+        for item in items:
             item = item.strip().split("\n")[0]
-            if item.lower() not in seen and 4 < len(item) < 200:
-                seen.add(item.lower())
+            if item.lower() not in seen_ps and 4 < len(item) < 200:
+                seen_ps.add(item.lower())
                 ps.append({"track": "", "title": item})
 
     result["problem_statements"] = ps[:20]
     return result
 
 
-# ══════════════════════════════════════════════════════════════════════════════
-# MERGE: LLM results take precedence, regex fills gaps
-# ══════════════════════════════════════════════════════════════════════════════
-
-def merge_results(llm: dict | None, regex: dict) -> tuple[dict, bool]:
-    """
-    Prefer LLM values; fall back to regex for any blank field.
-    Returns (merged_dict, llm_was_used).
-    """
-    if llm is None:
-        return regex, False
-
-    merged = {}
-    date_fields = [
-        "registration_deadline", "submission_deadline",
-        "result_date", "start_date", "end_date",
-    ]
-    for f in date_fields:
-        merged[f] = llm.get(f) or regex.get(f, "")
-
-    merged["prize_pool"] = llm.get("prize_pool") or regex.get("prize_pool", "")
-
-    # team_size: use LLM unless it's the bare default and regex found something
-    llm_ts  = llm.get("team_size",  {"min": 1, "max": 4})
-    regex_ts = regex.get("team_size", {"min": 1, "max": 4})
-    if llm_ts == {"min": 1, "max": 4} and regex_ts != {"min": 1, "max": 4}:
-        merged["team_size"] = regex_ts
-    else:
-        merged["team_size"] = llm_ts
-
-    # problem_statements: prefer LLM list; fall back to regex
-    llm_ps   = llm.get("problem_statements",  [])
-    regex_ps = regex.get("problem_statements", [])
-    merged["problem_statements"] = llm_ps if llm_ps else regex_ps
-
-    return merged, True
-
+# ============================================================
+# PLAYWRIGHT SCRAPER — gets innerText + meta from rendered page
+# ============================================================
 
-# ══════════════════════════════════════════════════════════════════════════════
-# PLATFORM-SPECIFIC JS EXTRACTION SCRIPTS
-# ══════════════════════════════════════════════════════════════════════════════
-
-# Generic script — works for Devpost, Unstop, DoraHacks, Other
-GENERIC_EXTRACT_SCRIPT = """() => {
+EXTRACT_SCRIPT = """() => {
     const getMeta = (name) => {
         const el = document.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
         return el ? el.getAttribute('content') || '' : '';
     };
+
+    // Name: try multiple selectors
     const nameSelectors = [
-        'h1', '.hackathon-name', '.event-name', '.challenge-title',
-        '#challenge-title', '.opp-title', '[class*="hackathon-title"]',
-        '[class*="event-title"]', '[class*="challenge-name"]',
+        'h1',
+        '.hackathon-name', '.event-name', '.challenge-title',
+        '#challenge-title', '.opp-title',
     ];
     let name = '';
     for (const sel of nameSelectors) {
         const el = document.querySelector(sel);
         if (el && el.textContent.trim().length > 2) {
-            name = el.textContent.trim(); break;
+            name = el.textContent.trim();
+            break;
         }
     }
     name = name || getMeta('og:title') || document.title.split('|')[0].trim();
 
+    // Banner
     const banner = getMeta('og:image') || '';
+
+    // Description
     let description = getMeta('og:description') || getMeta('description') || '';
+
+    // Full page text for parsing
     const bodyText = document.body.innerText;
 
-    // Devpost themes
+    // For Devpost: extract themes from tag links
     const themes = [];
     document.querySelectorAll('a[href*="themes"]').forEach(a => {
         const t = a.textContent.trim();
         if (t && t.length > 2 && t.length < 100) themes.push(t);
     });
 
-    // Prize sidebar (Devpost)
+    // Devpost sidebar prize text
     let sidebarPrize = '';
     document.querySelectorAll('a[href*="prizes"], .prize, [class*="prize"]').forEach(el => {
         const t = el.textContent.trim();
         if (t && t.length > 2) sidebarPrize += t + ' ';
     });
 
-    // Resource links
+    // Resource links: PDFs, Google Drive, problem statements, rules, guidelines
     const resourceLinks = [];
     const seenHrefs = new Set();
-    const kws = ['problem','statement','pdf','rule','guideline','brochure',
-                 'document','brief','challenge','track','theme','schedule','timeline'];
+    const linkKeywords = ['problem', 'statement', 'pdf', 'rule', 'guideline', 'brochure', 'document', 'brief', 'challenge', 'track', 'theme', 'schedule', 'timeline'];
     document.querySelectorAll('a[href]').forEach(a => {
         const href = a.href || '';
         const text = a.textContent.trim();
-        const hl = href.toLowerCase(), tl = text.toLowerCase();
+        const hrefLower = href.toLowerCase();
+        const textLower = text.toLowerCase();
         if (seenHrefs.has(href) || !href || href === '#') return;
-        const isPdf      = hl.endsWith('.pdf') || hl.includes('/pdf');
-        const isDrive    = hl.includes('drive.google.com') || hl.includes('docs.google.com');
-        const isDropbox  = hl.includes('dropbox.com');
-        const isRelevant = kws.some(kw => tl.includes(kw) || hl.includes(kw));
-        if (isPdf || isDrive || isDropbox || isRelevant) {
-            seenHrefs.add(href);
-            resourceLinks.push({
-                text: text.substring(0, 150) || 'Document',
-                url: href,
-                type: isPdf ? 'pdf' : isDrive ? 'google_drive' : isDropbox ? 'dropbox' : 'link',
-            });
-        }
-    });
-
-    return {
-        name: name.substring(0, 200),
-        description: description.substring(0, 2000),
-        banner_url: banner,
-        bodyText: bodyText.substring(0, 30000),
-        themes,
-        sidebarPrize: sidebarPrize.trim(),
-        resourceLinks: resourceLinks.slice(0, 30),
-    };
-}"""
-
 
-# Devfolio-specific: clicks "About" tab to expose full description + dates
-DEVFOLIO_EXTRACT_SCRIPT = """async () => {
-    // Try clicking the About/Overview tab if present
-    const tabSelectors = ['a[href*="about"]', 'button[aria-label*="about" i]',
-                          '[role="tab"]', 'nav a'];
-    for (const sel of tabSelectors) {
-        const tabs = document.querySelectorAll(sel);
-        for (const tab of tabs) {
-            if (/about|overview/i.test(tab.textContent)) {
-                tab.click();
-                await new Promise(r => setTimeout(r, 1000));
-                break;
-            }
-        }
-    }
+        const isPdf = hrefLower.endsWith('.pdf') || hrefLower.includes('/pdf');
+        const isDrive = hrefLower.includes('drive.google.com') || hrefLower.includes('docs.google.com');
+        const isDropbox = hrefLower.includes('dropbox.com');
+        const isRelevant = linkKeywords.some(kw => textLower.includes(kw) || hrefLower.includes(kw));
 
-    const getMeta = (name) => {
-        const el = document.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
-        return el ? el.getAttribute('content') || '' : '';
-    };
-
-    let name = '';
-    for (const sel of ['h1', '.sc-hackathon-title', '[class*="title"]']) {
-        const el = document.querySelector(sel);
-        if (el && el.textContent.trim().length > 2) { name = el.textContent.trim(); break; }
-    }
-    name = name || getMeta('og:title') || document.title.split('|')[0].trim();
-
-    const bodyText = document.body.innerText;
-    const banner = getMeta('og:image') || '';
-    const description = getMeta('og:description') || getMeta('description') || '';
-
-    // Resource links
-    const resourceLinks = [];
-    const seenHrefs = new Set();
-    const kws = ['problem','statement','pdf','rule','guideline','brochure',
-                 'document','brief','challenge','track','theme','schedule','timeline'];
-    document.querySelectorAll('a[href]').forEach(a => {
-        const href = a.href || '';
-        const text = a.textContent.trim();
-        const hl = href.toLowerCase(), tl = text.toLowerCase();
-        if (seenHrefs.has(href) || !href || href === '#') return;
-        const isPdf      = hl.endsWith('.pdf') || hl.includes('/pdf');
-        const isDrive    = hl.includes('drive.google.com') || hl.includes('docs.google.com');
-        const isDropbox  = hl.includes('dropbox.com');
-        const isRelevant = kws.some(kw => tl.includes(kw) || hl.includes(kw));
         if (isPdf || isDrive || isDropbox || isRelevant) {
             seenHrefs.add(href);
             resourceLinks.push({
@@ -529,255 +372,148 @@ DEVFOLIO_EXTRACT_SCRIPT = """async () => {
         description: description.substring(0, 2000),
         banner_url: banner,
         bodyText: bodyText.substring(0, 30000),
-        themes: [],
-        sidebarPrize: '',
+        themes: themes,
+        sidebarPrize: sidebarPrize.trim(),
         resourceLinks: resourceLinks.slice(0, 30),
     };
 }"""
 
 
-# MLH: static listing — we grab individual event pages
-MLH_EXTRACT_SCRIPT = """() => {
-    const getMeta = (name) => {
-        const el = document.querySelector(`meta[property="${name}"], meta[name="${name}"]`);
-        return el ? el.getAttribute('content') || '' : '';
-    };
-    let name = getMeta('og:title') || document.title.split('|')[0].trim();
-    const banner = getMeta('og:image') || '';
-    const description = getMeta('og:description') || '';
-    const bodyText = document.body.innerText;
-
-    const resourceLinks = [];
-    const seenHrefs = new Set();
-    document.querySelectorAll('a[href]').forEach(a => {
-        const href = a.href || '';
-        const text = a.textContent.trim();
-        const hl = href.toLowerCase();
-        if (seenHrefs.has(href) || !href || href === '#') return;
-        if (hl.endsWith('.pdf') || hl.includes('drive.google.com')) {
-            seenHrefs.add(href);
-            resourceLinks.push({ text: text.substring(0, 150) || 'Document', url: href, type: 'pdf' });
-        }
-    });
-
-    return {
-        name: name.substring(0, 200),
-        description: description.substring(0, 2000),
-        banner_url: banner,
-        bodyText: bodyText.substring(0, 30000),
-        themes: [],
-        sidebarPrize: '',
-        resourceLinks: resourceLinks.slice(0, 20),
-    };
-}"""
-
-
-def get_extract_script(platform: str) -> str:
-    if platform == "Devfolio":
-        return DEVFOLIO_EXTRACT_SCRIPT
-    if platform == "MLH":
-        return MLH_EXTRACT_SCRIPT
-    return GENERIC_EXTRACT_SCRIPT
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# PLAYWRIGHT SCRAPER
-# ══════════════════════════════════════════════════════════════════════════════
-
-EMPTY_DATA = {
-    "name": "", "description": "", "banner_url": "",
-    "bodyText": "", "themes": [], "sidebarPrize": "", "resourceLinks": [],
-}
-
-
-async def safe_evaluate(page, script: str, fallback_script: str = None) -> dict:
-    """
-    Evaluate JS safely with retry on 'Execution context was destroyed'
-    caused by Devfolio /overview redirecting to / mid-scrape.
-    """
-    for attempt in range(3):
-        try:
-            try:
-                await page.wait_for_load_state("networkidle", timeout=8000)
-            except Exception:
-                pass
-            return await page.evaluate(script)
-        except Exception as e:
-            err = str(e)
-            print(f"[Scraper] evaluate attempt {attempt + 1} failed: {err[:150]}")
-            if "Execution context was destroyed" in err or "Frame was detached" in err:
-                print("[Scraper] Redirect detected — waiting for page to settle...")
-                try:
-                    await page.wait_for_load_state("domcontentloaded", timeout=12000)
-                    await asyncio.sleep(2)
-                except Exception:
-                    await asyncio.sleep(3)
-                continue
-            if fallback_script and attempt == 1:
-                print("[Scraper] Switching to generic fallback script...")
-                script = fallback_script
-                continue
-            break
-    print("[Scraper] All evaluate attempts exhausted — returning empty data")
-    return EMPTY_DATA
-
-
-async def scrape_with_playwright(url: str, platform: str) -> dict:
-    global browser
-    if browser is None:
-        return {"scrape_success": False, "error": "Browser not initialized"}
-
-    context = await browser.new_context(
-        user_agent=(
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/125.0.0.0 Safari/537.36"
-        ),
-        viewport={"width": 1920, "height": 1080},
-    )
-
-    try:
-        page = await context.new_page()
-        print(f"[Scraper] → {url}  (platform={platform})")
-
-        # Devfolio /overview redirects to / — wait for "load" so the redirect
-        # finishes before we evaluate JS.
-        wait_until = "load" if platform in ("Devfolio", "MLH") else "domcontentloaded"
-        try:
-            await page.goto(url, wait_until=wait_until, timeout=30000)
-        except Exception as e:
-            if "Timeout" in str(e):
-                print(f"[Scraper] goto timeout ({wait_until}) — proceeding anyway")
-            else:
-                raise
-
-        wait_map = {"Unstop": 9, "DoraHacks": 8, "Devfolio": 8, "MLH": 4}
-        wait_sec = wait_map.get(platform, 5)
-        print(f"[Scraper] Waiting {wait_sec}s for JS rendering...")
-        await page.wait_for_timeout(wait_sec * 1000)
-
-        for frac in [0.33, 0.66, 1.0, 0.0]:
-            try:
-                await page.evaluate(f"window.scrollTo(0, document.body.scrollHeight * {frac})")
-            except Exception:
-                pass
-            await asyncio.sleep(0.6)
-
-        await asyncio.sleep(1.0)
-
-        primary_script  = get_extract_script(platform)
-        fallback_script = GENERIC_EXTRACT_SCRIPT if primary_script != GENERIC_EXTRACT_SCRIPT else None
-        data = await safe_evaluate(page, primary_script, fallback_script)
-
-        body_text = data.get("bodyText", "")
-        print(f"[Scraper] bodyText={len(body_text)} chars, name='{data.get('name','')}'")
-
-        # ── Extraction pipeline ───────────────────────────────────────────────
-        # 1. Regex extraction (fast, always runs)
-        regex_result = regex_extract(body_text, platform)
-
-        # 2. Groq LLM extraction (slower, enriches results)
-        llm_result = groq_extract(body_text, platform)
-
-        # 3. Merge: LLM wins, regex fills gaps
-        merged, llm_used = merge_results(llm_result, regex_result)
-
-        # 4. Platform-specific post-processing
-        # Devpost: inject sidebar themes if PS list is empty
-        themes = data.get("themes", [])
-        if themes and not merged["problem_statements"]:
-            seen = set()
-            for t in themes:
-                if t.lower() not in seen:
-                    seen.add(t.lower())
-                    merged["problem_statements"].append({"track": "Theme", "title": t})
-
-        # Devpost: sidebar prize fallback
-        sidebar_prize = data.get("sidebarPrize", "")
-        if not merged["prize_pool"] and sidebar_prize:
-            for pat in [r"(\$[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M))?)", r"(₹[\d,]+)"]:
-                m = re.search(pat, sidebar_prize)
-                if m:
-                    merged["prize_pool"] = m.group(1)
-                    break
-            if not merged["prize_pool"]:
-                merged["prize_pool"] = sidebar_prize[:100]
-
-        return {
-            "name":         data.get("name", ""),
-            "description":  data.get("description", ""),
-            "banner_url":   data.get("banner_url", ""),
-            "resource_links": data.get("resourceLinks", []),
-            "scrape_success": bool(data.get("name") and len(data.get("name", "")) > 2),
-            "llm_used":     llm_used,
-            **merged,
-        }
-
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        return {"scrape_success": False, "error": str(e)}
-    finally:
-        await context.close()
-
-
-# ══════════════════════════════════════════════════════════════════════════════
-# APP LIFECYCLE
-# ══════════════════════════════════════════════════════════════════════════════
-
 @app.on_event("startup")
 async def startup() -> None:
     global playwright, browser
     from playwright.async_api import async_playwright
+
     playwright = await async_playwright().start()
     browser = await playwright.chromium.launch(
         headless=True,
-        args=["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],
+        args=["--no-sandbox", "--disable-setuid-sandbox"],
     )
-    groq_ready = "✓" if get_groq() else "✗ (set GROQ_API_KEY for LLM enrichment)"
-    print(f"[Scraper] Playwright ready. Groq={groq_ready}")
+    print("[Scraper] Playwright browser initialized")
 
 
 @app.on_event("shutdown")
 async def shutdown() -> None:
     global playwright, browser
+
     try:
-        if browser:  await browser.close()
+        if browser is not None:
+            await browser.close()
+            print("[Scraper] Browser closed")
     finally:
         browser = None
+
     try:
-        if playwright: await playwright.stop()
+        if playwright is not None:
+            await playwright.stop()
+            print("[Scraper] Playwright stopped")
     finally:
         playwright = None
 
+async def scrape_with_playwright(url: str, platform: str) -> dict:
+    """Scrape using Playwright — renders JS, grabs full innerText for parsing."""
+    global browser
+    try:
+        if browser is None:
+            return {
+                "scrape_success": False,
+                "error": "Browser is not initialized. Service startup failed.",
+            }
 
-# ══════════════════════════════════════════════════════════════════════════════
-# ROUTES
-# ══════════════════════════════════════════════════════════════════════════════
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+        )
 
-@app.get("/")
-async def root():
-    return {
-        "status": "ok",
-        "service": "HackTrack Scraper v4",
-        "groq_enabled": get_groq() is not None,
-        "platforms": ["Devfolio", "Devpost", "Unstop", "DoraHacks", "MLH", "HackerEarth", "HackerRank"],
-    }
+        try:
+            page = await context.new_page()
+
+            print(f"[Scraper] Navigating to {url} (platform: {platform})")
+            await page.goto(url, wait_until="domcontentloaded", timeout=20000)
+
+            # Wait for JS rendering — longer for SPAs
+            wait_time = 8 if platform in ("Unstop",) else 5
+            print(f"[Scraper] Waiting {wait_time}s for JS rendering...")
+            await page.wait_for_timeout(wait_time * 1000)
+
+            # Scroll to trigger lazy content
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight * 2 / 3)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await asyncio.sleep(1)
+            await page.evaluate("window.scrollTo(0, 0)")
+            await asyncio.sleep(0.5)
+
+            # Extract structured + raw text data
+            data = await page.evaluate(EXTRACT_SCRIPT)
+
+            body_text = data.get("bodyText", "")
+            name = data.get("name", "")
+            description = data.get("description", "")
+
+            print(f"[Scraper] Extracted name: '{name}', bodyText length: {len(body_text)}")
+
+            # Parse all fields from full innerText
+            extracted = extract_all_from_text(body_text, platform)
+
+            # Devpost themes from sidebar tags
+            themes = data.get("themes", [])
+            if themes and not extracted["problem_statements"]:
+                seen = set()
+                for t in themes:
+                    if t.lower() not in seen:
+                        seen.add(t.lower())
+                        extracted["problem_statements"].append({"track": "Theme", "title": t})
+
+            # Sidebar prize fallback (Devpost)
+            if not extracted["prize_pool"] and data.get("sidebarPrize"):
+                prize_text = data["sidebarPrize"]
+                for pat in [r"(\$[\d,]+(?:\.\d+)?(?:\s*(?:K|k|M))?)", r"(₹[\d,]+)"]:
+                    m = re.search(pat, prize_text)
+                    if m:
+                        extracted["prize_pool"] = m.group(1)
+                        break
+                if not extracted["prize_pool"]:
+                    extracted["prize_pool"] = prize_text[:100]
+
+            return {
+                "name": name,
+                "description": description,
+                "banner_url": data.get("banner_url", ""),
+                "scrape_success": bool(name and len(name) > 2),
+                "resource_links": data.get("resourceLinks", []),
+                **extracted,
+            }
+        finally:
+            await context.close()
 
+    except Exception as e:
+        print(f"[Scraper] Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return {"scrape_success": False, "error": str(e)}
+
+
+# ============================================================
+# API ROUTES
+# ============================================================
 
-@app.get("/health")
-async def health():
-    return {"status": "ok", "timestamp": datetime.utcnow().isoformat()}
+@app.get("/")
+async def root():
+    return {"status": "ok", "service": "HackTrack Scraper v3"}
 
 
 @app.post("/scrape", response_model=ScrapeResponse)
 async def scrape(request: ScrapeRequest):
     url = request.url.strip()
     platform = detect_platform(url)
-    print(f"\n[Scraper] === {url}  platform={platform} ===")
+    print(f"\n[Scraper] === New scrape request: {url} (platform={platform}) ===")
 
     try:
         data = await scrape_with_playwright(url, platform)
+
         response = ScrapeResponse(
             name=data.get("name", ""),
             platform=platform,
@@ -794,15 +530,11 @@ async def scrape(request: ScrapeRequest):
             resource_links=data.get("resource_links", []),
             scrape_success=data.get("scrape_success", False),
             url=url,
-            llm_used=data.get("llm_used", False),
-        )
-        print(
-            f"[Scraper] Done: name='{response.name}' "
-            f"reg={response.registration_deadline} sub={response.submission_deadline} "
-            f"prize='{response.prize_pool}' ps={len(response.problem_statements)} "
-            f"llm={response.llm_used}"
         )
+
+        print(f"[Scraper] Result: name='{response.name}', dates=({response.start_date}, {response.end_date}, reg={response.registration_deadline}, sub={response.submission_deadline}), prize='{response.prize_pool}', team={response.team_size}, ps={len(response.problem_statements)}")
         return response
+
     except Exception as e:
         print(f"[Scraper] Endpoint error: {e}")
-        return ScrapeResponse(platform=platform, url=url, scrape_success=False)
+        return ScrapeResponse(platform=platform, url=url, scrape_success=False)