app.py

# app.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
SaveNest — 'Fetch Everything' Crawler API
=========================================
- Enumerates product URLs via sitemaps (Shopify / Woo + generic sitemap discovery).
- Optional fallback BFS enumerator for stores without sitemaps (lightweight, bounded).
- Scrapes product details with JSON-LD first; CSS fallback; optional Playwright (Firefox) for JS-heavy pages (Daraz etc).
- Stateless on the server: supports paginated crawling (cursor/limit). You can persist the results in Firebase/Firestore from your app.
Endpoints:
- GET /health
- GET /            -> intro JSON
- GET /crawl       -> enumerate + scrape entire store (paginated)
  params:
      store        : domain OR key (e.g. "alfatah.pk", "daraz.pk")
      limit        : number of product pages to fetch this call (default 100, max 500)
      cursor       : resume offset (int; default 0)
      mode         : "full" (default) or "urls" (only URLs, no scraping)
      use_js       : 0/1 override for Playwright for this request (default: env USE_PLAYWRIGHT)
- GET /search      -> per-query live search (kept from previous build)
Designed for HuggingFace Spaces free tier:
- Restrict limit per call to avoid timeouts; use `cursor` to continue next page.
- Uses a gentle rate limiter per host.
"""

import asyncio, time, re, json, os, gzip, io
from typing import Optional, Dict, Any, List, Tuple, Iterable
from urllib.parse import urljoin, urlparse, quote_plus
from dataclasses import dataclass
from fastapi.middleware.cors import CORSMiddleware

import os
import httpx
from bs4 import BeautifulSoup
from fastapi import FastAPI, HTTPException, Query
from fastapi.responses import JSONResponse, HTMLResponse
from rapidfuzz import fuzz
import logging

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    datefmt='%H:%M:%S'
)
logger = logging.getLogger(__name__)
# ---------------------- Runtime config (via env) ----------------------
USE_PLAYWRIGHT_DEFAULT = os.getenv("USE_PLAYWRIGHT", "1") == "1"  # enable JS fallback by default now
PER_HOST_GAP = float(os.getenv("PER_HOST_GAP", "2"))            # politeness per host (seconds)
REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "30"))       # seconds
TTL_SECONDS = int(os.getenv("TTL_SECONDS", "180"))                # page TTL cache
MAX_CONCURRENCY = int(os.getenv("MAX_CONCURRENCY", "5"))          # parallel HTTP requests
CRAWL_LIMIT_CAP = int(os.getenv("CRAWL_LIMIT_CAP", "500"))        # per-call max items
MAX_FALLBACK_BFS_PAGES = int(os.getenv("MAX_FALLBACK_BFS_PAGES", "50"))  # BFS page cap if no sitemap

# ---------------------- Store registry (extend as needed) ------------
# Each entry supports:
#  base: canonical base url
#  platform: "shopify" | "woo" | "custom"
#  product_link_patterns: regex patterns to recognize product pages
#  product_fallback: CSS selectors for title/price if no JSON-LD
#  sitemap_hints: list of potential sitemap endpoints
#  bfs_seeds: optional seeds for BFS (when no sitemap is available)
# ---------------------- Store registry (extend as needed) ------------
STORES: List[Dict[str, Any]] = [
    # Existing — keep as-is
    {"base": "https://alfatah.pk", "platform": "shopify"},

    # NEW STORES from API #2
    {
        "base": "https://qne.com.pk",
        "platform": "custom",
        "search_url_template": "{base}/search?type=product&q={query}",
        "product_link_patterns": [r"/products?/[-a-z0-9]+", r"/p/[-a-z0-9]+"],
        "product_fallback": {
            "title": "h1.product-title, h1.entry-title, .product-name",
            "price": ".price, .amount, span.price, .woocommerce-Price-amount",
            "image": ".product-image img, img[src*='product'], .attachment-woocommerce_thumbnail"
        }
    },
    {
        "base": "https://springs.com.pk",
        "platform": "custom",
        "search_url_template": "{base}/search?q={query}",
        "product_link_patterns": [r"/products?/[-a-z0-9]+"],
        "product_fallback": {
            "title": "h1.product_title, h1.entry-title, h1[itemprop='name']",
            "price": ".price ins .amount, .price > span, .woocommerce-Price-amount",
            "image": ".woocommerce-product-gallery__image img, img.wp-post-image, img.attachment-product"
        }
    },
    {
        "base": "https://vmart.pk",
        "platform": "custom",
        "search_url_template": "{base}/search?q={query}",
        "product_link_patterns": [r"/products?/[-a-z0-9]+"],
        "product_fallback": {
            "title": "h1.product-title, .product-name, h1[itemprop='name']",
            "price": ".price, .amount, span.price-current",
            "image": "img.product-image, .main-image img, img[src*='product']"
        }
    },
    {
        "base": "https://grocerapp.pk",
        "platform": "custom",
        "search_url_template": "{base}/search?q={query}",
        "sitemap_hints": [
            "https://grocerapp.pk/sitemap_index.xml",
            "https://grocerapp.pk/sitemap_categories.xml",
            "https://grocerapp.pk/sitemap_subcategories.xml",
        ],
        "product_link_patterns": [r"/products/[-a-z0-9]+-\d+/?$"],
        "product_fallback": {
            "title": "h1.product-title, .product-detail h1, h1[itemprop='name']",
            "price": ".price-current, .amount, .sale-price",
            "image": ".product-image img, img.main-product-image, img[src*='product']"
        }
    },
]
STORE_DISPLAY_NAMES = {
    "alfatah.pk": "Al-Fatah",
    "qne.com.pk": "QnE",
    "springs.com.pk": "Springs Store",
    "vmart.pk": "Vmart",
    "grocerapp.pk": "GrocerApp",
}

# ---------------------- Helpers --------------------------------------
def extract_og_image(soup: BeautifulSoup, page_url: str) -> Optional[str]:
    # 1) <meta property="og:image" ...>
    meta = soup.find("meta", attrs={"property": "og:image"}) or soup.find("meta", attrs={"name": "og:image"})
    if meta and meta.get("content"):
        return urljoin(page_url, meta["content"].strip())
    # 2) <link rel="image_src" href="...">
    link = soup.find("link", rel=lambda v: v and "image_src" in v)
    if link and link.get("href"):
        return urljoin(page_url, link["href"].strip())
    return None

def d(u: str) -> str:
    return urlparse(u).netloc.lower().replace("www.", "")

def abs_url(base: str, href: str) -> str:
    return urljoin(base.rstrip("/") + "/", href)

def _norm(s: str) -> str:
    return re.sub(r"[^a-z0-9\s]+", " ", (s or "").lower()).strip()

def clean_price(text: str) -> Optional[float]:
    if not text:
        return None
    t = text.replace(",", "")
    m = re.search(r"(\d+(?:\.\d{1,2})?)", t)
    return float(m.group(1)) if m else None

def guess_store(domain_or_key: str) -> Optional[Dict[str, Any]]:
    key = domain_or_key.strip().lower()
    for s in STORES:
        if key in d(s["base"]) or key in s["base"].lower():
            return s
    return None

# politeness limiter
_last_hit: Dict[str, float] = {}
async def rate_limit(host: str, min_gap: float = PER_HOST_GAP):
    last = _last_hit.get(host, 0.0)
    dt = time.time() - last
    if dt < min_gap:
        await asyncio.sleep(min_gap - dt)
    _last_hit[host] = time.time()

BROWSER_HEADERS = {
    "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                   "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
    "Cache-Control": "no-cache", "Pragma": "no-cache",
}

# in-memory TTL cache
_cache: Dict[str, Tuple[float, Optional[str]]] = {}
def cache_get(url: str) -> Optional[str]:
    v = _cache.get(url)
    if not v:
        return None
    ts, body = v
    if time.time() - ts > TTL_SECONDS:
        _cache.pop(url, None)
        return None
    return body

def cache_put(url: str, body: Optional[str]):
    _cache[url] = (time.time(), body)

async def fetch_text(client: httpx.AsyncClient, url: str, referer: Optional[str] = None) -> Optional[str]:
    cached = cache_get(url)
    if cached is not None:
        return cached
    try:
        await rate_limit(urlparse(url).netloc)
        headers = dict(BROWSER_HEADERS)
        if referer:
            headers["Referer"] = referer
        r = await client.get(url, headers=headers, timeout=REQUEST_TIMEOUT, follow_redirects=True)
        if r.status_code != 200:
            cache_put(url, None)
            return None
        # gunzip transparently if needed
        content = r.content
        if r.headers.get("Content-Encoding", "").lower() == "gzip" or url.endswith(".gz"):
            try:
                content = gzip.decompress(content)
            except Exception:
                pass
        text = content.decode(r.encoding or "utf-8", errors="ignore")
        cache_put(url, text)
        return text
    except Exception:
        cache_put(url, None)
        return None

# optional Playwright (Firefox) renderer
async def fetch_js(url: str) -> Optional[str]:
    try:
        from playwright.async_api import async_playwright
    except Exception:
        return None
    try:
        async with async_playwright() as p:
            browser = await p.firefox.launch(headless=True)
            page = await browser.new_page(user_agent=BROWSER_HEADERS["User-Agent"])
            await page.goto(url, wait_until="domcontentloaded", timeout=int((REQUEST_TIMEOUT+10) * 1000))
            await page.wait_for_timeout(1200)
            html = await page.content()
            await browser.close()
            return html
    except Exception:
        return None

# ---------------------- Parsing product pages -------------------------
@dataclass
class Offer:
    source: str
    title: str
    price: Optional[float]
    currency: Optional[str]
    url: str
    image: Optional[str] = None
    in_stock: Optional[bool] = None
    image_url: Optional[str] = None

SERVICE_BLACKLIST = [
    "service", "services", "installation", "repair", "fixing", "cleaning",
    "alignment", "movers", "packers", "examination", "inspection", "consultation"
]

def looks_like_service(title: str) -> bool:
    t = (title or "").lower()
    return any(w in t for w in SERVICE_BLACKLIST)

def parse_ldjson(html: str, page_url: str) -> List[Offer]:
    soup = BeautifulSoup(html or "", "html.parser")
    out: List[Offer] = []

    for tag in soup.find_all("script", type="application/ld+json"):
        raw = tag.string or ""
        if not raw.strip():
            continue
        try:
            data = json.loads(raw)
        except Exception:
            continue

        blocks = data if isinstance(data, list) else [data]

        for b in blocks:
            if not isinstance(b, dict):
                continue

            typ = b.get("@type")
            is_product = (isinstance(typ, list) and "Product" in typ) or (typ == "Product")
            if not is_product:
                continue

            name = (b.get("name") or "").strip()
            url = b.get("url") or page_url

            # ✅ Improved image extraction
            img = b.get("image")
            if isinstance(img, dict):
                img = img.get("url") or img.get("@id")
            elif isinstance(img, list) and img:
                img = img[0]
            image_url = str(img) if isinstance(img, str) else None

            # Handle offers
            offs = b.get("offers")
            arr = offs if isinstance(offs, list) else [offs] if offs else []

            if not arr:
                # ✅ Fallback: check priceSpecification or aggregateOffer
                ps = b.get("priceSpecification")
                if isinstance(ps, dict):
                    price = clean_price(str(ps.get("price") or ""))
                    cur = ps.get("priceCurrency")
                    if name and price is not None:
                        out.append(Offer(
                            source=d(page_url),
                            title=name,
                            price=price,
                            currency=cur,
                            url=url,
                            image_url=image_url
                        ))
                agg = b.get("aggregateOffer")
                if isinstance(agg, dict):
                    price = clean_price(str(agg.get("lowPrice") or agg.get("price") or ""))
                    cur = agg.get("priceCurrency")
                    if name and price is not None:
                        out.append(Offer(
                            source=d(page_url),
                            title=name,
                            price=price,
                            currency=cur,
                            url=url,
                            image_url=image_url
                        ))

            for o in arr:
                if not isinstance(o, dict):
                    continue

                price = o.get("price")
                if price is None and isinstance(o.get("priceSpecification"), dict):
                    price = o["priceSpecification"].get("price")

                currency = o.get("priceCurrency") or (o.get("priceSpecification") or {}).get("priceCurrency")

                avail = o.get("availability")
                in_stock = None
                if isinstance(avail, str):
                    in_stock = "OutOfStock" not in avail and "SoldOut" not in avail

                p = clean_price(str(price) if price is not None else "")
                if name and p is not None:
                    out.append(Offer(
                        source=d(page_url),
                        title=name,
                        price=p,
                        currency=currency,
                        url=url,
                        in_stock=in_stock,
                        image_url=image_url
                    ))

    return out
    
def css_text(soup: BeautifulSoup, sel: Optional[str]) -> Optional[str]:
    if not sel:
        return None
    node = soup.select_one(sel)
    return node.get_text(" ", strip=True) if node else None

def css_attr(soup: BeautifulSoup, sel: Optional[str], attr: str) -> Optional[str]:
    if not sel:
        return None
    node = soup.select_one(sel)
    return node.get(attr) if node else None

def parse_product_fallback(html: str, page_url: str, fallback: Dict[str, str]) -> Optional[Offer]:
    soup = BeautifulSoup(html or "", "html.parser")
    title = css_text(soup, fallback.get("title"))
    price_text = css_text(soup, fallback.get("price"))
    price = clean_price(price_text or "")
    
    # ✅ Improved image fallback
    image_url = css_attr(soup, fallback.get("image", "img[src]"), "src")
    if not image_url:
        image_url = extract_og_image(soup, page_url)

    if title and price is not None:
        return Offer(
            source=d(page_url),
            title=title,
            price=price,
            currency=None,
            url=page_url,
            image_url=image_url  # ✅ Updated field
        )
    return None

async def parse_product(
    client: httpx.AsyncClient,
    store: Dict[str, Any],
    url: str,
    use_js: bool,
    fetch_func=fetch_text  # ← NEW: custom fetcher, defaults to global one for backward compatibility
) -> List[Offer]:
    html = await fetch_func(client, url, referer=store["base"])
    if not html and use_js:
        html = await fetch_js(url)
    if not html:
        return []

    offers = parse_ldjson(html, url)
    if offers:
        return offers

    # CSS fallback by platform
    if store["platform"] == "shopify":
        fallback = {
            "title": "h1.product__title, h1.product-title, h1.page-title",
            "price": ".price-item--regular, .price .price__regular, span.price",
            "image": "img[src]"
        }
    elif store["platform"] == "woo":
        fallback = {
            "title": "h1.product_title, h1.entry-title",
            "price": "p.price, span.price, bdi",
            "image": "img[src]"
        }
    else:
        fallback = store.get("product_fallback", {
            "title": "h1, h1.product-title",
            "price": "span.price, .price",
            "image": "img[src]"
        })

    one = parse_product_fallback(html, url, fallback) if fallback else None
    return [one] if one else []

# ---------------------- Sitemap discovery & enumeration ---------------
SITEMAP_HINTS_DEFAULT = [
    "/sitemap.xml",
    "/sitemap_index.xml",
    "/sitemap-index.xml",
]

async def discover_sitemaps(client: httpx.AsyncClient, base: str, store: Dict[str, Any]) -> List[str]:
    results: List[str] = []
    # robots.txt
    robots = await fetch_text(client, abs_url(base, "/robots.txt"))
    if robots:
        for line in robots.splitlines():
            if line.lower().startswith("sitemap:"):
                sm = line.split(":", 1)[1].strip()
                if sm and sm not in results:
                    results.append(sm)
    # hints from store
    for h in store.get("sitemap_hints", []):
        results.append(h)
    # defaults
    for h in SITEMAP_HINTS_DEFAULT:
        results.append(abs_url(base, h))
    # uniq, keep order
    seen = set(); out = []
    for u in results:
        if u not in seen:
            seen.add(u); out.append(u)
    return out

def _extract_xml_urls(xml_text: str) -> List[str]:
    urls: List[str] = []
    soup = BeautifulSoup(xml_text or "", "xml")  # needs 'lxml' or bs4's xml parser
    for loc in soup.find_all("loc"):
        if loc and loc.text:
            urls.append(loc.text.strip())
    return urls

async def iter_sitemap_tree(client: httpx.AsyncClient, sitemap_url: str, max_nodes: int = 1000) -> List[str]:
    """Return flattened list of URLs from a (possibly nested) sitemap index."""
    urls: List[str] = []
    seen: set = set()
    queue: List[str] = [sitemap_url]
    while queue and len(seen) < max_nodes:
        u = queue.pop(0)
        if u in seen:
            continue
        seen.add(u)
        txt = await fetch_text(client, u)
        if not txt:
            continue
        locs = _extract_xml_urls(txt)
        if any(u2.endswith(".xml") or "/sitemap" in u2 for u2 in locs):
            # child sitemaps
            for c in locs:
                if c not in seen and len(seen) + len(queue) < max_nodes:
                    queue.append(c)
        else:
            urls.extend(locs)
    return urls

def _match_product(url: str, store: Dict[str, Any]) -> bool:
    pats = store.get("product_link_patterns")
    if pats:
        return any(re.search(p, url) for p in pats)
    # platform defaults
    if store["platform"] == "shopify":
        return re.search(r"/products?/[-a-z0-9]+/?$", url) is not None
    if store["platform"] == "woo":
        return re.search(r"/product/[-a-z0-9]+/?$", url) is not None
    return True  # best effort

async def enumerate_products_via_sitemaps(client: httpx.AsyncClient, store: Dict[str, Any]) -> List[str]:
    base = store["base"]
    roots = await discover_sitemaps(client, base, store)
    all_urls: List[str] = []
    for sm in roots:
        items = await iter_sitemap_tree(client, sm, max_nodes=5000)
        all_urls.extend(items)
    # filter to same domain + product patterns
    dd = d(base)
    prods = []
    seen = set()
    for u in all_urls:
        if d(u).endswith(dd) and _match_product(u, store):
            if u not in seen:
                seen.add(u); prods.append(u)
    return prods

# ---------------------- Fallback BFS (bounded) ------------------------
async def enumerate_products_via_bfs(client: httpx.AsyncClient, store: Dict[str, Any]) -> List[str]:
    base = store["base"].rstrip("/")
    seeds = store.get("bfs_seeds") or [base]
    dd = d(base)
    to_visit = list({abs_url(base, s) for s in seeds})
    visited = set()
    found: List[str] = []
    pages = 0
    while to_visit and pages < MAX_FALLBACK_BFS_PAGES and len(found) < 8000:
        url = to_visit.pop(0)
        if url in visited:
            continue
        visited.add(url)
        pages += 1
        html = await fetch_text(client, url, referer=base)
        if not html:
            continue
        soup = BeautifulSoup(html, "html.parser")
        # harvest links
        for a in soup.find_all("a", href=True):
            href = abs_url(base, a["href"])
            if d(href).endswith(dd):
                if _match_product(href, store):
                    if href not in found:
                        found.append(href)
                elif href not in visited and len(to_visit) + pages < MAX_FALLBACK_BFS_PAGES * 20:
                    # enqueue further category/listing pages conservatively
                    if re.search(r"(category|catalog|search|page=|/c/|/s/)", href, re.I):
                        to_visit.append(href)
    return found

# ---------------------- Query expansion (reused search) ---------------
async def harvest_shopify_suggestions(client: httpx.AsyncClient, stores: List[dict], query: str, limit_per_store: int = 10) -> List[str]:
    suggestions: List[str] = []
    for s in stores:
        if s.get("platform") != "shopify":
            continue
        base = s["base"].rstrip("/")
        sug_url = f"{base}/search/suggest.json?q={quote_plus(query)}&resources[type]=product&resources[limit]={limit_per_store}"
        txt = await fetch_text(client, sug_url, referer=base)
        if not txt:
            continue
        try:
            data = json.loads(txt)
            prods = data.get("resources", {}).get("results", {}).get("products", [])
            for p in prods:
                title = p.get("title") or p.get("handle") or ""
                if title:
                    suggestions.append(title.strip())
        except Exception:
            continue
    # de-dupe case-insensitive
    seen = set(); uniq = []
    for s in suggestions:
        k = s.lower()
        if k not in seen:
            seen.add(k); uniq.append(s)
    return uniq

# ---------------------- FastAPI app -----------------------------------
app = FastAPI(title="SaveNest Everything Crawler API", version="2.0")
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # or restrict to your app’s domain(s)
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.get("/", response_class=HTMLResponse)
async def root():
    return """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>SaveNest Crawler API</title>
    <link href="https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&family=Syne:wght@400;600;700;800&display=swap" rel="stylesheet">
    <style>
        :root {
            --bg: #0a0f0a;
            --surface: #111811;
            --surface2: #162016;
            --border: #1e2e1e;
            --green: #2dff6e;
            --green-dim: #1a9940;
            --green-glow: rgba(45,255,110,0.12);
            --amber: #ffb347;
            --red: #ff5f5f;
            --text: #e8f5e8;
            --muted: #6b8f6b;
            --code-bg: #0d150d;
        }

        * { margin:0; padding:0; box-sizing:border-box; }

        body {
            background: var(--bg);
            color: var(--text);
            font-family: 'Syne', sans-serif;
            min-height: 100vh;
            overflow-x: hidden;
        }

        body::before {
            content: '';
            position: fixed;
            inset: 0;
            background-image:
                linear-gradient(rgba(45,255,110,0.03) 1px, transparent 1px),
                linear-gradient(90deg, rgba(45,255,110,0.03) 1px, transparent 1px);
            background-size: 40px 40px;
            pointer-events: none;
            z-index: 0;
        }

        .wrap { position: relative; z-index: 1; max-width: 1100px; margin: 0 auto; padding: 0 24px; }

        header {
            border-bottom: 1px solid var(--border);
            padding: 28px 0;
            position: sticky;
            top: 0;
            background: rgba(10,15,10,0.92);
            backdrop-filter: blur(12px);
            z-index: 100;
        }
        header .wrap { display: flex; align-items: center; gap: 16px; }
        .logo-icon {
            width: 40px; height: 40px;
            background: var(--green);
            border-radius: 10px;
            display: flex; align-items: center; justify-content: center;
            font-size: 20px;
            box-shadow: 0 0 20px rgba(45,255,110,0.4);
        }
        .logo-text { font-size: 1.4rem; font-weight: 800; letter-spacing: -0.02em; }
        .logo-text span { color: var(--green); }
        .version-badge {
            margin-left: auto;
            font-family: 'Space Mono', monospace;
            font-size: 0.7rem;
            padding: 4px 10px;
            border: 1px solid var(--green-dim);
            border-radius: 20px;
            color: var(--green);
            background: var(--green-glow);
        }
        .live-dot {
            display: inline-block;
            width: 7px; height: 7px;
            background: var(--green);
            border-radius: 50%;
            margin-right: 6px;
            animation: pulse 1.8s ease-in-out infinite;
            box-shadow: 0 0 8px var(--green);
        }
        @keyframes pulse {
            0%,100% { opacity:1; transform:scale(1); }
            50% { opacity:0.5; transform:scale(0.8); }
        }

        .hero { padding: 80px 0 60px; text-align: center; }
        .hero-tag {
            display: inline-block;
            font-family: 'Space Mono', monospace;
            font-size: 0.72rem;
            letter-spacing: 0.15em;
            text-transform: uppercase;
            color: var(--green);
            background: var(--green-glow);
            border: 1px solid var(--green-dim);
            padding: 6px 16px;
            border-radius: 20px;
            margin-bottom: 28px;
        }
        .hero h1 {
            font-size: clamp(2.4rem, 6vw, 4rem);
            font-weight: 800;
            line-height: 1.1;
            letter-spacing: -0.03em;
            margin-bottom: 20px;
        }
        .hero h1 .accent { color: var(--green); }
        .hero p {
            font-size: 1.1rem;
            color: var(--muted);
            max-width: 560px;
            margin: 0 auto 40px;
            line-height: 1.7;
        }

        .stat-row {
            display: flex;
            justify-content: center;
            gap: 32px;
            flex-wrap: wrap;
        }
        .stat { text-align: center; }
        .stat-num {
            font-family: 'Space Mono', monospace;
            font-size: 2rem;
            font-weight: 700;
            color: var(--green);
            display: block;
        }
        .stat-label {
            font-size: 0.8rem;
            color: var(--muted);
            text-transform: uppercase;
            letter-spacing: 0.1em;
        }

        .section-title {
            font-family: 'Space Mono', monospace;
            font-size: 0.72rem;
            text-transform: uppercase;
            letter-spacing: 0.2em;
            color: var(--green-dim);
            margin-bottom: 20px;
            display: flex;
            align-items: center;
            gap: 12px;
        }
        .section-title::after {
            content: '';
            flex: 1;
            height: 1px;
            background: var(--border);
        }

        .endpoints { padding: 60px 0; }

        .endpoint-card {
            background: var(--surface);
            border: 1px solid var(--border);
            border-radius: 16px;
            margin-bottom: 24px;
            overflow: hidden;
            transition: border-color 0.2s;
        }
        .endpoint-card:hover { border-color: var(--green-dim); }

        .endpoint-header {
            padding: 20px 24px;
            display: flex;
            align-items: center;
            gap: 14px;
            cursor: pointer;
            user-select: none;
        }
        .method {
            font-family: 'Space Mono', monospace;
            font-size: 0.72rem;
            font-weight: 700;
            padding: 4px 10px;
            border-radius: 6px;
            letter-spacing: 0.05em;
        }
        .method.get { background: rgba(45,255,110,0.15); color: var(--green); border: 1px solid var(--green-dim); }

        .endpoint-path {
            font-family: 'Space Mono', monospace;
            font-size: 1rem;
            color: var(--text);
        }
        .endpoint-desc {
            font-size: 0.85rem;
            color: var(--muted);
            margin-left: auto;
        }
        .chevron {
            color: var(--muted);
            font-size: 0.9rem;
            transition: transform 0.2s;
            margin-left: 8px;
        }
        .endpoint-card.open .chevron { transform: rotate(180deg); }

        .endpoint-body {
            display: none;
            padding: 0 24px 24px;
            border-top: 1px solid var(--border);
        }
        .endpoint-card.open .endpoint-body { display: block; }

        .params-table {
            width: 100%;
            border-collapse: collapse;
            margin: 20px 0;
            font-size: 0.88rem;
        }
        .params-table th {
            font-family: 'Space Mono', monospace;
            font-size: 0.7rem;
            text-transform: uppercase;
            letter-spacing: 0.1em;
            color: var(--muted);
            text-align: left;
            padding: 8px 12px;
            border-bottom: 1px solid var(--border);
        }
        .params-table td {
            padding: 10px 12px;
            border-bottom: 1px solid rgba(30,46,30,0.5);
            vertical-align: top;
        }
        .params-table tr:last-child td { border-bottom: none; }
        .param-name { font-family: 'Space Mono', monospace; color: var(--green); font-size: 0.85rem; }
        .param-type { color: var(--amber); font-family: 'Space Mono', monospace; font-size: 0.8rem; }
        .required-badge {
            font-size: 0.65rem;
            padding: 2px 6px;
            border-radius: 4px;
            background: rgba(255,95,95,0.15);
            color: var(--red);
            border: 1px solid rgba(255,95,95,0.3);
            margin-left: 6px;
        }
        .optional-badge {
            font-size: 0.65rem;
            padding: 2px 6px;
            border-radius: 4px;
            background: rgba(107,143,107,0.15);
            color: var(--muted);
            border: 1px solid var(--border);
            margin-left: 6px;
        }

        .code-block {
            background: var(--code-bg);
            border: 1px solid var(--border);
            border-radius: 10px;
            padding: 16px 20px;
            font-family: 'Space Mono', monospace;
            font-size: 0.82rem;
            line-height: 1.7;
            overflow-x: auto;
            position: relative;
            margin: 12px 0;
        }
        .code-block .key { color: var(--amber); }
        .code-block .val { color: #a8d5a2; }
        .code-block .str { color: #72c17a; }
        .code-block .num { color: #6bbfff; }
        .code-block .comment { color: var(--muted); }
        .copy-btn {
            position: absolute;
            top: 10px; right: 10px;
            font-family: 'Space Mono', monospace;
            font-size: 0.65rem;
            padding: 4px 10px;
            border: 1px solid var(--border);
            border-radius: 6px;
            background: var(--surface);
            color: var(--muted);
            cursor: pointer;
            transition: all 0.2s;
        }
        .copy-btn:hover { border-color: var(--green-dim); color: var(--green); }

        .tester { padding: 60px 0; }

        .tester-card {
            background: var(--surface);
            border: 1px solid var(--border);
            border-radius: 20px;
            overflow: hidden;
        }
        .tester-tabs {
            display: flex;
            border-bottom: 1px solid var(--border);
        }
        .tab-btn {
            padding: 16px 24px;
            font-family: 'Space Mono', monospace;
            font-size: 0.78rem;
            letter-spacing: 0.05em;
            border: none;
            background: transparent;
            color: var(--muted);
            cursor: pointer;
            border-bottom: 2px solid transparent;
            transition: all 0.2s;
        }
        .tab-btn.active { color: var(--green); border-bottom-color: var(--green); }
        .tab-btn:hover:not(.active) { color: var(--text); }

        .tab-pane { display: none; padding: 28px; }
        .tab-pane.active { display: block; }

        .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 16px; }
        @media(max-width:600px) { .form-row { grid-template-columns: 1fr; } }

        .form-group { display: flex; flex-direction: column; gap: 8px; }
        .form-group label {
            font-family: 'Space Mono', monospace;
            font-size: 0.72rem;
            text-transform: uppercase;
            letter-spacing: 0.1em;
            color: var(--muted);
        }
        .form-group input, .form-group select {
            background: var(--code-bg);
            border: 1px solid var(--border);
            border-radius: 8px;
            padding: 10px 14px;
            color: var(--text);
            font-family: 'Space Mono', monospace;
            font-size: 0.88rem;
            outline: none;
            transition: border-color 0.2s;
        }
        .form-group input:focus, .form-group select:focus { border-color: var(--green-dim); }
        .form-group input::placeholder { color: var(--muted); }

        .run-btn {
            display: flex;
            align-items: center;
            gap: 10px;
            padding: 13px 28px;
            background: var(--green);
            color: #0a0f0a;
            border: none;
            border-radius: 10px;
            font-family: 'Syne', sans-serif;
            font-size: 0.95rem;
            font-weight: 700;
            cursor: pointer;
            transition: all 0.2s;
            box-shadow: 0 0 20px rgba(45,255,110,0.3);
        }
        .run-btn:hover { background: #50ff88; box-shadow: 0 0 30px rgba(45,255,110,0.5); transform: translateY(-1px); }
        .run-btn:active { transform: translateY(0); }
        .run-btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }

        .response-area {
            margin-top: 24px;
            background: var(--code-bg);
            border: 1px solid var(--border);
            border-radius: 12px;
            overflow: hidden;
        }
        .response-header {
            display: flex;
            align-items: center;
            justify-content: space-between;
            padding: 12px 18px;
            border-bottom: 1px solid var(--border);
        }
        .response-label {
            font-family: 'Space Mono', monospace;
            font-size: 0.7rem;
            text-transform: uppercase;
            letter-spacing: 0.12em;
            color: var(--muted);
        }
        .status-pill {
            font-family: 'Space Mono', monospace;
            font-size: 0.7rem;
            padding: 3px 10px;
            border-radius: 10px;
        }
        .status-200 { background: rgba(45,255,110,0.15); color: var(--green); }
        .status-err { background: rgba(255,95,95,0.15); color: var(--red); }
        .status-loading { background: rgba(255,179,71,0.15); color: var(--amber); }

        #response-output, #crawl-output, #health-output {
            padding: 18px;
            font-family: 'Space Mono', monospace;
            font-size: 0.8rem;
            line-height: 1.7;
            max-height: 440px;
            overflow-y: auto;
            white-space: pre-wrap;
            word-break: break-all;
            color: #a8d5a2;
        }

        .placeholder-msg {
            color: var(--muted);
            font-style: italic;
            font-size: 0.82rem;
        }

        .stores { padding: 40px 0 60px; }
        .stores-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; }
        .store-chip {
            display: flex;
            align-items: center;
            gap: 10px;
            padding: 14px 18px;
            background: var(--surface);
            border: 1px solid var(--border);
            border-radius: 12px;
            font-size: 0.88rem;
            transition: all 0.2s;
            cursor: default;
        }
        .store-chip:hover { border-color: var(--green-dim); background: var(--surface2); }
        .store-dot { width: 8px; height: 8px; background: var(--green); border-radius: 50%; box-shadow: 0 0 6px var(--green); }
        .store-key { font-family: 'Space Mono', monospace; font-size: 0.7rem; color: var(--muted); margin-left: auto; }

        footer {
            border-top: 1px solid var(--border);
            padding: 32px 0;
            text-align: center;
        }
        footer p { font-size: 0.82rem; color: var(--muted); }
        footer a { color: var(--green); text-decoration: none; }

        .spin {
            display: inline-block;
            width: 16px; height: 16px;
            border: 2px solid rgba(10,15,10,0.3);
            border-top-color: #0a0f0a;
            border-radius: 50%;
            animation: spin 0.7s linear infinite;
        }
        @keyframes spin { to { transform: rotate(360deg); } }

        .fade-up {
            opacity: 0;
            transform: translateY(24px);
            animation: fadeUp 0.6s ease forwards;
        }
        @keyframes fadeUp { to { opacity:1; transform:translateY(0); } }
        .delay-1 { animation-delay: 0.1s; }
        .delay-2 { animation-delay: 0.2s; }
        .delay-3 { animation-delay: 0.3s; }
        .delay-4 { animation-delay: 0.4s; }
    </style>
</head>
<body>

<header>
    <div class="wrap">
        <div class="logo-icon">🛒</div>
        <div class="logo-text">Save<span>Nest</span> API</div>
        <div class="version-badge"><span class="live-dot"></span>v2.0 Live</div>
        <a href="/documentation" target="_blank" style="font-family:'Space Mono',monospace;font-size:0.68rem;padding:4px 12px;border-radius:20px;border:1px solid #2dff6e;color:#2dff6e;background:rgba(45,255,110,0.08);text-decoration:none;white-space:nowrap;">📖 Docs</a>
    </div>
</header>

<main>
    <div class="wrap">

        <section class="hero">
            <div class="hero-tag fade-up">🕷️ Grocery Price Intelligence</div>
            <h1 class="fade-up delay-1">
                Crawl. Compare.<br><span class="accent">Save Smarter.</span>
            </h1>
            <p class="fade-up delay-2">
                Real-time product price scraping across Pakistan's top grocery stores.
                Sitemap enumeration, BFS fallback, JSON-LD parsing — all in one API.
            </p>
            <div class="stat-row fade-up delay-3">
                <div class="stat">
                    <span class="stat-num">5</span>
                    <span class="stat-label">Stores</span>
                </div>
                <div class="stat">
                    <span class="stat-num">500</span>
                    <span class="stat-label">Max / Call</span>
                </div>
                <div class="stat">
                    <span class="stat-num">3</span>
                    <span class="stat-label">Endpoints</span>
                </div>
                <div class="stat">
                    <span class="stat-num">PKR</span>
                    <span class="stat-label">Currency</span>
                </div>
            </div>
        </section>

        <section class="stores fade-up delay-4">
            <div class="section-title">Supported Stores</div>
            <div class="stores-grid">
                <div class="store-chip"><div class="store-dot"></div> Al-Fatah <span class="store-key">alfatah.pk</span></div>
                <div class="store-chip"><div class="store-dot"></div> QnE <span class="store-key">qne.com.pk</span></div>
                <div class="store-chip"><div class="store-dot"></div> Springs <span class="store-key">springs.com.pk</span></div>
                <div class="store-chip"><div class="store-dot"></div> Vmart <span class="store-key">vmart.pk</span></div>
                <div class="store-chip"><div class="store-dot"></div> GrocerApp <span class="store-key">grocerapp.pk</span></div>
            </div>
        </section>

        <section class="endpoints">
            <div class="section-title">Endpoints</div>

            <div class="endpoint-card" onclick="toggleCard(this)">
                <div class="endpoint-header">
                    <span class="method get">GET</span>
                    <span class="endpoint-path">/health</span>
                    <span class="endpoint-desc">API status check</span>
                    <span class="chevron">▾</span>
                </div>
                <div class="endpoint-body">
                    <p style="color:var(--muted);font-size:0.88rem;margin:16px 0 12px;">Returns API operational status.</p>
                    <div class="code-block">
                        <button class="copy-btn" onclick="copyCode(this)">copy</button>
                        <span class="comment">// Response</span>
{
  "ok": true
}
                    </div>
                </div>
            </div>

            <div class="endpoint-card" onclick="toggleCard(this)">
                <div class="endpoint-header">
                    <span class="method get">GET</span>
                    <span class="endpoint-path">/search</span>
                    <span class="endpoint-desc">Live product search across all stores</span>
                    <span class="chevron">▾</span>
                </div>
                <div class="endpoint-body">
                    <p style="color:var(--muted);font-size:0.88rem;margin:16px 0 12px;">
                        Search for a product across all registered stores simultaneously. Returns deduplicated, price-sorted results.
                    </p>
                    <table class="params-table">
                        <thead><tr><th>Parameter</th><th>Type</th><th>Description</th></tr></thead>
                        <tbody>
                            <tr>
                                <td><span class="param-name">q</span><span class="required-badge">required</span></td>
                                <td><span class="param-type">string</span></td>
                                <td style="color:var(--muted)">Product keyword (e.g., "milk", "bread", "laptop")</td>
                            </tr>
                            <tr>
                                <td><span class="param-name">limit</span><span class="optional-badge">optional</span></td>
                                <td><span class="param-type">integer</span></td>
                                <td style="color:var(--muted)">Max results (1–100, default: 40)</td>
                            </tr>
                        </tbody>
                    </table>
                    <div class="code-block">
                        <button class="copy-btn" onclick="copyCode(this)">copy</button>
                        <span class="comment">// GET /search?q=milk&limit=10</span>
{
  "query": "milk",
  "count": 4,
  "offers": [
    {
      "source": "Al-Fatah",
      "title": "Olpers Full Cream Milk 1L",
      "price": 285.0,
      "currency": "PKR",
      "url": "https://alfatah.pk/products/...",
      "image": "https://...",
      "in_stock": true
    }
  ]
}
                    </div>
                </div>
            </div>

            <div class="endpoint-card" onclick="toggleCard(this)">
                <div class="endpoint-header">
                    <span class="method get">GET</span>
                    <span class="endpoint-path">/crawl</span>
                    <span class="endpoint-desc">Full store enumeration + scraping</span>
                    <span class="chevron">▾</span>
                </div>
                <div class="endpoint-body">
                    <p style="color:var(--muted);font-size:0.88rem;margin:16px 0 12px;">
                        Enumerate ALL product URLs from a store via sitemaps (BFS fallback) and optionally scrape details.
                        Use <code style="color:var(--green);font-family:'Space Mono',monospace">cursor</code> for pagination.
                    </p>
                    <table class="params-table">
                        <thead><tr><th>Parameter</th><th>Type</th><th>Description</th></tr></thead>
                        <tbody>
                            <tr>
                                <td><span class="param-name">store</span><span class="required-badge">required</span></td>
                                <td><span class="param-type">string</span></td>
                                <td style="color:var(--muted)">Domain key, e.g. "alfatah.pk"</td>
                            </tr>
                            <tr>
                                <td><span class="param-name">limit</span><span class="optional-badge">optional</span></td>
                                <td><span class="param-type">integer</span></td>
                                <td style="color:var(--muted)">Items per call (1–500, default: 100)</td>
                            </tr>
                            <tr>
                                <td><span class="param-name">cursor</span><span class="optional-badge">optional</span></td>
                                <td><span class="param-type">integer</span></td>
                                <td style="color:var(--muted)">Resume offset for pagination (default: 0)</td>
                            </tr>
                            <tr>
                                <td><span class="param-name">mode</span><span class="optional-badge">optional</span></td>
                                <td><span class="param-type">string</span></td>
                                <td style="color:var(--muted)">"full" (scrape) or "urls" (enumerate only)</td>
                            </tr>
                            <tr>
                                <td><span class="param-name">use_js</span><span class="optional-badge">optional</span></td>
                                <td><span class="param-type">0|1</span></td>
                                <td style="color:var(--muted)">Override Playwright JS rendering</td>
                            </tr>
                        </tbody>
                    </table>
                    <div class="code-block">
                        <button class="copy-btn" onclick="copyCode(this)">copy</button>
                        <span class="comment">// GET /crawl?store=alfatah.pk&limit=50&cursor=0</span>
{
  "store": "alfatah.pk",
  "count": 48,
  "items": [ /* array of product objects */ ],
  "next_cursor": 50,
  "total_urls": 1240,
  "enumeration": "sitemap_or_bfs",
  "js_fallback": true
}
                    </div>
                </div>
            </div>
        </section>

        <section class="tester">
            <div class="section-title">Live API Tester</div>
            <div class="tester-card">
                <div class="tester-tabs">
                    <button class="tab-btn active" onclick="switchTab('search-tab', this)">🔍 /search</button>
                    <button class="tab-btn" onclick="switchTab('crawl-tab', this)">🕷️ /crawl</button>
                    <button class="tab-btn" onclick="switchTab('health-tab', this)">💚 /health</button>
                </div>

                <div id="search-tab" class="tab-pane active">
                    <div class="form-row">
                        <div class="form-group" style="grid-column:1/-1">
                            <label>Search Query</label>
                            <input type="text" id="search-q" placeholder="e.g. milk, bread, rice..." value="milk">
                        </div>
                    </div>
                    <div class="form-row">
                        <div class="form-group">
                            <label>Limit (1–100)</label>
                            <input type="number" id="search-limit" value="10" min="1" max="100">
                        </div>
                    </div>
                    <button class="run-btn" id="search-run-btn" onclick="runSearch()">
                        <span id="search-btn-icon">▶</span> Run Request
                    </button>
                    <div class="response-area" id="search-response-area" style="display:none">
                        <div class="response-header">
                            <span class="response-label">Response</span>
                            <span class="status-pill" id="search-status"></span>
                        </div>
                        <div id="response-output"><span class="placeholder-msg">Response will appear here...</span></div>
                    </div>
                </div>

                <div id="crawl-tab" class="tab-pane">
                    <div class="form-row">
                        <div class="form-group">
                            <label>Store</label>
                            <select id="crawl-store">
                                <option value="alfatah.pk">alfatah.pk — Al-Fatah</option>
                                <option value="qne.com.pk">qne.com.pk — QnE</option>
                                <option value="springs.com.pk">springs.com.pk — Springs</option>
                                <option value="vmart.pk">vmart.pk — Vmart</option>
                                <option value="grocerapp.pk">grocerapp.pk — GrocerApp</option>
                            </select>
                        </div>
                        <div class="form-group">
                            <label>Mode</label>
                            <select id="crawl-mode">
                                <option value="urls">urls — enumerate only (faster)</option>
                                <option value="full">full — enumerate + scrape</option>
                            </select>
                        </div>
                    </div>
                    <div class="form-row">
                        <div class="form-group">
                            <label>Limit (1–500)</label>
                            <input type="number" id="crawl-limit" value="20" min="1" max="500">
                        </div>
                        <div class="form-group">
                            <label>Cursor (offset)</label>
                            <input type="number" id="crawl-cursor" value="0" min="0">
                        </div>
                    </div>
                    <button class="run-btn" id="crawl-run-btn" onclick="runCrawl()">
                        <span id="crawl-btn-icon">▶</span> Run Request
                    </button>
                    <div class="response-area" id="crawl-response-area" style="display:none">
                        <div class="response-header">
                            <span class="response-label">Response</span>
                            <span class="status-pill" id="crawl-status"></span>
                        </div>
                        <div id="crawl-output"><span class="placeholder-msg">Response will appear here...</span></div>
                    </div>
                </div>

                <div id="health-tab" class="tab-pane">
                    <p style="color:var(--muted);font-size:0.9rem;margin-bottom:20px;">Check if the API is running and operational.</p>
                    <button class="run-btn" id="health-run-btn" onclick="runHealth()">
                        <span id="health-btn-icon">▶</span> Check Health
                    </button>
                    <div class="response-area" id="health-response-area" style="display:none">
                        <div class="response-header">
                            <span class="response-label">Response</span>
                            <span class="status-pill" id="health-status"></span>
                        </div>
                        <div id="health-output"><span class="placeholder-msg">Response will appear here...</span></div>
                    </div>
                </div>
            </div>
        </section>

    </div>
</main>

<footer>
    <div class="wrap">
        <p>SaveNest Crawler API v2.0 · Built with FastAPI + Playwright ·
            <a href="/docs">Swagger Docs</a> ·
            <a href="/health">Health</a> ·
            <a href="/documentation" style="color:#2dff6e">📖 Developer Docs</a>
        </p>
    </div>
</footer>

<script>
    const BASE = window.location.origin;

    function toggleCard(card) {
        const isOpen = card.classList.contains('open');
        document.querySelectorAll('.endpoint-card').forEach(c => c.classList.remove('open'));
        if (!isOpen) card.classList.add('open');
    }

    function switchTab(tabId, btn) {
        document.querySelectorAll('.tab-pane').forEach(p => p.classList.remove('active'));
        document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active'));
        document.getElementById(tabId).classList.add('active');
        btn.classList.add('active');
    }

    function copyCode(btn) {
        const codeBlock = btn.parentElement;
        const lines = codeBlock.innerText.split('\\n');
        const text = lines
            .filter(line => line.trim() !== 'copy')
            .join('\\n')
            .trim();
        navigator.clipboard.writeText(text).then(() => {
            const original = btn.textContent;
            btn.textContent = '✓ copied';
            setTimeout(() => { btn.textContent = original; }, 1800);
        }).catch(() => {});
    }

    function setLoading(btnId, iconId, isLoading) {
        const btn = document.getElementById(btnId);
        const icon = document.getElementById(iconId);
        btn.disabled = isLoading;
        icon.innerHTML = isLoading ? '<span class="spin"></span>' : '▶';
    }

    function showResponse(areaId, statusId, outputId, status, data) {
        const area = document.getElementById(areaId);
        const statusEl = document.getElementById(statusId);
        const output = document.getElementById(outputId);

        area.style.display = 'block';

        if (status === 200) {
            statusEl.className = 'status-pill status-200';
            statusEl.textContent = '200 OK';
        } else {
            statusEl.className = 'status-pill status-err';
            statusEl.textContent = status + ' Error';
        }

        output.textContent = JSON.stringify(data, null, 2);
    }

    async function runSearch() {
        const q = document.getElementById('search-q').value.trim();
        const limit = document.getElementById('search-limit').value;

        if (!q) {
            alert('Please enter a search query');
            return;
        }

        setLoading('search-run-btn', 'search-btn-icon', true);

        const area = document.getElementById('search-response-area');
        const statusEl = document.getElementById('search-status');
        const output = document.getElementById('response-output');

        area.style.display = 'block';
        statusEl.className = 'status-pill status-loading';
        statusEl.textContent = 'Fetching...';
        output.textContent = 'Searching across stores, please wait...';

        try {
            const url = `${BASE}/search?q=${encodeURIComponent(q)}&limit=${limit}`;
            const res = await fetch(url);
            const data = await res.json();
            showResponse('search-response-area', 'search-status', 'response-output', res.status, data);
        } catch (err) {
            showResponse('search-response-area', 'search-status', 'response-output', 500, { error: err.message });
        } finally {
            setLoading('search-run-btn', 'search-btn-icon', false);
        }
    }

    async function runCrawl() {
        const store = document.getElementById('crawl-store').value;
        const mode = document.getElementById('crawl-mode').value;
        const limit = document.getElementById('crawl-limit').value;
        const cursor = document.getElementById('crawl-cursor').value;

        setLoading('crawl-run-btn', 'crawl-btn-icon', true);

        const area = document.getElementById('crawl-response-area');
        const statusEl = document.getElementById('crawl-status');
        const output = document.getElementById('crawl-output');

        area.style.display = 'block';
        statusEl.className = 'status-pill status-loading';
        statusEl.textContent = 'Crawling...';
        output.textContent = 'Enumerating product URLs from store...';

        try {
            const params = new URLSearchParams({
                store,
                mode,
                limit,
                cursor
            });
            const url = `${BASE}/crawl?${params}`;
            const res = await fetch(url);
            const data = await res.json();
            showResponse('crawl-response-area', 'crawl-status', 'crawl-output', res.status, data);
        } catch (err) {
            showResponse('crawl-response-area', 'crawl-status', 'crawl-output', 500, { error: err.message });
        } finally {
            setLoading('crawl-run-btn', 'crawl-btn-icon', false);
        }
    }

    async function runHealth() {
        setLoading('health-run-btn', 'health-btn-icon', true);

        const area = document.getElementById('health-response-area');
        const statusEl = document.getElementById('health-status');
        const output = document.getElementById('health-output');

        area.style.display = 'block';

        try {
            const res = await fetch(`${BASE}/health`);
            const data = await res.json();
            showResponse('health-response-area', 'health-status', 'health-output', res.status, data);
        } catch (err) {
            showResponse('health-response-area', 'health-status', 'health-output', 500, { error: err.message });
        } finally {
            setLoading('health-run-btn', 'health-btn-icon', false);
        }
    }
</script>
</body>
</html>
    """
    
@app.get("/health")
async def health():
    return {"ok": True}

# SaveNest app.py  
@app.get("/documentation")
async def documentation():
    from fastapi.responses import RedirectResponse
    return RedirectResponse("https://aero-woad.vercel.app/savenest-docs.html")
    
        
@app.get("/crawl")
async def crawl(
    store: str = Query(..., description="Domain or key, e.g., 'alfatah.pk', 'daraz.pk'"),
    limit: int = Query(100, ge=1, le=CRAWL_LIMIT_CAP),
    cursor: int = Query(0, ge=0),
    mode: str = Query("full", pattern="^(full|urls)$"),
    use_js: Optional[int] = Query(None, description="Override USE_PLAYWRIGHT (1 or 0)"),
):
    """
    Enumerate ALL product URLs for a store (via sitemaps, else BFS) and optionally scrape details.
    Pagination: call repeatedly with `cursor=next_cursor` until next_cursor is None.
    """
    s = guess_store(store)
    if not s:
        raise HTTPException(400, f"Unknown store: {store}")

    use_js_effective = USE_PLAYWRIGHT_DEFAULT if use_js is None else (use_js == 1)

    async with httpx.AsyncClient(follow_redirects=True, timeout=REQUEST_TIMEOUT) as client:
        # 1) enumerate product URLs
        urls = await enumerate_products_via_sitemaps(client, s)
        if not urls:
            urls = await enumerate_products_via_bfs(client, s)

        if not urls:
            return {"store": store, "count": 0, "items": [], "next_cursor": None, "enumeration": "none"}

        # 2) paginate
        start = cursor
        end = min(len(urls), start + limit)
        batch = urls[start:end]

        if mode == "urls":
            return {
                "store": store,
                "count": len(batch),
                "items": [{"url": u} for u in batch],
                "next_cursor": (end if end < len(urls) else None),
                "total_urls": len(urls),
                "enumeration": "sitemap_or_bfs"
            }

        # 3) scrape product pages (concurrently, politely)
        sem = asyncio.Semaphore(MAX_CONCURRENCY)
        async def _one(u: str):
            async with sem:
                try:
                    return await parse_product(client, s, u, use_js_effective)
                except Exception:
                    return []

        tasks = [_one(u) for u in batch]
        parsed = await asyncio.gather(*tasks, return_exceptions=True)

    # flatten + clean
    offers: List[Offer] = []
    for r in parsed:
        if isinstance(r, Exception) or not r:
            continue
        offers.extend(r)

    def _ok(o: Offer) -> bool:
        return o.title and (o.price is not None) and not looks_like_service(o.title)

    items = [{
        "source": STORE_DISPLAY_NAMES.get(o.source, o.source.replace(".pk", "").title()),
        "title": o.title,
        "price": o.price,
        "currency": o.currency or "PKR",
        "url": o.url,
        "image": o.image_url,
        "in_stock": o.in_stock,
        
    } for o in offers if _ok(o)]

    # de-duplicate by normalized title+source keep cheapest
    keyd: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for it in items:
        k = (it["source"], _norm(it["title"]))
        if k not in keyd or (it["price"] or 1e18) < (keyd[k]["price"] or 1e18):
            keyd[k] = it
    items = list(keyd.values())
    items.sort(key=lambda x: (x["price"] if x["price"] is not None else 1e18, x["title"].lower()))

    return {
        "store": store,
        "count": len(items),
        "items": items,
        "next_cursor": (end if end < len(urls) else None),
        "total_urls": len(urls),
        "enumeration": "sitemap_or_bfs",
        "js_fallback": use_js_effective,
    }

# (kept) lightweight live search from previous build, handy for ad-hoc queries
@app.get("/search")
async def search(
    q: str = Query(..., description="Product keyword (e.g., 'bread', 'milk', 'laptop')"),
    limit: int = Query(40, ge=1, le=100),
):
    """
    Search for products across all stores.
    ✅ FIX #1: Per-request local cache (no global pollution)
    ✅ FIX #2: Proper search_url_template usage
    ✅ Comprehensive logging for debugging
    """
    request_id = f"{int(time.time() * 1000) % 100000}"
    logger.info(f"[{request_id}] ========== NEW SEARCH REQUEST ==========")
    logger.info(f"[{request_id}] Query: '{q}' | Limit: {limit}")
    
    q = q.strip()
    if not q:
        raise HTTPException(status_code=400, detail="Missing query")

    # 🔧 FIX #1: Per-request local cache and rate limiter
    local_cache: Dict[str, Tuple[float, Optional[str]]] = {}
    local_last_hit: Dict[str, float] = {}
    logger.info(f"[{request_id}] 🆕 Fresh local cache created (global cache has {len(_cache)} entries but will be ignored for product parsing)")
    logger.info(f"[{request_id}] 🆕 Created fresh local cache for this request")

    def cache_get(url: str) -> Optional[str]:
        v = local_cache.get(url)
        if not v:
            return None
        ts, body = v
        if time.time() - ts > TTL_SECONDS:
            local_cache.pop(url, None)
            logger.debug(f"[{request_id}]   ⏰ Cache expired for {url}")
            return None
        logger.debug(f"[{request_id}]   💾 Cache hit for {url}")
        return body

    def cache_put(url: str, body: Optional[str]):
        local_cache[url] = (time.time(), body)
        status = "✅ cached" if body else "❌ cached as None"
        logger.debug(f"[{request_id}]   {status}: {url}")

    async def rate_limit_local(host: str, min_gap: float = PER_HOST_GAP):
        last = local_last_hit.get(host, 0.0)
        dt = time.time() - last
        if dt < min_gap:
            sleep_time = min_gap - dt
            logger.debug(f"[{request_id}]   ⏳ Rate limiting {host}: sleeping {sleep_time:.2f}s")
            await asyncio.sleep(sleep_time)
        local_last_hit[host] = time.time()

    async def fetch_text_local(client: httpx.AsyncClient, url: str, referer: Optional[str] = None) -> Optional[str]:
        cached = cache_get(url)
        if cached is not None:
            return cached
        
        try:
            await rate_limit_local(urlparse(url).netloc)
            headers = dict(BROWSER_HEADERS)
            if referer:
                headers["Referer"] = referer
            
            logger.debug(f"[{request_id}]   🌐 Fetching: {url}")
            r = await client.get(url, headers=headers, timeout=REQUEST_TIMEOUT, follow_redirects=True)
            
            if r.status_code != 200:
                logger.warning(f"[{request_id}]   ⚠️ HTTP {r.status_code} for {url}")
                cache_put(url, None)
                return None
            
            content = r.content
            if r.headers.get("Content-Encoding", "").lower() == "gzip" or url.endswith(".gz"):
                try:
                    content = gzip.decompress(content)
                except Exception as e:
                    logger.warning(f"[{request_id}]   ⚠️ Gunzip failed for {url}: {e}")
            
            text = content.decode(r.encoding or "utf-8", errors="ignore")
            logger.debug(f"[{request_id}]   ✅ Fetched {len(text)} bytes from {url}")
            cache_put(url, text)
            return text
            
        except Exception as e:
            logger.error(f"[{request_id}]   ❌ Fetch error for {url}: {e}")
            cache_put(url, None)
            return None

    async with httpx.AsyncClient(follow_redirects=True, timeout=REQUEST_TIMEOUT) as client:
        async def _links_for_store(s: Dict[str, Any]) -> List[str]:
            store_name = d(s["base"])
            logger.info(f"[{request_id}] 🏪 Processing store: {store_name} ({s['platform']})")
            
            links: List[str] = []
            base = s["base"].rstrip("/")
            
            # ---------- Shopify ----------
            if s["platform"] == "shopify":
                # 1) Try suggest.json
                sug_url = f"{base}/search/suggest.json?q={quote_plus(q)}&resources[type]=product&resources[limit]=10"
                logger.info(f"[{request_id}]   → Shopify suggest: {sug_url}")
                
                txt = await fetch_text_local(client, sug_url, referer=base)
                
                if txt:
                    logger.info(f"[{request_id}]   ✅ Suggest API response: {len(txt)} bytes")
                    try:
                        data = json.loads(txt)
                        prods = data.get("resources", {}).get("results", {}).get("products", [])
                        logger.info(f"[{request_id}]   📦 Found {len(prods)} products in suggest JSON")
                        
                        for p in prods:
                            if "handle" in p:
                                product_url = abs_url(base, f"/products/{p['handle']}")
                                links.append(product_url)
                    except Exception as e:
                        logger.error(f"[{request_id}]   ❌ JSON parse error: {e}")
                else:
                    logger.warning(f"[{request_id}]   ⚠️ No response from suggest API")

                # 2) Fallback: ?view=json
                if not links:
                    json_url = f"{base}/search?q={quote_plus(q)}&view=json"
                    logger.info(f"[{request_id}]   → Shopify JSON view fallback: {json_url}")
                    
                    txt = await fetch_text_local(client, json_url, referer=base)
                    if txt and txt.strip().startswith("{"):
                        try:
                            data = json.loads(txt)
                            prods = data.get("products", [])
                            logger.info(f"[{request_id}]   📦 Found {len(prods)} products in JSON view")
                            
                            for prod in prods:
                                if "url" in prod:
                                    links.append(abs_url(base, prod["url"]))
                        except Exception as e:
                            logger.error(f"[{request_id}]   ❌ JSON view parse error: {e}")

                # 3) Fallback: HTML scrape
                if not links:
                    search_url = f"{base}/search?q={quote_plus(q)}"
                    logger.info(f"[{request_id}]   → Shopify HTML fallback: {search_url}")
                    
                    html = await fetch_text_local(client, search_url, referer=base)
                    if html:
                        logger.info(f"[{request_id}]   ✅ HTML response: {len(html)} bytes")
                        soup = BeautifulSoup(html, "html.parser")
                        
                        for a in soup.find_all("a", href=True):
                            href = abs_url(base, a["href"])
                            if _match_product(href, s):
                                links.append(href)
                                if len(links) >= 10:
                                    break
                        
                        logger.info(f"[{request_id}]   📦 Scraped {len(links)} product links")
                    else:
                        logger.warning(f"[{request_id}]   ⚠️ No HTML response")

            # ---------- WooCommerce ----------
            elif s["platform"] == "woo":
                search_url = f"{base}/?s={quote_plus(q)}&post_type=product"
                logger.info(f"[{request_id}]   → WooCommerce: {search_url}")
                
                html = await fetch_text_local(client, search_url, referer=base)
                if html:
                    logger.info(f"[{request_id}]   ✅ HTML response: {len(html)} bytes")
                    soup = BeautifulSoup(html, "html.parser")
                    
                    for a in soup.find_all("a", href=True):
                        href = abs_url(base, a["href"])
                        if _match_product(href, s):
                            links.append(href)
                            if len(links) >= 10:
                                break
                    
                    logger.info(f"[{request_id}]   📦 Scraped {len(links)} product links")
                else:
                    logger.warning(f"[{request_id}]   ⚠️ No HTML response")

            # ---------- Custom ----------
            else:
                # 🔧 FIX #2: Use search_url_template if defined
                search_template = s.get("search_url_template")
                
                if search_template:
                    search_url = search_template.format(base=base, query=quote_plus(q))
                    logger.info(f"[{request_id}]   → Custom (template): {search_url}")
                else:
                    search_url = f"{base}/search?q={quote_plus(q)}"
                    logger.info(f"[{request_id}]   → Custom (generic): {search_url}")
                
                # Special Daraz handling
                if "daraz.pk" in base:
                    logger.info(f"[{request_id}]   🔧 Daraz: using JS rendering")
                    if USE_PLAYWRIGHT_DEFAULT:
                        html = await fetch_js(search_url)
                    else:
                        html = await fetch_text_local(client, search_url, referer=base)
                else:
                    html = await fetch_text_local(client, search_url, referer=base)
                
                if html:
                    logger.info(f"[{request_id}]   ✅ HTML response: {len(html)} bytes")
                    soup = BeautifulSoup(html, "html.parser")
                    
                    for a in soup.find_all("a", href=True):
                        href = abs_url(base, a["href"])
                        if _match_product(href, s):
                            links.append(href)
                            if len(links) >= 10:
                                break
                    
                    logger.info(f"[{request_id}]   📦 Scraped {len(links)} product links")
                else:
                    logger.warning(f"[{request_id}]   ⚠️ No HTML response")

            logger.info(f"[{request_id}] ✅ {store_name}: collected {len(links)} links, returning top {min(len(links), 6)}")
            return links[:6]

        # Gather links from all stores
        logger.info(f"[{request_id}] 🚀 Starting parallel search across {len(STORES)} stores...")
        link_lists = await asyncio.gather(*[_links_for_store(s) for s in STORES])
        
        # Flatten and deduplicate
        product_urls = []
        seen = set()
        for links in link_lists:
            for url in links:
                if url not in seen:
                    seen.add(url)
                    product_urls.append(url)
        
        logger.info(f"[{request_id}] 📊 Total unique product URLs: {len(product_urls)}")
        logger.info(f"[{request_id}] 💾 Local cache entries: {len(local_cache)}")
        logger.info(f"[{request_id}] 💾 Global cache entries: {len(_cache)} (should not be used)")
        
        product_urls = product_urls[:limit * 3]

        # Parse product pages
        sem = asyncio.Semaphore(MAX_CONCURRENCY)
        logger.info(f"[{request_id}] 🔧 Parsing {min(len(product_urls), limit)} product pages...")

        async def _one(u: str):
            async with sem:
                try:
                    store_config = guess_store(d(u)) or {"base": u, "platform": "custom"}
                    return await parse_product(
                        client,
                        store_config,
                        u,
                        USE_PLAYWRIGHT_DEFAULT,
                        fetch_func=fetch_text_local  # ← CRITICAL: use per-request cache
                    )
                except Exception as e:
                    logger.error(f"[{request_id}] ❌ Parse error for {u}: {e}")
                    return []

        parsed = await asyncio.gather(*[_one(u) for u in product_urls[:limit]], return_exceptions=True)
        logger.info(f"[{request_id}] ✅ Parsing complete")

    # Flatten + clean
    offers: List[Offer] = []
    for r in parsed:
        if isinstance(r, Exception) or not r:
            continue
        offers.extend(r)

    logger.info(f"[{request_id}] 📦 Total offers extracted: {len(offers)}")

    # Build items
    items = []
    for o in offers:
        if not o.title or o.price is None:
            continue
        
        items.append({
            "source": STORE_DISPLAY_NAMES.get(o.source, o.source.replace(".pk", "").title()),
            "title": o.title,
            "price": o.price,
            "currency": o.currency or "PKR",
            "url": o.url,
            "image": o.image_url,
            "in_stock": o.in_stock,
        })

    logger.info(f"[{request_id}] 🧹 After filtering: {len(items)} valid items")

    # De-duplicate by normalized title+source, keep cheapest
    keyd: Dict[Tuple[str, str], Dict[str, Any]] = {}
    for it in items:
        k = (it["source"], _norm(it["title"]))
        if k not in keyd or (it["price"] or 1e18) < (keyd[k]["price"] or 1e18):
            keyd[k] = it
    
    items = list(keyd.values())
    logger.info(f"[{request_id}] 🔄 After deduplication: {len(items)} unique items")
    
    items.sort(key=lambda x: (x["price"] if x["price"] is not None else 1e18, x["title"].lower()))

    logger.info(f"[{request_id}] ✅ RETURNING {len(items[:limit])} offers for '{q}'")
    logger.info(f"[{request_id}] ========== REQUEST COMPLETE ==========\n")
    
    return {"query": q, "count": len(items[:limit]), "offers": items[:limit]}