app/services/amazon/scraper_utils.py

"""Shared HTTP/parsing helpers for Amazon scrapers (avoids circular imports)."""
from __future__ import annotations

import random
import re
from typing import Optional

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
]


def get_headers(marketplace: str = "us") -> dict:
    """Browser-like headers. Force US locale for amazon.com even when callers are in UAE."""
    from app.services.amazon.marketplace import get_marketplace

    m = get_marketplace(marketplace)
    accept_lang = "en-US,en;q=0.9" if m.code == "us" else "en-AE,en;q=0.9,ar;q=0.8"
    return {
        "User-Agent": random.choice(USER_AGENTS),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": accept_lang,
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Cache-Control": "max-age=0",
    }


def parse_number(text: str) -> Optional[float]:
    if not text:
        return None
    cleaned = re.sub(r"[^\d.]", "", text.replace(",", ""))
    try:
        return float(cleaned) if cleaned else None
    except ValueError:
        return None


def parse_int(text: str) -> Optional[int]:
    val = parse_number(text)
    return int(val) if val is not None else None


def normalize_price(price: Optional[float], marketplace: str = "us") -> Optional[float]:
    """Normalize scraped price. Skip PKR heuristic for AED (prices often > 500)."""
    if price is None:
        return None
    if marketplace and marketplace.lower() in ("ae", "uae"):
        return round(price, 2)
    if price > 500:
        usd = round(price / 278, 2)
        print(f"[scraper] Price {price} looks like PKR, converting to USD: ${usd}")
        return usd
    return round(price, 2)


def parse_amazon_price(soup, marketplace: str = "us") -> Optional[float]:
    """Parse full price including cents (USD or AED)."""
    if soup is None:
        return None
    max_price = 50000 if (marketplace or "us").lower() in ("ae", "uae") else 500

    # Hidden full price e.g. "$136.99" / "AED 499.00"
    for el in soup.select("span.a-offscreen"):
        val = parse_number(el.get_text())
        if val and 0 < val < max_price:
            return round(val, 2)

    # Whole + fraction spans (a-price-whole often drops cents if read alone)
    for root in (
        soup.find("div", {"id": "corePrice_feature_div"}),
        soup.find("div", {"id": "corePriceDisplay_desktop_feature_div"}),
        soup.find("div", {"id": "buybox"}),
        soup,
    ):
        if root is None:
            continue
        whole_el = root.find("span", class_="a-price-whole")
        if not whole_el:
            continue
        whole_digits = re.sub(r"[^\d]", "", whole_el.get_text(strip=True))
        frac_el = root.find("span", class_="a-price-fraction")
        frac = (frac_el.get_text(strip=True) if frac_el else "00")[:2].ljust(2, "0")
        if whole_digits:
            try:
                return round(float(f"{whole_digits}.{frac}"), 2)
            except ValueError:
                pass

    for tag, attrs in (
        ("span", {"id": "price_inside_buybox"}),
        ("span", {"id": "priceblock_ourprice"}),
        ("span", {"id": "priceblock_dealprice"}),
    ):
        el = soup.find(tag, attrs)
        if el:
            val = parse_number(el.get_text())
            if val and val > 0:
                return round(normalize_price(val, marketplace) or val, 2)

    return None