"""Shared HTTP/parsing helpers for Amazon scrapers (avoids circular imports).""" from __future__ import annotations import random import re from typing import Optional USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0", ] def get_headers(marketplace: str = "us") -> dict: """Browser-like headers. Force US locale for amazon.com even when callers are in UAE.""" from app.services.amazon.marketplace import get_marketplace m = get_marketplace(marketplace) accept_lang = "en-US,en;q=0.9" if m.code == "us" else "en-AE,en;q=0.9,ar;q=0.8" return { "User-Agent": random.choice(USER_AGENTS), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": accept_lang, "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Cache-Control": "max-age=0", } def parse_number(text: str) -> Optional[float]: if not text: return None cleaned = re.sub(r"[^\d.]", "", text.replace(",", "")) try: return float(cleaned) if cleaned else None except ValueError: return None def parse_int(text: str) -> Optional[int]: val = parse_number(text) return int(val) if val is not None else None def normalize_price(price: Optional[float], marketplace: str = "us") -> Optional[float]: """Normalize scraped price. Skip PKR heuristic for AED (prices often > 500).""" if price is None: return None if marketplace and marketplace.lower() in ("ae", "uae"): return round(price, 2) if price > 500: usd = round(price / 278, 2) print(f"[scraper] Price {price} looks like PKR, converting to USD: ${usd}") return usd return round(price, 2) def parse_amazon_price(soup, marketplace: str = "us") -> Optional[float]: """Parse full price including cents (USD or AED).""" if soup is None: return None max_price = 50000 if (marketplace or "us").lower() in ("ae", "uae") else 500 # Hidden full price e.g. "$136.99" / "AED 499.00" for el in soup.select("span.a-offscreen"): val = parse_number(el.get_text()) if val and 0 < val < max_price: return round(val, 2) # Whole + fraction spans (a-price-whole often drops cents if read alone) for root in ( soup.find("div", {"id": "corePrice_feature_div"}), soup.find("div", {"id": "corePriceDisplay_desktop_feature_div"}), soup.find("div", {"id": "buybox"}), soup, ): if root is None: continue whole_el = root.find("span", class_="a-price-whole") if not whole_el: continue whole_digits = re.sub(r"[^\d]", "", whole_el.get_text(strip=True)) frac_el = root.find("span", class_="a-price-fraction") frac = (frac_el.get_text(strip=True) if frac_el else "00")[:2].ljust(2, "0") if whole_digits: try: return round(float(f"{whole_digits}.{frac}"), 2) except ValueError: pass for tag, attrs in ( ("span", {"id": "price_inside_buybox"}), ("span", {"id": "priceblock_ourprice"}), ("span", {"id": "priceblock_dealprice"}), ): el = soup.find(tag, attrs) if el: val = parse_number(el.get_text()) if val and val > 0: return round(normalize_price(val, marketplace) or val, 2) return None