Spaces:
Running
Running
| """Shared HTTP/parsing helpers for Amazon scrapers (avoids circular imports).""" | |
| from __future__ import annotations | |
| import random | |
| import re | |
| from typing import Optional | |
| USER_AGENTS = [ | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", | |
| "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0", | |
| ] | |
| def get_headers(marketplace: str = "us") -> dict: | |
| """Browser-like headers. Force US locale for amazon.com even when callers are in UAE.""" | |
| from app.services.amazon.marketplace import get_marketplace | |
| m = get_marketplace(marketplace) | |
| accept_lang = "en-US,en;q=0.9" if m.code == "us" else "en-AE,en;q=0.9,ar;q=0.8" | |
| return { | |
| "User-Agent": random.choice(USER_AGENTS), | |
| "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", | |
| "Accept-Language": accept_lang, | |
| "Accept-Encoding": "gzip, deflate, br", | |
| "DNT": "1", | |
| "Connection": "keep-alive", | |
| "Upgrade-Insecure-Requests": "1", | |
| "Sec-Fetch-Dest": "document", | |
| "Sec-Fetch-Mode": "navigate", | |
| "Sec-Fetch-Site": "none", | |
| "Cache-Control": "max-age=0", | |
| } | |
| def parse_number(text: str) -> Optional[float]: | |
| if not text: | |
| return None | |
| cleaned = re.sub(r"[^\d.]", "", text.replace(",", "")) | |
| try: | |
| return float(cleaned) if cleaned else None | |
| except ValueError: | |
| return None | |
| def parse_int(text: str) -> Optional[int]: | |
| val = parse_number(text) | |
| return int(val) if val is not None else None | |
| def normalize_price(price: Optional[float], marketplace: str = "us") -> Optional[float]: | |
| """Normalize scraped price. Skip PKR heuristic for AED (prices often > 500).""" | |
| if price is None: | |
| return None | |
| if marketplace and marketplace.lower() in ("ae", "uae"): | |
| return round(price, 2) | |
| if price > 500: | |
| usd = round(price / 278, 2) | |
| print(f"[scraper] Price {price} looks like PKR, converting to USD: ${usd}") | |
| return usd | |
| return round(price, 2) | |
| def parse_amazon_price(soup, marketplace: str = "us") -> Optional[float]: | |
| """Parse full price including cents (USD or AED).""" | |
| if soup is None: | |
| return None | |
| max_price = 50000 if (marketplace or "us").lower() in ("ae", "uae") else 500 | |
| # Hidden full price e.g. "$136.99" / "AED 499.00" | |
| for el in soup.select("span.a-offscreen"): | |
| val = parse_number(el.get_text()) | |
| if val and 0 < val < max_price: | |
| return round(val, 2) | |
| # Whole + fraction spans (a-price-whole often drops cents if read alone) | |
| for root in ( | |
| soup.find("div", {"id": "corePrice_feature_div"}), | |
| soup.find("div", {"id": "corePriceDisplay_desktop_feature_div"}), | |
| soup.find("div", {"id": "buybox"}), | |
| soup, | |
| ): | |
| if root is None: | |
| continue | |
| whole_el = root.find("span", class_="a-price-whole") | |
| if not whole_el: | |
| continue | |
| whole_digits = re.sub(r"[^\d]", "", whole_el.get_text(strip=True)) | |
| frac_el = root.find("span", class_="a-price-fraction") | |
| frac = (frac_el.get_text(strip=True) if frac_el else "00")[:2].ljust(2, "0") | |
| if whole_digits: | |
| try: | |
| return round(float(f"{whole_digits}.{frac}"), 2) | |
| except ValueError: | |
| pass | |
| for tag, attrs in ( | |
| ("span", {"id": "price_inside_buybox"}), | |
| ("span", {"id": "priceblock_ourprice"}), | |
| ("span", {"id": "priceblock_dealprice"}), | |
| ): | |
| el = soup.find(tag, attrs) | |
| if el: | |
| val = parse_number(el.get_text()) | |
| if val and val > 0: | |
| return round(normalize_price(val, marketplace) or val, 2) | |
| return None | |