"""Fetch retail list prices for verified platform listings. API-first where possible (Google Books, Apple iTunes). HTML fallback with US-locale headers for Amazon and other US retailers. Open Library shows Free. """ from __future__ import annotations import json import re from dataclasses import dataclass import httpx import structlog from app.services.book_url_scraper import ( _API_TIMEOUT, _browser_headers, _google_books_api_params, _parse_amazon_html, _score_itunes_item, ) logger = structlog.get_logger(__name__) _PRICE_TIMEOUT = 12.0 # Goodreads is discovery-only — no single list price on the page. _NO_PRICE_PLATFORMS = frozenset({"goodreads"}) @dataclass class PriceContext: """Optional catalog hints when listing URL alone is insufficient.""" isbn13: str | None = None title: str | None = None author: str | None = None @dataclass class PlatformPrice: """Normalized list price for a retailer listing.""" formatted: str amount: float | None = None currency: str = "USD" source: str = "unknown" def format_price(amount: float, currency: str = "USD") -> str: """Format amount with a common currency symbol when known.""" currency = (currency or "USD").upper() symbols = {"USD": "$", "GBP": "£", "EUR": "€", "CAD": "CA$", "AUD": "A$", "PKR": "PKR "} sym = symbols.get(currency) if sym == "$": return f"${amount:.2f}" if sym and sym != "PKR ": return f"{sym}{amount:.2f}" if sym == "PKR ": return f"PKR {amount:.2f}" return f"{amount:.2f} {currency}" def _retailer_headers(platform_id: str, variant: str = "us") -> dict[str, str]: """Browser headers tuned per retailer (US locale for Amazon pricing).""" headers = _browser_headers() headers["Accept-Language"] = "en-US,en;q=0.9" if platform_id == "amazon" or variant == "us": headers["Cookie"] = "i18n-prefs=USD; lc-main=en_US; sp-cdn=J4F7" if variant == "mobile": headers["User-Agent"] = ( "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" ) return headers def _extract_amazon_price(html: str) -> PlatformPrice | None: """Parse Amazon list price — USD and international currency labels.""" parsed = _parse_amazon_html(html) raw = (parsed.get("price") or "").strip() if raw: m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", raw.replace(",", "")) if m: amount = float(m.group(1)) return PlatformPrice( formatted=raw if raw.startswith("$") else format_price(amount, "USD"), amount=amount, currency="USD", source="html", ) patterns: list[tuple[str, str]] = [ (r'class="[^"]*a-offscreen[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})', "USD"), (r'apex-pricetopay-accessibility-label">\s*USD ([0-9,.]+)', "USD"), (r'apex-pricetopay-accessibility-label">\s*([A-Z]{3}) ([0-9,.]+)', "MULTI"), (r'"priceToPay"\s*:\s*\{[^}]*"amount"\s*:\s*([0-9.]+)', "USD"), (r'"priceAmount"\s*:\s*([0-9.]+)', "USD"), (r'class="a-price-whole">(\d+)(\d+)', "SPLIT"), ] for pat, kind in patterns: m = re.search(pat, html, re.IGNORECASE | re.DOTALL) if not m: continue if kind == "SPLIT": amount = float(f"{m.group(1)}.{m.group(2)}") return PlatformPrice(formatted=format_price(amount, "USD"), amount=amount, currency="USD", source="html") if kind == "MULTI": currency = m.group(1).upper() amount = float(m.group(2).replace(",", "")) return PlatformPrice(formatted=format_price(amount, currency), amount=amount, currency=currency, source="html") amount = float(m.group(1).replace(",", "")) return PlatformPrice(formatted=format_price(amount, kind), amount=amount, currency=kind, source="html") return None def _price_from_json_ld(html: str) -> PlatformPrice | None: """Extract offer price from schema.org JSON-LD on retailer pages.""" for m in re.finditer( r']*type=["\']application/ld\+json["\'][^>]*>(.*?)', html, re.DOTALL | re.IGNORECASE, ): try: data = json.loads(m.group(1)) except json.JSONDecodeError: continue items = data if isinstance(data, list) else [data] for item in items: offers = item.get("offers") if isinstance(offers, dict): offers = [offers] for offer in offers or []: if not isinstance(offer, dict): continue raw = offer.get("price") if raw is None: continue try: amount = float(str(raw).replace(",", "")) except ValueError: continue currency = str(offer.get("priceCurrency") or "USD").upper() return PlatformPrice( formatted=format_price(amount, currency), amount=amount, currency=currency, source="html", ) return None def _price_from_patterns(platform_id: str, html: str) -> PlatformPrice | None: """Platform-specific HTML price patterns (after JSON-LD).""" patterns: dict[str, list[str]] = { "barnes_noble": [ r'"currentPrice"\s*:\s*([0-9]+\.?[0-9]*)', r'"salePrice"\s*:\s*([0-9]+\.?[0-9]*)', r'"listPrice"\s*:\s*([0-9]+\.?[0-9]*)', r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', r'class="[^"]*price[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})', r'itemprop="price"[^>]*content="([0-9.]+)"', ], "abebooks": [ r'itemprop="price"[^>]*content="([0-9.]+)"', r'class="[^"]*item-price[^"]*"[^>]*>\s*US\$\s*([0-9]+\.[0-9]{2})', ], "kobo": [ r'itemprop="price"[^>]*content="([0-9.]+)"', r'"Price"\s*:\s*"?([0-9]+\.?[0-9]*)"?', r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', ], "thriftbooks": [ r'class="[^"]*Item-price[^"]*">\s*\$([0-9]+\.[0-9]{2})', r'class="[^"]*AllEditionsItem-price[^"]*">\s*\$([0-9]+\.[0-9]{2})', r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', ], "bookshop": [ r'itemprop="price"[^>]*content="([0-9.]+)"', r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', r'data-price="([0-9.]+)"', ], } for pat in patterns.get(platform_id, []): m = re.search(pat, html, re.IGNORECASE | re.DOTALL) if m: amount = float(m.group(1)) return PlatformPrice( formatted=format_price(amount, "USD"), amount=amount, currency="USD", source="html", ) return None def extract_price_from_html(platform_id: str, html: str) -> PlatformPrice | None: """Parse list price from retailer HTML.""" if platform_id == "amazon": return _extract_amazon_price(html) return _price_from_json_ld(html) or _price_from_patterns(platform_id, html) def _google_volume_id(listing_url: str) -> str | None: m = re.search(r"[?&]id=([^&]+)", listing_url) return m.group(1) if m else None def _apple_book_id(listing_url: str) -> str | None: m = re.search(r"/id(\d+)", listing_url) return m.group(1) if m else None def _sale_info_price(sale: dict) -> PlatformPrice | None: """Parse Google Books saleInfo block.""" for key in ("listPrice", "retailPrice"): lp = sale.get(key) or {} if lp.get("amount") is None: continue amount = float(lp["amount"]) currency = str(lp.get("currencyCode") or "USD").upper() return PlatformPrice( formatted=format_price(amount, currency), amount=amount, currency=currency, source="api", ) return None async def fetch_google_books_price( client: httpx.AsyncClient, listing_url: str, ctx: PriceContext | None = None, ) -> PlatformPrice | None: """Google Books API saleInfo — volume ID then ISBN search fallback.""" volume_id = _google_volume_id(listing_url) isbn = (ctx.isbn13 if ctx else None) or "" try: if volume_id: r = await client.get( f"https://www.googleapis.com/books/v1/volumes/{volume_id}", params={**_google_books_api_params(), "country": "US"}, timeout=_API_TIMEOUT, ) if r.status_code == 200: hit = _sale_info_price(r.json().get("saleInfo") or {}) if hit: return hit if isbn: r = await client.get( "https://www.googleapis.com/books/v1/volumes", params={ **_google_books_api_params(q=f"isbn:{isbn}", maxResults="5"), "country": "US", }, timeout=_API_TIMEOUT, ) if r.status_code == 200: for item in r.json().get("items", []): hit = _sale_info_price(item.get("saleInfo") or {}) if hit: return hit except Exception as exc: logger.debug("Google Books price fetch failed", error=str(exc)) return None async def fetch_apple_books_price( client: httpx.AsyncClient, listing_url: str, ctx: PriceContext | None = None, ) -> PlatformPrice | None: """Apple iTunes Lookup API with title search fallback.""" book_id = _apple_book_id(listing_url) def _from_item(item: dict) -> PlatformPrice | None: if item.get("formattedPrice"): formatted = str(item["formattedPrice"]) m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", formatted.replace(",", "")) amount = float(m.group(1)) if m else None return PlatformPrice( formatted=formatted, amount=amount, currency=str(item.get("currency") or "USD").upper(), source="api", ) if item.get("trackPrice") is not None: amount = float(item["trackPrice"]) currency = str(item.get("currency") or "USD").upper() return PlatformPrice( formatted=format_price(amount, currency), amount=amount, currency=currency, source="api", ) return None try: if book_id: for params in ({"id": book_id, "entity": "ebook"}, {"id": book_id}): r = await client.get( "https://itunes.apple.com/lookup", params=params, timeout=_API_TIMEOUT, ) if r.status_code != 200: continue results = r.json().get("results") or [] if results: hit = _from_item(results[0]) if hit: return hit title = (ctx.title if ctx else None) or "" author = (ctx.author if ctx else None) or "" if title: term = f"{title} {author}".strip() r = await client.get( "https://itunes.apple.com/search", params={"term": term, "entity": "ebook", "limit": "8", "country": "US"}, timeout=_API_TIMEOUT, ) if r.status_code == 200: results = r.json().get("results") or [] if book_id: for item in results: if str(item.get("trackId")) == book_id: hit = _from_item(item) if hit: return hit if results: best = max(results, key=lambda i: _score_itunes_item(i, title, author)) if _score_itunes_item(best, title, author) >= 4.0: hit = _from_item(best) if hit: return hit except Exception as exc: logger.debug("Apple Books price fetch failed", error=str(exc)) return None async def _fetch_html_price( client: httpx.AsyncClient, platform_id: str, listing_url: str, ) -> PlatformPrice | None: """GET product page with US then mobile headers.""" variants = ("us", "mobile") if platform_id in { "amazon", "kobo", "thriftbooks", "bookshop", } else ("us",) for variant in variants: try: r = await client.get( listing_url, headers=_retailer_headers(platform_id, variant), timeout=_PRICE_TIMEOUT, ) if r.status_code != 200 or len(r.text) < 500: continue hit = extract_price_from_html(platform_id, r.text) if hit: return hit except Exception as exc: logger.debug( "HTML price fetch failed", platform=platform_id, variant=variant, error=str(exc)[:120], ) return None async def fetch_listing_price( client: httpx.AsyncClient, platform_id: str, listing_url: str, ctx: PriceContext | None = None, credential: "EffectiveCredential | None" = None, ) -> PlatformPrice | None: """Fetch list price for one verified platform listing.""" if not listing_url: return None if platform_id == "open_library": return PlatformPrice(formatted="Free", amount=0.0, currency="USD", source="catalog") if platform_id in _NO_PRICE_PLATFORMS: return None # Priority 1: SuperAdmin / environment API credential if credential is not None: from app.services.platform_api_registry import fetch_price_via_credential hit = await fetch_price_via_credential( client, credential, platform_id, listing_url, ctx, ) if hit: return hit # Priority 2: built-in free logic if platform_id == "google_books": return await fetch_google_books_price(client, listing_url, ctx) if platform_id == "apple_books": return await fetch_apple_books_price(client, listing_url, ctx) return await _fetch_html_price(client, platform_id, listing_url)