Spaces:
Running
Running
| """Fetch retail list prices for verified platform listings. | |
| API-first where possible (Google Books, Apple iTunes). HTML fallback with | |
| US-locale headers for Amazon and other US retailers. Open Library shows Free. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from dataclasses import dataclass | |
| import httpx | |
| import structlog | |
| from app.services.book_url_scraper import ( | |
| _API_TIMEOUT, | |
| _browser_headers, | |
| _google_books_api_params, | |
| _parse_amazon_html, | |
| _score_itunes_item, | |
| ) | |
| logger = structlog.get_logger(__name__) | |
| _PRICE_TIMEOUT = 12.0 | |
| # Goodreads is discovery-only — no single list price on the page. | |
| _NO_PRICE_PLATFORMS = frozenset({"goodreads"}) | |
| class PriceContext: | |
| """Optional catalog hints when listing URL alone is insufficient.""" | |
| isbn13: str | None = None | |
| title: str | None = None | |
| author: str | None = None | |
| class PlatformPrice: | |
| """Normalized list price for a retailer listing.""" | |
| formatted: str | |
| amount: float | None = None | |
| currency: str = "USD" | |
| source: str = "unknown" | |
| def format_price(amount: float, currency: str = "USD") -> str: | |
| """Format amount with a common currency symbol when known.""" | |
| currency = (currency or "USD").upper() | |
| symbols = {"USD": "$", "GBP": "£", "EUR": "€", "CAD": "CA$", "AUD": "A$", "PKR": "PKR "} | |
| sym = symbols.get(currency) | |
| if sym == "$": | |
| return f"${amount:.2f}" | |
| if sym and sym != "PKR ": | |
| return f"{sym}{amount:.2f}" | |
| if sym == "PKR ": | |
| return f"PKR {amount:.2f}" | |
| return f"{amount:.2f} {currency}" | |
| def _retailer_headers(platform_id: str, variant: str = "us") -> dict[str, str]: | |
| """Browser headers tuned per retailer (US locale for Amazon pricing).""" | |
| headers = _browser_headers() | |
| headers["Accept-Language"] = "en-US,en;q=0.9" | |
| if platform_id == "amazon" or variant == "us": | |
| headers["Cookie"] = "i18n-prefs=USD; lc-main=en_US; sp-cdn=J4F7" | |
| if variant == "mobile": | |
| headers["User-Agent"] = ( | |
| "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " | |
| "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" | |
| ) | |
| return headers | |
| def _extract_amazon_price(html: str) -> PlatformPrice | None: | |
| """Parse Amazon list price — USD and international currency labels.""" | |
| parsed = _parse_amazon_html(html) | |
| raw = (parsed.get("price") or "").strip() | |
| if raw: | |
| m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", raw.replace(",", "")) | |
| if m: | |
| amount = float(m.group(1)) | |
| return PlatformPrice( | |
| formatted=raw if raw.startswith("$") else format_price(amount, "USD"), | |
| amount=amount, | |
| currency="USD", | |
| source="html", | |
| ) | |
| patterns: list[tuple[str, str]] = [ | |
| (r'class="[^"]*a-offscreen[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})', "USD"), | |
| (r'apex-pricetopay-accessibility-label">\s*USD ([0-9,.]+)', "USD"), | |
| (r'apex-pricetopay-accessibility-label">\s*([A-Z]{3}) ([0-9,.]+)', "MULTI"), | |
| (r'"priceToPay"\s*:\s*\{[^}]*"amount"\s*:\s*([0-9.]+)', "USD"), | |
| (r'"priceAmount"\s*:\s*([0-9.]+)', "USD"), | |
| (r'class="a-price-whole">(\d+)</span><span class="a-price-fraction">(\d+)', "SPLIT"), | |
| ] | |
| for pat, kind in patterns: | |
| m = re.search(pat, html, re.IGNORECASE | re.DOTALL) | |
| if not m: | |
| continue | |
| if kind == "SPLIT": | |
| amount = float(f"{m.group(1)}.{m.group(2)}") | |
| return PlatformPrice(formatted=format_price(amount, "USD"), amount=amount, currency="USD", source="html") | |
| if kind == "MULTI": | |
| currency = m.group(1).upper() | |
| amount = float(m.group(2).replace(",", "")) | |
| return PlatformPrice(formatted=format_price(amount, currency), amount=amount, currency=currency, source="html") | |
| amount = float(m.group(1).replace(",", "")) | |
| return PlatformPrice(formatted=format_price(amount, kind), amount=amount, currency=kind, source="html") | |
| return None | |
| def _price_from_json_ld(html: str) -> PlatformPrice | None: | |
| """Extract offer price from schema.org JSON-LD on retailer pages.""" | |
| for m in re.finditer( | |
| r'<script[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', | |
| html, | |
| re.DOTALL | re.IGNORECASE, | |
| ): | |
| try: | |
| data = json.loads(m.group(1)) | |
| except json.JSONDecodeError: | |
| continue | |
| items = data if isinstance(data, list) else [data] | |
| for item in items: | |
| offers = item.get("offers") | |
| if isinstance(offers, dict): | |
| offers = [offers] | |
| for offer in offers or []: | |
| if not isinstance(offer, dict): | |
| continue | |
| raw = offer.get("price") | |
| if raw is None: | |
| continue | |
| try: | |
| amount = float(str(raw).replace(",", "")) | |
| except ValueError: | |
| continue | |
| currency = str(offer.get("priceCurrency") or "USD").upper() | |
| return PlatformPrice( | |
| formatted=format_price(amount, currency), | |
| amount=amount, | |
| currency=currency, | |
| source="html", | |
| ) | |
| return None | |
| def _price_from_patterns(platform_id: str, html: str) -> PlatformPrice | None: | |
| """Platform-specific HTML price patterns (after JSON-LD).""" | |
| patterns: dict[str, list[str]] = { | |
| "barnes_noble": [ | |
| r'"currentPrice"\s*:\s*([0-9]+\.?[0-9]*)', | |
| r'"salePrice"\s*:\s*([0-9]+\.?[0-9]*)', | |
| r'"listPrice"\s*:\s*([0-9]+\.?[0-9]*)', | |
| r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', | |
| r'class="[^"]*price[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})', | |
| r'itemprop="price"[^>]*content="([0-9.]+)"', | |
| ], | |
| "abebooks": [ | |
| r'itemprop="price"[^>]*content="([0-9.]+)"', | |
| r'class="[^"]*item-price[^"]*"[^>]*>\s*US\$\s*([0-9]+\.[0-9]{2})', | |
| ], | |
| "kobo": [ | |
| r'itemprop="price"[^>]*content="([0-9.]+)"', | |
| r'"Price"\s*:\s*"?([0-9]+\.?[0-9]*)"?', | |
| r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', | |
| ], | |
| "thriftbooks": [ | |
| r'class="[^"]*Item-price[^"]*">\s*\$([0-9]+\.[0-9]{2})', | |
| r'class="[^"]*AllEditionsItem-price[^"]*">\s*\$([0-9]+\.[0-9]{2})', | |
| r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', | |
| ], | |
| "bookshop": [ | |
| r'itemprop="price"[^>]*content="([0-9.]+)"', | |
| r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?', | |
| r'data-price="([0-9.]+)"', | |
| ], | |
| } | |
| for pat in patterns.get(platform_id, []): | |
| m = re.search(pat, html, re.IGNORECASE | re.DOTALL) | |
| if m: | |
| amount = float(m.group(1)) | |
| return PlatformPrice( | |
| formatted=format_price(amount, "USD"), | |
| amount=amount, | |
| currency="USD", | |
| source="html", | |
| ) | |
| return None | |
| def extract_price_from_html(platform_id: str, html: str) -> PlatformPrice | None: | |
| """Parse list price from retailer HTML.""" | |
| if platform_id == "amazon": | |
| return _extract_amazon_price(html) | |
| return _price_from_json_ld(html) or _price_from_patterns(platform_id, html) | |
| def _google_volume_id(listing_url: str) -> str | None: | |
| m = re.search(r"[?&]id=([^&]+)", listing_url) | |
| return m.group(1) if m else None | |
| def _apple_book_id(listing_url: str) -> str | None: | |
| m = re.search(r"/id(\d+)", listing_url) | |
| return m.group(1) if m else None | |
| def _sale_info_price(sale: dict) -> PlatformPrice | None: | |
| """Parse Google Books saleInfo block.""" | |
| for key in ("listPrice", "retailPrice"): | |
| lp = sale.get(key) or {} | |
| if lp.get("amount") is None: | |
| continue | |
| amount = float(lp["amount"]) | |
| currency = str(lp.get("currencyCode") or "USD").upper() | |
| return PlatformPrice( | |
| formatted=format_price(amount, currency), | |
| amount=amount, | |
| currency=currency, | |
| source="api", | |
| ) | |
| return None | |
| async def fetch_google_books_price( | |
| client: httpx.AsyncClient, | |
| listing_url: str, | |
| ctx: PriceContext | None = None, | |
| ) -> PlatformPrice | None: | |
| """Google Books API saleInfo — volume ID then ISBN search fallback.""" | |
| volume_id = _google_volume_id(listing_url) | |
| isbn = (ctx.isbn13 if ctx else None) or "" | |
| try: | |
| if volume_id: | |
| r = await client.get( | |
| f"https://www.googleapis.com/books/v1/volumes/{volume_id}", | |
| params={**_google_books_api_params(), "country": "US"}, | |
| timeout=_API_TIMEOUT, | |
| ) | |
| if r.status_code == 200: | |
| hit = _sale_info_price(r.json().get("saleInfo") or {}) | |
| if hit: | |
| return hit | |
| if isbn: | |
| r = await client.get( | |
| "https://www.googleapis.com/books/v1/volumes", | |
| params={ | |
| **_google_books_api_params(q=f"isbn:{isbn}", maxResults="5"), | |
| "country": "US", | |
| }, | |
| timeout=_API_TIMEOUT, | |
| ) | |
| if r.status_code == 200: | |
| for item in r.json().get("items", []): | |
| hit = _sale_info_price(item.get("saleInfo") or {}) | |
| if hit: | |
| return hit | |
| except Exception as exc: | |
| logger.debug("Google Books price fetch failed", error=str(exc)) | |
| return None | |
| async def fetch_apple_books_price( | |
| client: httpx.AsyncClient, | |
| listing_url: str, | |
| ctx: PriceContext | None = None, | |
| ) -> PlatformPrice | None: | |
| """Apple iTunes Lookup API with title search fallback.""" | |
| book_id = _apple_book_id(listing_url) | |
| def _from_item(item: dict) -> PlatformPrice | None: | |
| if item.get("formattedPrice"): | |
| formatted = str(item["formattedPrice"]) | |
| m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", formatted.replace(",", "")) | |
| amount = float(m.group(1)) if m else None | |
| return PlatformPrice( | |
| formatted=formatted, | |
| amount=amount, | |
| currency=str(item.get("currency") or "USD").upper(), | |
| source="api", | |
| ) | |
| if item.get("trackPrice") is not None: | |
| amount = float(item["trackPrice"]) | |
| currency = str(item.get("currency") or "USD").upper() | |
| return PlatformPrice( | |
| formatted=format_price(amount, currency), | |
| amount=amount, | |
| currency=currency, | |
| source="api", | |
| ) | |
| return None | |
| try: | |
| if book_id: | |
| for params in ({"id": book_id, "entity": "ebook"}, {"id": book_id}): | |
| r = await client.get( | |
| "https://itunes.apple.com/lookup", | |
| params=params, | |
| timeout=_API_TIMEOUT, | |
| ) | |
| if r.status_code != 200: | |
| continue | |
| results = r.json().get("results") or [] | |
| if results: | |
| hit = _from_item(results[0]) | |
| if hit: | |
| return hit | |
| title = (ctx.title if ctx else None) or "" | |
| author = (ctx.author if ctx else None) or "" | |
| if title: | |
| term = f"{title} {author}".strip() | |
| r = await client.get( | |
| "https://itunes.apple.com/search", | |
| params={"term": term, "entity": "ebook", "limit": "8", "country": "US"}, | |
| timeout=_API_TIMEOUT, | |
| ) | |
| if r.status_code == 200: | |
| results = r.json().get("results") or [] | |
| if book_id: | |
| for item in results: | |
| if str(item.get("trackId")) == book_id: | |
| hit = _from_item(item) | |
| if hit: | |
| return hit | |
| if results: | |
| best = max(results, key=lambda i: _score_itunes_item(i, title, author)) | |
| if _score_itunes_item(best, title, author) >= 4.0: | |
| hit = _from_item(best) | |
| if hit: | |
| return hit | |
| except Exception as exc: | |
| logger.debug("Apple Books price fetch failed", error=str(exc)) | |
| return None | |
| async def _fetch_html_price( | |
| client: httpx.AsyncClient, | |
| platform_id: str, | |
| listing_url: str, | |
| ) -> PlatformPrice | None: | |
| """GET product page with US then mobile headers.""" | |
| variants = ("us", "mobile") if platform_id in { | |
| "amazon", "kobo", "thriftbooks", "bookshop", | |
| } else ("us",) | |
| for variant in variants: | |
| try: | |
| r = await client.get( | |
| listing_url, | |
| headers=_retailer_headers(platform_id, variant), | |
| timeout=_PRICE_TIMEOUT, | |
| ) | |
| if r.status_code != 200 or len(r.text) < 500: | |
| continue | |
| hit = extract_price_from_html(platform_id, r.text) | |
| if hit: | |
| return hit | |
| except Exception as exc: | |
| logger.debug( | |
| "HTML price fetch failed", | |
| platform=platform_id, | |
| variant=variant, | |
| error=str(exc)[:120], | |
| ) | |
| return None | |
| async def fetch_listing_price( | |
| client: httpx.AsyncClient, | |
| platform_id: str, | |
| listing_url: str, | |
| ctx: PriceContext | None = None, | |
| credential: "EffectiveCredential | None" = None, | |
| ) -> PlatformPrice | None: | |
| """Fetch list price for one verified platform listing.""" | |
| if not listing_url: | |
| return None | |
| if platform_id == "open_library": | |
| return PlatformPrice(formatted="Free", amount=0.0, currency="USD", source="catalog") | |
| if platform_id in _NO_PRICE_PLATFORMS: | |
| return None | |
| # Priority 1: SuperAdmin / environment API credential | |
| if credential is not None: | |
| from app.services.platform_api_registry import fetch_price_via_credential | |
| hit = await fetch_price_via_credential( | |
| client, credential, platform_id, listing_url, ctx, | |
| ) | |
| if hit: | |
| return hit | |
| # Priority 2: built-in free logic | |
| if platform_id == "google_books": | |
| return await fetch_google_books_price(client, listing_url, ctx) | |
| if platform_id == "apple_books": | |
| return await fetch_apple_books_price(client, listing_url, ctx) | |
| return await _fetch_html_price(client, platform_id, listing_url) | |