Arag / app /services /platform_pricing.py
AuthorBot
Add SuperAdmin retailer API keys with priority over env and free scraping.
ad1b7c6
Raw
History Blame Contribute Delete
14.7 kB
"""Fetch retail list prices for verified platform listings.
API-first where possible (Google Books, Apple iTunes). HTML fallback with
US-locale headers for Amazon and other US retailers. Open Library shows Free.
"""
from __future__ import annotations
import json
import re
from dataclasses import dataclass
import httpx
import structlog
from app.services.book_url_scraper import (
_API_TIMEOUT,
_browser_headers,
_google_books_api_params,
_parse_amazon_html,
_score_itunes_item,
)
logger = structlog.get_logger(__name__)
_PRICE_TIMEOUT = 12.0
# Goodreads is discovery-only — no single list price on the page.
_NO_PRICE_PLATFORMS = frozenset({"goodreads"})
@dataclass
class PriceContext:
"""Optional catalog hints when listing URL alone is insufficient."""
isbn13: str | None = None
title: str | None = None
author: str | None = None
@dataclass
class PlatformPrice:
"""Normalized list price for a retailer listing."""
formatted: str
amount: float | None = None
currency: str = "USD"
source: str = "unknown"
def format_price(amount: float, currency: str = "USD") -> str:
"""Format amount with a common currency symbol when known."""
currency = (currency or "USD").upper()
symbols = {"USD": "$", "GBP": "£", "EUR": "€", "CAD": "CA$", "AUD": "A$", "PKR": "PKR "}
sym = symbols.get(currency)
if sym == "$":
return f"${amount:.2f}"
if sym and sym != "PKR ":
return f"{sym}{amount:.2f}"
if sym == "PKR ":
return f"PKR {amount:.2f}"
return f"{amount:.2f} {currency}"
def _retailer_headers(platform_id: str, variant: str = "us") -> dict[str, str]:
"""Browser headers tuned per retailer (US locale for Amazon pricing)."""
headers = _browser_headers()
headers["Accept-Language"] = "en-US,en;q=0.9"
if platform_id == "amazon" or variant == "us":
headers["Cookie"] = "i18n-prefs=USD; lc-main=en_US; sp-cdn=J4F7"
if variant == "mobile":
headers["User-Agent"] = (
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
)
return headers
def _extract_amazon_price(html: str) -> PlatformPrice | None:
"""Parse Amazon list price — USD and international currency labels."""
parsed = _parse_amazon_html(html)
raw = (parsed.get("price") or "").strip()
if raw:
m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", raw.replace(",", ""))
if m:
amount = float(m.group(1))
return PlatformPrice(
formatted=raw if raw.startswith("$") else format_price(amount, "USD"),
amount=amount,
currency="USD",
source="html",
)
patterns: list[tuple[str, str]] = [
(r'class="[^"]*a-offscreen[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})', "USD"),
(r'apex-pricetopay-accessibility-label">\s*USD ([0-9,.]+)', "USD"),
(r'apex-pricetopay-accessibility-label">\s*([A-Z]{3}) ([0-9,.]+)', "MULTI"),
(r'"priceToPay"\s*:\s*\{[^}]*"amount"\s*:\s*([0-9.]+)', "USD"),
(r'"priceAmount"\s*:\s*([0-9.]+)', "USD"),
(r'class="a-price-whole">(\d+)</span><span class="a-price-fraction">(\d+)', "SPLIT"),
]
for pat, kind in patterns:
m = re.search(pat, html, re.IGNORECASE | re.DOTALL)
if not m:
continue
if kind == "SPLIT":
amount = float(f"{m.group(1)}.{m.group(2)}")
return PlatformPrice(formatted=format_price(amount, "USD"), amount=amount, currency="USD", source="html")
if kind == "MULTI":
currency = m.group(1).upper()
amount = float(m.group(2).replace(",", ""))
return PlatformPrice(formatted=format_price(amount, currency), amount=amount, currency=currency, source="html")
amount = float(m.group(1).replace(",", ""))
return PlatformPrice(formatted=format_price(amount, kind), amount=amount, currency=kind, source="html")
return None
def _price_from_json_ld(html: str) -> PlatformPrice | None:
"""Extract offer price from schema.org JSON-LD on retailer pages."""
for m in re.finditer(
r'<script[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
html,
re.DOTALL | re.IGNORECASE,
):
try:
data = json.loads(m.group(1))
except json.JSONDecodeError:
continue
items = data if isinstance(data, list) else [data]
for item in items:
offers = item.get("offers")
if isinstance(offers, dict):
offers = [offers]
for offer in offers or []:
if not isinstance(offer, dict):
continue
raw = offer.get("price")
if raw is None:
continue
try:
amount = float(str(raw).replace(",", ""))
except ValueError:
continue
currency = str(offer.get("priceCurrency") or "USD").upper()
return PlatformPrice(
formatted=format_price(amount, currency),
amount=amount,
currency=currency,
source="html",
)
return None
def _price_from_patterns(platform_id: str, html: str) -> PlatformPrice | None:
"""Platform-specific HTML price patterns (after JSON-LD)."""
patterns: dict[str, list[str]] = {
"barnes_noble": [
r'"currentPrice"\s*:\s*([0-9]+\.?[0-9]*)',
r'"salePrice"\s*:\s*([0-9]+\.?[0-9]*)',
r'"listPrice"\s*:\s*([0-9]+\.?[0-9]*)',
r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?',
r'class="[^"]*price[^"]*"[^>]*>\s*\$([0-9]+\.[0-9]{2})',
r'itemprop="price"[^>]*content="([0-9.]+)"',
],
"abebooks": [
r'itemprop="price"[^>]*content="([0-9.]+)"',
r'class="[^"]*item-price[^"]*"[^>]*>\s*US\$\s*([0-9]+\.[0-9]{2})',
],
"kobo": [
r'itemprop="price"[^>]*content="([0-9.]+)"',
r'"Price"\s*:\s*"?([0-9]+\.?[0-9]*)"?',
r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?',
],
"thriftbooks": [
r'class="[^"]*Item-price[^"]*">\s*\$([0-9]+\.[0-9]{2})',
r'class="[^"]*AllEditionsItem-price[^"]*">\s*\$([0-9]+\.[0-9]{2})',
r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?',
],
"bookshop": [
r'itemprop="price"[^>]*content="([0-9.]+)"',
r'"price"\s*:\s*"?([0-9]+\.[0-9]{2})"?',
r'data-price="([0-9.]+)"',
],
}
for pat in patterns.get(platform_id, []):
m = re.search(pat, html, re.IGNORECASE | re.DOTALL)
if m:
amount = float(m.group(1))
return PlatformPrice(
formatted=format_price(amount, "USD"),
amount=amount,
currency="USD",
source="html",
)
return None
def extract_price_from_html(platform_id: str, html: str) -> PlatformPrice | None:
"""Parse list price from retailer HTML."""
if platform_id == "amazon":
return _extract_amazon_price(html)
return _price_from_json_ld(html) or _price_from_patterns(platform_id, html)
def _google_volume_id(listing_url: str) -> str | None:
m = re.search(r"[?&]id=([^&]+)", listing_url)
return m.group(1) if m else None
def _apple_book_id(listing_url: str) -> str | None:
m = re.search(r"/id(\d+)", listing_url)
return m.group(1) if m else None
def _sale_info_price(sale: dict) -> PlatformPrice | None:
"""Parse Google Books saleInfo block."""
for key in ("listPrice", "retailPrice"):
lp = sale.get(key) or {}
if lp.get("amount") is None:
continue
amount = float(lp["amount"])
currency = str(lp.get("currencyCode") or "USD").upper()
return PlatformPrice(
formatted=format_price(amount, currency),
amount=amount,
currency=currency,
source="api",
)
return None
async def fetch_google_books_price(
client: httpx.AsyncClient,
listing_url: str,
ctx: PriceContext | None = None,
) -> PlatformPrice | None:
"""Google Books API saleInfo — volume ID then ISBN search fallback."""
volume_id = _google_volume_id(listing_url)
isbn = (ctx.isbn13 if ctx else None) or ""
try:
if volume_id:
r = await client.get(
f"https://www.googleapis.com/books/v1/volumes/{volume_id}",
params={**_google_books_api_params(), "country": "US"},
timeout=_API_TIMEOUT,
)
if r.status_code == 200:
hit = _sale_info_price(r.json().get("saleInfo") or {})
if hit:
return hit
if isbn:
r = await client.get(
"https://www.googleapis.com/books/v1/volumes",
params={
**_google_books_api_params(q=f"isbn:{isbn}", maxResults="5"),
"country": "US",
},
timeout=_API_TIMEOUT,
)
if r.status_code == 200:
for item in r.json().get("items", []):
hit = _sale_info_price(item.get("saleInfo") or {})
if hit:
return hit
except Exception as exc:
logger.debug("Google Books price fetch failed", error=str(exc))
return None
async def fetch_apple_books_price(
client: httpx.AsyncClient,
listing_url: str,
ctx: PriceContext | None = None,
) -> PlatformPrice | None:
"""Apple iTunes Lookup API with title search fallback."""
book_id = _apple_book_id(listing_url)
def _from_item(item: dict) -> PlatformPrice | None:
if item.get("formattedPrice"):
formatted = str(item["formattedPrice"])
m = re.search(r"([0-9]+(?:\.[0-9]{2})?)", formatted.replace(",", ""))
amount = float(m.group(1)) if m else None
return PlatformPrice(
formatted=formatted,
amount=amount,
currency=str(item.get("currency") or "USD").upper(),
source="api",
)
if item.get("trackPrice") is not None:
amount = float(item["trackPrice"])
currency = str(item.get("currency") or "USD").upper()
return PlatformPrice(
formatted=format_price(amount, currency),
amount=amount,
currency=currency,
source="api",
)
return None
try:
if book_id:
for params in ({"id": book_id, "entity": "ebook"}, {"id": book_id}):
r = await client.get(
"https://itunes.apple.com/lookup",
params=params,
timeout=_API_TIMEOUT,
)
if r.status_code != 200:
continue
results = r.json().get("results") or []
if results:
hit = _from_item(results[0])
if hit:
return hit
title = (ctx.title if ctx else None) or ""
author = (ctx.author if ctx else None) or ""
if title:
term = f"{title} {author}".strip()
r = await client.get(
"https://itunes.apple.com/search",
params={"term": term, "entity": "ebook", "limit": "8", "country": "US"},
timeout=_API_TIMEOUT,
)
if r.status_code == 200:
results = r.json().get("results") or []
if book_id:
for item in results:
if str(item.get("trackId")) == book_id:
hit = _from_item(item)
if hit:
return hit
if results:
best = max(results, key=lambda i: _score_itunes_item(i, title, author))
if _score_itunes_item(best, title, author) >= 4.0:
hit = _from_item(best)
if hit:
return hit
except Exception as exc:
logger.debug("Apple Books price fetch failed", error=str(exc))
return None
async def _fetch_html_price(
client: httpx.AsyncClient,
platform_id: str,
listing_url: str,
) -> PlatformPrice | None:
"""GET product page with US then mobile headers."""
variants = ("us", "mobile") if platform_id in {
"amazon", "kobo", "thriftbooks", "bookshop",
} else ("us",)
for variant in variants:
try:
r = await client.get(
listing_url,
headers=_retailer_headers(platform_id, variant),
timeout=_PRICE_TIMEOUT,
)
if r.status_code != 200 or len(r.text) < 500:
continue
hit = extract_price_from_html(platform_id, r.text)
if hit:
return hit
except Exception as exc:
logger.debug(
"HTML price fetch failed",
platform=platform_id,
variant=variant,
error=str(exc)[:120],
)
return None
async def fetch_listing_price(
client: httpx.AsyncClient,
platform_id: str,
listing_url: str,
ctx: PriceContext | None = None,
credential: "EffectiveCredential | None" = None,
) -> PlatformPrice | None:
"""Fetch list price for one verified platform listing."""
if not listing_url:
return None
if platform_id == "open_library":
return PlatformPrice(formatted="Free", amount=0.0, currency="USD", source="catalog")
if platform_id in _NO_PRICE_PLATFORMS:
return None
# Priority 1: SuperAdmin / environment API credential
if credential is not None:
from app.services.platform_api_registry import fetch_price_via_credential
hit = await fetch_price_via_credential(
client, credential, platform_id, listing_url, ctx,
)
if hit:
return hit
# Priority 2: built-in free logic
if platform_id == "google_books":
return await fetch_google_books_price(client, listing_url, ctx)
if platform_id == "apple_books":
return await fetch_apple_books_price(client, listing_url, ctx)
return await _fetch_html_price(client, platform_id, listing_url)