Arag / app /services /catalog_hub.py
AuthorBot
Harden platform scan when Goodreads or ISBN metadata is partial.
4d2ac8b
Raw
History Blame Contribute Delete
11.3 kB
"""API-first catalog hub — resolve book identity and platform listings.
Uses only public JSON APIs (Google Books, Open Library, Goodreads autocomplete,
Apple iTunes) plus Goodreads buy links. No retailer HTML scraping.
"""
from __future__ import annotations
import asyncio
import urllib.parse
from dataclasses import dataclass, field
import httpx
import structlog
from app.services.book_url_scraper import (
_google_by_isbn,
_google_search_best,
_normalize_isbn,
_score_itunes_item,
)
from app.services.isbn_catalog import (
classify_retailer_url,
enrich_retailer_links_from_isbn,
extract_amazon_asin,
is_confirmed_isbn13,
lookup_open_library,
resolve_google_books_html,
sanitize_platform_url,
)
logger = structlog.get_logger(__name__)
_TIMEOUT = 8.0
VERIFIED_THRESHOLD = 0.85
LIKELY_THRESHOLD = 0.70
# Import platform IDs that map to scored platform IDs
SOURCE_PLATFORM_ALIASES: dict[str, str] = {
"google_play": "google_books",
}
@dataclass
class CatalogSnapshot:
"""Canonical book identity from public catalog APIs."""
title: str = ""
author: str = ""
isbn13: str | None = None
isbn10: str | None = None
asin: str | None = None
gb_volume_id: str | None = None
gb_confidence: float = 0.0
ol_key: str | None = None
ol_confidence: float = 0.0
goodreads_id: str | None = None
goodreads_url: str | None = None
goodreads_confidence: float = 0.0
apple_url: str | None = None
apple_confidence: float = 0.0
retailer_links: dict[str, str] = field(default_factory=dict)
@property
def confirmed_isbn13(self) -> str | None:
"""ISBN-13 confirmed by at least one catalog API or valid import."""
if self.isbn13 and (self.gb_confidence >= LIKELY_THRESHOLD or self.ol_confidence >= LIKELY_THRESHOLD):
return self.isbn13
if self.isbn13 and self.gb_volume_id and self.ol_key:
return self.isbn13
return None
def google_books_url(self) -> str | None:
if self.gb_volume_id:
return f"https://books.google.com/books?id={self.gb_volume_id}"
return None
def open_library_url(self) -> str | None:
if self.isbn13:
return f"https://openlibrary.org/isbn/{self.isbn13}"
if self.ol_key:
return f"https://openlibrary.org{self.ol_key}"
return None
def amazon_product_url(self) -> str | None:
if self.asin:
return f"https://www.amazon.com/dp/{self.asin}"
if self.isbn10 and self.isbn10.isdigit():
return f"https://www.amazon.com/dp/{self.isbn10}"
return None
@dataclass
class PlatformHit:
"""Discovery result for one retailer."""
confidence: float
listing_url: str | None
isbn_match: bool = False
def normalize_source_platform(platform_id: str | None) -> str | None:
"""Map import platform IDs to scored platform IDs."""
if not platform_id:
return None
return SOURCE_PLATFORM_ALIASES.get(platform_id, platform_id)
def isbn_retailer_urls(isbn13: str) -> dict[str, str]:
"""ISBN-targeted lookup URLs — honest search, not fake product pages."""
q = urllib.parse.quote(isbn13)
return {
"barnes_noble": f"https://www.barnesandnoble.com/s/{q}",
"kobo": f"https://www.kobo.com/us/en/search?query={q}",
"abebooks": f"https://www.abebooks.com/servlet/SearchResults?isbn={q}",
"thriftbooks": f"https://www.thriftbooks.com/browse/?b.search={q}",
"bookshop": f"https://bookshop.org/search?keywords={q}",
}
async def _resolve_google(
client: httpx.AsyncClient,
title: str,
author: str,
isbn: str | None,
) -> dict:
"""Google Books API lookup with HTML fallback when quota is exceeded."""
from app.services.platform_presence import title_match_score
gb: dict = {}
if isbn:
gb = await _google_by_isbn(isbn, client, author_hint=author)
if not gb.get("title"):
gb = await _google_search_best(title, author, client)
if not gb.get("title") and is_confirmed_isbn13(isbn):
gb = await resolve_google_books_html(client, isbn, title, author)
if not gb.get("title"):
return {}
conf = title_match_score(title, gb["title"], author, gb.get("author", ""))
isbn_match = bool(
isbn and gb.get("isbn") and _normalize_isbn(isbn) == _normalize_isbn(gb["isbn"])
)
if isbn_match or gb.get("isbn_match"):
conf = max(conf, 0.95)
return {
"confidence": gb.get("confidence", conf),
"volume_id": gb.get("volume_id") or "",
"isbn13": _normalize_isbn(gb.get("isbn13") or gb.get("isbn") or ""),
"isbn10": _normalize_isbn(gb.get("isbn10") or ""),
"title": gb.get("title", ""),
"author": gb.get("author", ""),
"isbn_match": isbn_match or bool(gb.get("isbn_match")),
}
async def _resolve_apple(
client: httpx.AsyncClient,
title: str,
author: str,
) -> dict:
"""Apple iTunes Search API lookup."""
from app.services.platform_presence import search_title, title_match_score
core = search_title(title) or title
if not core:
return {}
term = f"{core} {author}".strip() if author else core
try:
resp = await client.get(
"https://itunes.apple.com/search",
params={"term": term, "entity": "ebook", "limit": "8", "country": "US"},
timeout=_TIMEOUT,
)
if resp.status_code != 200:
return {}
results = resp.json().get("results", [])
if not results:
return {}
best = max(results, key=lambda item: _score_itunes_item(item, core, author))
if _score_itunes_item(best, core, author) < 4.0:
return {}
found_title = best.get("trackName") or ""
conf = title_match_score(core, found_title, author, best.get("artistName") or "")
if conf < LIKELY_THRESHOLD:
return {}
return {"confidence": conf, "url": best.get("trackViewUrl")}
except Exception as exc:
logger.debug("Apple catalog lookup failed", error=str(exc))
return {}
async def build_catalog_snapshot(
client: httpx.AsyncClient,
*,
title: str,
author: str,
isbn: str | None = None,
asin: str | None = None,
goodreads_links: dict[str, str] | None = None,
goodreads_url: str | None = None,
goodreads_confidence: float = 0.0,
resolved_author: str = "",
skip_enrich: bool = False,
) -> CatalogSnapshot:
"""Resolve book identity from public APIs in parallel."""
from app.services.platform_presence import search_title
core_title = search_title(title) or title.strip()
norm_isbn = _normalize_isbn(isbn) if isbn else None
if is_confirmed_isbn13(norm_isbn):
catalog_isbn = norm_isbn
else:
catalog_isbn = None
norm_asin = (asin or "").upper() or None
if not norm_asin and isbn and len(_normalize_isbn(isbn) or "") == 10:
candidate = (isbn or "").upper()
if candidate.startswith("B0"):
norm_asin = candidate
gb_task = asyncio.create_task(_resolve_google(client, core_title, author, catalog_isbn))
ol_task = asyncio.create_task(
lookup_open_library(client, core_title, author, catalog_isbn)
)
apple_task = asyncio.create_task(_resolve_apple(client, core_title, author))
gb, ol, apple = await asyncio.gather(gb_task, ol_task, apple_task)
snap = CatalogSnapshot(title=core_title, author=resolved_author or author)
snap.retailer_links = dict(goodreads_links or {})
if gb:
snap.gb_volume_id = gb.get("volume_id") or None
snap.gb_confidence = gb.get("confidence", 0.0)
if gb.get("isbn13"):
snap.isbn13 = gb["isbn13"]
if gb.get("isbn10"):
snap.isbn10 = gb["isbn10"]
if gb.get("author") and not snap.author:
snap.author = gb["author"]
if ol:
snap.ol_key = ol.get("work_key")
snap.ol_confidence = ol.get("confidence", 0.0)
if ol.get("isbn13") and not snap.isbn13:
snap.isbn13 = ol["isbn13"]
if ol.get("author") and not snap.author:
snap.author = ol["author"]
if is_confirmed_isbn13(norm_isbn):
snap.isbn13 = norm_isbn
elif catalog_isbn and not snap.isbn13:
snap.isbn13 = catalog_isbn
if norm_asin:
snap.asin = norm_asin
elif isbn:
extracted = extract_amazon_asin(isbn) or (
isbn.upper() if isbn.upper().startswith("B0") and len(isbn) == 10 else None
)
if extracted:
snap.asin = extracted
if apple:
snap.apple_url = apple.get("url")
snap.apple_confidence = apple.get("confidence", 0.0)
if goodreads_url:
snap.goodreads_url = goodreads_url
snap.goodreads_confidence = max(goodreads_confidence, 0.95)
if not skip_enrich and (
is_confirmed_isbn13(snap.isbn13) or (core_title and (snap.author or author))
):
snap.retailer_links = await enrich_retailer_links_from_isbn(
client,
snap.retailer_links,
isbn13=snap.isbn13,
title=core_title,
author=snap.author or author,
)
return snap
def discover_platform_hit(
platform_id: str,
snap: CatalogSnapshot,
*,
source_platform: str | None = None,
source_url: str | None = None,
buy_url: str | None = None,
) -> PlatformHit:
"""Return a hit only when the book is definitively listed (product page or API)."""
src = normalize_source_platform(source_platform)
if src == platform_id and (source_url or buy_url):
url = sanitize_platform_url(buy_url or source_url or "", platform_id)
if url and classify_retailer_url(url, platform_id) == "product":
return PlatformHit(1.0, url, isbn_match=True)
if platform_id == "google_books":
if snap.gb_volume_id and snap.gb_confidence >= VERIFIED_THRESHOLD:
return PlatformHit(
snap.gb_confidence,
snap.google_books_url(),
snap.gb_confidence >= 0.95,
)
if platform_id == "open_library" and snap.ol_confidence >= VERIFIED_THRESHOLD:
url = snap.open_library_url()
if url:
return PlatformHit(snap.ol_confidence, url)
if (
platform_id == "goodreads"
and snap.goodreads_url
and snap.goodreads_confidence >= VERIFIED_THRESHOLD
):
return PlatformHit(snap.goodreads_confidence, snap.goodreads_url)
if (
platform_id == "apple_books"
and snap.apple_url
and snap.apple_confidence >= VERIFIED_THRESHOLD
):
return PlatformHit(snap.apple_confidence, snap.apple_url)
gr_url = snap.retailer_links.get(platform_id)
if gr_url:
clean = sanitize_platform_url(gr_url, platform_id)
if clean and classify_retailer_url(clean, platform_id) == "product":
return PlatformHit(0.95, clean)
if platform_id == "amazon" and snap.asin:
amazon_url = snap.amazon_product_url()
if amazon_url:
return PlatformHit(0.98, amazon_url, isbn_match=True)
return PlatformHit(0.0, None)