"""Unit tests for book URL metadata extraction and idempotent re-fetch.""" from __future__ import annotations import json import pytest from app.services.book_url_scraper import ( BookURLMetadata, _amazon_html_looks_blocked, _amazon_product_urls, _extract_isbn_pair, _title_author_from_url_slug, _isbn10_to_isbn13, _isbn13_to_isbn10, _merge_isbn_fields, _normalize_import_url, _parse_amazon_html, _parse_amazon_breadcrumb_genre, _score, fetch_book_metadata, ) AMAZON_FIXTURE = """ Sample Novel
Jane Author

A gripping story about books.

ISBN-13978-0593128251 ISBN-100593128257 PublisherDel Rey (February 4, 2020) LanguageEnglish Print length496 pages EditionFirst Edition FormatPaperback """ class TestIsbnHelpers: def test_extract_isbn_pair_from_labeled_bullets(self): i10, i13 = _extract_isbn_pair("ISBN-13: 978-0593128251 and ISBN-10: 0593128257") assert i10 == "0593128257" assert i13 == "9780593128251" def test_isbn10_to_isbn13_roundtrip(self): i13 = _isbn10_to_isbn13("0593128257") assert i13 == "9780593128251" assert _isbn13_to_isbn10(i13) == "0593128257" def test_merge_isbn_fields_from_single_isbn10(self): result = {"isbn": "0593128257"} _merge_isbn_fields(result) assert result["isbn10"] == "0593128257" assert result["isbn13"] == "9780593128251" assert result["isbn"] == "9780593128251" class TestAmazonHtmlParsing: def test_parse_amazon_html_extracts_extended_fields(self): parsed = _parse_amazon_html(AMAZON_FIXTURE) assert parsed["title"] == "Sample Novel" assert parsed["author"] == "Jane Author" assert parsed["isbn13"] == "9780593128251" assert parsed["isbn10"] == "0593128257" assert parsed["publisher"].startswith("Del Rey") assert parsed["publish_year"] == "2020" assert parsed["language"] == "English" assert parsed["page_count"] == 496 assert parsed["edition"] == "First Edition" assert "paperback" in parsed["book_format"].lower() AMAZON_KINDLE_FIXTURE = """ Lethal Lines (Eden Mercer K-9 Mystery Thriller Book 3)
376 pages
Contains real page numbers based on the print edition (ISBN B0DLBDZBLF).
English
November 1, 2024
Book 3 of 10
Eden Mercer K-9 Mystery Thriller
""" class TestAmazonKindleFixture: def test_kindle_fixture_genre_isbn_asin(self): parsed = _parse_amazon_html(AMAZON_KINDLE_FIXTURE) assert parsed["title"] == "Lethal Lines" assert parsed["author"] == "Paige Black" assert "Kidnapping" in parsed.get("genre", "") or "Crime Fiction" in parsed.get("genre", "") assert parsed.get("page_count") == 376 assert parsed.get("language") == "English" assert parsed.get("publish_year") == "2024" assert "Eden Mercer" in parsed.get("series", "") def test_kindle_meta_scores_100_with_asin_identifier(self): meta = BookURLMetadata( title="Lethal Lines", author="Paige Black", description="x" * 50, cover_url="https://example.com/c.jpg", isbn="B0DHWS3P6T", asin="B0DHWS3P6T", genre="Kidnapping, Crime Fiction", about_author="Bio " * 10, page_count=376, language="English", book_format="Kindle Edition", series="Book 3 of 10: Eden Mercer K-9 Mystery Thriller", ) assert _score(meta) == 1.0 def test_amazon_html_looks_blocked_on_captcha_page(self): html = 'Amazon.comopfcaptcha.amazon.com Continue shopping' assert _amazon_html_looks_blocked(html) is True def test_amazon_product_urls_prefers_gp_product(self): urls = _amazon_product_urls( "https://www.amazon.com/foo/dp/B0DHWS3P6T?tag=x", "B0DHWS3P6T", ) assert urls[0] == "https://www.amazon.com/gp/product/B0DHWS3P6T" assert any("gp/aw/d" in u for u in urls) def test_amazon_slug_from_kindle_url(self): title, author = _title_author_from_url_slug( "https://www.amazon.com/Lethal-Lines-Mercer-Mystery-Thriller-ebook/dp/B0DHWS3P6T", ) assert "Lethal Lines" in title assert author == "" def test_normalize_import_url_strips_tracking_and_canonicalizes_amazon(self): raw = "https://www.amazon.com/dp/0593128257/ref=sr_1_1?tag=foo-20&dib_tag=se&keywords=book" norm = _normalize_import_url(raw) assert norm.startswith("https://amazon.com/dp/0593128257") assert "tag=" not in norm assert _normalize_import_url(raw) == _normalize_import_url(norm + "?tag=other-20") class TestScoring: def test_score_counts_isbn13_and_page_count(self): meta = BookURLMetadata( title="A Book", author="An Author", description="x" * 50, cover_url="https://example.com/cover.jpg", isbn13="9780593128251", isbn10="0593128257", page_count=300, language="English", book_format="Paperback", publisher="Del Rey", price="$12.99", genre="Fiction", about_author="Bio " * 10, ) assert _score(meta) == 1.0 def test_kindle_listing_scores_100_without_isbn_or_publisher(self): meta = BookURLMetadata( title="Lethal Lines", author="Paige Black", description="x" * 50, cover_url="https://example.com/cover.jpg", asin="B0DHWS3P6T", genre="Thriller", about_author="Bio " * 10, page_count=376, language="English", book_format="Kindle Edition", series="Book 3 of 10: Eden Mercer K-9 Mystery Thriller", ) assert _score(meta) == 1.0 def test_paperback_missing_publisher_scores_below_100(self): meta = BookURLMetadata( title="A Book", author="An Author", description="x" * 50, cover_url="https://example.com/cover.jpg", isbn13="9780593128251", page_count=300, language="English", book_format="Paperback", price="$12.99", ) assert _score(meta) < 1.0 class FakeRedis: def __init__(self) -> None: self.store: dict[str, str] = {} async def get(self, key: str): return self.store.get(key) async def setex(self, key: str, ttl: int, value: str): self.store[key] = value @pytest.mark.asyncio async def test_repeat_fetch_uses_cache_fallback_when_live_fetch_degrades(monkeypatch): good = BookURLMetadata( title="Cached Title", author="Cached Author", description="A" * 60, cover_url="https://example.com/cover.jpg", isbn13="9780593128251", isbn10="0593128257", platform="amazon", source_url="https://amazon.com/dp/0593128257", buy_url="https://amazon.com/dp/0593128257", confidence=0.95, ) redis = FakeRedis() key = "book_v3:" + "x" # overwritten below after we know real key from hashlib import sha256 url = "https://amazon.com/dp/0593128257" cache_key = "book_v3:" + sha256(_normalize_import_url(url).encode()).hexdigest()[:20] redis.store[cache_key] = json.dumps({ "title": good.title, "author": good.author, "about_author": "", "description": good.description, "genre": "", "isbn": good.isbn13, "isbn10": good.isbn10, "isbn13": good.isbn13, "asin": "", "publisher": "", "publish_year": "", "edition": "", "book_format": "", "series": "", "language": "", "page_count": 0, "price": "", "rating": "", "rating_count": "", "cover_url": good.cover_url, "platform": "amazon", "source_url": url, "buy_url": url, "confidence": good.confidence, "error": "", "warnings": [], }) async def _empty_resolve(*_args, **_kwargs): return {"warnings": ["blocked"]} monkeypatch.setattr("app.services.book_url_scraper._resolve", _empty_resolve) meta = await fetch_book_metadata(url, redis=redis, force_refresh=True) assert meta.title == "Cached Title" assert meta.confidence == good.confidence assert any("previous successful result" in w.lower() for w in meta.warnings)