Spaces:
Running
Running
| """Unit tests for book URL metadata extraction and idempotent re-fetch.""" | |
| from __future__ import annotations | |
| import json | |
| import pytest | |
| from app.services.book_url_scraper import ( | |
| BookURLMetadata, | |
| _amazon_html_looks_blocked, | |
| _amazon_product_urls, | |
| _extract_isbn_pair, | |
| _title_author_from_url_slug, | |
| _isbn10_to_isbn13, | |
| _isbn13_to_isbn10, | |
| _merge_isbn_fields, | |
| _normalize_import_url, | |
| _parse_amazon_html, | |
| _parse_amazon_breadcrumb_genre, | |
| _score, | |
| fetch_book_metadata, | |
| ) | |
| AMAZON_FIXTURE = """ | |
| <html><body> | |
| <span id="productTitle">Sample Novel</span> | |
| <div id="bylineInfo"><a class="contributorNameID">Jane Author</a></div> | |
| <div id="bookDescription_feature_div"><p>A gripping story about books.</p></div> | |
| <span>ISBN-13</span><span>978-0593128251</span> | |
| <span>ISBN-10</span><span>0593128257</span> | |
| <span>Publisher</span><span><span>Del Rey (February 4, 2020)</span></span> | |
| <span>Language</span><span>English</span> | |
| <span>Print length</span><span>496 pages</span> | |
| <span>Edition</span><span>First Edition</span> | |
| <span>Format</span><span>Paperback</span> | |
| <script type="application/ld+json">{ | |
| "@type": "Book", | |
| "name": "Sample Novel", | |
| "author": {"name": "Jane Author"}, | |
| "isbn": "9780593128251", | |
| "numberOfPages": 496, | |
| "inLanguage": "en" | |
| }</script> | |
| </body></html> | |
| """ | |
| class TestIsbnHelpers: | |
| def test_extract_isbn_pair_from_labeled_bullets(self): | |
| i10, i13 = _extract_isbn_pair("ISBN-13: 978-0593128251 and ISBN-10: 0593128257") | |
| assert i10 == "0593128257" | |
| assert i13 == "9780593128251" | |
| def test_isbn10_to_isbn13_roundtrip(self): | |
| i13 = _isbn10_to_isbn13("0593128257") | |
| assert i13 == "9780593128251" | |
| assert _isbn13_to_isbn10(i13) == "0593128257" | |
| def test_merge_isbn_fields_from_single_isbn10(self): | |
| result = {"isbn": "0593128257"} | |
| _merge_isbn_fields(result) | |
| assert result["isbn10"] == "0593128257" | |
| assert result["isbn13"] == "9780593128251" | |
| assert result["isbn"] == "9780593128251" | |
| class TestAmazonHtmlParsing: | |
| def test_parse_amazon_html_extracts_extended_fields(self): | |
| parsed = _parse_amazon_html(AMAZON_FIXTURE) | |
| assert parsed["title"] == "Sample Novel" | |
| assert parsed["author"] == "Jane Author" | |
| assert parsed["isbn13"] == "9780593128251" | |
| assert parsed["isbn10"] == "0593128257" | |
| assert parsed["publisher"].startswith("Del Rey") | |
| assert parsed["publish_year"] == "2020" | |
| assert parsed["language"] == "English" | |
| assert parsed["page_count"] == 496 | |
| assert parsed["edition"] == "First Edition" | |
| assert "paperback" in parsed["book_format"].lower() | |
| AMAZON_KINDLE_FIXTURE = """ | |
| <meta name="title" content="Amazon.com: Lethal Lines (Eden Mercer K-9 Mystery Thriller Book 3) eBook : Black, Paige: Kindle Store"/> | |
| <span id="productTitle">Lethal Lines (Eden Mercer K-9 Mystery Thriller Book 3)</span> | |
| <div id="wayfinding-breadcrumbs_feature_div"><ul> | |
| <li><a>Kindle Store</a></li><li><a>Kindle eBooks</a></li> | |
| <li><a>Mystery, Thriller & Suspense</a></li><li><a>Crime Fiction</a></li> | |
| <li><a>Kidnapping</a></li></ul></div> | |
| <div id="rpi-attribute-book_details-ebook_pages" class="rpi-attribute-content"> | |
| <div class="rpi-attribute-value"><span>376 pages</span></div> | |
| </div> | |
| Contains real page numbers based on the print edition (ISBN B0DLBDZBLF). | |
| <div id="rpi-attribute-language" class="rpi-attribute-content"> | |
| <div class="rpi-attribute-value"><span>English</span></div> | |
| </div> | |
| <div id="rpi-attribute-book_details-publication_date" class="rpi-attribute-content"> | |
| <div class="rpi-attribute-value"><span>November 1, 2024</span></div> | |
| </div> | |
| <div id="rpi-attribute-book_details-series" class="rpi-attribute-content"> | |
| <span class="rpi-attribute-label"><span>Book 3 of 10</span></span> | |
| <div class="rpi-attribute-value"><span>Eden Mercer K-9 Mystery Thriller</span></div> | |
| </div> | |
| <input type="hidden" id="ASIN" value="B0DHWS3P6T"> | |
| """ | |
| class TestAmazonKindleFixture: | |
| def test_kindle_fixture_genre_isbn_asin(self): | |
| parsed = _parse_amazon_html(AMAZON_KINDLE_FIXTURE) | |
| assert parsed["title"] == "Lethal Lines" | |
| assert parsed["author"] == "Paige Black" | |
| assert "Kidnapping" in parsed.get("genre", "") or "Crime Fiction" in parsed.get("genre", "") | |
| assert parsed.get("page_count") == 376 | |
| assert parsed.get("language") == "English" | |
| assert parsed.get("publish_year") == "2024" | |
| assert "Eden Mercer" in parsed.get("series", "") | |
| def test_kindle_meta_scores_100_with_asin_identifier(self): | |
| meta = BookURLMetadata( | |
| title="Lethal Lines", | |
| author="Paige Black", | |
| description="x" * 50, | |
| cover_url="https://example.com/c.jpg", | |
| isbn="B0DHWS3P6T", | |
| asin="B0DHWS3P6T", | |
| genre="Kidnapping, Crime Fiction", | |
| about_author="Bio " * 10, | |
| page_count=376, | |
| language="English", | |
| book_format="Kindle Edition", | |
| series="Book 3 of 10: Eden Mercer K-9 Mystery Thriller", | |
| ) | |
| assert _score(meta) == 1.0 | |
| def test_amazon_html_looks_blocked_on_captcha_page(self): | |
| html = '<html><title>Amazon.com</title>opfcaptcha.amazon.com Continue shopping</html>' | |
| assert _amazon_html_looks_blocked(html) is True | |
| def test_amazon_product_urls_prefers_gp_product(self): | |
| urls = _amazon_product_urls( | |
| "https://www.amazon.com/foo/dp/B0DHWS3P6T?tag=x", | |
| "B0DHWS3P6T", | |
| ) | |
| assert urls[0] == "https://www.amazon.com/gp/product/B0DHWS3P6T" | |
| assert any("gp/aw/d" in u for u in urls) | |
| def test_amazon_slug_from_kindle_url(self): | |
| title, author = _title_author_from_url_slug( | |
| "https://www.amazon.com/Lethal-Lines-Mercer-Mystery-Thriller-ebook/dp/B0DHWS3P6T", | |
| ) | |
| assert "Lethal Lines" in title | |
| assert author == "" | |
| def test_normalize_import_url_strips_tracking_and_canonicalizes_amazon(self): | |
| raw = "https://www.amazon.com/dp/0593128257/ref=sr_1_1?tag=foo-20&dib_tag=se&keywords=book" | |
| norm = _normalize_import_url(raw) | |
| assert norm.startswith("https://amazon.com/dp/0593128257") | |
| assert "tag=" not in norm | |
| assert _normalize_import_url(raw) == _normalize_import_url(norm + "?tag=other-20") | |
| class TestScoring: | |
| def test_score_counts_isbn13_and_page_count(self): | |
| meta = BookURLMetadata( | |
| title="A Book", | |
| author="An Author", | |
| description="x" * 50, | |
| cover_url="https://example.com/cover.jpg", | |
| isbn13="9780593128251", | |
| isbn10="0593128257", | |
| page_count=300, | |
| language="English", | |
| book_format="Paperback", | |
| publisher="Del Rey", | |
| price="$12.99", | |
| genre="Fiction", | |
| about_author="Bio " * 10, | |
| ) | |
| assert _score(meta) == 1.0 | |
| def test_kindle_listing_scores_100_without_isbn_or_publisher(self): | |
| meta = BookURLMetadata( | |
| title="Lethal Lines", | |
| author="Paige Black", | |
| description="x" * 50, | |
| cover_url="https://example.com/cover.jpg", | |
| asin="B0DHWS3P6T", | |
| genre="Thriller", | |
| about_author="Bio " * 10, | |
| page_count=376, | |
| language="English", | |
| book_format="Kindle Edition", | |
| series="Book 3 of 10: Eden Mercer K-9 Mystery Thriller", | |
| ) | |
| assert _score(meta) == 1.0 | |
| def test_paperback_missing_publisher_scores_below_100(self): | |
| meta = BookURLMetadata( | |
| title="A Book", | |
| author="An Author", | |
| description="x" * 50, | |
| cover_url="https://example.com/cover.jpg", | |
| isbn13="9780593128251", | |
| page_count=300, | |
| language="English", | |
| book_format="Paperback", | |
| price="$12.99", | |
| ) | |
| assert _score(meta) < 1.0 | |
| class FakeRedis: | |
| def __init__(self) -> None: | |
| self.store: dict[str, str] = {} | |
| async def get(self, key: str): | |
| return self.store.get(key) | |
| async def setex(self, key: str, ttl: int, value: str): | |
| self.store[key] = value | |
| async def test_repeat_fetch_uses_cache_fallback_when_live_fetch_degrades(monkeypatch): | |
| good = BookURLMetadata( | |
| title="Cached Title", | |
| author="Cached Author", | |
| description="A" * 60, | |
| cover_url="https://example.com/cover.jpg", | |
| isbn13="9780593128251", | |
| isbn10="0593128257", | |
| platform="amazon", | |
| source_url="https://amazon.com/dp/0593128257", | |
| buy_url="https://amazon.com/dp/0593128257", | |
| confidence=0.95, | |
| ) | |
| redis = FakeRedis() | |
| key = "book_v3:" + "x" # overwritten below after we know real key | |
| from hashlib import sha256 | |
| url = "https://amazon.com/dp/0593128257" | |
| cache_key = "book_v3:" + sha256(_normalize_import_url(url).encode()).hexdigest()[:20] | |
| redis.store[cache_key] = json.dumps({ | |
| "title": good.title, | |
| "author": good.author, | |
| "about_author": "", | |
| "description": good.description, | |
| "genre": "", | |
| "isbn": good.isbn13, | |
| "isbn10": good.isbn10, | |
| "isbn13": good.isbn13, | |
| "asin": "", | |
| "publisher": "", | |
| "publish_year": "", | |
| "edition": "", | |
| "book_format": "", | |
| "series": "", | |
| "language": "", | |
| "page_count": 0, | |
| "price": "", | |
| "rating": "", | |
| "rating_count": "", | |
| "cover_url": good.cover_url, | |
| "platform": "amazon", | |
| "source_url": url, | |
| "buy_url": url, | |
| "confidence": good.confidence, | |
| "error": "", | |
| "warnings": [], | |
| }) | |
| async def _empty_resolve(*_args, **_kwargs): | |
| return {"warnings": ["blocked"]} | |
| monkeypatch.setattr("app.services.book_url_scraper._resolve", _empty_resolve) | |
| meta = await fetch_book_metadata(url, redis=redis, force_refresh=True) | |
| assert meta.title == "Cached Title" | |
| assert meta.confidence == good.confidence | |
| assert any("previous successful result" in w.lower() for w in meta.warnings) | |