OpenCode commited on
Commit
bbb9a5b
·
1 Parent(s): 477ef5f

feat: add PDF OCR reader support

Browse files
app.py CHANGED
@@ -35,7 +35,8 @@ from search_indexes import (
35
  resolve_postings,
36
  )
37
  from reader_assets import (
38
- READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE, decode_reader_assets,
 
39
  )
40
  from data_formats import (
41
  decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
@@ -1461,12 +1462,14 @@ READER_ISOLATION_HEADERS = {
1461
  }
1462
  READER_DOCUMENT_MAX_BYTES = 512 * 1024 * 1024
1463
  READER_RESOURCE_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
1464
- READER_RESOURCE_MAX_BYTES = 16 * 1024 * 1024
1465
  READER_BUCKET_CACHE_TTL_SECONDS = 3600
1466
  READER_BUCKET_CACHE_MAX_BYTES = 32 * 1024 * 1024
1467
  READER_BUCKET_CACHE_MAX_ITEM_BYTES = 8 * 1024 * 1024
1468
  READER_BUCKET_MAX_CONCURRENT_READS = 8
1469
  READER_BUCKET_MAX_INFLIGHT = 32
 
 
1470
  READER_ARCHIVE_MAX_ENTRIES = 20_000
1471
  READER_ARCHIVE_MAX_EXPANDED_BYTES = 1536 * 1024 * 1024
1472
  READER_ARCHIVE_MAX_COMPRESSION_RATIO = 200
@@ -1608,15 +1611,17 @@ def validate_reader_archive(archive: bytes) -> None:
1608
  READER_RESOURCE_TYPES = {
1609
  "json": "application/json",
1610
  "css": "text/css", "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png",
1611
- "gif": "image/gif", "webp": "image/webp", "svg": "image/svg+xml", "ttf": "font/ttf",
1612
  "otf": "font/otf", "woff": "font/woff", "woff2": "font/woff2", "mp3": "audio/mpeg",
1613
- "mp4": "video/mp4",
1614
  }
1615
  def initialize_reader_bucket_state(target_app: FastAPI) -> None:
1616
  target_app.state.reader_bucket_cache = OrderedDict()
1617
  target_app.state.reader_bucket_cache_bytes = 0
1618
  target_app.state.reader_bucket_inflight = {}
1619
  target_app.state.reader_bucket_semaphore = asyncio.Semaphore(READER_BUCKET_MAX_CONCURRENT_READS)
 
 
1620
 
1621
 
1622
  async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
@@ -1628,14 +1633,24 @@ async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
1628
  await asyncio.gather(*tasks, return_exceptions=True)
1629
  target_app.state.reader_bucket_cache.clear()
1630
  target_app.state.reader_bucket_cache_bytes = 0
 
 
1631
 
1632
 
1633
- def is_reader_bucket_cacheable(path: str) -> bool:
1634
- return path.endswith("/page-manifest.json") or path.endswith("/pages/page-000001.webp")
 
 
1635
 
1636
 
1637
- def read_reader_bucket_resource(path: str) -> bytes:
1638
- uri = f"hf://buckets/vomebook/pdf-pages/{path}"
 
 
 
 
 
 
1639
  with HfFileSystem().open(uri, "rb") as stream:
1640
  payload = stream.read(READER_RESOURCE_MAX_BYTES + 1)
1641
  if len(payload) > READER_RESOURCE_MAX_BYTES:
@@ -1643,54 +1658,116 @@ def read_reader_bucket_resource(path: str) -> bytes:
1643
  return payload
1644
 
1645
 
1646
- async def get_reader_bucket_resource(path: str) -> bytes:
1647
  state = app.state
1648
- if is_reader_bucket_cacheable(path):
1649
- cached = state.reader_bucket_cache.get(path)
 
1650
  if cached:
1651
  cached_at, payload = cached
1652
  if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
1653
- state.reader_bucket_cache.move_to_end(path)
1654
  return payload
1655
- state.reader_bucket_cache.pop(path)
1656
  state.reader_bucket_cache_bytes -= len(payload)
1657
 
1658
- task = state.reader_bucket_inflight.get(path)
1659
  if task is None:
1660
  if len(state.reader_bucket_inflight) >= READER_BUCKET_MAX_INFLIGHT:
1661
  raise RuntimeError("too many reader bucket requests")
1662
  async def load() -> bytes:
1663
  try:
1664
  async with state.reader_bucket_semaphore:
1665
- payload = await asyncio.to_thread(read_reader_bucket_resource, path)
1666
- if is_reader_bucket_cacheable(path) and len(payload) <= READER_BUCKET_CACHE_MAX_ITEM_BYTES:
1667
- previous = state.reader_bucket_cache.pop(path, None)
 
 
 
1668
  if previous:
1669
  state.reader_bucket_cache_bytes -= len(previous[1])
1670
- state.reader_bucket_cache[path] = (time.monotonic(), payload)
1671
  state.reader_bucket_cache_bytes += len(payload)
1672
  while state.reader_bucket_cache_bytes > READER_BUCKET_CACHE_MAX_BYTES:
1673
  _, (_, evicted) = state.reader_bucket_cache.popitem(last=False)
1674
  state.reader_bucket_cache_bytes -= len(evicted)
1675
  return payload
1676
  finally:
1677
- if state.reader_bucket_inflight.get(path) is asyncio.current_task():
1678
- state.reader_bucket_inflight.pop(path, None)
1679
 
1680
  task = asyncio.create_task(load())
1681
  task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
1682
- state.reader_bucket_inflight[path] = task
1683
  return await asyncio.shield(task)
1684
 
1685
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1686
  @app.api_route("/api/reader-bucket-resource", methods=["GET", "HEAD"])
1687
- async def api_reader_bucket_resource(path: str = Query(...), request: Request = None):
1688
  resource_path = unquote(unquote(path)).replace("\\", "/").lstrip("/")
1689
- if not READER_BUCKET_PATH_RE.fullmatch(resource_path):
 
 
1690
  return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400,
1691
  headers={"Cache-Control": "no-store"})
1692
  try:
1693
- payload = await get_reader_bucket_resource(resource_path)
1694
  except FileNotFoundError:
1695
  return JSONResponse({"error": "阅读资源不存在"}, status_code=404,
1696
  headers={"Cache-Control": "no-store"})
@@ -1867,6 +1944,61 @@ async def api_reader_resolve(id: str = Query(..., min_length=13, max_length=13))
1867
  return JSONResponse({"error": "阅读文件不存在"}, status_code=404)
1868
 
1869
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1870
  @app.api_route("/api/reader-content", methods=["GET", "HEAD"])
1871
  async def api_reader_content(request: Request, url: Optional[str] = Query(default=None), id: Optional[str] = Query(default=None)):
1872
  if id and not url:
@@ -1885,9 +2017,11 @@ async def api_reader_content(request: Request, url: Optional[str] = Query(defaul
1885
  return JSONResponse({"error": str(exc)}, status_code=403)
1886
  if not parsed.scheme and not parsed.netloc and parsed.path == "/api/reader-bucket-resource" and not parsed.fragment:
1887
  parameters = parse_qs(parsed.query)
1888
- if set(parameters) != {"path"} or len(parameters["path"]) != 1:
 
 
1889
  return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
1890
- return await api_reader_bucket_resource(parameters["path"][0], request)
1891
  try:
1892
  target_url = validate_reader_source_url(url)
1893
  except ValueError as exc:
 
35
  resolve_postings,
36
  )
37
  from reader_assets import (
38
+ READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE, READER_PDF_RANGE_BUCKET,
39
+ READER_PDF_RANGE_PATH_RE, decode_reader_assets,
40
  )
41
  from data_formats import (
42
  decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
 
1462
  }
1463
  READER_DOCUMENT_MAX_BYTES = 512 * 1024 * 1024
1464
  READER_RESOURCE_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
1465
+ READER_RESOURCE_MAX_BYTES = 64 * 1024 * 1024
1466
  READER_BUCKET_CACHE_TTL_SECONDS = 3600
1467
  READER_BUCKET_CACHE_MAX_BYTES = 32 * 1024 * 1024
1468
  READER_BUCKET_CACHE_MAX_ITEM_BYTES = 8 * 1024 * 1024
1469
  READER_BUCKET_MAX_CONCURRENT_READS = 8
1470
  READER_BUCKET_MAX_INFLIGHT = 32
1471
+ READER_OCR_CACHE_MAX_BYTES = 32 * 1024 * 1024
1472
+ READER_OCR_CACHE_MAX_ITEMS = 4
1473
  READER_ARCHIVE_MAX_ENTRIES = 20_000
1474
  READER_ARCHIVE_MAX_EXPANDED_BYTES = 1536 * 1024 * 1024
1475
  READER_ARCHIVE_MAX_COMPRESSION_RATIO = 200
 
1611
  READER_RESOURCE_TYPES = {
1612
  "json": "application/json",
1613
  "css": "text/css", "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png",
1614
+ "gif": "image/gif", "webp": "image/webp", "jxl": "image/jxl", "svg": "image/svg+xml", "ttf": "font/ttf",
1615
  "otf": "font/otf", "woff": "font/woff", "woff2": "font/woff2", "mp3": "audio/mpeg",
1616
+ "mp4": "video/mp4", "gz": "application/gzip",
1617
  }
1618
  def initialize_reader_bucket_state(target_app: FastAPI) -> None:
1619
  target_app.state.reader_bucket_cache = OrderedDict()
1620
  target_app.state.reader_bucket_cache_bytes = 0
1621
  target_app.state.reader_bucket_inflight = {}
1622
  target_app.state.reader_bucket_semaphore = asyncio.Semaphore(READER_BUCKET_MAX_CONCURRENT_READS)
1623
+ target_app.state.reader_ocr_cache = OrderedDict()
1624
+ target_app.state.reader_ocr_cache_bytes = 0
1625
 
1626
 
1627
  async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
 
1633
  await asyncio.gather(*tasks, return_exceptions=True)
1634
  target_app.state.reader_bucket_cache.clear()
1635
  target_app.state.reader_bucket_cache_bytes = 0
1636
+ target_app.state.reader_ocr_cache.clear()
1637
+ target_app.state.reader_ocr_cache_bytes = 0
1638
 
1639
 
1640
+ def is_reader_bucket_cacheable(bucket: str, path: str) -> bool:
1641
+ return (bucket == "vomebook/pdf-pages" and
1642
+ (path.endswith("/page-manifest.json") or path.endswith("/ocr-manifest.json")
1643
+ or path.endswith("/ocr/book-text.json.gz") or path.endswith("/pages/page-000001.webp")))
1644
 
1645
 
1646
+ def reader_bucket_cache_key(bucket: str, path: str):
1647
+ # Keep the default bucket namespace compatible with the existing cache;
1648
+ # isolate the optional optimized-PDF bucket by namespace.
1649
+ return path if bucket == "vomebook/pdf-pages" else (bucket, path)
1650
+
1651
+
1652
+ def read_reader_bucket_resource(path: str, bucket: str = "vomebook/pdf-pages") -> bytes:
1653
+ uri = f"hf://buckets/{bucket}/{path}"
1654
  with HfFileSystem().open(uri, "rb") as stream:
1655
  payload = stream.read(READER_RESOURCE_MAX_BYTES + 1)
1656
  if len(payload) > READER_RESOURCE_MAX_BYTES:
 
1658
  return payload
1659
 
1660
 
1661
+ async def get_reader_bucket_resource(path: str, bucket: str = "vomebook/pdf-pages") -> bytes:
1662
  state = app.state
1663
+ cache_key = path if bucket == "vomebook/pdf-pages" else (bucket, path)
1664
+ if is_reader_bucket_cacheable(bucket, path):
1665
+ cached = state.reader_bucket_cache.get(cache_key)
1666
  if cached:
1667
  cached_at, payload = cached
1668
  if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
1669
+ state.reader_bucket_cache.move_to_end(cache_key)
1670
  return payload
1671
+ state.reader_bucket_cache.pop(cache_key)
1672
  state.reader_bucket_cache_bytes -= len(payload)
1673
 
1674
+ task = state.reader_bucket_inflight.get(cache_key)
1675
  if task is None:
1676
  if len(state.reader_bucket_inflight) >= READER_BUCKET_MAX_INFLIGHT:
1677
  raise RuntimeError("too many reader bucket requests")
1678
  async def load() -> bytes:
1679
  try:
1680
  async with state.reader_bucket_semaphore:
1681
+ loader = (lambda: read_reader_bucket_resource(path)
1682
+ if bucket == "vomebook/pdf-pages"
1683
+ else read_reader_bucket_resource(path, bucket))
1684
+ payload = await asyncio.to_thread(loader)
1685
+ if is_reader_bucket_cacheable(bucket, path) and len(payload) <= READER_BUCKET_CACHE_MAX_ITEM_BYTES:
1686
+ previous = state.reader_bucket_cache.pop(cache_key, None)
1687
  if previous:
1688
  state.reader_bucket_cache_bytes -= len(previous[1])
1689
+ state.reader_bucket_cache[cache_key] = (time.monotonic(), payload)
1690
  state.reader_bucket_cache_bytes += len(payload)
1691
  while state.reader_bucket_cache_bytes > READER_BUCKET_CACHE_MAX_BYTES:
1692
  _, (_, evicted) = state.reader_bucket_cache.popitem(last=False)
1693
  state.reader_bucket_cache_bytes -= len(evicted)
1694
  return payload
1695
  finally:
1696
+ if state.reader_bucket_inflight.get(cache_key) is asyncio.current_task():
1697
+ state.reader_bucket_inflight.pop(cache_key, None)
1698
 
1699
  task = asyncio.create_task(load())
1700
  task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
1701
+ state.reader_bucket_inflight[cache_key] = task
1702
  return await asyncio.shield(task)
1703
 
1704
 
1705
+ def parse_reader_ocr_book(raw: bytes) -> tuple[list[tuple[int, str, str]], int]:
1706
+ """Decode and validate the compact per-page OCR text cache off the event loop."""
1707
+ with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream:
1708
+ decoded = stream.read(READER_RESOURCE_MAX_BYTES + 1)
1709
+ if len(decoded) > READER_RESOURCE_MAX_BYTES:
1710
+ raise ValueError("PDF OCR text index is too large")
1711
+ payload = json.loads(decoded)
1712
+ pages = payload.get("pages") if isinstance(payload, dict) else None
1713
+ if (not isinstance(pages, list) or len(pages) > 200000
1714
+ or payload.get("version") != 1 or payload.get("kind") != "pdf-ocr-book-text"):
1715
+ raise ValueError("invalid PDF OCR text index")
1716
+ output = []
1717
+ total_bytes = 0
1718
+ previous = 0
1719
+ for item in pages:
1720
+ if (not isinstance(item, dict) or not isinstance(item.get("page"), int)
1721
+ or item["page"] < 1 or item["page"] <= previous
1722
+ or not isinstance(item.get("text"), str)):
1723
+ raise ValueError("invalid PDF OCR text page")
1724
+ text = item["text"]
1725
+ total_bytes += len(text.encode("utf-8"))
1726
+ if total_bytes > READER_RESOURCE_MAX_BYTES:
1727
+ raise ValueError("PDF OCR text index is too large")
1728
+ output.append((item["page"], text, text.casefold()))
1729
+ previous = item["page"]
1730
+ return output, total_bytes
1731
+
1732
+
1733
+ async def get_reader_ocr_book(path: str) -> list[tuple[int, str, str]]:
1734
+ state = app.state
1735
+ if not hasattr(state, "reader_ocr_cache"):
1736
+ state.reader_ocr_cache = OrderedDict()
1737
+ state.reader_ocr_cache_bytes = 0
1738
+ cached = state.reader_ocr_cache.get(path)
1739
+ if cached:
1740
+ cached_at, pages, size = cached
1741
+ if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
1742
+ state.reader_ocr_cache.move_to_end(path)
1743
+ return pages
1744
+ state.reader_ocr_cache.pop(path)
1745
+ state.reader_ocr_cache_bytes -= size
1746
+ raw = await get_reader_bucket_resource(path)
1747
+ pages, size = await asyncio.to_thread(parse_reader_ocr_book, raw)
1748
+ if size <= READER_OCR_CACHE_MAX_BYTES:
1749
+ previous = state.reader_ocr_cache.pop(path, None)
1750
+ if previous:
1751
+ state.reader_ocr_cache_bytes -= previous[2]
1752
+ state.reader_ocr_cache[path] = (time.monotonic(), pages, size)
1753
+ state.reader_ocr_cache_bytes += size
1754
+ while (len(state.reader_ocr_cache) > READER_OCR_CACHE_MAX_ITEMS
1755
+ or state.reader_ocr_cache_bytes > READER_OCR_CACHE_MAX_BYTES):
1756
+ _, (_, _, evicted_size) = state.reader_ocr_cache.popitem(last=False)
1757
+ state.reader_ocr_cache_bytes -= evicted_size
1758
+ return pages
1759
+
1760
+
1761
  @app.api_route("/api/reader-bucket-resource", methods=["GET", "HEAD"])
1762
+ async def api_reader_bucket_resource(path: str = Query(...), bucket: str = "vomebook/pdf-pages", request: Request = None):
1763
  resource_path = unquote(unquote(path)).replace("\\", "/").lstrip("/")
1764
+ valid = ((bucket == "vomebook/pdf-pages" and READER_BUCKET_PATH_RE.fullmatch(resource_path))
1765
+ or (bucket == READER_PDF_RANGE_BUCKET and READER_PDF_RANGE_PATH_RE.fullmatch(resource_path)))
1766
+ if not valid:
1767
  return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400,
1768
  headers={"Cache-Control": "no-store"})
1769
  try:
1770
+ payload = await get_reader_bucket_resource(resource_path, bucket)
1771
  except FileNotFoundError:
1772
  return JSONResponse({"error": "阅读资源不存在"}, status_code=404,
1773
  headers={"Cache-Control": "no-store"})
 
1944
  return JSONResponse({"error": "阅读文件不存在"}, status_code=404)
1945
 
1946
 
1947
+ @app.get("/api/reader-ocr-search")
1948
+ async def api_reader_ocr_search(
1949
+ id: str = Query(..., min_length=13, max_length=13),
1950
+ q: str = Query(..., min_length=1, max_length=256),
1951
+ page: int = Query(1, ge=1, le=1000000),
1952
+ page_size: int = Query(50, ge=1, le=100),
1953
+ ):
1954
+ """Search one PDF's immutable OCR text without loading it into app memory."""
1955
+ resolved = await api_reader_resolve(id)
1956
+ if resolved.status_code != 200:
1957
+ return resolved
1958
+ try:
1959
+ data = json.loads(resolved.body)
1960
+ manifest_url = str(data.get("ocr_manifest") or "")
1961
+ parsed = urlparse(manifest_url)
1962
+ parameters = parse_qs(parsed.query)
1963
+ manifest_path = parameters.get("path", [""])[0]
1964
+ if (parsed.path != "/api/reader-bucket-resource" or set(parameters) != {"path"}
1965
+ or not READER_BUCKET_PATH_RE.fullmatch(manifest_path)
1966
+ or not manifest_path.endswith("/ocr-manifest.json")):
1967
+ return JSONResponse({"error": "本书暂无 OCR 索引"}, status_code=404)
1968
+ manifest = json.loads(await get_reader_bucket_resource(manifest_path))
1969
+ book = manifest.get("book_text") if isinstance(manifest, dict) else None
1970
+ book_path = book.get("path") if isinstance(book, dict) else ""
1971
+ if (not isinstance(book_path, str) or not READER_BUCKET_PATH_RE.fullmatch(book_path)
1972
+ or not book_path.endswith("/book-text.json.gz")):
1973
+ return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
1974
+ pages = await get_reader_ocr_book(book_path)
1975
+ needle = q.casefold()
1976
+ matches = []
1977
+ for page_number, text, folded in pages:
1978
+ start = 0
1979
+ while True:
1980
+ found = folded.find(needle, start)
1981
+ if found < 0:
1982
+ break
1983
+ end = found + len(q)
1984
+ snippet_start = max(0, found - 72)
1985
+ snippet_end = min(len(text), end + 88)
1986
+ matches.append({
1987
+ "page": page_number, "start": found, "length": len(q),
1988
+ "snippet": text[snippet_start:snippet_end],
1989
+ "snippet_start": found - snippet_start,
1990
+ })
1991
+ start = end
1992
+ total = len(matches)
1993
+ offset = (page - 1) * page_size
1994
+ return JSONResponse({
1995
+ "version": 1, "page": page, "page_size": page_size, "total": total,
1996
+ "results": matches[offset:offset + page_size],
1997
+ }, headers={"Cache-Control": "no-store"})
1998
+ except (ValueError, TypeError, KeyError, EOFError, gzip.BadGzipFile, zlib.error):
1999
+ return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
2000
+
2001
+
2002
  @app.api_route("/api/reader-content", methods=["GET", "HEAD"])
2003
  async def api_reader_content(request: Request, url: Optional[str] = Query(default=None), id: Optional[str] = Query(default=None)):
2004
  if id and not url:
 
2017
  return JSONResponse({"error": str(exc)}, status_code=403)
2018
  if not parsed.scheme and not parsed.netloc and parsed.path == "/api/reader-bucket-resource" and not parsed.fragment:
2019
  parameters = parse_qs(parsed.query)
2020
+ if (set(parameters) - {"path", "bucket"} or set(parameters) < {"path"}
2021
+ or len(parameters["path"]) != 1
2022
+ or len(parameters.get("bucket", ["x"])) != 1):
2023
  return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
2024
+ return await api_reader_bucket_resource(parameters["path"][0], parameters.get("bucket", ["vomebook/pdf-pages"])[0], request)
2025
  try:
2026
  target_url = validate_reader_source_url(url)
2027
  except ValueError as exc:
reader_assets.py CHANGED
@@ -18,7 +18,11 @@ READER_ASSET_FILENAMES = {
18
  "v": {"video.mp4"},
19
  }
20
  READER_BUCKET_PATH_RE = re.compile(
21
- r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[0-9a-f]{16}/(?:page-manifest\.json|pages/page-[0-9]{6}\.webp)$"
 
 
 
 
22
  )
23
 
24
 
@@ -39,7 +43,7 @@ def decode_reader_assets(
39
  raise ValueError("invalid reader assets sidecar")
40
  clean = {}
41
  for key, entry in files.items():
42
- if not isinstance(key, str) or not isinstance(entry, dict) or entry.get("s") not in (2, 4):
43
  continue
44
  if entry.get("s") == 2:
45
  path = entry.get("p")
@@ -54,6 +58,7 @@ def decode_reader_assets(
54
  continue
55
  chapter_manifest = entry.get("c")
56
  fallback = entry.get("f")
 
57
  if chapter_manifest is not None and (
58
  mode not in {"e", "p"}
59
  or not isinstance(chapter_manifest, str)
@@ -67,14 +72,25 @@ def decode_reader_assets(
67
  or not fallback.endswith("/document.pdf")
68
  ):
69
  continue
70
- bucket = entry.get("b")
71
- if bucket is not None and (
72
- bucket != "vomebook/pdf-pages"
73
- or mode != "p"
74
- or not path.endswith("/page-manifest.json")
75
- or not READER_BUCKET_PATH_RE.fullmatch(path)
76
  ):
77
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  clean[key] = {
79
  "s": 2,
80
  "m": mode,
@@ -82,7 +98,16 @@ def decode_reader_assets(
82
  **({"b": bucket} if bucket else {}),
83
  **({"c": chapter_manifest} if chapter_manifest else {}),
84
  **({"f": fallback} if fallback else {}),
 
 
 
85
  }
86
  else:
87
- clean[key] = {"s": 4}
 
 
 
 
 
 
88
  return {"v": 1, "f": clean}
 
18
  "v": {"video.mp4"},
19
  }
20
  READER_BUCKET_PATH_RE = re.compile(
21
+ r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[0-9a-f]{16}/(?:page-manifest\.json|ocr-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl)|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz))$"
22
+ )
23
+ READER_PDF_RANGE_BUCKET = "vomebook/pdf-optimized"
24
+ READER_PDF_RANGE_PATH_RE = re.compile(
25
+ r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[a-z0-9-]+/document\.pdf$"
26
  )
27
 
28
 
 
43
  raise ValueError("invalid reader assets sidecar")
44
  clean = {}
45
  for key, entry in files.items():
46
+ if not isinstance(key, str) or not isinstance(entry, dict) or entry.get("s") not in (2, 3, 4):
47
  continue
48
  if entry.get("s") == 2:
49
  path = entry.get("p")
 
58
  continue
59
  chapter_manifest = entry.get("c")
60
  fallback = entry.get("f")
61
+ ocr_manifest = entry.get("o")
62
  if chapter_manifest is not None and (
63
  mode not in {"e", "p"}
64
  or not isinstance(chapter_manifest, str)
 
72
  or not fallback.endswith("/document.pdf")
73
  ):
74
  continue
75
+ if ocr_manifest is not None and (
76
+ not isinstance(ocr_manifest, str)
77
+ or not READER_ASSET_PATH_RE.fullmatch(ocr_manifest)
78
+ or not ocr_manifest.endswith("/ocr-manifest.json")
 
 
79
  ):
80
  continue
81
+ if entry.get("b") is not None and entry.get("b") not in {"vomebook/pdf-pages", READER_PDF_RANGE_BUCKET}:
82
+ continue
83
+ if entry.get("ob") is not None and entry.get("ob") != "vomebook/pdf-pages":
84
+ continue
85
+ bucket = entry.get("b")
86
+ if bucket == "vomebook/pdf-pages" and (
87
+ mode != "p" or not path.endswith("/page-manifest.json")
88
+ or not READER_BUCKET_PATH_RE.fullmatch(path)):
89
+ continue
90
+ if bucket == READER_PDF_RANGE_BUCKET and (
91
+ mode != "p" or not path.endswith("/document.pdf")
92
+ or not READER_PDF_RANGE_PATH_RE.fullmatch(path)):
93
+ continue
94
  clean[key] = {
95
  "s": 2,
96
  "m": mode,
 
98
  **({"b": bucket} if bucket else {}),
99
  **({"c": chapter_manifest} if chapter_manifest else {}),
100
  **({"f": fallback} if fallback else {}),
101
+ **({"o": ocr_manifest} if ocr_manifest else {}),
102
+ **({"ob": "vomebook/pdf-pages"} if entry.get("ob") else {}),
103
+ **({"om": entry.get("om")} if isinstance(entry.get("om"), str) else {}),
104
  }
105
  else:
106
+ ocr_manifest = entry.get("o")
107
+ if (isinstance(ocr_manifest, str) and READER_ASSET_PATH_RE.fullmatch(ocr_manifest)
108
+ and ocr_manifest.endswith("/ocr-manifest.json")):
109
+ clean[key] = {"s": 3, "m": "p", "o": ocr_manifest, "b": "vomebook/pdf-pages",
110
+ **({"om": entry.get("om")} if isinstance(entry.get("om"), str) else {})}
111
+ else:
112
+ clean[key] = {"s": 4}
113
  return {"v": 1, "f": clean}
record_metadata.py CHANGED
@@ -73,8 +73,11 @@ def enrich_browser_entry(repo: str, path: str, entry: dict) -> dict:
73
 
74
  def reader_asset_url(asset: dict) -> str:
75
  path = quote(str(asset.get("p") or ""), safe="/")
76
- if asset.get("b") == "vomebook/pdf-pages":
77
- return f"/api/reader-bucket-resource?path={path}"
 
 
 
78
  return READER_ASSETS_BASE + path
79
 
80
 
@@ -94,9 +97,25 @@ def reader_extension_for_asset(asset: dict, original_extension: str = "") -> str
94
 
95
 
96
  def reader_asset_fields(asset: dict, original_extension: str) -> dict:
 
 
 
 
 
 
 
 
 
 
97
  if asset.get("s") != 2 or asset.get("m") not in READER_ASSET_MODES or not asset.get("p"):
98
- return {}
99
- fields = {"url": reader_asset_url(asset), "extension": reader_extension_for_asset(asset, original_extension)}
 
 
 
 
 
 
100
  for key, field in (("c", "chapter_manifest"), ("f", "fallback")):
101
  if asset.get(key):
102
  fields[field] = f"{READER_ASSETS_BASE}{asset[key]}"
@@ -108,7 +127,8 @@ def enrich_reader_record(record: dict, asset: dict) -> dict:
108
  fields = reader_asset_fields(asset, record.get("Extension", ""))
109
  if fields:
110
  for source, target in (("url", "ReaderLink"), ("extension", "ReaderExtension"),
111
- ("chapter_manifest", "ReaderChapterManifest"), ("fallback", "ReaderFallback")):
 
112
  if source in fields:
113
  result[target] = fields[source]
114
  result["DownloadLink"] = record.get("Link") or build_record_link(record)
 
73
 
74
  def reader_asset_url(asset: dict) -> str:
75
  path = quote(str(asset.get("p") or ""), safe="/")
76
+ bucket = asset.get("b")
77
+ if bucket in {"vomebook/pdf-pages", "vomebook/pdf-optimized"}:
78
+ prefix = ("/api/reader-bucket-resource?path=" if bucket == "vomebook/pdf-pages"
79
+ else f"/api/reader-bucket-resource?bucket={quote(bucket, safe='')}&path=")
80
+ return prefix + path
81
  return READER_ASSETS_BASE + path
82
 
83
 
 
97
 
98
 
99
  def reader_asset_fields(asset: dict, original_extension: str) -> dict:
100
+ ocr_path = str(asset.get("o") or "")
101
+ fields = {}
102
+ if ocr_path and ocr_path.endswith("/ocr-manifest.json"):
103
+ fields["ocr_manifest"] = (
104
+ f"https://voiceofml-search.hf.space/api/reader-bucket-resource?path={quote(ocr_path, safe='/')}"
105
+ if asset.get("b") == "vomebook/pdf-pages" or asset.get("ob") == "vomebook/pdf-pages" else f"{READER_ASSETS_BASE}{ocr_path}"
106
+ )
107
+ fields["ocr_mode"] = str(asset.get("om") or "")
108
+ if asset.get("s") == 3:
109
+ return fields
110
  if asset.get("s") != 2 or asset.get("m") not in READER_ASSET_MODES or not asset.get("p"):
111
+ return fields
112
+ fields.update({"url": reader_asset_url(asset), "extension": reader_extension_for_asset(asset, original_extension)})
113
+ if ocr_path and ocr_path.endswith("/ocr-manifest.json"):
114
+ fields["ocr_manifest"] = (
115
+ f"https://voiceofml-search.hf.space/api/reader-bucket-resource?path={quote(ocr_path, safe='/')}"
116
+ if asset.get("b") == "vomebook/pdf-pages" or asset.get("ob") == "vomebook/pdf-pages" else f"{READER_ASSETS_BASE}{ocr_path}"
117
+ )
118
+ fields["ocr_mode"] = str(asset.get("om") or "")
119
  for key, field in (("c", "chapter_manifest"), ("f", "fallback")):
120
  if asset.get(key):
121
  fields[field] = f"{READER_ASSETS_BASE}{asset[key]}"
 
127
  fields = reader_asset_fields(asset, record.get("Extension", ""))
128
  if fields:
129
  for source, target in (("url", "ReaderLink"), ("extension", "ReaderExtension"),
130
+ ("chapter_manifest", "ReaderChapterManifest"), ("fallback", "ReaderFallback"),
131
+ ("ocr_manifest", "ReaderOcrManifest"), ("ocr_mode", "ReaderOcrMode")):
132
  if source in fields:
133
  result[target] = fields[source]
134
  result["DownloadLink"] = record.get("Link") or build_record_link(record)
static/reader-contract.js CHANGED
@@ -212,16 +212,19 @@
212
 
213
  const assetRoot = String.raw`objects/[0-9a-f]{2}/[0-9a-f]{64}/`;
214
  const assetVersion = String.raw`(?:[0-9a-f]{16}/)?`;
215
- const assetPages = String.raw`(?:page-manifest\.json|pages/page-[0-9]{6}\.webp)`;
 
216
  const assetDocument = String.raw`(?:linearized\.pdf|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4)`;
217
  const assetChapters = String.raw`(?:chapter-manifest\.json|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz))`;
218
  const assetPrimaryPattern = new RegExp(
219
  `^${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?${assetDocument})$`
220
  );
221
  const assetSourcePattern = new RegExp(
222
- `^(?:pdf_manifest\\.json|${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?(?:${assetDocument}|${assetChapters})))$`
223
  );
224
- const bucketPathPattern = new RegExp(`^${assetRoot}${assetVersion}${assetPages}$`);
 
 
225
  const versionedBucketPathPattern = new RegExp(`^${assetRoot}[0-9a-f]{16}/${assetPages}$`);
226
  const assetBase = "https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/";
227
  const assetModes = Object.freeze({
@@ -233,7 +236,8 @@
233
  v: "video"
234
  });
235
 
236
- function isBucketPath(path, versioned = false) {
 
237
  return (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path);
238
  }
239
 
@@ -267,16 +271,21 @@
267
  }
268
  if (!path.endsWith("/page-manifest.json")) return null;
269
  const rootPath = path.slice(0, -"/page-manifest.json".length);
 
270
  return {
271
  root: rootPath,
 
 
 
 
 
 
272
  pageUrl(page) {
273
  if (!Number.isInteger(page) || page < 1 || page > 999999)
274
  throw new Error("PDF_PAGE_INVALID");
275
  const target = new URL(url.href);
276
  const filename = `pages/page-${String(page).padStart(6, "0")}.webp`;
277
- if (bucket) target.searchParams.set("path", `${rootPath}/${filename}`);
278
- else target.pathname = target.pathname.replace(/page-manifest\.json$/, filename);
279
- return target.href;
280
  }
281
  };
282
  } catch (_) {
@@ -292,13 +301,21 @@
292
 
293
  function assetFields(asset, bucketBase) {
294
  const path = String(asset?.p || "");
295
- const bucket = isBucketPath(path, true);
 
 
 
 
 
 
 
 
296
  if (
297
  !asset ||
298
  asset.s !== 2 ||
299
  !Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
300
  !assetPrimaryPattern.test(path) ||
301
- (bucket && asset.b !== "vomebook/pdf-pages")
302
  )
303
  return null;
304
  const nativeExtension =
@@ -310,10 +327,14 @@
310
  ? "pdf-pages"
311
  : assetModes[asset.m];
312
  return {
313
- ReaderLink: bucket ? `${bucketBase}?path=${encodeURIComponent(path)}` : assetBase + path,
 
 
314
  ReaderExtension: asset.c ? "epub-chapters" : extension,
315
  ReaderChapterManifest: asset.c ? assetBase + asset.c : "",
316
- ReaderFallback: asset.f ? assetBase + asset.f : ""
 
 
317
  };
318
  }
319
 
@@ -340,6 +361,8 @@
340
  params.set("download", record.DownloadLink || record.downloadLink);
341
  if (!shortId && (record.OcrUrl || record.ocrUrl))
342
  params.set("ocr", record.OcrUrl || record.ocrUrl);
 
 
343
  if (!shortId && (record.ReaderFallback || record.readerFallback))
344
  params.set("fallback", record.ReaderFallback || record.readerFallback);
345
  if (
 
212
 
213
  const assetRoot = String.raw`objects/[0-9a-f]{2}/[0-9a-f]{64}/`;
214
  const assetVersion = String.raw`(?:[0-9a-f]{16}/)?`;
215
+ const assetPages = String.raw`(?:page-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl))`;
216
+ const assetOcr = String.raw`(?:ocr-manifest\.json|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz))`;
217
  const assetDocument = String.raw`(?:linearized\.pdf|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4)`;
218
  const assetChapters = String.raw`(?:chapter-manifest\.json|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz))`;
219
  const assetPrimaryPattern = new RegExp(
220
  `^${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?${assetDocument})$`
221
  );
222
  const assetSourcePattern = new RegExp(
223
+ `^(?:pdf_manifest\\.json|${assetRoot}${assetVersion}(?:${assetPages}|${assetOcr}|(?:[a-z0-9-]+/)?(?:${assetDocument}|${assetChapters})))$`
224
  );
225
+ const bucketPathPattern = new RegExp(`^${assetRoot}${assetVersion}(?:${assetPages}|${assetOcr})$`);
226
+ const optimizedBucket = "vomebook/pdf-optimized";
227
+ const optimizedPathPattern = new RegExp(`^${assetRoot}(?:[a-z0-9-]+/)?document\\.pdf$`);
228
  const versionedBucketPathPattern = new RegExp(`^${assetRoot}[0-9a-f]{16}/${assetPages}$`);
229
  const assetBase = "https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/";
230
  const assetModes = Object.freeze({
 
236
  v: "video"
237
  });
238
 
239
+ function isBucketPath(path, versioned = false, bucket = "vomebook/pdf-pages") {
240
+ if (bucket === optimizedBucket) return optimizedPathPattern.test(path);
241
  return (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path);
242
  }
243
 
 
271
  }
272
  if (!path.endsWith("/page-manifest.json")) return null;
273
  const rootPath = path.slice(0, -"/page-manifest.json".length);
274
+ const pathPrefix = bucket ? "" : url.pathname.slice(0, -path.length);
275
  return {
276
  root: rootPath,
277
+ assetUrl(relativePath) {
278
+ const target = new URL(url.href);
279
+ if (bucket) target.searchParams.set("path", relativePath);
280
+ else target.pathname = `${pathPrefix}${relativePath}`;
281
+ return target.href;
282
+ },
283
  pageUrl(page) {
284
  if (!Number.isInteger(page) || page < 1 || page > 999999)
285
  throw new Error("PDF_PAGE_INVALID");
286
  const target = new URL(url.href);
287
  const filename = `pages/page-${String(page).padStart(6, "0")}.webp`;
288
+ return this.assetUrl(`${rootPath}/${filename}`);
 
 
289
  }
290
  };
291
  } catch (_) {
 
301
 
302
  function assetFields(asset, bucketBase) {
303
  const path = String(asset?.p || "");
304
+ const ocrPath = String(asset?.o || "");
305
+ const ocrBucket = asset?.b === "vomebook/pdf-pages" || asset?.ob === "vomebook/pdf-pages";
306
+ const ocrUrl = ocrPath.endsWith("/ocr-manifest.json") && ocrBucket && isBucketPath(ocrPath, true)
307
+ ? `${bucketBase}?path=${encodeURIComponent(ocrPath)}`
308
+ : "";
309
+ if (asset?.s === 3 && ocrUrl)
310
+ return { ReaderOcrManifest: ocrUrl, ReaderOcrMode: String(asset.om || "") };
311
+ const bucketName = String(asset?.b || "vomebook/pdf-pages");
312
+ const bucket = isBucketPath(path, true, bucketName);
313
  if (
314
  !asset ||
315
  asset.s !== 2 ||
316
  !Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
317
  !assetPrimaryPattern.test(path) ||
318
+ (bucket && !["vomebook/pdf-pages", optimizedBucket].includes(asset?.b))
319
  )
320
  return null;
321
  const nativeExtension =
 
327
  ? "pdf-pages"
328
  : assetModes[asset.m];
329
  return {
330
+ ReaderLink: bucket
331
+ ? `${bucketBase}?${asset.b === optimizedBucket ? `bucket=${encodeURIComponent(asset.b)}&` : ""}path=${encodeURIComponent(path)}`
332
+ : assetBase + path,
333
  ReaderExtension: asset.c ? "epub-chapters" : extension,
334
  ReaderChapterManifest: asset.c ? assetBase + asset.c : "",
335
+ ReaderFallback: asset.f ? assetBase + asset.f : "",
336
+ ReaderOcrManifest: ocrUrl || (ocrPath.endsWith("/ocr-manifest.json") ? assetBase + ocrPath : ""),
337
+ ReaderOcrMode: String(asset.om || "")
338
  };
339
  }
340
 
 
361
  params.set("download", record.DownloadLink || record.downloadLink);
362
  if (!shortId && (record.OcrUrl || record.ocrUrl))
363
  params.set("ocr", record.OcrUrl || record.ocrUrl);
364
+ if (!shortId && (record.ReaderOcrManifest || record.readerOcrManifest))
365
+ params.set("ocr_manifest", record.ReaderOcrManifest || record.readerOcrManifest);
366
  if (!shortId && (record.ReaderFallback || record.readerFallback))
367
  params.set("fallback", record.ReaderFallback || record.readerFallback);
368
  if (
static/reader-pdf-text.js CHANGED
@@ -24,3 +24,22 @@ export function populatePdfTextLayer(layer, items, pdfViewport) {
24
  if (!layer.textContent.trim()) layer.textContent = "此页没有可提取文本";
25
  } else layer.textContent = plainText.trim() || "此页没有可提取文本";
26
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  if (!layer.textContent.trim()) layer.textContent = "此页没有可提取文本";
25
  } else layer.textContent = plainText.trim() || "此页没有可提取文本";
26
  }
27
+
28
+ export function populateOcrTextLayer(layer, blocks) {
29
+ const fragment = layer.ownerDocument.createDocumentFragment();
30
+ for (const block of blocks || []) {
31
+ const text = String(block?.t || "");
32
+ const box = Array.isArray(block?.b) ? block.b : [];
33
+ if (!text || box.length !== 4) continue;
34
+ const span = layer.ownerDocument.createElement("span");
35
+ span.textContent = text;
36
+ span.dataset.ocrText = text;
37
+ span.style.left = `${Math.max(0, Math.min(1, Number(box[0]) || 0)) * 100}%`;
38
+ span.style.top = `${Math.max(0, Math.min(1, Number(box[1]) || 0)) * 100}%`;
39
+ span.style.width = `${Math.max(0, Math.min(1, Number(box[2]) - Number(box[0])) * 100)}%`;
40
+ span.style.height = `${Math.max(0.5, Math.min(1, Number(box[3]) - Number(box[1])) * 100)}%`;
41
+ span.style.fontSize = `${Math.max(8, Math.min(72, Math.abs(Number(box[3]) - Number(box[1])) * 100))}%`;
42
+ fragment.appendChild(span);
43
+ }
44
+ layer.replaceChildren(fragment);
45
+ }
static/reader-security.js CHANGED
@@ -265,6 +265,52 @@
265
  }
266
  return manifest;
267
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
  function isZipContainer(extension, bytes) {
269
  return (
270
  ["epub", "fbz"].includes(String(extension || "").toLowerCase()) ||
@@ -280,6 +326,8 @@
280
  inspectZip,
281
  validateChapterManifest,
282
  validatePdfPageManifest,
 
 
283
  isZipContainer
284
  });
285
  })(typeof self !== "undefined" ? self : globalThis);
 
265
  }
266
  return manifest;
267
  }
268
+ const PDF_OCR_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?ocr\/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz)$/;
269
+ const PDF_OCR_MANIFEST_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?ocr-manifest\.json$/;
270
+ const PDF_OCR_PAGE_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?pages\/page-[0-9]{6}\.(?:webp|jxl)$/;
271
+ const PDF_OCR_ROOT = /^(objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16})?)/;
272
+ function validatePdfOcrManifest(manifest) {
273
+ if (!manifest || manifest.version !== 1 || manifest.kind !== "pdf-ocr" ||
274
+ !Number.isInteger(manifest.page_count) || manifest.page_count < 1 ||
275
+ manifest.page_count > LIMITS.pdfPages || !Array.isArray(manifest.pages) ||
276
+ manifest.pages.length !== manifest.page_count)
277
+ throw error("PDF_OCR_MANIFEST_INVALID");
278
+ let root = "";
279
+ for (const [index, page] of manifest.pages.entries()) {
280
+ const expected = String(index + 1).padStart(6, "0");
281
+ const pageRoot = typeof page?.o === "string" ? PDF_OCR_ROOT.exec(page.o)?.[1] || "" : "";
282
+ if (!page || page.p !== index + 1 || !PDF_OCR_PATH.test(page.o) ||
283
+ page.o !== `${pageRoot}/ocr/page-${expected}.json.gz` ||
284
+ (page.w !== undefined && (typeof page.w !== "string" || !PDF_OCR_PAGE_PATH.test(page.w) ||
285
+ page.w !== `${pageRoot}/pages/page-${expected}.webp`)) ||
286
+ (page.j !== undefined && (typeof page.j !== "string" || !PDF_OCR_PAGE_PATH.test(page.j) ||
287
+ page.j !== `${pageRoot}/pages/page-${expected}.jxl`)) ||
288
+ (root && root !== pageRoot))
289
+ throw error("PDF_OCR_MANIFEST_INVALID");
290
+ root = pageRoot;
291
+ }
292
+ if (!manifest.book_text || typeof manifest.book_text.path !== "string" ||
293
+ !PDF_OCR_PATH.test(manifest.book_text.path) || manifest.book_text.path !== `${root}/ocr/book-text.json.gz` ||
294
+ (manifest.page_manifest !== undefined &&
295
+ (!manifest.page_manifest || typeof manifest.page_manifest.path !== "string" ||
296
+ !PDF_OCR_MANIFEST_PATH.test(manifest.page_manifest.path.replace("page-manifest.json", "ocr-manifest.json")))))
297
+ throw error("PDF_OCR_MANIFEST_INVALID");
298
+ return manifest;
299
+ }
300
+ function validatePdfOcrPage(payload, pageNumber) {
301
+ if (!payload || payload.version !== 1 || payload.kind !== "pdf-ocr-page" ||
302
+ payload.page !== pageNumber || !Array.isArray(payload.blocks) || payload.blocks.length > 10000 ||
303
+ !Number.isFinite(Number(payload.width)) || !Number.isFinite(Number(payload.height)))
304
+ throw error("PDF_OCR_PAGE_INVALID");
305
+ for (const block of payload.blocks) {
306
+ const box = block?.b;
307
+ if (!block || typeof block.t !== "string" || block.t.length > 4096 ||
308
+ !Array.isArray(box) || box.length !== 4 || box.some((value) => !Number.isFinite(Number(value))) ||
309
+ box[0] < 0 || box[1] < 0 || box[2] > 1 || box[3] > 1 || box[2] < box[0] || box[3] < box[1])
310
+ throw error("PDF_OCR_PAGE_INVALID");
311
+ }
312
+ return payload;
313
+ }
314
  function isZipContainer(extension, bytes) {
315
  return (
316
  ["epub", "fbz"].includes(String(extension || "").toLowerCase()) ||
 
326
  inspectZip,
327
  validateChapterManifest,
328
  validatePdfPageManifest,
329
+ validatePdfOcrManifest,
330
+ validatePdfOcrPage,
331
  isZipContainer
332
  });
333
  })(typeof self !== "undefined" ? self : globalThis);
static/reader.css CHANGED
@@ -869,7 +869,7 @@ body {
869
  position: absolute;
870
  inset: 0;
871
  overflow: hidden;
872
- pointer-events: none;
873
  white-space: pre;
874
  border: 0;
875
  color: transparent;
@@ -879,6 +879,8 @@ body {
879
  color: transparent;
880
  white-space: pre;
881
  transform-origin: left bottom;
 
 
882
  }
883
  .reader-pdf-text span.full-search-highlight {
884
  color: #1d2023;
 
869
  position: absolute;
870
  inset: 0;
871
  overflow: hidden;
872
+ pointer-events: auto;
873
  white-space: pre;
874
  border: 0;
875
  color: transparent;
 
879
  color: transparent;
880
  white-space: pre;
881
  transform-origin: left bottom;
882
+ user-select: text;
883
+ cursor: text;
884
  }
885
  .reader-pdf-text span.full-search-highlight {
886
  color: #1d2023;
static/reader.js CHANGED
@@ -6,7 +6,7 @@ import "/static/reader-scroll-anchor.js";
6
  import "/static/reader-section-virtualizer.js";
7
  import "/static/reader-runtime.js";
8
  import "/static/reader-format-adapters.js";
9
- import { populatePdfTextLayer } from "/static/reader-pdf-text.js";
10
  // Engines and Reader lifecycle.
11
  const PDFJS_URL = VoiceOfMLReaderResources.vendorUrl("pdf", "/static/");
12
  const PDFJS_WORKER_URL = "/static/pdf-worker-wrapper.mjs";
@@ -296,6 +296,10 @@ updateDocumentState({
296
  });
297
  function applyReaderMetadata(data) {
298
  if (data) resolvedReaderData = { ...(resolvedReaderData || {}), ...data };
 
 
 
 
299
  readerRuntime.update("source", { metadata: resolvedReaderData });
300
  const title = String(resolvedReaderData?.title || params.get("title") || sourceName).trim();
301
  const suffix = String(
@@ -311,6 +315,7 @@ function applyReaderMetadata(data) {
311
  }
312
  let fallbackUrl = params.get("fallback") || "";
313
  const ocrUrl = params.get("ocr") || "";
 
314
  let returnUrl = params.get("return") || "";
315
  const readerPathLabel = params.get("path") || "";
316
  const folderReturnUrl = params.get("folder_url") || "";
@@ -364,11 +369,16 @@ let bookmarkInvoker = null;
364
  const bookmarkInertSiblings = new Map();
365
  let pdfDocument = null;
366
  let pdfPageManifest = null;
 
 
367
  let pdfActiveRenders = 0;
368
  let pdfShellsReady = Promise.resolve();
369
  const pdfRenderWaiters = [];
 
 
370
  trackReaderResource(() => {
371
  for (const waiter of pdfRenderWaiters.splice(0)) waiter.reject(readerAbortError());
 
372
  });
373
  let epubRendition = null;
374
  let epubBook = null;
@@ -2018,10 +2028,11 @@ function validSource(raw) {
2018
  try {
2019
  const url = new URL(raw);
2020
  const bucketPath = url.searchParams.get("path") || "";
 
2021
  if (
2022
  url.origin === "https://voiceofml-search.hf.space" &&
2023
  url.pathname === "/api/reader-bucket-resource" &&
2024
- VoiceOfMLReader.isBucketPath(bucketPath)
2025
  )
2026
  return true;
2027
  if (url.protocol !== "https:" || !["huggingface.co", "hf-mirror.com"].includes(url.hostname))
@@ -2175,12 +2186,60 @@ function pdfManifestSource() {
2175
  }
2176
  function pdfPageEntry(page) {
2177
  if (!pdfPageManifest || page < 1 || page > pdfPageManifest.pageCount) return null;
2178
- return (
2179
  pdfPageManifest.entries?.[page - 1] || {
2180
  page,
2181
  path: `${pdfPageManifest.root}/pages/page-${String(page).padStart(6, "0")}.webp`
2182
  }
2183
  );
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2184
  }
2185
  function trimPdfManifestImages(protectedShell) {
2186
  const images = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
@@ -2193,6 +2252,10 @@ function trimPdfManifestImages(protectedShell) {
2193
  );
2194
  for (const shell of images.slice(25)) {
2195
  shell.querySelector("img")?.remove();
 
 
 
 
2196
  shell.dataset.renderState = "idle";
2197
  }
2198
  }
@@ -2355,6 +2418,7 @@ async function renderPdfPages(prepared) {
2355
  const totalPages = manifest.page_count;
2356
  if (!source) throw new Error("PDF_MANIFEST_INVALID");
2357
  pdfPageManifest = { ...source, pageCount: totalPages };
 
2358
  updateDocumentState({ pageCount: totalPages });
2359
  pageInput.max = String(totalPages);
2360
  document.querySelector("#page-total").textContent = `/ ${totalPages}`;
@@ -2370,7 +2434,10 @@ async function renderPdfPages(prepared) {
2370
  const observer = new IntersectionObserver(
2371
  (items) => {
2372
  if (readerAbortController.signal.aborted) return;
2373
- for (const entry of items) if (entry.isIntersecting) renderPdfInBackground(entry.target);
 
 
 
2374
  },
2375
  { root: viewport, rootMargin: "1200px 0px" }
2376
  );
@@ -2414,7 +2481,7 @@ async function renderPdf(prepared) {
2414
  (entries) =>
2415
  entries.forEach((entry) => {
2416
  if (readerAbortController.signal.aborted) return;
2417
- entry.target.dataset.renderVisible = entry.isIntersecting ? "1" : "0";
2418
  if (entry.isIntersecting) renderPdfInBackground(entry.target);
2419
  }),
2420
  { root: document.querySelector("#viewport"), rootMargin: "1200px 0px" }
@@ -2490,7 +2557,12 @@ async function renderPdf(prepared) {
2490
  assertReaderActive();
2491
  }
2492
 
2493
- function renderPdfInBackground(shell, force = false) {
 
 
 
 
 
2494
  if (readerAbortController.signal.aborted) return;
2495
  if (shell._backgroundRender) {
2496
  if (force) shell.dataset.pendingRerender = "1";
@@ -2498,8 +2570,8 @@ function renderPdfInBackground(shell, force = false) {
2498
  }
2499
  const task =
2500
  capability.mode === "pdf-pages"
2501
- ? renderPdfManifestShell(shell, force, true)
2502
- : renderPdfShell(shell, force);
2503
  shell._backgroundRender = task;
2504
  task
2505
  .catch((error) => {
@@ -2539,9 +2611,13 @@ function renderPdfManifestShell(shell, force = false, priority = false) {
2539
  image.decoding = "async";
2540
  shell.appendChild(image);
2541
  }
2542
- const target = pdfPageManifest.pageUrl(entry.page);
2543
- await new Promise((resolve, reject) => {
2544
- let settled = false,
 
 
 
 
2545
  timeout = 0,
2546
  untrack = () => {};
2547
  const finish = (error) => {
@@ -2561,13 +2637,23 @@ function renderPdfManifestShell(shell, force = false, priority = false) {
2561
  () => finish(new Error("PDF page image timeout")),
2562
  READER_PROXY_TIMEOUT_MS
2563
  );
2564
- image.onload = () => finish();
2565
- image.onerror = () => finish(new Error(`PDF page ${entry.page} image failed`));
2566
- if (image.src !== target) image.src = target;
2567
- if (image.complete && image.naturalWidth) finish();
 
 
 
 
 
 
2568
  });
2569
  assertReaderActive();
2570
  shell.style.aspectRatio = `${image.naturalWidth || 1} / ${image.naturalHeight || 1}`;
 
 
 
 
2571
  image.classList.add("ready");
2572
  shell.dataset.renderState = "rendered";
2573
  shell.dataset.renderUsedAt = String(Date.now());
@@ -2674,8 +2760,17 @@ async function renderPdfText(page, shell) {
2674
  function highlightPdfText(shell, query = searchState.query) {
2675
  if (!shell || readerAbortController.signal.aborted || !query) return;
2676
  const pattern = new RegExp(fullSearchEscape(query), "iu");
2677
- for (const span of shell.querySelectorAll(".reader-pdf-text span"))
2678
- if (pattern.test(span.textContent) && !span.classList.contains("full-search-highlight")) {
 
 
 
 
 
 
 
 
 
2679
  span.classList.add("full-search-highlight");
2680
  fullSearchActiveMarks.push(span);
2681
  }
@@ -2706,6 +2801,31 @@ function releasePdfRenderSlot() {
2706
  const waiter = pdfRenderWaiters.shift();
2707
  if (waiter) waiter.resolve();
2708
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2709
  function trimPdfCanvases(protectedShell) {
2710
  const limit = matchMedia("(max-width: 700px)").matches ? 7 : 11;
2711
  const rendered = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
@@ -2737,6 +2857,10 @@ function trimPdfCanvases(protectedShell) {
2737
  canvas.width = 0;
2738
  canvas.height = 0;
2739
  canvas.classList.remove("ready");
 
 
 
 
2740
  shell.dataset.renderState = "idle";
2741
  }
2742
  }
@@ -3412,6 +3536,8 @@ function disposeFormatResources(mode) {
3412
  if (["pdf", "pdf-pages"].includes(mode)) {
3413
  pdfDocument = null;
3414
  pdfPageManifest = null;
 
 
3415
  pdfFirstPagePreload?.removeAttribute("src");
3416
  pdfFirstPagePreload = null;
3417
  for (const canvas of content.querySelectorAll(".reader-page canvas")) {
@@ -4895,6 +5021,47 @@ async function navigateFoliateSearchResult(result, generation) {
4895
  updateProgressTools();
4896
  scheduleSave();
4897
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4898
  async function fullSearchPdfMatches(query, generation) {
4899
  const pdf = pdfDocument,
4900
  output = [],
@@ -4964,7 +5131,14 @@ async function runFullSearch() {
4964
  }
4965
  fullSearchStatus.textContent = "正在搜索正文…";
4966
  try {
4967
- if (capability.mode === "pdf")
 
 
 
 
 
 
 
4968
  publishSearchResults(generation, await fullSearchPdfMatches(query, generation));
4969
  else if (capability.mode === "foliate")
4970
  publishSearchResults(generation, await fullSearchFoliateMatches(query, generation));
 
6
  import "/static/reader-section-virtualizer.js";
7
  import "/static/reader-runtime.js";
8
  import "/static/reader-format-adapters.js";
9
+ import { populatePdfTextLayer, populateOcrTextLayer } from "/static/reader-pdf-text.js";
10
  // Engines and Reader lifecycle.
11
  const PDFJS_URL = VoiceOfMLReaderResources.vendorUrl("pdf", "/static/");
12
  const PDFJS_WORKER_URL = "/static/pdf-worker-wrapper.mjs";
 
296
  });
297
  function applyReaderMetadata(data) {
298
  if (data) resolvedReaderData = { ...(resolvedReaderData || {}), ...data };
299
+ if (data?.ReaderOcrManifest || data?.ocr_manifest) {
300
+ ocrManifestUrl = data.ReaderOcrManifest || data.ocr_manifest;
301
+ params.set("ocr_manifest", ocrManifestUrl);
302
+ }
303
  readerRuntime.update("source", { metadata: resolvedReaderData });
304
  const title = String(resolvedReaderData?.title || params.get("title") || sourceName).trim();
305
  const suffix = String(
 
315
  }
316
  let fallbackUrl = params.get("fallback") || "";
317
  const ocrUrl = params.get("ocr") || "";
318
+ let ocrManifestUrl = params.get("ocr_manifest") || "";
319
  let returnUrl = params.get("return") || "";
320
  const readerPathLabel = params.get("path") || "";
321
  const folderReturnUrl = params.get("folder_url") || "";
 
369
  const bookmarkInertSiblings = new Map();
370
  let pdfDocument = null;
371
  let pdfPageManifest = null;
372
+ let pdfOcrManifest = null;
373
+ let pdfOcrManifestPromise = null;
374
  let pdfActiveRenders = 0;
375
  let pdfShellsReady = Promise.resolve();
376
  const pdfRenderWaiters = [];
377
+ let pdfActiveTextLoads = 0;
378
+ const pdfTextWaiters = [];
379
  trackReaderResource(() => {
380
  for (const waiter of pdfRenderWaiters.splice(0)) waiter.reject(readerAbortError());
381
+ for (const waiter of pdfTextWaiters.splice(0)) waiter.reject(readerAbortError());
382
  });
383
  let epubRendition = null;
384
  let epubBook = null;
 
2028
  try {
2029
  const url = new URL(raw);
2030
  const bucketPath = url.searchParams.get("path") || "";
2031
+ const bucketName = url.searchParams.get("bucket") || "vomebook/pdf-pages";
2032
  if (
2033
  url.origin === "https://voiceofml-search.hf.space" &&
2034
  url.pathname === "/api/reader-bucket-resource" &&
2035
+ VoiceOfMLReader.isBucketPath(bucketPath, true, bucketName)
2036
  )
2037
  return true;
2038
  if (url.protocol !== "https:" || !["huggingface.co", "hf-mirror.com"].includes(url.hostname))
 
2186
  }
2187
  function pdfPageEntry(page) {
2188
  if (!pdfPageManifest || page < 1 || page > pdfPageManifest.pageCount) return null;
2189
+ const pageEntry = (
2190
  pdfPageManifest.entries?.[page - 1] || {
2191
  page,
2192
  path: `${pdfPageManifest.root}/pages/page-${String(page).padStart(6, "0")}.webp`
2193
  }
2194
  );
2195
+ const ocrEntry = pdfOcrManifest?.pages?.[page - 1];
2196
+ return ocrEntry ? { ...pageEntry, ...ocrEntry, page } : pageEntry;
2197
+ }
2198
+
2199
+ async function readPdfOcrJson(url, limit = VoiceOfMLReaderSecurity.LIMITS.chapterBytes) {
2200
+ const response = await fetchReaderUrl(url);
2201
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
2202
+ let bytes = await VoiceOfMLReaderSecurity.readBytes(response, limit);
2203
+ if (/\.gz$/i.test(url)) {
2204
+ if (typeof DecompressionStream !== "function") throw new Error("PDF_OCR_GZIP_UNSUPPORTED");
2205
+ const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip"));
2206
+ bytes = await VoiceOfMLReaderSecurity.readBytes(new Response(stream), limit);
2207
+ }
2208
+ return JSON.parse(new TextDecoder().decode(bytes));
2209
+ }
2210
+
2211
+ async function loadPdfOcrManifest() {
2212
+ if (pdfOcrManifest) return pdfOcrManifest;
2213
+ if (!ocrManifestUrl) return null;
2214
+ if (!pdfOcrManifestPromise) {
2215
+ pdfOcrManifestPromise = readPdfOcrJson(ocrManifestUrl, VoiceOfMLReaderSecurity.LIMITS.manifestBytes).then((manifest) => {
2216
+ pdfOcrManifest = VoiceOfMLReaderSecurity.validatePdfOcrManifest(manifest);
2217
+ return pdfOcrManifest;
2218
+ });
2219
+ }
2220
+ try { return await pdfOcrManifestPromise; }
2221
+ catch (error) { pdfOcrManifestPromise = null; throw error; }
2222
+ }
2223
+
2224
+ async function renderPdfOcrText(shell, entry) {
2225
+ if (shell.dataset.textReady === "1" || !entry?.o || !pdfPageManifest) return;
2226
+ const layer = shell.querySelector(".reader-pdf-text");
2227
+ if (!layer) return;
2228
+ await acquirePdfTextSlot(shell.dataset.renderVisible === "1");
2229
+ try {
2230
+ const payload = await readPdfOcrJson(pdfPageManifest.assetUrl(entry.o));
2231
+ assertReaderActive();
2232
+ VoiceOfMLReaderSecurity.validatePdfOcrPage(payload, Number(shell.dataset.page));
2233
+ populateOcrTextLayer(layer, payload.blocks);
2234
+ shell._bookmarkTextItems = payload.blocks.map((block) => ({ text: block.t, y: block.b?.[1] || 0 }));
2235
+ shell.dataset.textReady = "1";
2236
+ highlightPdfText(shell);
2237
+ } catch (error) {
2238
+ if (!readerAbortController.signal.aborted)
2239
+ console.warn(`PDF page ${shell.dataset.page} OCR text failed`, error);
2240
+ } finally {
2241
+ releasePdfTextSlot();
2242
+ }
2243
  }
2244
  function trimPdfManifestImages(protectedShell) {
2245
  const images = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
 
2252
  );
2253
  for (const shell of images.slice(25)) {
2254
  shell.querySelector("img")?.remove();
2255
+ const textLayer = shell.querySelector(".reader-pdf-text");
2256
+ if (textLayer) textLayer.replaceChildren();
2257
+ shell._bookmarkTextItems = [];
2258
+ shell.dataset.textReady = "0";
2259
  shell.dataset.renderState = "idle";
2260
  }
2261
  }
 
2418
  const totalPages = manifest.page_count;
2419
  if (!source) throw new Error("PDF_MANIFEST_INVALID");
2420
  pdfPageManifest = { ...source, pageCount: totalPages };
2421
+ if (ocrManifestUrl) loadPdfOcrManifest().catch(() => {});
2422
  updateDocumentState({ pageCount: totalPages });
2423
  pageInput.max = String(totalPages);
2424
  document.querySelector("#page-total").textContent = `/ ${totalPages}`;
 
2434
  const observer = new IntersectionObserver(
2435
  (items) => {
2436
  if (readerAbortController.signal.aborted) return;
2437
+ for (const entry of items) if (entry.isIntersecting) {
2438
+ entry.target.dataset.renderVisible = isPdfPageVisible(entry.target) ? "1" : "0";
2439
+ renderPdfInBackground(entry.target);
2440
+ }
2441
  },
2442
  { root: viewport, rootMargin: "1200px 0px" }
2443
  );
 
2481
  (entries) =>
2482
  entries.forEach((entry) => {
2483
  if (readerAbortController.signal.aborted) return;
2484
+ entry.target.dataset.renderVisible = entry.isIntersecting && isPdfPageVisible(entry.target) ? "1" : "0";
2485
  if (entry.isIntersecting) renderPdfInBackground(entry.target);
2486
  }),
2487
  { root: document.querySelector("#viewport"), rootMargin: "1200px 0px" }
 
2557
  assertReaderActive();
2558
  }
2559
 
2560
+ function isPdfPageVisible(shell) {
2561
+ const pageRect = shell?.getBoundingClientRect?.();
2562
+ const viewportRect = viewport?.getBoundingClientRect?.();
2563
+ return !!pageRect && !!viewportRect && pageRect.bottom > viewportRect.top && pageRect.top < viewportRect.bottom;
2564
+ }
2565
+ function renderPdfInBackground(shell, force = false, priority = isPdfPageVisible(shell)) {
2566
  if (readerAbortController.signal.aborted) return;
2567
  if (shell._backgroundRender) {
2568
  if (force) shell.dataset.pendingRerender = "1";
 
2570
  }
2571
  const task =
2572
  capability.mode === "pdf-pages"
2573
+ ? renderPdfManifestShell(shell, force, priority)
2574
+ : renderPdfShell(shell, force, priority);
2575
  shell._backgroundRender = task;
2576
  task
2577
  .catch((error) => {
 
2611
  image.decoding = "async";
2612
  shell.appendChild(image);
2613
  }
2614
+ const fallbackPath = entry.w || `${pdfPageManifest.root}/pages/page-${String(entry.page).padStart(6, "0")}.webp`;
2615
+ const targets = [...new Set([entry.j, fallbackPath].filter(Boolean))].map((path) =>
2616
+ pdfPageManifest.assetUrl(path)
2617
+ );
2618
+ await new Promise((resolve, reject) => {
2619
+ let settled = false,
2620
+ targetIndex = 0,
2621
  timeout = 0,
2622
  untrack = () => {};
2623
  const finish = (error) => {
 
2637
  () => finish(new Error("PDF page image timeout")),
2638
  READER_PROXY_TIMEOUT_MS
2639
  );
2640
+ image.onload = () => finish();
2641
+ image.onerror = () => {
2642
+ if (targetIndex + 1 < targets.length) {
2643
+ targetIndex += 1;
2644
+ image.src = targets[targetIndex];
2645
+ } else finish(new Error(`PDF page ${entry.page} image failed`));
2646
+ };
2647
+ if (!targets.length) return finish(new Error(`PDF page ${entry.page} image missing`));
2648
+ if (image.src !== targets[targetIndex]) image.src = targets[targetIndex];
2649
+ if (image.complete && image.naturalWidth) finish();
2650
  });
2651
  assertReaderActive();
2652
  shell.style.aspectRatio = `${image.naturalWidth || 1} / ${image.naturalHeight || 1}`;
2653
+ const loadText = pdfOcrManifestPromise
2654
+ ? pdfOcrManifestPromise.then(() => renderPdfOcrText(shell, pdfPageEntry(Number(shell.dataset.page))))
2655
+ : renderPdfOcrText(shell, pdfPageEntry(Number(shell.dataset.page)));
2656
+ loadText.catch(() => {});
2657
  image.classList.add("ready");
2658
  shell.dataset.renderState = "rendered";
2659
  shell.dataset.renderUsedAt = String(Date.now());
 
2760
  function highlightPdfText(shell, query = searchState.query) {
2761
  if (!shell || readerAbortController.signal.aborted || !query) return;
2762
  const pattern = new RegExp(fullSearchEscape(query), "iu");
2763
+ const spans = [...shell.querySelectorAll(".reader-pdf-text span")];
2764
+ let text = "";
2765
+ const ranges = spans.map((span) => {
2766
+ const start = text.length;
2767
+ text += `${span.textContent || ""} `;
2768
+ return { span, start, end: text.length };
2769
+ });
2770
+ const hit = text.search(pattern);
2771
+ for (const { span, start, end } of ranges)
2772
+ if (!span.classList.contains("full-search-highlight") &&
2773
+ ((hit >= 0 && end > hit && start < hit + query.length) || pattern.test(span.textContent))) {
2774
  span.classList.add("full-search-highlight");
2775
  fullSearchActiveMarks.push(span);
2776
  }
 
2801
  const waiter = pdfRenderWaiters.shift();
2802
  if (waiter) waiter.resolve();
2803
  }
2804
+ function acquirePdfTextSlot(priority = false) {
2805
+ if (readerAbortController.signal.aborted) return Promise.reject(readerAbortError());
2806
+ const limit = matchMedia("(max-width: 700px)").matches ? 1 : 2;
2807
+ if (pdfActiveTextLoads < limit) {
2808
+ pdfActiveTextLoads++;
2809
+ return Promise.resolve();
2810
+ }
2811
+ return new Promise((resolve, reject) => {
2812
+ const waiter = {
2813
+ resolve: () => {
2814
+ pdfActiveTextLoads++;
2815
+ resolve();
2816
+ },
2817
+ reject
2818
+ };
2819
+ if (priority) pdfTextWaiters.unshift(waiter);
2820
+ else pdfTextWaiters.push(waiter);
2821
+ });
2822
+ }
2823
+ function releasePdfTextSlot() {
2824
+ pdfActiveTextLoads = Math.max(0, pdfActiveTextLoads - 1);
2825
+ if (readerAbortController.signal.aborted) return;
2826
+ const waiter = pdfTextWaiters.shift();
2827
+ if (waiter) waiter.resolve();
2828
+ }
2829
  function trimPdfCanvases(protectedShell) {
2830
  const limit = matchMedia("(max-width: 700px)").matches ? 7 : 11;
2831
  const rendered = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
 
2857
  canvas.width = 0;
2858
  canvas.height = 0;
2859
  canvas.classList.remove("ready");
2860
+ const textLayer = shell.querySelector(".reader-pdf-text");
2861
+ if (textLayer) textLayer.replaceChildren();
2862
+ shell._bookmarkTextItems = [];
2863
+ shell.dataset.textReady = "0";
2864
  shell.dataset.renderState = "idle";
2865
  }
2866
  }
 
3536
  if (["pdf", "pdf-pages"].includes(mode)) {
3537
  pdfDocument = null;
3538
  pdfPageManifest = null;
3539
+ pdfOcrManifest = null;
3540
+ pdfOcrManifestPromise = null;
3541
  pdfFirstPagePreload?.removeAttribute("src");
3542
  pdfFirstPagePreload = null;
3543
  for (const canvas of content.querySelectorAll(".reader-page canvas")) {
 
5021
  updateProgressTools();
5022
  scheduleSave();
5023
  }
5024
+ async function fullSearchPdfPageMatches(query, generation) {
5025
+ const manifest = await loadPdfOcrManifest();
5026
+ if (!manifest?.book_text?.path || !pdfPageManifest) return [];
5027
+ const book = await readPdfOcrJson(pdfPageManifest.assetUrl(manifest.book_text.path), VoiceOfMLReaderSecurity.LIMITS.chapterTotalBytes);
5028
+ const output = [], pattern = new RegExp(fullSearchEscape(query), "giu");
5029
+ for (const page of book.pages || []) {
5030
+ if (!isReaderGenerationCurrent("search", generation)) return [];
5031
+ const text = String(page.text || "");
5032
+ for (const match of text.matchAll(pattern)) {
5033
+ if (output.length >= 100) break;
5034
+ output.push({
5035
+ location: `第 ${page.page} 页`,
5036
+ snippet: fullSearchSnippet(text, match.index, match[0].length),
5037
+ activate: (navigationGeneration) => goToPage(page.page, navigationGeneration)
5038
+ });
5039
+ }
5040
+ if (output.length >= 100) break;
5041
+ }
5042
+ return output;
5043
+ }
5044
+
5045
+ async function fullSearchPdfApiMatches(query, generation) {
5046
+ if (!readerId) throw new Error("PDF_OCR_API_ID_MISSING");
5047
+ const url = `https://voiceofml-search.hf.space/api/reader-ocr-search?id=${encodeURIComponent(readerId)}&q=${encodeURIComponent(query)}&page=1&page_size=100`;
5048
+ const response = await fetchWithReaderTimeout(url);
5049
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
5050
+ const payload = await response.json();
5051
+ if (!payload || payload.version !== 1 || !Array.isArray(payload.results))
5052
+ throw new Error("PDF_OCR_API_INVALID");
5053
+ return payload.results.map((item) => {
5054
+ const text = String(item.snippet || "");
5055
+ const start = Math.max(0, Number(item.snippet_start) || 0);
5056
+ const length = Math.max(0, Number(item.length) || query.length);
5057
+ return {
5058
+ location: `第 ${item.page} 页`,
5059
+ snippet: fullSearchSnippet(text, start, length),
5060
+ activate: (navigationGeneration) => goToPage(item.page, navigationGeneration)
5061
+ };
5062
+ });
5063
+ }
5064
+
5065
  async function fullSearchPdfMatches(query, generation) {
5066
  const pdf = pdfDocument,
5067
  output = [],
 
5131
  }
5132
  fullSearchStatus.textContent = "正在搜索正文…";
5133
  try {
5134
+ if (capability.mode === "pdf-pages") {
5135
+ try {
5136
+ publishSearchResults(generation, await fullSearchPdfApiMatches(query, generation));
5137
+ } catch (error) {
5138
+ publishSearchResults(generation, await fullSearchPdfPageMatches(query, generation));
5139
+ }
5140
+ }
5141
+ else if (capability.mode === "pdf")
5142
  publishSearchResults(generation, await fullSearchPdfMatches(query, generation));
5143
  else if (capability.mode === "foliate")
5144
  publishSearchResults(generation, await fullSearchFoliateMatches(query, generation));
tests/test_fastapi_endpoints.py CHANGED
@@ -1,6 +1,7 @@
1
  from __future__ import annotations
2
 
3
  import copy
 
4
  import json
5
  import tempfile
6
  import time
@@ -41,6 +42,30 @@ class FastApiEndpointTests(unittest.TestCase):
41
  self.assertRegex(response.headers["server-timing"], r"queue;dur=\d+\.\d+, search;dur=\d+\.\d+")
42
  self.assertNotIn("readme", response.headers["server-timing"])
43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  def test_search_endpoint_rejects_malformed_json(self):
45
  self.assertEqual(self.client.post("/api/search", content="not-json", headers={"content-type": "application/json"}).status_code, 422)
46
 
 
1
  from __future__ import annotations
2
 
3
  import copy
4
+ import gzip
5
  import json
6
  import tempfile
7
  import time
 
42
  self.assertRegex(response.headers["server-timing"], r"queue;dur=\d+\.\d+, search;dur=\d+\.\d+")
43
  self.assertNotIn("readme", response.headers["server-timing"])
44
 
45
+ def test_reader_ocr_search_returns_exact_page_hits(self):
46
+ root = "objects/aa/" + "a" * 64 + "/" + "b" * 16
47
+ manifest_path = root + "/ocr-manifest.json"
48
+ book_path = root + "/ocr/book-text.json.gz"
49
+ manifest = {"version": 1, "kind": "pdf-ocr", "page_count": 2,
50
+ "pages": [{"p": 1, "o": root + "/ocr/page-000001.json.gz"},
51
+ {"p": 2, "o": root + "/ocr/page-000002.json.gz"}],
52
+ "book_text": {"path": book_path}}
53
+ book = gzip.compress(json.dumps({"version": 1, "kind": "pdf-ocr-book-text", "pages": [
54
+ {"page": 1, "text": "第一章 手机阅读"},
55
+ {"page": 2, "text": "第二章 手机搜索"},
56
+ ]}).encode())
57
+
58
+ async def resource(path):
59
+ return json.dumps(manifest).encode() if path == manifest_path else book
60
+
61
+ resolved = app.JSONResponse({"ocr_manifest": "https://voiceofml-search.hf.space/api/reader-bucket-resource?path=" + manifest_path})
62
+ with mock.patch.object(app, "api_reader_resolve", mock.AsyncMock(return_value=resolved)), \
63
+ mock.patch.object(app, "get_reader_bucket_resource", side_effect=resource):
64
+ response = self.client.get("/api/reader-ocr-search", params={"id": "a" * 13, "q": "手机"})
65
+ self.assertEqual(response.status_code, 200)
66
+ self.assertEqual(response.json()["total"], 2)
67
+ self.assertEqual([item["page"] for item in response.json()["results"]], [1, 2])
68
+
69
  def test_search_endpoint_rejects_malformed_json(self):
70
  self.assertEqual(self.client.post("/api/search", content="not-json", headers={"content-type": "application/json"}).status_code, 422)
71
 
tests/test_reader_pdf_loading.py CHANGED
@@ -12,7 +12,7 @@ from tests.test_reader_performance import (
12
  PlaywrightError, StaticHandler, minimal_pdf, static_server, sync_playwright,
13
  )
14
 
15
- MODULE = '**/static/vendor/pdf.min.f80490490320.mjs*'
16
  SOURCE = 'https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/cold-retry.pdf'
17
 
18
 
 
12
  PlaywrightError, StaticHandler, minimal_pdf, static_server, sync_playwright,
13
  )
14
 
15
+ MODULE = '**/static/vendor/pdf.min.*.mjs*'
16
  SOURCE = 'https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/cold-retry.pdf'
17
 
18
 
tests/test_reader_performance.py CHANGED
@@ -7,6 +7,7 @@ import io
7
  import json
8
  import mimetypes
9
  import pathlib
 
10
  import threading
11
  import time
12
  import unittest
@@ -23,6 +24,43 @@ except ImportError:
23
 
24
 
25
  ROOT = pathlib.Path(__file__).resolve().parents[1]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  SOURCE_URL = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/performance.pdf"
27
  PDF_MODULE = """
28
  export const GlobalWorkerOptions = {};
@@ -260,7 +298,7 @@ class ReaderPerformanceTest(unittest.TestCase):
260
  self.page = self.context.new_page()
261
  self.pdf_requested_at = None
262
  self.page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
263
- self.page.route("**/static/vendor/pdf.min.f80490490320.mjs", self.route_pdf)
264
  self.page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
265
 
266
  def tearDown(self):
@@ -474,7 +512,7 @@ class ReaderPerformanceTest(unittest.TestCase):
474
  self.assertEqual(first_page.locator(".reader-pdf-text").text_content(), "Accessible PDF text")
475
 
476
  def test_scanned_pdf_bookmark_has_empty_excerpt(self):
477
- self.page.route('**/static/vendor/pdf.min.f80490490320.mjs', lambda route: route.fulfill(
478
  content_type='text/javascript', body=PDF_MODULE.replace("[{ str: 'Accessible PDF text', hasEOL: false }]", '[]')))
479
  self.open_pdf()
480
  self.page.wait_for_function("document.querySelector('.reader-pdf-text')?.textContent.includes('此页没有可提取文本')")
@@ -583,7 +621,7 @@ class ReaderPerformanceTest(unittest.TestCase):
583
  mobile_context = self.browser.new_context(viewport={"width": 390, "height": 844})
584
  mobile_page = mobile_context.new_page()
585
  mobile_page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
586
- mobile_page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
587
  mobile_page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
588
  query = urllib.parse.quote(SOURCE_URL, safe="")
589
  mobile_page.goto(f"{self.origin}/static/reader.html?url={query}&ext=pdf&title=Performance", wait_until="domcontentloaded")
@@ -663,13 +701,13 @@ class ReaderPerformanceTest(unittest.TestCase):
663
  context = self.browser.new_context(viewport={"width": 390, "height": 844})
664
  page = context.new_page()
665
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
666
- page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
667
  if extension == "md":
668
- page.route("**/static/vendor/marked.min.b147274a9ce2.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
669
- page.route("**/static/vendor/purify.min.f263b05369e0.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
670
  if extension == "docx":
671
- page.route("**/static/vendor/jszip.min.7f839b2d4688.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=JSZIP_SCRIPT))
672
- page.route("**/static/vendor/docx-preview.min.051ef503f267.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=DOCX_SCRIPT))
673
  page.route("**/api/reader-content**", lambda route, _request, content_type=content_type, body=body: route.fulfill(status=200, content_type=content_type, body=body))
674
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/matrix.{extension}"
675
  if extension == "docx":
@@ -1126,8 +1164,8 @@ class ReaderPerformanceTest(unittest.TestCase):
1126
  self.addCleanup(context.close)
1127
  page = context.new_page()
1128
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1129
- page.route("**/static/vendor/jszip.min.7f839b2d4688.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.JSZip=function(){window.__archiveParserStarted=true};"))
1130
- page.route("**/static/vendor/docx-preview.min.051ef503f267.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.docx={renderAsync(){window.__archiveParserStarted=true}};"))
1131
  page.route("**/static/foliate-reader/view.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="customElements.define('foliate-view', class extends HTMLElement { open() { window.__archiveParserStarted=true; throw new Error('parser must not start'); } });"))
1132
  page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/octet-stream", body=payload))
1133
  source = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/bomb.epub" if extension == "epub" else f"https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/objects/aa/{'a' * 64}/docx-native-v1/document.docx"
@@ -1154,14 +1192,14 @@ class ReaderPerformanceTest(unittest.TestCase):
1154
  context = self.browser.new_context(viewport={"width": 390, "height": 844}); page = context.new_page()
1155
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1156
  if extension == "md":
1157
- page.route("**/static/vendor/marked.min.b147274a9ce2.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.marked={parse:()=>'<h1>Markdown one</h1><p style=\"height:1000px\"></p><h2>Markdown two</h2>'};")); page.route("**/static/vendor/purify.min.f263b05369e0.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
1158
  page.route("**/api/reader-content**", lambda route, _request, body=body: route.fulfill(status=200, content_type="text/html" if extension == "html" else "text/markdown", body=body))
1159
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/toc.{extension}"; page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=TOC", wait_until="domcontentloaded"); page.locator("#history").click(); page.locator("#toc-list .panel-item-main").nth(1).click()
1160
  page.wait_for_function("() => document.querySelector('iframe') ? document.querySelector('iframe').contentWindow.scrollY > 0 : document.querySelector('#viewport').scrollTop > 0"); context.close()
1161
 
1162
  def test_pdf_outline_click_navigates_to_declared_page(self):
1163
  self.page.add_init_script("window.__pdfOutlineEnabled = true")
1164
- module = PDF_MODULE.replace("[{title: '第一章', dest: [{}], items: []}]", "[{title: '第三章', dest: [{}], items: []}]").replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: () => Promise.resolve(2)"); self.page.unroute("**/static/vendor/pdf.min.f80490490320.mjs"); self.page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module)); self.open_pdf(); self.page.locator("#history").click(); self.page.locator("#toc-list .panel-item-main").click(); self.page.wait_for_function("() => document.querySelector('#page-number').value === '3'")
1165
 
1166
  def test_pdf_outline_marks_current_entry_when_page_changes(self):
1167
  self.page.add_init_script("window.__pdfOutlineEnabled = true")
@@ -1169,8 +1207,8 @@ class ReaderPerformanceTest(unittest.TestCase):
1169
  "[{title: '第一章', dest: [{}], items: []}]",
1170
  "[{title: '第一章', dest: [{page: 0}], items: []}, {title: '第二章', dest: [{page: 2}], items: []}]",
1171
  ).replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: (ref) => Promise.resolve(ref.page || 0)")
1172
- self.page.unroute("**/static/vendor/pdf.min.f80490490320.mjs")
1173
- self.page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module))
1174
  self.open_pdf()
1175
  self.page.locator("#history").click()
1176
  self.page.locator("#toc-list .toc-item").nth(1).wait_for()
@@ -1186,7 +1224,7 @@ class ReaderPerformanceTest(unittest.TestCase):
1186
  def test_pdf_stale_navigation_and_search_keep_latest_state(self):
1187
  errors = []
1188
  self.page.on("pageerror", lambda error: errors.append(str(error)))
1189
- self.page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route: route.fulfill(
1190
  status=200, content_type="text/javascript", body=PDF_TASK_MODULE,
1191
  ))
1192
  self.open_pdf()
@@ -1249,7 +1287,7 @@ class ReaderPerformanceTest(unittest.TestCase):
1249
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1250
  requests = []
1251
  page.on("request", lambda request: requests.append(request.url) if "reader-content" in request.url or request.url == SOURCE_URL else None)
1252
- page.route("**/static/vendor/pdf.min.f80490490320.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_TASK_MODULE))
1253
  page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(SOURCE_URL, safe='')}&ext=pdf", wait_until="domcontentloaded")
1254
  page.wait_for_function("() => Boolean(window.__pdfProbe)")
1255
  if phase == "rendering":
@@ -2007,8 +2045,8 @@ class ReaderPerformanceTest(unittest.TestCase):
2007
  context = self.browser.new_context(viewport={"width": 390, "height": 844})
2008
  page = context.new_page()
2009
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
2010
- page.route("**/static/vendor/marked.min.b147274a9ce2.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
2011
- page.route("**/static/vendor/purify.min.f263b05369e0.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
2012
  page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="text/markdown", body=b"# Markdown readable"))
2013
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/readme.{extension}"
2014
  page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=Markdown", wait_until="domcontentloaded")
@@ -2113,11 +2151,11 @@ class ReaderPerformanceTest(unittest.TestCase):
2113
  return fulfill
2114
 
2115
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
2116
- page.route("**/static/vendor/marked.min.b147274a9ce2.js", timed("marked", "text/javascript", MARKED_SCRIPT))
2117
- page.route("**/static/vendor/purify.min.f263b05369e0.js", timed("purify", "text/javascript", PURIFY_SCRIPT))
2118
- page.route("**/static/vendor/jszip.min.7f839b2d4688.js", timed("jszip", "text/javascript", JSZIP_SCRIPT))
2119
  page.route("**/static/vendor/epub.min.06eae1574510.js", timed("epub", "text/javascript", EPUB_SCRIPT))
2120
- page.route("**/static/vendor/docx-preview.min.051ef503f267.js", timed("docx", "text/javascript", DOCX_SCRIPT))
2121
  content_type = "image/png" if extension == "png" else "application/octet-stream"
2122
  body = PNG_BYTES if extension == "png" else minimal_docx() if extension == "docx" else minimal_epub() if extension == "epub" else b"Reader benchmark content"
2123
  page.route("**/api/reader-content**", timed("content", content_type, body))
@@ -2151,9 +2189,9 @@ class ReaderPerformanceTest(unittest.TestCase):
2151
  def test_cold_cache_first_read_with_real_format_engines(self):
2152
  cases = [
2153
  ("txt", b"Cold TXT readable", "text/plain", "已加载", ".reader-text", []),
2154
- ("md", b"# Cold Markdown", "text/markdown", "已加载", ".reader-markdown", ["marked.min.b147274a9ce2.js", "purify.min.f263b05369e0.js"]),
2155
- ("pdf", minimal_pdf(), "application/pdf", "1 页", ".reader-page canvas.ready", ["pdf.min.f80490490320.mjs", "pdf.worker.min.8ab0e5e30031.mjs"]),
2156
- ("docx", minimal_docx(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "1 页", ".docx-body", ["jszip.min.7f839b2d4688.js", "docx-preview.min.051ef503f267.js"]),
2157
  ("png", PNG_BYTES, "image/png", "图片", ".reader-image", []),
2158
  ]
2159
  results = []
@@ -2164,6 +2202,7 @@ class ReaderPerformanceTest(unittest.TestCase):
2164
  page = context.new_page(); session = context.new_cdp_session(page)
2165
  session.send("Network.enable"); session.send("Network.setCacheDisabled", {"cacheDisabled": True})
2166
  responses = []; page.on("response", lambda response: responses.append(response))
 
2167
 
2168
  def serve_document(route):
2169
  route.fulfill(status=200, content_type=content_type, body=document)
@@ -2192,7 +2231,11 @@ class ReaderPerformanceTest(unittest.TestCase):
2192
  entry = next((item for item in entries if item["name"].endswith(engine)), None)
2193
  self.assertIsNotNone(entry, engine)
2194
  self.assertGreater(entry["transferSize"], 0, f"{engine} was not transferred on a cold load")
2195
- byte_count = len(document) + sum((ROOT / "static/vendor" / engine).stat().st_size for engine in engines)
 
 
 
 
2196
  results.append((extension, elapsed, byte_count, len(responses)))
2197
  finally:
2198
  context.close()
 
7
  import json
8
  import mimetypes
9
  import pathlib
10
+ import re
11
  import threading
12
  import time
13
  import unittest
 
24
 
25
 
26
  ROOT = pathlib.Path(__file__).resolve().parents[1]
27
+
28
+
29
+ def vendor_name(key, pattern):
30
+ resources = (ROOT / "static/reader-resources.js").read_text(encoding="utf-8")
31
+ match = re.search(rf"\b{re.escape(key)}:\s*\{{.*?sha256:\s*\"([0-9a-f]+)\"", resources, re.S)
32
+ if not match:
33
+ raise RuntimeError(f"missing vendor digest for {key!r}")
34
+ return pattern.replace("*", match.group(1)[:12], 1)
35
+
36
+
37
+ VENDOR_FILES = {
38
+ "pdf": vendor_name("pdf", "pdf.min.*.mjs"),
39
+ "pdf_worker": vendor_name("pdfWorker", "pdf.worker.min.*.mjs"),
40
+ "marked": vendor_name("marked", "marked.min.*.js"),
41
+ "purify": vendor_name("purify", "purify.min.*.js"),
42
+ "jszip": vendor_name("jszip", "jszip.min.*.js"),
43
+ "docx": vendor_name("docx", "docx-preview.min.*.js"),
44
+ }
45
+ VENDOR_STEMS = {
46
+ "pdf": "pdf.min.*.mjs",
47
+ "pdf_worker": "pdf.worker.min.*.mjs",
48
+ "marked": "marked.min.*.js",
49
+ "purify": "purify.min.*.js",
50
+ "jszip": "jszip.min.*.js",
51
+ "docx": "docx-preview.min.*.js",
52
+ }
53
+
54
+
55
+ def route_local_vendor_fallback(page):
56
+ for key, requested in VENDOR_FILES.items():
57
+ candidates = sorted((ROOT / "static/vendor").glob(VENDOR_STEMS[key]))
58
+ if not candidates:
59
+ continue
60
+ page.route(
61
+ f"**/static/vendor/{requested}",
62
+ lambda route, _request, path=str(candidates[-1]): route.fulfill(path=path),
63
+ )
64
  SOURCE_URL = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/performance.pdf"
65
  PDF_MODULE = """
66
  export const GlobalWorkerOptions = {};
 
298
  self.page = self.context.new_page()
299
  self.pdf_requested_at = None
300
  self.page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
301
+ self.page.route("**/static/vendor/pdf.min.*.mjs", self.route_pdf)
302
  self.page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
303
 
304
  def tearDown(self):
 
512
  self.assertEqual(first_page.locator(".reader-pdf-text").text_content(), "Accessible PDF text")
513
 
514
  def test_scanned_pdf_bookmark_has_empty_excerpt(self):
515
+ self.page.route('**/static/vendor/pdf.min.*.mjs', lambda route: route.fulfill(
516
  content_type='text/javascript', body=PDF_MODULE.replace("[{ str: 'Accessible PDF text', hasEOL: false }]", '[]')))
517
  self.open_pdf()
518
  self.page.wait_for_function("document.querySelector('.reader-pdf-text')?.textContent.includes('此页没有可提取文本')")
 
621
  mobile_context = self.browser.new_context(viewport={"width": 390, "height": 844})
622
  mobile_page = mobile_context.new_page()
623
  mobile_page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
624
+ mobile_page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
625
  mobile_page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
626
  query = urllib.parse.quote(SOURCE_URL, safe="")
627
  mobile_page.goto(f"{self.origin}/static/reader.html?url={query}&ext=pdf&title=Performance", wait_until="domcontentloaded")
 
701
  context = self.browser.new_context(viewport={"width": 390, "height": 844})
702
  page = context.new_page()
703
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
704
+ page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
705
  if extension == "md":
706
+ page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
707
+ page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
708
  if extension == "docx":
709
+ page.route("**/static/vendor/jszip.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=JSZIP_SCRIPT))
710
+ page.route("**/static/vendor/docx-preview.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=DOCX_SCRIPT))
711
  page.route("**/api/reader-content**", lambda route, _request, content_type=content_type, body=body: route.fulfill(status=200, content_type=content_type, body=body))
712
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/matrix.{extension}"
713
  if extension == "docx":
 
1164
  self.addCleanup(context.close)
1165
  page = context.new_page()
1166
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1167
+ page.route("**/static/vendor/jszip.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.JSZip=function(){window.__archiveParserStarted=true};"))
1168
+ page.route("**/static/vendor/docx-preview.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.docx={renderAsync(){window.__archiveParserStarted=true}};"))
1169
  page.route("**/static/foliate-reader/view.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="customElements.define('foliate-view', class extends HTMLElement { open() { window.__archiveParserStarted=true; throw new Error('parser must not start'); } });"))
1170
  page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/octet-stream", body=payload))
1171
  source = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/bomb.epub" if extension == "epub" else f"https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/objects/aa/{'a' * 64}/docx-native-v1/document.docx"
 
1192
  context = self.browser.new_context(viewport={"width": 390, "height": 844}); page = context.new_page()
1193
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1194
  if extension == "md":
1195
+ page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.marked={parse:()=>'<h1>Markdown one</h1><p style=\"height:1000px\"></p><h2>Markdown two</h2>'};")); page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
1196
  page.route("**/api/reader-content**", lambda route, _request, body=body: route.fulfill(status=200, content_type="text/html" if extension == "html" else "text/markdown", body=body))
1197
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/toc.{extension}"; page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=TOC", wait_until="domcontentloaded"); page.locator("#history").click(); page.locator("#toc-list .panel-item-main").nth(1).click()
1198
  page.wait_for_function("() => document.querySelector('iframe') ? document.querySelector('iframe').contentWindow.scrollY > 0 : document.querySelector('#viewport').scrollTop > 0"); context.close()
1199
 
1200
  def test_pdf_outline_click_navigates_to_declared_page(self):
1201
  self.page.add_init_script("window.__pdfOutlineEnabled = true")
1202
+ module = PDF_MODULE.replace("[{title: '第一章', dest: [{}], items: []}]", "[{title: '第三章', dest: [{}], items: []}]").replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: () => Promise.resolve(2)"); self.page.unroute("**/static/vendor/pdf.min.*.mjs"); self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module)); self.open_pdf(); self.page.locator("#history").click(); self.page.locator("#toc-list .panel-item-main").click(); self.page.wait_for_function("() => document.querySelector('#page-number').value === '3'")
1203
 
1204
  def test_pdf_outline_marks_current_entry_when_page_changes(self):
1205
  self.page.add_init_script("window.__pdfOutlineEnabled = true")
 
1207
  "[{title: '第一章', dest: [{}], items: []}]",
1208
  "[{title: '第一章', dest: [{page: 0}], items: []}, {title: '第二章', dest: [{page: 2}], items: []}]",
1209
  ).replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: (ref) => Promise.resolve(ref.page || 0)")
1210
+ self.page.unroute("**/static/vendor/pdf.min.*.mjs")
1211
+ self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module))
1212
  self.open_pdf()
1213
  self.page.locator("#history").click()
1214
  self.page.locator("#toc-list .toc-item").nth(1).wait_for()
 
1224
  def test_pdf_stale_navigation_and_search_keep_latest_state(self):
1225
  errors = []
1226
  self.page.on("pageerror", lambda error: errors.append(str(error)))
1227
+ self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(
1228
  status=200, content_type="text/javascript", body=PDF_TASK_MODULE,
1229
  ))
1230
  self.open_pdf()
 
1287
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
1288
  requests = []
1289
  page.on("request", lambda request: requests.append(request.url) if "reader-content" in request.url or request.url == SOURCE_URL else None)
1290
+ page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_TASK_MODULE))
1291
  page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(SOURCE_URL, safe='')}&ext=pdf", wait_until="domcontentloaded")
1292
  page.wait_for_function("() => Boolean(window.__pdfProbe)")
1293
  if phase == "rendering":
 
2045
  context = self.browser.new_context(viewport={"width": 390, "height": 844})
2046
  page = context.new_page()
2047
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
2048
+ page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
2049
+ page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
2050
  page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="text/markdown", body=b"# Markdown readable"))
2051
  source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/readme.{extension}"
2052
  page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=Markdown", wait_until="domcontentloaded")
 
2151
  return fulfill
2152
 
2153
  page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
2154
+ page.route("**/static/vendor/marked.min.*.js", timed("marked", "text/javascript", MARKED_SCRIPT))
2155
+ page.route("**/static/vendor/purify.min.*.js", timed("purify", "text/javascript", PURIFY_SCRIPT))
2156
+ page.route("**/static/vendor/jszip.min.*.js", timed("jszip", "text/javascript", JSZIP_SCRIPT))
2157
  page.route("**/static/vendor/epub.min.06eae1574510.js", timed("epub", "text/javascript", EPUB_SCRIPT))
2158
+ page.route("**/static/vendor/docx-preview.min.*.js", timed("docx", "text/javascript", DOCX_SCRIPT))
2159
  content_type = "image/png" if extension == "png" else "application/octet-stream"
2160
  body = PNG_BYTES if extension == "png" else minimal_docx() if extension == "docx" else minimal_epub() if extension == "epub" else b"Reader benchmark content"
2161
  page.route("**/api/reader-content**", timed("content", content_type, body))
 
2189
  def test_cold_cache_first_read_with_real_format_engines(self):
2190
  cases = [
2191
  ("txt", b"Cold TXT readable", "text/plain", "已加载", ".reader-text", []),
2192
+ ("md", b"# Cold Markdown", "text/markdown", "已加载", ".reader-markdown", [VENDOR_FILES["marked"], VENDOR_FILES["purify"]]),
2193
+ ("pdf", minimal_pdf(), "application/pdf", "1 页", ".reader-page canvas.ready", [VENDOR_FILES["pdf"], VENDOR_FILES["pdf_worker"]]),
2194
+ ("docx", minimal_docx(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "1 页", ".docx-body", [VENDOR_FILES["jszip"], VENDOR_FILES["docx"]]),
2195
  ("png", PNG_BYTES, "image/png", "图片", ".reader-image", []),
2196
  ]
2197
  results = []
 
2202
  page = context.new_page(); session = context.new_cdp_session(page)
2203
  session.send("Network.enable"); session.send("Network.setCacheDisabled", {"cacheDisabled": True})
2204
  responses = []; page.on("response", lambda response: responses.append(response))
2205
+ route_local_vendor_fallback(page)
2206
 
2207
  def serve_document(route):
2208
  route.fulfill(status=200, content_type=content_type, body=document)
 
2231
  entry = next((item for item in entries if item["name"].endswith(engine)), None)
2232
  self.assertIsNotNone(entry, engine)
2233
  self.assertGreater(entry["transferSize"], 0, f"{engine} was not transferred on a cold load")
2234
+ byte_count = len(document) + sum(
2235
+ (ROOT / "static/vendor" / engine).stat().st_size
2236
+ if (ROOT / "static/vendor" / engine).exists() else 0
2237
+ for engine in engines
2238
+ )
2239
  results.append((extension, elapsed, byte_count, len(responses)))
2240
  finally:
2241
  context.close()
upstream_http.py CHANGED
@@ -9,7 +9,7 @@ ALLOWED_DOWNLOAD_HOSTS = {"huggingface.co", "hf-mirror.com", "hf.co"}
9
  ALLOWED_DOWNLOAD_HOST_SUFFIXES = (".hf.co", ".huggingface.co", ".xethub.hf.co")
10
  MAX_REDIRECTS = 5
11
  READER_ASSET_PATH_RE = re.compile(
12
- r"objects/[0-9a-f]{2}/[0-9a-f]{64}/(?:[0-9a-f]{16}/)?(?:page-manifest\.json|pages/page-[0-9]{6}\.webp|(?:[a-z0-9-]+/)?(?:linearized\.pdf|chapter-manifest\.json|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz)))"
13
  )
14
  READER_ASSET_SOURCE_RE = re.compile(
15
  r"/datasets/vomebook/Reader-Assets/resolve/main/(?:pdf_manifest\.json|" + READER_ASSET_PATH_RE.pattern + r")"
 
9
  ALLOWED_DOWNLOAD_HOST_SUFFIXES = (".hf.co", ".huggingface.co", ".xethub.hf.co")
10
  MAX_REDIRECTS = 5
11
  READER_ASSET_PATH_RE = re.compile(
12
+ r"objects/[0-9a-f]{2}/[0-9a-f]{64}/(?:[0-9a-f]{16}/)?(?:page-manifest\.json|ocr-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl)|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz)|(?:[a-z0-9-]+/)?(?:linearized\.pdf|chapter-manifest\.json|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz)))"
13
  )
14
  READER_ASSET_SOURCE_RE = re.compile(
15
  r"/datasets/vomebook/Reader-Assets/resolve/main/(?:pdf_manifest\.json|" + READER_ASSET_PATH_RE.pattern + r")"