OpenCode commited on
Commit ·
bbb9a5b
1
Parent(s): 477ef5f
feat: add PDF OCR reader support
Browse files- app.py +160 -26
- reader_assets.py +34 -9
- record_metadata.py +25 -5
- static/reader-contract.js +34 -11
- static/reader-pdf-text.js +19 -0
- static/reader-security.js +48 -0
- static/reader.css +3 -1
- static/reader.js +192 -18
- tests/test_fastapi_endpoints.py +25 -0
- tests/test_reader_pdf_loading.py +1 -1
- tests/test_reader_performance.py +69 -26
- upstream_http.py +1 -1
app.py
CHANGED
|
@@ -35,7 +35,8 @@ from search_indexes import (
|
|
| 35 |
resolve_postings,
|
| 36 |
)
|
| 37 |
from reader_assets import (
|
| 38 |
-
READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE,
|
|
|
|
| 39 |
)
|
| 40 |
from data_formats import (
|
| 41 |
decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
|
|
@@ -1461,12 +1462,14 @@ READER_ISOLATION_HEADERS = {
|
|
| 1461 |
}
|
| 1462 |
READER_DOCUMENT_MAX_BYTES = 512 * 1024 * 1024
|
| 1463 |
READER_RESOURCE_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
|
| 1464 |
-
READER_RESOURCE_MAX_BYTES =
|
| 1465 |
READER_BUCKET_CACHE_TTL_SECONDS = 3600
|
| 1466 |
READER_BUCKET_CACHE_MAX_BYTES = 32 * 1024 * 1024
|
| 1467 |
READER_BUCKET_CACHE_MAX_ITEM_BYTES = 8 * 1024 * 1024
|
| 1468 |
READER_BUCKET_MAX_CONCURRENT_READS = 8
|
| 1469 |
READER_BUCKET_MAX_INFLIGHT = 32
|
|
|
|
|
|
|
| 1470 |
READER_ARCHIVE_MAX_ENTRIES = 20_000
|
| 1471 |
READER_ARCHIVE_MAX_EXPANDED_BYTES = 1536 * 1024 * 1024
|
| 1472 |
READER_ARCHIVE_MAX_COMPRESSION_RATIO = 200
|
|
@@ -1608,15 +1611,17 @@ def validate_reader_archive(archive: bytes) -> None:
|
|
| 1608 |
READER_RESOURCE_TYPES = {
|
| 1609 |
"json": "application/json",
|
| 1610 |
"css": "text/css", "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png",
|
| 1611 |
-
"gif": "image/gif", "webp": "image/webp", "svg": "image/svg+xml", "ttf": "font/ttf",
|
| 1612 |
"otf": "font/otf", "woff": "font/woff", "woff2": "font/woff2", "mp3": "audio/mpeg",
|
| 1613 |
-
"mp4": "video/mp4",
|
| 1614 |
}
|
| 1615 |
def initialize_reader_bucket_state(target_app: FastAPI) -> None:
|
| 1616 |
target_app.state.reader_bucket_cache = OrderedDict()
|
| 1617 |
target_app.state.reader_bucket_cache_bytes = 0
|
| 1618 |
target_app.state.reader_bucket_inflight = {}
|
| 1619 |
target_app.state.reader_bucket_semaphore = asyncio.Semaphore(READER_BUCKET_MAX_CONCURRENT_READS)
|
|
|
|
|
|
|
| 1620 |
|
| 1621 |
|
| 1622 |
async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
|
|
@@ -1628,14 +1633,24 @@ async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
|
|
| 1628 |
await asyncio.gather(*tasks, return_exceptions=True)
|
| 1629 |
target_app.state.reader_bucket_cache.clear()
|
| 1630 |
target_app.state.reader_bucket_cache_bytes = 0
|
|
|
|
|
|
|
| 1631 |
|
| 1632 |
|
| 1633 |
-
def is_reader_bucket_cacheable(path: str) -> bool:
|
| 1634 |
-
return
|
|
|
|
|
|
|
| 1635 |
|
| 1636 |
|
| 1637 |
-
def
|
| 1638 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1639 |
with HfFileSystem().open(uri, "rb") as stream:
|
| 1640 |
payload = stream.read(READER_RESOURCE_MAX_BYTES + 1)
|
| 1641 |
if len(payload) > READER_RESOURCE_MAX_BYTES:
|
|
@@ -1643,54 +1658,116 @@ def read_reader_bucket_resource(path: str) -> bytes:
|
|
| 1643 |
return payload
|
| 1644 |
|
| 1645 |
|
| 1646 |
-
async def get_reader_bucket_resource(path: str) -> bytes:
|
| 1647 |
state = app.state
|
| 1648 |
-
if
|
| 1649 |
-
|
|
|
|
| 1650 |
if cached:
|
| 1651 |
cached_at, payload = cached
|
| 1652 |
if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
|
| 1653 |
-
state.reader_bucket_cache.move_to_end(
|
| 1654 |
return payload
|
| 1655 |
-
state.reader_bucket_cache.pop(
|
| 1656 |
state.reader_bucket_cache_bytes -= len(payload)
|
| 1657 |
|
| 1658 |
-
task = state.reader_bucket_inflight.get(
|
| 1659 |
if task is None:
|
| 1660 |
if len(state.reader_bucket_inflight) >= READER_BUCKET_MAX_INFLIGHT:
|
| 1661 |
raise RuntimeError("too many reader bucket requests")
|
| 1662 |
async def load() -> bytes:
|
| 1663 |
try:
|
| 1664 |
async with state.reader_bucket_semaphore:
|
| 1665 |
-
|
| 1666 |
-
|
| 1667 |
-
|
|
|
|
|
|
|
|
|
|
| 1668 |
if previous:
|
| 1669 |
state.reader_bucket_cache_bytes -= len(previous[1])
|
| 1670 |
-
state.reader_bucket_cache[
|
| 1671 |
state.reader_bucket_cache_bytes += len(payload)
|
| 1672 |
while state.reader_bucket_cache_bytes > READER_BUCKET_CACHE_MAX_BYTES:
|
| 1673 |
_, (_, evicted) = state.reader_bucket_cache.popitem(last=False)
|
| 1674 |
state.reader_bucket_cache_bytes -= len(evicted)
|
| 1675 |
return payload
|
| 1676 |
finally:
|
| 1677 |
-
if state.reader_bucket_inflight.get(
|
| 1678 |
-
state.reader_bucket_inflight.pop(
|
| 1679 |
|
| 1680 |
task = asyncio.create_task(load())
|
| 1681 |
task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
|
| 1682 |
-
state.reader_bucket_inflight[
|
| 1683 |
return await asyncio.shield(task)
|
| 1684 |
|
| 1685 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1686 |
@app.api_route("/api/reader-bucket-resource", methods=["GET", "HEAD"])
|
| 1687 |
-
async def api_reader_bucket_resource(path: str = Query(...), request: Request = None):
|
| 1688 |
resource_path = unquote(unquote(path)).replace("\\", "/").lstrip("/")
|
| 1689 |
-
|
|
|
|
|
|
|
| 1690 |
return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400,
|
| 1691 |
headers={"Cache-Control": "no-store"})
|
| 1692 |
try:
|
| 1693 |
-
payload = await get_reader_bucket_resource(resource_path)
|
| 1694 |
except FileNotFoundError:
|
| 1695 |
return JSONResponse({"error": "阅读资源不存在"}, status_code=404,
|
| 1696 |
headers={"Cache-Control": "no-store"})
|
|
@@ -1867,6 +1944,61 @@ async def api_reader_resolve(id: str = Query(..., min_length=13, max_length=13))
|
|
| 1867 |
return JSONResponse({"error": "阅读文件不存在"}, status_code=404)
|
| 1868 |
|
| 1869 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1870 |
@app.api_route("/api/reader-content", methods=["GET", "HEAD"])
|
| 1871 |
async def api_reader_content(request: Request, url: Optional[str] = Query(default=None), id: Optional[str] = Query(default=None)):
|
| 1872 |
if id and not url:
|
|
@@ -1885,9 +2017,11 @@ async def api_reader_content(request: Request, url: Optional[str] = Query(defaul
|
|
| 1885 |
return JSONResponse({"error": str(exc)}, status_code=403)
|
| 1886 |
if not parsed.scheme and not parsed.netloc and parsed.path == "/api/reader-bucket-resource" and not parsed.fragment:
|
| 1887 |
parameters = parse_qs(parsed.query)
|
| 1888 |
-
if set(parameters)
|
|
|
|
|
|
|
| 1889 |
return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
|
| 1890 |
-
return await api_reader_bucket_resource(parameters["path"][0], request)
|
| 1891 |
try:
|
| 1892 |
target_url = validate_reader_source_url(url)
|
| 1893 |
except ValueError as exc:
|
|
|
|
| 35 |
resolve_postings,
|
| 36 |
)
|
| 37 |
from reader_assets import (
|
| 38 |
+
READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE, READER_PDF_RANGE_BUCKET,
|
| 39 |
+
READER_PDF_RANGE_PATH_RE, decode_reader_assets,
|
| 40 |
)
|
| 41 |
from data_formats import (
|
| 42 |
decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
|
|
|
|
| 1462 |
}
|
| 1463 |
READER_DOCUMENT_MAX_BYTES = 512 * 1024 * 1024
|
| 1464 |
READER_RESOURCE_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
|
| 1465 |
+
READER_RESOURCE_MAX_BYTES = 64 * 1024 * 1024
|
| 1466 |
READER_BUCKET_CACHE_TTL_SECONDS = 3600
|
| 1467 |
READER_BUCKET_CACHE_MAX_BYTES = 32 * 1024 * 1024
|
| 1468 |
READER_BUCKET_CACHE_MAX_ITEM_BYTES = 8 * 1024 * 1024
|
| 1469 |
READER_BUCKET_MAX_CONCURRENT_READS = 8
|
| 1470 |
READER_BUCKET_MAX_INFLIGHT = 32
|
| 1471 |
+
READER_OCR_CACHE_MAX_BYTES = 32 * 1024 * 1024
|
| 1472 |
+
READER_OCR_CACHE_MAX_ITEMS = 4
|
| 1473 |
READER_ARCHIVE_MAX_ENTRIES = 20_000
|
| 1474 |
READER_ARCHIVE_MAX_EXPANDED_BYTES = 1536 * 1024 * 1024
|
| 1475 |
READER_ARCHIVE_MAX_COMPRESSION_RATIO = 200
|
|
|
|
| 1611 |
READER_RESOURCE_TYPES = {
|
| 1612 |
"json": "application/json",
|
| 1613 |
"css": "text/css", "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png",
|
| 1614 |
+
"gif": "image/gif", "webp": "image/webp", "jxl": "image/jxl", "svg": "image/svg+xml", "ttf": "font/ttf",
|
| 1615 |
"otf": "font/otf", "woff": "font/woff", "woff2": "font/woff2", "mp3": "audio/mpeg",
|
| 1616 |
+
"mp4": "video/mp4", "gz": "application/gzip",
|
| 1617 |
}
|
| 1618 |
def initialize_reader_bucket_state(target_app: FastAPI) -> None:
|
| 1619 |
target_app.state.reader_bucket_cache = OrderedDict()
|
| 1620 |
target_app.state.reader_bucket_cache_bytes = 0
|
| 1621 |
target_app.state.reader_bucket_inflight = {}
|
| 1622 |
target_app.state.reader_bucket_semaphore = asyncio.Semaphore(READER_BUCKET_MAX_CONCURRENT_READS)
|
| 1623 |
+
target_app.state.reader_ocr_cache = OrderedDict()
|
| 1624 |
+
target_app.state.reader_ocr_cache_bytes = 0
|
| 1625 |
|
| 1626 |
|
| 1627 |
async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
|
|
|
|
| 1633 |
await asyncio.gather(*tasks, return_exceptions=True)
|
| 1634 |
target_app.state.reader_bucket_cache.clear()
|
| 1635 |
target_app.state.reader_bucket_cache_bytes = 0
|
| 1636 |
+
target_app.state.reader_ocr_cache.clear()
|
| 1637 |
+
target_app.state.reader_ocr_cache_bytes = 0
|
| 1638 |
|
| 1639 |
|
| 1640 |
+
def is_reader_bucket_cacheable(bucket: str, path: str) -> bool:
|
| 1641 |
+
return (bucket == "vomebook/pdf-pages" and
|
| 1642 |
+
(path.endswith("/page-manifest.json") or path.endswith("/ocr-manifest.json")
|
| 1643 |
+
or path.endswith("/ocr/book-text.json.gz") or path.endswith("/pages/page-000001.webp")))
|
| 1644 |
|
| 1645 |
|
| 1646 |
+
def reader_bucket_cache_key(bucket: str, path: str):
|
| 1647 |
+
# Keep the default bucket namespace compatible with the existing cache;
|
| 1648 |
+
# isolate the optional optimized-PDF bucket by namespace.
|
| 1649 |
+
return path if bucket == "vomebook/pdf-pages" else (bucket, path)
|
| 1650 |
+
|
| 1651 |
+
|
| 1652 |
+
def read_reader_bucket_resource(path: str, bucket: str = "vomebook/pdf-pages") -> bytes:
|
| 1653 |
+
uri = f"hf://buckets/{bucket}/{path}"
|
| 1654 |
with HfFileSystem().open(uri, "rb") as stream:
|
| 1655 |
payload = stream.read(READER_RESOURCE_MAX_BYTES + 1)
|
| 1656 |
if len(payload) > READER_RESOURCE_MAX_BYTES:
|
|
|
|
| 1658 |
return payload
|
| 1659 |
|
| 1660 |
|
| 1661 |
+
async def get_reader_bucket_resource(path: str, bucket: str = "vomebook/pdf-pages") -> bytes:
|
| 1662 |
state = app.state
|
| 1663 |
+
cache_key = path if bucket == "vomebook/pdf-pages" else (bucket, path)
|
| 1664 |
+
if is_reader_bucket_cacheable(bucket, path):
|
| 1665 |
+
cached = state.reader_bucket_cache.get(cache_key)
|
| 1666 |
if cached:
|
| 1667 |
cached_at, payload = cached
|
| 1668 |
if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
|
| 1669 |
+
state.reader_bucket_cache.move_to_end(cache_key)
|
| 1670 |
return payload
|
| 1671 |
+
state.reader_bucket_cache.pop(cache_key)
|
| 1672 |
state.reader_bucket_cache_bytes -= len(payload)
|
| 1673 |
|
| 1674 |
+
task = state.reader_bucket_inflight.get(cache_key)
|
| 1675 |
if task is None:
|
| 1676 |
if len(state.reader_bucket_inflight) >= READER_BUCKET_MAX_INFLIGHT:
|
| 1677 |
raise RuntimeError("too many reader bucket requests")
|
| 1678 |
async def load() -> bytes:
|
| 1679 |
try:
|
| 1680 |
async with state.reader_bucket_semaphore:
|
| 1681 |
+
loader = (lambda: read_reader_bucket_resource(path)
|
| 1682 |
+
if bucket == "vomebook/pdf-pages"
|
| 1683 |
+
else read_reader_bucket_resource(path, bucket))
|
| 1684 |
+
payload = await asyncio.to_thread(loader)
|
| 1685 |
+
if is_reader_bucket_cacheable(bucket, path) and len(payload) <= READER_BUCKET_CACHE_MAX_ITEM_BYTES:
|
| 1686 |
+
previous = state.reader_bucket_cache.pop(cache_key, None)
|
| 1687 |
if previous:
|
| 1688 |
state.reader_bucket_cache_bytes -= len(previous[1])
|
| 1689 |
+
state.reader_bucket_cache[cache_key] = (time.monotonic(), payload)
|
| 1690 |
state.reader_bucket_cache_bytes += len(payload)
|
| 1691 |
while state.reader_bucket_cache_bytes > READER_BUCKET_CACHE_MAX_BYTES:
|
| 1692 |
_, (_, evicted) = state.reader_bucket_cache.popitem(last=False)
|
| 1693 |
state.reader_bucket_cache_bytes -= len(evicted)
|
| 1694 |
return payload
|
| 1695 |
finally:
|
| 1696 |
+
if state.reader_bucket_inflight.get(cache_key) is asyncio.current_task():
|
| 1697 |
+
state.reader_bucket_inflight.pop(cache_key, None)
|
| 1698 |
|
| 1699 |
task = asyncio.create_task(load())
|
| 1700 |
task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
|
| 1701 |
+
state.reader_bucket_inflight[cache_key] = task
|
| 1702 |
return await asyncio.shield(task)
|
| 1703 |
|
| 1704 |
|
| 1705 |
+
def parse_reader_ocr_book(raw: bytes) -> tuple[list[tuple[int, str, str]], int]:
|
| 1706 |
+
"""Decode and validate the compact per-page OCR text cache off the event loop."""
|
| 1707 |
+
with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream:
|
| 1708 |
+
decoded = stream.read(READER_RESOURCE_MAX_BYTES + 1)
|
| 1709 |
+
if len(decoded) > READER_RESOURCE_MAX_BYTES:
|
| 1710 |
+
raise ValueError("PDF OCR text index is too large")
|
| 1711 |
+
payload = json.loads(decoded)
|
| 1712 |
+
pages = payload.get("pages") if isinstance(payload, dict) else None
|
| 1713 |
+
if (not isinstance(pages, list) or len(pages) > 200000
|
| 1714 |
+
or payload.get("version") != 1 or payload.get("kind") != "pdf-ocr-book-text"):
|
| 1715 |
+
raise ValueError("invalid PDF OCR text index")
|
| 1716 |
+
output = []
|
| 1717 |
+
total_bytes = 0
|
| 1718 |
+
previous = 0
|
| 1719 |
+
for item in pages:
|
| 1720 |
+
if (not isinstance(item, dict) or not isinstance(item.get("page"), int)
|
| 1721 |
+
or item["page"] < 1 or item["page"] <= previous
|
| 1722 |
+
or not isinstance(item.get("text"), str)):
|
| 1723 |
+
raise ValueError("invalid PDF OCR text page")
|
| 1724 |
+
text = item["text"]
|
| 1725 |
+
total_bytes += len(text.encode("utf-8"))
|
| 1726 |
+
if total_bytes > READER_RESOURCE_MAX_BYTES:
|
| 1727 |
+
raise ValueError("PDF OCR text index is too large")
|
| 1728 |
+
output.append((item["page"], text, text.casefold()))
|
| 1729 |
+
previous = item["page"]
|
| 1730 |
+
return output, total_bytes
|
| 1731 |
+
|
| 1732 |
+
|
| 1733 |
+
async def get_reader_ocr_book(path: str) -> list[tuple[int, str, str]]:
|
| 1734 |
+
state = app.state
|
| 1735 |
+
if not hasattr(state, "reader_ocr_cache"):
|
| 1736 |
+
state.reader_ocr_cache = OrderedDict()
|
| 1737 |
+
state.reader_ocr_cache_bytes = 0
|
| 1738 |
+
cached = state.reader_ocr_cache.get(path)
|
| 1739 |
+
if cached:
|
| 1740 |
+
cached_at, pages, size = cached
|
| 1741 |
+
if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
|
| 1742 |
+
state.reader_ocr_cache.move_to_end(path)
|
| 1743 |
+
return pages
|
| 1744 |
+
state.reader_ocr_cache.pop(path)
|
| 1745 |
+
state.reader_ocr_cache_bytes -= size
|
| 1746 |
+
raw = await get_reader_bucket_resource(path)
|
| 1747 |
+
pages, size = await asyncio.to_thread(parse_reader_ocr_book, raw)
|
| 1748 |
+
if size <= READER_OCR_CACHE_MAX_BYTES:
|
| 1749 |
+
previous = state.reader_ocr_cache.pop(path, None)
|
| 1750 |
+
if previous:
|
| 1751 |
+
state.reader_ocr_cache_bytes -= previous[2]
|
| 1752 |
+
state.reader_ocr_cache[path] = (time.monotonic(), pages, size)
|
| 1753 |
+
state.reader_ocr_cache_bytes += size
|
| 1754 |
+
while (len(state.reader_ocr_cache) > READER_OCR_CACHE_MAX_ITEMS
|
| 1755 |
+
or state.reader_ocr_cache_bytes > READER_OCR_CACHE_MAX_BYTES):
|
| 1756 |
+
_, (_, _, evicted_size) = state.reader_ocr_cache.popitem(last=False)
|
| 1757 |
+
state.reader_ocr_cache_bytes -= evicted_size
|
| 1758 |
+
return pages
|
| 1759 |
+
|
| 1760 |
+
|
| 1761 |
@app.api_route("/api/reader-bucket-resource", methods=["GET", "HEAD"])
|
| 1762 |
+
async def api_reader_bucket_resource(path: str = Query(...), bucket: str = "vomebook/pdf-pages", request: Request = None):
|
| 1763 |
resource_path = unquote(unquote(path)).replace("\\", "/").lstrip("/")
|
| 1764 |
+
valid = ((bucket == "vomebook/pdf-pages" and READER_BUCKET_PATH_RE.fullmatch(resource_path))
|
| 1765 |
+
or (bucket == READER_PDF_RANGE_BUCKET and READER_PDF_RANGE_PATH_RE.fullmatch(resource_path)))
|
| 1766 |
+
if not valid:
|
| 1767 |
return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400,
|
| 1768 |
headers={"Cache-Control": "no-store"})
|
| 1769 |
try:
|
| 1770 |
+
payload = await get_reader_bucket_resource(resource_path, bucket)
|
| 1771 |
except FileNotFoundError:
|
| 1772 |
return JSONResponse({"error": "阅读资源不存在"}, status_code=404,
|
| 1773 |
headers={"Cache-Control": "no-store"})
|
|
|
|
| 1944 |
return JSONResponse({"error": "阅读文件不存在"}, status_code=404)
|
| 1945 |
|
| 1946 |
|
| 1947 |
+
@app.get("/api/reader-ocr-search")
|
| 1948 |
+
async def api_reader_ocr_search(
|
| 1949 |
+
id: str = Query(..., min_length=13, max_length=13),
|
| 1950 |
+
q: str = Query(..., min_length=1, max_length=256),
|
| 1951 |
+
page: int = Query(1, ge=1, le=1000000),
|
| 1952 |
+
page_size: int = Query(50, ge=1, le=100),
|
| 1953 |
+
):
|
| 1954 |
+
"""Search one PDF's immutable OCR text without loading it into app memory."""
|
| 1955 |
+
resolved = await api_reader_resolve(id)
|
| 1956 |
+
if resolved.status_code != 200:
|
| 1957 |
+
return resolved
|
| 1958 |
+
try:
|
| 1959 |
+
data = json.loads(resolved.body)
|
| 1960 |
+
manifest_url = str(data.get("ocr_manifest") or "")
|
| 1961 |
+
parsed = urlparse(manifest_url)
|
| 1962 |
+
parameters = parse_qs(parsed.query)
|
| 1963 |
+
manifest_path = parameters.get("path", [""])[0]
|
| 1964 |
+
if (parsed.path != "/api/reader-bucket-resource" or set(parameters) != {"path"}
|
| 1965 |
+
or not READER_BUCKET_PATH_RE.fullmatch(manifest_path)
|
| 1966 |
+
or not manifest_path.endswith("/ocr-manifest.json")):
|
| 1967 |
+
return JSONResponse({"error": "本书暂无 OCR 索引"}, status_code=404)
|
| 1968 |
+
manifest = json.loads(await get_reader_bucket_resource(manifest_path))
|
| 1969 |
+
book = manifest.get("book_text") if isinstance(manifest, dict) else None
|
| 1970 |
+
book_path = book.get("path") if isinstance(book, dict) else ""
|
| 1971 |
+
if (not isinstance(book_path, str) or not READER_BUCKET_PATH_RE.fullmatch(book_path)
|
| 1972 |
+
or not book_path.endswith("/book-text.json.gz")):
|
| 1973 |
+
return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
|
| 1974 |
+
pages = await get_reader_ocr_book(book_path)
|
| 1975 |
+
needle = q.casefold()
|
| 1976 |
+
matches = []
|
| 1977 |
+
for page_number, text, folded in pages:
|
| 1978 |
+
start = 0
|
| 1979 |
+
while True:
|
| 1980 |
+
found = folded.find(needle, start)
|
| 1981 |
+
if found < 0:
|
| 1982 |
+
break
|
| 1983 |
+
end = found + len(q)
|
| 1984 |
+
snippet_start = max(0, found - 72)
|
| 1985 |
+
snippet_end = min(len(text), end + 88)
|
| 1986 |
+
matches.append({
|
| 1987 |
+
"page": page_number, "start": found, "length": len(q),
|
| 1988 |
+
"snippet": text[snippet_start:snippet_end],
|
| 1989 |
+
"snippet_start": found - snippet_start,
|
| 1990 |
+
})
|
| 1991 |
+
start = end
|
| 1992 |
+
total = len(matches)
|
| 1993 |
+
offset = (page - 1) * page_size
|
| 1994 |
+
return JSONResponse({
|
| 1995 |
+
"version": 1, "page": page, "page_size": page_size, "total": total,
|
| 1996 |
+
"results": matches[offset:offset + page_size],
|
| 1997 |
+
}, headers={"Cache-Control": "no-store"})
|
| 1998 |
+
except (ValueError, TypeError, KeyError, EOFError, gzip.BadGzipFile, zlib.error):
|
| 1999 |
+
return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
|
| 2000 |
+
|
| 2001 |
+
|
| 2002 |
@app.api_route("/api/reader-content", methods=["GET", "HEAD"])
|
| 2003 |
async def api_reader_content(request: Request, url: Optional[str] = Query(default=None), id: Optional[str] = Query(default=None)):
|
| 2004 |
if id and not url:
|
|
|
|
| 2017 |
return JSONResponse({"error": str(exc)}, status_code=403)
|
| 2018 |
if not parsed.scheme and not parsed.netloc and parsed.path == "/api/reader-bucket-resource" and not parsed.fragment:
|
| 2019 |
parameters = parse_qs(parsed.query)
|
| 2020 |
+
if (set(parameters) - {"path", "bucket"} or set(parameters) < {"path"}
|
| 2021 |
+
or len(parameters["path"]) != 1
|
| 2022 |
+
or len(parameters.get("bucket", ["x"])) != 1):
|
| 2023 |
return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
|
| 2024 |
+
return await api_reader_bucket_resource(parameters["path"][0], parameters.get("bucket", ["vomebook/pdf-pages"])[0], request)
|
| 2025 |
try:
|
| 2026 |
target_url = validate_reader_source_url(url)
|
| 2027 |
except ValueError as exc:
|
reader_assets.py
CHANGED
|
@@ -18,7 +18,11 @@ READER_ASSET_FILENAMES = {
|
|
| 18 |
"v": {"video.mp4"},
|
| 19 |
}
|
| 20 |
READER_BUCKET_PATH_RE = re.compile(
|
| 21 |
-
r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[0-9a-f]{16}/(?:page-manifest\.json|pages/page-[0-9]{6}\.webp)$"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
)
|
| 23 |
|
| 24 |
|
|
@@ -39,7 +43,7 @@ def decode_reader_assets(
|
|
| 39 |
raise ValueError("invalid reader assets sidecar")
|
| 40 |
clean = {}
|
| 41 |
for key, entry in files.items():
|
| 42 |
-
if not isinstance(key, str) or not isinstance(entry, dict) or entry.get("s") not in (2, 4):
|
| 43 |
continue
|
| 44 |
if entry.get("s") == 2:
|
| 45 |
path = entry.get("p")
|
|
@@ -54,6 +58,7 @@ def decode_reader_assets(
|
|
| 54 |
continue
|
| 55 |
chapter_manifest = entry.get("c")
|
| 56 |
fallback = entry.get("f")
|
|
|
|
| 57 |
if chapter_manifest is not None and (
|
| 58 |
mode not in {"e", "p"}
|
| 59 |
or not isinstance(chapter_manifest, str)
|
|
@@ -67,14 +72,25 @@ def decode_reader_assets(
|
|
| 67 |
or not fallback.endswith("/document.pdf")
|
| 68 |
):
|
| 69 |
continue
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
or
|
| 74 |
-
or not path.endswith("/page-manifest.json")
|
| 75 |
-
or not READER_BUCKET_PATH_RE.fullmatch(path)
|
| 76 |
):
|
| 77 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
clean[key] = {
|
| 79 |
"s": 2,
|
| 80 |
"m": mode,
|
|
@@ -82,7 +98,16 @@ def decode_reader_assets(
|
|
| 82 |
**({"b": bucket} if bucket else {}),
|
| 83 |
**({"c": chapter_manifest} if chapter_manifest else {}),
|
| 84 |
**({"f": fallback} if fallback else {}),
|
|
|
|
|
|
|
|
|
|
| 85 |
}
|
| 86 |
else:
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
return {"v": 1, "f": clean}
|
|
|
|
| 18 |
"v": {"video.mp4"},
|
| 19 |
}
|
| 20 |
READER_BUCKET_PATH_RE = re.compile(
|
| 21 |
+
r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[0-9a-f]{16}/(?:page-manifest\.json|ocr-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl)|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz))$"
|
| 22 |
+
)
|
| 23 |
+
READER_PDF_RANGE_BUCKET = "vomebook/pdf-optimized"
|
| 24 |
+
READER_PDF_RANGE_PATH_RE = re.compile(
|
| 25 |
+
r"^objects/[0-9a-f]{2}/[0-9a-f]{64}/[a-z0-9-]+/document\.pdf$"
|
| 26 |
)
|
| 27 |
|
| 28 |
|
|
|
|
| 43 |
raise ValueError("invalid reader assets sidecar")
|
| 44 |
clean = {}
|
| 45 |
for key, entry in files.items():
|
| 46 |
+
if not isinstance(key, str) or not isinstance(entry, dict) or entry.get("s") not in (2, 3, 4):
|
| 47 |
continue
|
| 48 |
if entry.get("s") == 2:
|
| 49 |
path = entry.get("p")
|
|
|
|
| 58 |
continue
|
| 59 |
chapter_manifest = entry.get("c")
|
| 60 |
fallback = entry.get("f")
|
| 61 |
+
ocr_manifest = entry.get("o")
|
| 62 |
if chapter_manifest is not None and (
|
| 63 |
mode not in {"e", "p"}
|
| 64 |
or not isinstance(chapter_manifest, str)
|
|
|
|
| 72 |
or not fallback.endswith("/document.pdf")
|
| 73 |
):
|
| 74 |
continue
|
| 75 |
+
if ocr_manifest is not None and (
|
| 76 |
+
not isinstance(ocr_manifest, str)
|
| 77 |
+
or not READER_ASSET_PATH_RE.fullmatch(ocr_manifest)
|
| 78 |
+
or not ocr_manifest.endswith("/ocr-manifest.json")
|
|
|
|
|
|
|
| 79 |
):
|
| 80 |
continue
|
| 81 |
+
if entry.get("b") is not None and entry.get("b") not in {"vomebook/pdf-pages", READER_PDF_RANGE_BUCKET}:
|
| 82 |
+
continue
|
| 83 |
+
if entry.get("ob") is not None and entry.get("ob") != "vomebook/pdf-pages":
|
| 84 |
+
continue
|
| 85 |
+
bucket = entry.get("b")
|
| 86 |
+
if bucket == "vomebook/pdf-pages" and (
|
| 87 |
+
mode != "p" or not path.endswith("/page-manifest.json")
|
| 88 |
+
or not READER_BUCKET_PATH_RE.fullmatch(path)):
|
| 89 |
+
continue
|
| 90 |
+
if bucket == READER_PDF_RANGE_BUCKET and (
|
| 91 |
+
mode != "p" or not path.endswith("/document.pdf")
|
| 92 |
+
or not READER_PDF_RANGE_PATH_RE.fullmatch(path)):
|
| 93 |
+
continue
|
| 94 |
clean[key] = {
|
| 95 |
"s": 2,
|
| 96 |
"m": mode,
|
|
|
|
| 98 |
**({"b": bucket} if bucket else {}),
|
| 99 |
**({"c": chapter_manifest} if chapter_manifest else {}),
|
| 100 |
**({"f": fallback} if fallback else {}),
|
| 101 |
+
**({"o": ocr_manifest} if ocr_manifest else {}),
|
| 102 |
+
**({"ob": "vomebook/pdf-pages"} if entry.get("ob") else {}),
|
| 103 |
+
**({"om": entry.get("om")} if isinstance(entry.get("om"), str) else {}),
|
| 104 |
}
|
| 105 |
else:
|
| 106 |
+
ocr_manifest = entry.get("o")
|
| 107 |
+
if (isinstance(ocr_manifest, str) and READER_ASSET_PATH_RE.fullmatch(ocr_manifest)
|
| 108 |
+
and ocr_manifest.endswith("/ocr-manifest.json")):
|
| 109 |
+
clean[key] = {"s": 3, "m": "p", "o": ocr_manifest, "b": "vomebook/pdf-pages",
|
| 110 |
+
**({"om": entry.get("om")} if isinstance(entry.get("om"), str) else {})}
|
| 111 |
+
else:
|
| 112 |
+
clean[key] = {"s": 4}
|
| 113 |
return {"v": 1, "f": clean}
|
record_metadata.py
CHANGED
|
@@ -73,8 +73,11 @@ def enrich_browser_entry(repo: str, path: str, entry: dict) -> dict:
|
|
| 73 |
|
| 74 |
def reader_asset_url(asset: dict) -> str:
|
| 75 |
path = quote(str(asset.get("p") or ""), safe="/")
|
| 76 |
-
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
| 78 |
return READER_ASSETS_BASE + path
|
| 79 |
|
| 80 |
|
|
@@ -94,9 +97,25 @@ def reader_extension_for_asset(asset: dict, original_extension: str = "") -> str
|
|
| 94 |
|
| 95 |
|
| 96 |
def reader_asset_fields(asset: dict, original_extension: str) -> dict:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
if asset.get("s") != 2 or asset.get("m") not in READER_ASSET_MODES or not asset.get("p"):
|
| 98 |
-
return
|
| 99 |
-
fields
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
for key, field in (("c", "chapter_manifest"), ("f", "fallback")):
|
| 101 |
if asset.get(key):
|
| 102 |
fields[field] = f"{READER_ASSETS_BASE}{asset[key]}"
|
|
@@ -108,7 +127,8 @@ def enrich_reader_record(record: dict, asset: dict) -> dict:
|
|
| 108 |
fields = reader_asset_fields(asset, record.get("Extension", ""))
|
| 109 |
if fields:
|
| 110 |
for source, target in (("url", "ReaderLink"), ("extension", "ReaderExtension"),
|
| 111 |
-
("chapter_manifest", "ReaderChapterManifest"), ("fallback", "ReaderFallback")
|
|
|
|
| 112 |
if source in fields:
|
| 113 |
result[target] = fields[source]
|
| 114 |
result["DownloadLink"] = record.get("Link") or build_record_link(record)
|
|
|
|
| 73 |
|
| 74 |
def reader_asset_url(asset: dict) -> str:
|
| 75 |
path = quote(str(asset.get("p") or ""), safe="/")
|
| 76 |
+
bucket = asset.get("b")
|
| 77 |
+
if bucket in {"vomebook/pdf-pages", "vomebook/pdf-optimized"}:
|
| 78 |
+
prefix = ("/api/reader-bucket-resource?path=" if bucket == "vomebook/pdf-pages"
|
| 79 |
+
else f"/api/reader-bucket-resource?bucket={quote(bucket, safe='')}&path=")
|
| 80 |
+
return prefix + path
|
| 81 |
return READER_ASSETS_BASE + path
|
| 82 |
|
| 83 |
|
|
|
|
| 97 |
|
| 98 |
|
| 99 |
def reader_asset_fields(asset: dict, original_extension: str) -> dict:
|
| 100 |
+
ocr_path = str(asset.get("o") or "")
|
| 101 |
+
fields = {}
|
| 102 |
+
if ocr_path and ocr_path.endswith("/ocr-manifest.json"):
|
| 103 |
+
fields["ocr_manifest"] = (
|
| 104 |
+
f"https://voiceofml-search.hf.space/api/reader-bucket-resource?path={quote(ocr_path, safe='/')}"
|
| 105 |
+
if asset.get("b") == "vomebook/pdf-pages" or asset.get("ob") == "vomebook/pdf-pages" else f"{READER_ASSETS_BASE}{ocr_path}"
|
| 106 |
+
)
|
| 107 |
+
fields["ocr_mode"] = str(asset.get("om") or "")
|
| 108 |
+
if asset.get("s") == 3:
|
| 109 |
+
return fields
|
| 110 |
if asset.get("s") != 2 or asset.get("m") not in READER_ASSET_MODES or not asset.get("p"):
|
| 111 |
+
return fields
|
| 112 |
+
fields.update({"url": reader_asset_url(asset), "extension": reader_extension_for_asset(asset, original_extension)})
|
| 113 |
+
if ocr_path and ocr_path.endswith("/ocr-manifest.json"):
|
| 114 |
+
fields["ocr_manifest"] = (
|
| 115 |
+
f"https://voiceofml-search.hf.space/api/reader-bucket-resource?path={quote(ocr_path, safe='/')}"
|
| 116 |
+
if asset.get("b") == "vomebook/pdf-pages" or asset.get("ob") == "vomebook/pdf-pages" else f"{READER_ASSETS_BASE}{ocr_path}"
|
| 117 |
+
)
|
| 118 |
+
fields["ocr_mode"] = str(asset.get("om") or "")
|
| 119 |
for key, field in (("c", "chapter_manifest"), ("f", "fallback")):
|
| 120 |
if asset.get(key):
|
| 121 |
fields[field] = f"{READER_ASSETS_BASE}{asset[key]}"
|
|
|
|
| 127 |
fields = reader_asset_fields(asset, record.get("Extension", ""))
|
| 128 |
if fields:
|
| 129 |
for source, target in (("url", "ReaderLink"), ("extension", "ReaderExtension"),
|
| 130 |
+
("chapter_manifest", "ReaderChapterManifest"), ("fallback", "ReaderFallback"),
|
| 131 |
+
("ocr_manifest", "ReaderOcrManifest"), ("ocr_mode", "ReaderOcrMode")):
|
| 132 |
if source in fields:
|
| 133 |
result[target] = fields[source]
|
| 134 |
result["DownloadLink"] = record.get("Link") or build_record_link(record)
|
static/reader-contract.js
CHANGED
|
@@ -212,16 +212,19 @@
|
|
| 212 |
|
| 213 |
const assetRoot = String.raw`objects/[0-9a-f]{2}/[0-9a-f]{64}/`;
|
| 214 |
const assetVersion = String.raw`(?:[0-9a-f]{16}/)?`;
|
| 215 |
-
const assetPages = String.raw`(?:page-manifest\.json|pages/page-[0-9]{6}\.webp)`;
|
|
|
|
| 216 |
const assetDocument = String.raw`(?:linearized\.pdf|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4)`;
|
| 217 |
const assetChapters = String.raw`(?:chapter-manifest\.json|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz))`;
|
| 218 |
const assetPrimaryPattern = new RegExp(
|
| 219 |
`^${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?${assetDocument})$`
|
| 220 |
);
|
| 221 |
const assetSourcePattern = new RegExp(
|
| 222 |
-
`^(?:pdf_manifest\\.json|${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?(?:${assetDocument}|${assetChapters})))$`
|
| 223 |
);
|
| 224 |
-
const bucketPathPattern = new RegExp(`^${assetRoot}${assetVersion}${assetPages}$`);
|
|
|
|
|
|
|
| 225 |
const versionedBucketPathPattern = new RegExp(`^${assetRoot}[0-9a-f]{16}/${assetPages}$`);
|
| 226 |
const assetBase = "https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/";
|
| 227 |
const assetModes = Object.freeze({
|
|
@@ -233,7 +236,8 @@
|
|
| 233 |
v: "video"
|
| 234 |
});
|
| 235 |
|
| 236 |
-
function isBucketPath(path, versioned = false) {
|
|
|
|
| 237 |
return (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path);
|
| 238 |
}
|
| 239 |
|
|
@@ -267,16 +271,21 @@
|
|
| 267 |
}
|
| 268 |
if (!path.endsWith("/page-manifest.json")) return null;
|
| 269 |
const rootPath = path.slice(0, -"/page-manifest.json".length);
|
|
|
|
| 270 |
return {
|
| 271 |
root: rootPath,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
pageUrl(page) {
|
| 273 |
if (!Number.isInteger(page) || page < 1 || page > 999999)
|
| 274 |
throw new Error("PDF_PAGE_INVALID");
|
| 275 |
const target = new URL(url.href);
|
| 276 |
const filename = `pages/page-${String(page).padStart(6, "0")}.webp`;
|
| 277 |
-
|
| 278 |
-
else target.pathname = target.pathname.replace(/page-manifest\.json$/, filename);
|
| 279 |
-
return target.href;
|
| 280 |
}
|
| 281 |
};
|
| 282 |
} catch (_) {
|
|
@@ -292,13 +301,21 @@
|
|
| 292 |
|
| 293 |
function assetFields(asset, bucketBase) {
|
| 294 |
const path = String(asset?.p || "");
|
| 295 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
if (
|
| 297 |
!asset ||
|
| 298 |
asset.s !== 2 ||
|
| 299 |
!Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
|
| 300 |
!assetPrimaryPattern.test(path) ||
|
| 301 |
-
(bucket &&
|
| 302 |
)
|
| 303 |
return null;
|
| 304 |
const nativeExtension =
|
|
@@ -310,10 +327,14 @@
|
|
| 310 |
? "pdf-pages"
|
| 311 |
: assetModes[asset.m];
|
| 312 |
return {
|
| 313 |
-
ReaderLink: bucket
|
|
|
|
|
|
|
| 314 |
ReaderExtension: asset.c ? "epub-chapters" : extension,
|
| 315 |
ReaderChapterManifest: asset.c ? assetBase + asset.c : "",
|
| 316 |
-
ReaderFallback: asset.f ? assetBase + asset.f : ""
|
|
|
|
|
|
|
| 317 |
};
|
| 318 |
}
|
| 319 |
|
|
@@ -340,6 +361,8 @@
|
|
| 340 |
params.set("download", record.DownloadLink || record.downloadLink);
|
| 341 |
if (!shortId && (record.OcrUrl || record.ocrUrl))
|
| 342 |
params.set("ocr", record.OcrUrl || record.ocrUrl);
|
|
|
|
|
|
|
| 343 |
if (!shortId && (record.ReaderFallback || record.readerFallback))
|
| 344 |
params.set("fallback", record.ReaderFallback || record.readerFallback);
|
| 345 |
if (
|
|
|
|
| 212 |
|
| 213 |
const assetRoot = String.raw`objects/[0-9a-f]{2}/[0-9a-f]{64}/`;
|
| 214 |
const assetVersion = String.raw`(?:[0-9a-f]{16}/)?`;
|
| 215 |
+
const assetPages = String.raw`(?:page-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl))`;
|
| 216 |
+
const assetOcr = String.raw`(?:ocr-manifest\.json|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz))`;
|
| 217 |
const assetDocument = String.raw`(?:linearized\.pdf|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4)`;
|
| 218 |
const assetChapters = String.raw`(?:chapter-manifest\.json|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz))`;
|
| 219 |
const assetPrimaryPattern = new RegExp(
|
| 220 |
`^${assetRoot}${assetVersion}(?:${assetPages}|(?:[a-z0-9-]+/)?${assetDocument})$`
|
| 221 |
);
|
| 222 |
const assetSourcePattern = new RegExp(
|
| 223 |
+
`^(?:pdf_manifest\\.json|${assetRoot}${assetVersion}(?:${assetPages}|${assetOcr}|(?:[a-z0-9-]+/)?(?:${assetDocument}|${assetChapters})))$`
|
| 224 |
);
|
| 225 |
+
const bucketPathPattern = new RegExp(`^${assetRoot}${assetVersion}(?:${assetPages}|${assetOcr})$`);
|
| 226 |
+
const optimizedBucket = "vomebook/pdf-optimized";
|
| 227 |
+
const optimizedPathPattern = new RegExp(`^${assetRoot}(?:[a-z0-9-]+/)?document\\.pdf$`);
|
| 228 |
const versionedBucketPathPattern = new RegExp(`^${assetRoot}[0-9a-f]{16}/${assetPages}$`);
|
| 229 |
const assetBase = "https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/";
|
| 230 |
const assetModes = Object.freeze({
|
|
|
|
| 236 |
v: "video"
|
| 237 |
});
|
| 238 |
|
| 239 |
+
function isBucketPath(path, versioned = false, bucket = "vomebook/pdf-pages") {
|
| 240 |
+
if (bucket === optimizedBucket) return optimizedPathPattern.test(path);
|
| 241 |
return (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path);
|
| 242 |
}
|
| 243 |
|
|
|
|
| 271 |
}
|
| 272 |
if (!path.endsWith("/page-manifest.json")) return null;
|
| 273 |
const rootPath = path.slice(0, -"/page-manifest.json".length);
|
| 274 |
+
const pathPrefix = bucket ? "" : url.pathname.slice(0, -path.length);
|
| 275 |
return {
|
| 276 |
root: rootPath,
|
| 277 |
+
assetUrl(relativePath) {
|
| 278 |
+
const target = new URL(url.href);
|
| 279 |
+
if (bucket) target.searchParams.set("path", relativePath);
|
| 280 |
+
else target.pathname = `${pathPrefix}${relativePath}`;
|
| 281 |
+
return target.href;
|
| 282 |
+
},
|
| 283 |
pageUrl(page) {
|
| 284 |
if (!Number.isInteger(page) || page < 1 || page > 999999)
|
| 285 |
throw new Error("PDF_PAGE_INVALID");
|
| 286 |
const target = new URL(url.href);
|
| 287 |
const filename = `pages/page-${String(page).padStart(6, "0")}.webp`;
|
| 288 |
+
return this.assetUrl(`${rootPath}/${filename}`);
|
|
|
|
|
|
|
| 289 |
}
|
| 290 |
};
|
| 291 |
} catch (_) {
|
|
|
|
| 301 |
|
| 302 |
function assetFields(asset, bucketBase) {
|
| 303 |
const path = String(asset?.p || "");
|
| 304 |
+
const ocrPath = String(asset?.o || "");
|
| 305 |
+
const ocrBucket = asset?.b === "vomebook/pdf-pages" || asset?.ob === "vomebook/pdf-pages";
|
| 306 |
+
const ocrUrl = ocrPath.endsWith("/ocr-manifest.json") && ocrBucket && isBucketPath(ocrPath, true)
|
| 307 |
+
? `${bucketBase}?path=${encodeURIComponent(ocrPath)}`
|
| 308 |
+
: "";
|
| 309 |
+
if (asset?.s === 3 && ocrUrl)
|
| 310 |
+
return { ReaderOcrManifest: ocrUrl, ReaderOcrMode: String(asset.om || "") };
|
| 311 |
+
const bucketName = String(asset?.b || "vomebook/pdf-pages");
|
| 312 |
+
const bucket = isBucketPath(path, true, bucketName);
|
| 313 |
if (
|
| 314 |
!asset ||
|
| 315 |
asset.s !== 2 ||
|
| 316 |
!Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
|
| 317 |
!assetPrimaryPattern.test(path) ||
|
| 318 |
+
(bucket && !["vomebook/pdf-pages", optimizedBucket].includes(asset?.b))
|
| 319 |
)
|
| 320 |
return null;
|
| 321 |
const nativeExtension =
|
|
|
|
| 327 |
? "pdf-pages"
|
| 328 |
: assetModes[asset.m];
|
| 329 |
return {
|
| 330 |
+
ReaderLink: bucket
|
| 331 |
+
? `${bucketBase}?${asset.b === optimizedBucket ? `bucket=${encodeURIComponent(asset.b)}&` : ""}path=${encodeURIComponent(path)}`
|
| 332 |
+
: assetBase + path,
|
| 333 |
ReaderExtension: asset.c ? "epub-chapters" : extension,
|
| 334 |
ReaderChapterManifest: asset.c ? assetBase + asset.c : "",
|
| 335 |
+
ReaderFallback: asset.f ? assetBase + asset.f : "",
|
| 336 |
+
ReaderOcrManifest: ocrUrl || (ocrPath.endsWith("/ocr-manifest.json") ? assetBase + ocrPath : ""),
|
| 337 |
+
ReaderOcrMode: String(asset.om || "")
|
| 338 |
};
|
| 339 |
}
|
| 340 |
|
|
|
|
| 361 |
params.set("download", record.DownloadLink || record.downloadLink);
|
| 362 |
if (!shortId && (record.OcrUrl || record.ocrUrl))
|
| 363 |
params.set("ocr", record.OcrUrl || record.ocrUrl);
|
| 364 |
+
if (!shortId && (record.ReaderOcrManifest || record.readerOcrManifest))
|
| 365 |
+
params.set("ocr_manifest", record.ReaderOcrManifest || record.readerOcrManifest);
|
| 366 |
if (!shortId && (record.ReaderFallback || record.readerFallback))
|
| 367 |
params.set("fallback", record.ReaderFallback || record.readerFallback);
|
| 368 |
if (
|
static/reader-pdf-text.js
CHANGED
|
@@ -24,3 +24,22 @@ export function populatePdfTextLayer(layer, items, pdfViewport) {
|
|
| 24 |
if (!layer.textContent.trim()) layer.textContent = "此页没有可提取文本";
|
| 25 |
} else layer.textContent = plainText.trim() || "此页没有可提取文本";
|
| 26 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
if (!layer.textContent.trim()) layer.textContent = "此页没有可提取文本";
|
| 25 |
} else layer.textContent = plainText.trim() || "此页没有可提取文本";
|
| 26 |
}
|
| 27 |
+
|
| 28 |
+
export function populateOcrTextLayer(layer, blocks) {
|
| 29 |
+
const fragment = layer.ownerDocument.createDocumentFragment();
|
| 30 |
+
for (const block of blocks || []) {
|
| 31 |
+
const text = String(block?.t || "");
|
| 32 |
+
const box = Array.isArray(block?.b) ? block.b : [];
|
| 33 |
+
if (!text || box.length !== 4) continue;
|
| 34 |
+
const span = layer.ownerDocument.createElement("span");
|
| 35 |
+
span.textContent = text;
|
| 36 |
+
span.dataset.ocrText = text;
|
| 37 |
+
span.style.left = `${Math.max(0, Math.min(1, Number(box[0]) || 0)) * 100}%`;
|
| 38 |
+
span.style.top = `${Math.max(0, Math.min(1, Number(box[1]) || 0)) * 100}%`;
|
| 39 |
+
span.style.width = `${Math.max(0, Math.min(1, Number(box[2]) - Number(box[0])) * 100)}%`;
|
| 40 |
+
span.style.height = `${Math.max(0.5, Math.min(1, Number(box[3]) - Number(box[1])) * 100)}%`;
|
| 41 |
+
span.style.fontSize = `${Math.max(8, Math.min(72, Math.abs(Number(box[3]) - Number(box[1])) * 100))}%`;
|
| 42 |
+
fragment.appendChild(span);
|
| 43 |
+
}
|
| 44 |
+
layer.replaceChildren(fragment);
|
| 45 |
+
}
|
static/reader-security.js
CHANGED
|
@@ -265,6 +265,52 @@
|
|
| 265 |
}
|
| 266 |
return manifest;
|
| 267 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
function isZipContainer(extension, bytes) {
|
| 269 |
return (
|
| 270 |
["epub", "fbz"].includes(String(extension || "").toLowerCase()) ||
|
|
@@ -280,6 +326,8 @@
|
|
| 280 |
inspectZip,
|
| 281 |
validateChapterManifest,
|
| 282 |
validatePdfPageManifest,
|
|
|
|
|
|
|
| 283 |
isZipContainer
|
| 284 |
});
|
| 285 |
})(typeof self !== "undefined" ? self : globalThis);
|
|
|
|
| 265 |
}
|
| 266 |
return manifest;
|
| 267 |
}
|
| 268 |
+
const PDF_OCR_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?ocr\/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz)$/;
|
| 269 |
+
const PDF_OCR_MANIFEST_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?ocr-manifest\.json$/;
|
| 270 |
+
const PDF_OCR_PAGE_PATH = /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16}\/)?pages\/page-[0-9]{6}\.(?:webp|jxl)$/;
|
| 271 |
+
const PDF_OCR_ROOT = /^(objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/(?:[0-9a-f]{16})?)/;
|
| 272 |
+
function validatePdfOcrManifest(manifest) {
|
| 273 |
+
if (!manifest || manifest.version !== 1 || manifest.kind !== "pdf-ocr" ||
|
| 274 |
+
!Number.isInteger(manifest.page_count) || manifest.page_count < 1 ||
|
| 275 |
+
manifest.page_count > LIMITS.pdfPages || !Array.isArray(manifest.pages) ||
|
| 276 |
+
manifest.pages.length !== manifest.page_count)
|
| 277 |
+
throw error("PDF_OCR_MANIFEST_INVALID");
|
| 278 |
+
let root = "";
|
| 279 |
+
for (const [index, page] of manifest.pages.entries()) {
|
| 280 |
+
const expected = String(index + 1).padStart(6, "0");
|
| 281 |
+
const pageRoot = typeof page?.o === "string" ? PDF_OCR_ROOT.exec(page.o)?.[1] || "" : "";
|
| 282 |
+
if (!page || page.p !== index + 1 || !PDF_OCR_PATH.test(page.o) ||
|
| 283 |
+
page.o !== `${pageRoot}/ocr/page-${expected}.json.gz` ||
|
| 284 |
+
(page.w !== undefined && (typeof page.w !== "string" || !PDF_OCR_PAGE_PATH.test(page.w) ||
|
| 285 |
+
page.w !== `${pageRoot}/pages/page-${expected}.webp`)) ||
|
| 286 |
+
(page.j !== undefined && (typeof page.j !== "string" || !PDF_OCR_PAGE_PATH.test(page.j) ||
|
| 287 |
+
page.j !== `${pageRoot}/pages/page-${expected}.jxl`)) ||
|
| 288 |
+
(root && root !== pageRoot))
|
| 289 |
+
throw error("PDF_OCR_MANIFEST_INVALID");
|
| 290 |
+
root = pageRoot;
|
| 291 |
+
}
|
| 292 |
+
if (!manifest.book_text || typeof manifest.book_text.path !== "string" ||
|
| 293 |
+
!PDF_OCR_PATH.test(manifest.book_text.path) || manifest.book_text.path !== `${root}/ocr/book-text.json.gz` ||
|
| 294 |
+
(manifest.page_manifest !== undefined &&
|
| 295 |
+
(!manifest.page_manifest || typeof manifest.page_manifest.path !== "string" ||
|
| 296 |
+
!PDF_OCR_MANIFEST_PATH.test(manifest.page_manifest.path.replace("page-manifest.json", "ocr-manifest.json")))))
|
| 297 |
+
throw error("PDF_OCR_MANIFEST_INVALID");
|
| 298 |
+
return manifest;
|
| 299 |
+
}
|
| 300 |
+
function validatePdfOcrPage(payload, pageNumber) {
|
| 301 |
+
if (!payload || payload.version !== 1 || payload.kind !== "pdf-ocr-page" ||
|
| 302 |
+
payload.page !== pageNumber || !Array.isArray(payload.blocks) || payload.blocks.length > 10000 ||
|
| 303 |
+
!Number.isFinite(Number(payload.width)) || !Number.isFinite(Number(payload.height)))
|
| 304 |
+
throw error("PDF_OCR_PAGE_INVALID");
|
| 305 |
+
for (const block of payload.blocks) {
|
| 306 |
+
const box = block?.b;
|
| 307 |
+
if (!block || typeof block.t !== "string" || block.t.length > 4096 ||
|
| 308 |
+
!Array.isArray(box) || box.length !== 4 || box.some((value) => !Number.isFinite(Number(value))) ||
|
| 309 |
+
box[0] < 0 || box[1] < 0 || box[2] > 1 || box[3] > 1 || box[2] < box[0] || box[3] < box[1])
|
| 310 |
+
throw error("PDF_OCR_PAGE_INVALID");
|
| 311 |
+
}
|
| 312 |
+
return payload;
|
| 313 |
+
}
|
| 314 |
function isZipContainer(extension, bytes) {
|
| 315 |
return (
|
| 316 |
["epub", "fbz"].includes(String(extension || "").toLowerCase()) ||
|
|
|
|
| 326 |
inspectZip,
|
| 327 |
validateChapterManifest,
|
| 328 |
validatePdfPageManifest,
|
| 329 |
+
validatePdfOcrManifest,
|
| 330 |
+
validatePdfOcrPage,
|
| 331 |
isZipContainer
|
| 332 |
});
|
| 333 |
})(typeof self !== "undefined" ? self : globalThis);
|
static/reader.css
CHANGED
|
@@ -869,7 +869,7 @@ body {
|
|
| 869 |
position: absolute;
|
| 870 |
inset: 0;
|
| 871 |
overflow: hidden;
|
| 872 |
-
pointer-events:
|
| 873 |
white-space: pre;
|
| 874 |
border: 0;
|
| 875 |
color: transparent;
|
|
@@ -879,6 +879,8 @@ body {
|
|
| 879 |
color: transparent;
|
| 880 |
white-space: pre;
|
| 881 |
transform-origin: left bottom;
|
|
|
|
|
|
|
| 882 |
}
|
| 883 |
.reader-pdf-text span.full-search-highlight {
|
| 884 |
color: #1d2023;
|
|
|
|
| 869 |
position: absolute;
|
| 870 |
inset: 0;
|
| 871 |
overflow: hidden;
|
| 872 |
+
pointer-events: auto;
|
| 873 |
white-space: pre;
|
| 874 |
border: 0;
|
| 875 |
color: transparent;
|
|
|
|
| 879 |
color: transparent;
|
| 880 |
white-space: pre;
|
| 881 |
transform-origin: left bottom;
|
| 882 |
+
user-select: text;
|
| 883 |
+
cursor: text;
|
| 884 |
}
|
| 885 |
.reader-pdf-text span.full-search-highlight {
|
| 886 |
color: #1d2023;
|
static/reader.js
CHANGED
|
@@ -6,7 +6,7 @@ import "/static/reader-scroll-anchor.js";
|
|
| 6 |
import "/static/reader-section-virtualizer.js";
|
| 7 |
import "/static/reader-runtime.js";
|
| 8 |
import "/static/reader-format-adapters.js";
|
| 9 |
-
import { populatePdfTextLayer } from "/static/reader-pdf-text.js";
|
| 10 |
// Engines and Reader lifecycle.
|
| 11 |
const PDFJS_URL = VoiceOfMLReaderResources.vendorUrl("pdf", "/static/");
|
| 12 |
const PDFJS_WORKER_URL = "/static/pdf-worker-wrapper.mjs";
|
|
@@ -296,6 +296,10 @@ updateDocumentState({
|
|
| 296 |
});
|
| 297 |
function applyReaderMetadata(data) {
|
| 298 |
if (data) resolvedReaderData = { ...(resolvedReaderData || {}), ...data };
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
readerRuntime.update("source", { metadata: resolvedReaderData });
|
| 300 |
const title = String(resolvedReaderData?.title || params.get("title") || sourceName).trim();
|
| 301 |
const suffix = String(
|
|
@@ -311,6 +315,7 @@ function applyReaderMetadata(data) {
|
|
| 311 |
}
|
| 312 |
let fallbackUrl = params.get("fallback") || "";
|
| 313 |
const ocrUrl = params.get("ocr") || "";
|
|
|
|
| 314 |
let returnUrl = params.get("return") || "";
|
| 315 |
const readerPathLabel = params.get("path") || "";
|
| 316 |
const folderReturnUrl = params.get("folder_url") || "";
|
|
@@ -364,11 +369,16 @@ let bookmarkInvoker = null;
|
|
| 364 |
const bookmarkInertSiblings = new Map();
|
| 365 |
let pdfDocument = null;
|
| 366 |
let pdfPageManifest = null;
|
|
|
|
|
|
|
| 367 |
let pdfActiveRenders = 0;
|
| 368 |
let pdfShellsReady = Promise.resolve();
|
| 369 |
const pdfRenderWaiters = [];
|
|
|
|
|
|
|
| 370 |
trackReaderResource(() => {
|
| 371 |
for (const waiter of pdfRenderWaiters.splice(0)) waiter.reject(readerAbortError());
|
|
|
|
| 372 |
});
|
| 373 |
let epubRendition = null;
|
| 374 |
let epubBook = null;
|
|
@@ -2018,10 +2028,11 @@ function validSource(raw) {
|
|
| 2018 |
try {
|
| 2019 |
const url = new URL(raw);
|
| 2020 |
const bucketPath = url.searchParams.get("path") || "";
|
|
|
|
| 2021 |
if (
|
| 2022 |
url.origin === "https://voiceofml-search.hf.space" &&
|
| 2023 |
url.pathname === "/api/reader-bucket-resource" &&
|
| 2024 |
-
VoiceOfMLReader.isBucketPath(bucketPath)
|
| 2025 |
)
|
| 2026 |
return true;
|
| 2027 |
if (url.protocol !== "https:" || !["huggingface.co", "hf-mirror.com"].includes(url.hostname))
|
|
@@ -2175,12 +2186,60 @@ function pdfManifestSource() {
|
|
| 2175 |
}
|
| 2176 |
function pdfPageEntry(page) {
|
| 2177 |
if (!pdfPageManifest || page < 1 || page > pdfPageManifest.pageCount) return null;
|
| 2178 |
-
|
| 2179 |
pdfPageManifest.entries?.[page - 1] || {
|
| 2180 |
page,
|
| 2181 |
path: `${pdfPageManifest.root}/pages/page-${String(page).padStart(6, "0")}.webp`
|
| 2182 |
}
|
| 2183 |
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2184 |
}
|
| 2185 |
function trimPdfManifestImages(protectedShell) {
|
| 2186 |
const images = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
|
|
@@ -2193,6 +2252,10 @@ function trimPdfManifestImages(protectedShell) {
|
|
| 2193 |
);
|
| 2194 |
for (const shell of images.slice(25)) {
|
| 2195 |
shell.querySelector("img")?.remove();
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2196 |
shell.dataset.renderState = "idle";
|
| 2197 |
}
|
| 2198 |
}
|
|
@@ -2355,6 +2418,7 @@ async function renderPdfPages(prepared) {
|
|
| 2355 |
const totalPages = manifest.page_count;
|
| 2356 |
if (!source) throw new Error("PDF_MANIFEST_INVALID");
|
| 2357 |
pdfPageManifest = { ...source, pageCount: totalPages };
|
|
|
|
| 2358 |
updateDocumentState({ pageCount: totalPages });
|
| 2359 |
pageInput.max = String(totalPages);
|
| 2360 |
document.querySelector("#page-total").textContent = `/ ${totalPages}`;
|
|
@@ -2370,7 +2434,10 @@ async function renderPdfPages(prepared) {
|
|
| 2370 |
const observer = new IntersectionObserver(
|
| 2371 |
(items) => {
|
| 2372 |
if (readerAbortController.signal.aborted) return;
|
| 2373 |
-
for (const entry of items) if (entry.isIntersecting)
|
|
|
|
|
|
|
|
|
|
| 2374 |
},
|
| 2375 |
{ root: viewport, rootMargin: "1200px 0px" }
|
| 2376 |
);
|
|
@@ -2414,7 +2481,7 @@ async function renderPdf(prepared) {
|
|
| 2414 |
(entries) =>
|
| 2415 |
entries.forEach((entry) => {
|
| 2416 |
if (readerAbortController.signal.aborted) return;
|
| 2417 |
-
entry.target.dataset.renderVisible = entry.isIntersecting ? "1" : "0";
|
| 2418 |
if (entry.isIntersecting) renderPdfInBackground(entry.target);
|
| 2419 |
}),
|
| 2420 |
{ root: document.querySelector("#viewport"), rootMargin: "1200px 0px" }
|
|
@@ -2490,7 +2557,12 @@ async function renderPdf(prepared) {
|
|
| 2490 |
assertReaderActive();
|
| 2491 |
}
|
| 2492 |
|
| 2493 |
-
function
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2494 |
if (readerAbortController.signal.aborted) return;
|
| 2495 |
if (shell._backgroundRender) {
|
| 2496 |
if (force) shell.dataset.pendingRerender = "1";
|
|
@@ -2498,8 +2570,8 @@ function renderPdfInBackground(shell, force = false) {
|
|
| 2498 |
}
|
| 2499 |
const task =
|
| 2500 |
capability.mode === "pdf-pages"
|
| 2501 |
-
? renderPdfManifestShell(shell, force,
|
| 2502 |
-
: renderPdfShell(shell, force);
|
| 2503 |
shell._backgroundRender = task;
|
| 2504 |
task
|
| 2505 |
.catch((error) => {
|
|
@@ -2539,9 +2611,13 @@ function renderPdfManifestShell(shell, force = false, priority = false) {
|
|
| 2539 |
image.decoding = "async";
|
| 2540 |
shell.appendChild(image);
|
| 2541 |
}
|
| 2542 |
-
|
| 2543 |
-
|
| 2544 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2545 |
timeout = 0,
|
| 2546 |
untrack = () => {};
|
| 2547 |
const finish = (error) => {
|
|
@@ -2561,13 +2637,23 @@ function renderPdfManifestShell(shell, force = false, priority = false) {
|
|
| 2561 |
() => finish(new Error("PDF page image timeout")),
|
| 2562 |
READER_PROXY_TIMEOUT_MS
|
| 2563 |
);
|
| 2564 |
-
|
| 2565 |
-
|
| 2566 |
-
|
| 2567 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2568 |
});
|
| 2569 |
assertReaderActive();
|
| 2570 |
shell.style.aspectRatio = `${image.naturalWidth || 1} / ${image.naturalHeight || 1}`;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2571 |
image.classList.add("ready");
|
| 2572 |
shell.dataset.renderState = "rendered";
|
| 2573 |
shell.dataset.renderUsedAt = String(Date.now());
|
|
@@ -2674,8 +2760,17 @@ async function renderPdfText(page, shell) {
|
|
| 2674 |
function highlightPdfText(shell, query = searchState.query) {
|
| 2675 |
if (!shell || readerAbortController.signal.aborted || !query) return;
|
| 2676 |
const pattern = new RegExp(fullSearchEscape(query), "iu");
|
| 2677 |
-
|
| 2678 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2679 |
span.classList.add("full-search-highlight");
|
| 2680 |
fullSearchActiveMarks.push(span);
|
| 2681 |
}
|
|
@@ -2706,6 +2801,31 @@ function releasePdfRenderSlot() {
|
|
| 2706 |
const waiter = pdfRenderWaiters.shift();
|
| 2707 |
if (waiter) waiter.resolve();
|
| 2708 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2709 |
function trimPdfCanvases(protectedShell) {
|
| 2710 |
const limit = matchMedia("(max-width: 700px)").matches ? 7 : 11;
|
| 2711 |
const rendered = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
|
|
@@ -2737,6 +2857,10 @@ function trimPdfCanvases(protectedShell) {
|
|
| 2737 |
canvas.width = 0;
|
| 2738 |
canvas.height = 0;
|
| 2739 |
canvas.classList.remove("ready");
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2740 |
shell.dataset.renderState = "idle";
|
| 2741 |
}
|
| 2742 |
}
|
|
@@ -3412,6 +3536,8 @@ function disposeFormatResources(mode) {
|
|
| 3412 |
if (["pdf", "pdf-pages"].includes(mode)) {
|
| 3413 |
pdfDocument = null;
|
| 3414 |
pdfPageManifest = null;
|
|
|
|
|
|
|
| 3415 |
pdfFirstPagePreload?.removeAttribute("src");
|
| 3416 |
pdfFirstPagePreload = null;
|
| 3417 |
for (const canvas of content.querySelectorAll(".reader-page canvas")) {
|
|
@@ -4895,6 +5021,47 @@ async function navigateFoliateSearchResult(result, generation) {
|
|
| 4895 |
updateProgressTools();
|
| 4896 |
scheduleSave();
|
| 4897 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4898 |
async function fullSearchPdfMatches(query, generation) {
|
| 4899 |
const pdf = pdfDocument,
|
| 4900 |
output = [],
|
|
@@ -4964,7 +5131,14 @@ async function runFullSearch() {
|
|
| 4964 |
}
|
| 4965 |
fullSearchStatus.textContent = "正在搜索正文…";
|
| 4966 |
try {
|
| 4967 |
-
if (capability.mode === "pdf")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4968 |
publishSearchResults(generation, await fullSearchPdfMatches(query, generation));
|
| 4969 |
else if (capability.mode === "foliate")
|
| 4970 |
publishSearchResults(generation, await fullSearchFoliateMatches(query, generation));
|
|
|
|
| 6 |
import "/static/reader-section-virtualizer.js";
|
| 7 |
import "/static/reader-runtime.js";
|
| 8 |
import "/static/reader-format-adapters.js";
|
| 9 |
+
import { populatePdfTextLayer, populateOcrTextLayer } from "/static/reader-pdf-text.js";
|
| 10 |
// Engines and Reader lifecycle.
|
| 11 |
const PDFJS_URL = VoiceOfMLReaderResources.vendorUrl("pdf", "/static/");
|
| 12 |
const PDFJS_WORKER_URL = "/static/pdf-worker-wrapper.mjs";
|
|
|
|
| 296 |
});
|
| 297 |
function applyReaderMetadata(data) {
|
| 298 |
if (data) resolvedReaderData = { ...(resolvedReaderData || {}), ...data };
|
| 299 |
+
if (data?.ReaderOcrManifest || data?.ocr_manifest) {
|
| 300 |
+
ocrManifestUrl = data.ReaderOcrManifest || data.ocr_manifest;
|
| 301 |
+
params.set("ocr_manifest", ocrManifestUrl);
|
| 302 |
+
}
|
| 303 |
readerRuntime.update("source", { metadata: resolvedReaderData });
|
| 304 |
const title = String(resolvedReaderData?.title || params.get("title") || sourceName).trim();
|
| 305 |
const suffix = String(
|
|
|
|
| 315 |
}
|
| 316 |
let fallbackUrl = params.get("fallback") || "";
|
| 317 |
const ocrUrl = params.get("ocr") || "";
|
| 318 |
+
let ocrManifestUrl = params.get("ocr_manifest") || "";
|
| 319 |
let returnUrl = params.get("return") || "";
|
| 320 |
const readerPathLabel = params.get("path") || "";
|
| 321 |
const folderReturnUrl = params.get("folder_url") || "";
|
|
|
|
| 369 |
const bookmarkInertSiblings = new Map();
|
| 370 |
let pdfDocument = null;
|
| 371 |
let pdfPageManifest = null;
|
| 372 |
+
let pdfOcrManifest = null;
|
| 373 |
+
let pdfOcrManifestPromise = null;
|
| 374 |
let pdfActiveRenders = 0;
|
| 375 |
let pdfShellsReady = Promise.resolve();
|
| 376 |
const pdfRenderWaiters = [];
|
| 377 |
+
let pdfActiveTextLoads = 0;
|
| 378 |
+
const pdfTextWaiters = [];
|
| 379 |
trackReaderResource(() => {
|
| 380 |
for (const waiter of pdfRenderWaiters.splice(0)) waiter.reject(readerAbortError());
|
| 381 |
+
for (const waiter of pdfTextWaiters.splice(0)) waiter.reject(readerAbortError());
|
| 382 |
});
|
| 383 |
let epubRendition = null;
|
| 384 |
let epubBook = null;
|
|
|
|
| 2028 |
try {
|
| 2029 |
const url = new URL(raw);
|
| 2030 |
const bucketPath = url.searchParams.get("path") || "";
|
| 2031 |
+
const bucketName = url.searchParams.get("bucket") || "vomebook/pdf-pages";
|
| 2032 |
if (
|
| 2033 |
url.origin === "https://voiceofml-search.hf.space" &&
|
| 2034 |
url.pathname === "/api/reader-bucket-resource" &&
|
| 2035 |
+
VoiceOfMLReader.isBucketPath(bucketPath, true, bucketName)
|
| 2036 |
)
|
| 2037 |
return true;
|
| 2038 |
if (url.protocol !== "https:" || !["huggingface.co", "hf-mirror.com"].includes(url.hostname))
|
|
|
|
| 2186 |
}
|
| 2187 |
function pdfPageEntry(page) {
|
| 2188 |
if (!pdfPageManifest || page < 1 || page > pdfPageManifest.pageCount) return null;
|
| 2189 |
+
const pageEntry = (
|
| 2190 |
pdfPageManifest.entries?.[page - 1] || {
|
| 2191 |
page,
|
| 2192 |
path: `${pdfPageManifest.root}/pages/page-${String(page).padStart(6, "0")}.webp`
|
| 2193 |
}
|
| 2194 |
);
|
| 2195 |
+
const ocrEntry = pdfOcrManifest?.pages?.[page - 1];
|
| 2196 |
+
return ocrEntry ? { ...pageEntry, ...ocrEntry, page } : pageEntry;
|
| 2197 |
+
}
|
| 2198 |
+
|
| 2199 |
+
async function readPdfOcrJson(url, limit = VoiceOfMLReaderSecurity.LIMITS.chapterBytes) {
|
| 2200 |
+
const response = await fetchReaderUrl(url);
|
| 2201 |
+
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
| 2202 |
+
let bytes = await VoiceOfMLReaderSecurity.readBytes(response, limit);
|
| 2203 |
+
if (/\.gz$/i.test(url)) {
|
| 2204 |
+
if (typeof DecompressionStream !== "function") throw new Error("PDF_OCR_GZIP_UNSUPPORTED");
|
| 2205 |
+
const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip"));
|
| 2206 |
+
bytes = await VoiceOfMLReaderSecurity.readBytes(new Response(stream), limit);
|
| 2207 |
+
}
|
| 2208 |
+
return JSON.parse(new TextDecoder().decode(bytes));
|
| 2209 |
+
}
|
| 2210 |
+
|
| 2211 |
+
async function loadPdfOcrManifest() {
|
| 2212 |
+
if (pdfOcrManifest) return pdfOcrManifest;
|
| 2213 |
+
if (!ocrManifestUrl) return null;
|
| 2214 |
+
if (!pdfOcrManifestPromise) {
|
| 2215 |
+
pdfOcrManifestPromise = readPdfOcrJson(ocrManifestUrl, VoiceOfMLReaderSecurity.LIMITS.manifestBytes).then((manifest) => {
|
| 2216 |
+
pdfOcrManifest = VoiceOfMLReaderSecurity.validatePdfOcrManifest(manifest);
|
| 2217 |
+
return pdfOcrManifest;
|
| 2218 |
+
});
|
| 2219 |
+
}
|
| 2220 |
+
try { return await pdfOcrManifestPromise; }
|
| 2221 |
+
catch (error) { pdfOcrManifestPromise = null; throw error; }
|
| 2222 |
+
}
|
| 2223 |
+
|
| 2224 |
+
async function renderPdfOcrText(shell, entry) {
|
| 2225 |
+
if (shell.dataset.textReady === "1" || !entry?.o || !pdfPageManifest) return;
|
| 2226 |
+
const layer = shell.querySelector(".reader-pdf-text");
|
| 2227 |
+
if (!layer) return;
|
| 2228 |
+
await acquirePdfTextSlot(shell.dataset.renderVisible === "1");
|
| 2229 |
+
try {
|
| 2230 |
+
const payload = await readPdfOcrJson(pdfPageManifest.assetUrl(entry.o));
|
| 2231 |
+
assertReaderActive();
|
| 2232 |
+
VoiceOfMLReaderSecurity.validatePdfOcrPage(payload, Number(shell.dataset.page));
|
| 2233 |
+
populateOcrTextLayer(layer, payload.blocks);
|
| 2234 |
+
shell._bookmarkTextItems = payload.blocks.map((block) => ({ text: block.t, y: block.b?.[1] || 0 }));
|
| 2235 |
+
shell.dataset.textReady = "1";
|
| 2236 |
+
highlightPdfText(shell);
|
| 2237 |
+
} catch (error) {
|
| 2238 |
+
if (!readerAbortController.signal.aborted)
|
| 2239 |
+
console.warn(`PDF page ${shell.dataset.page} OCR text failed`, error);
|
| 2240 |
+
} finally {
|
| 2241 |
+
releasePdfTextSlot();
|
| 2242 |
+
}
|
| 2243 |
}
|
| 2244 |
function trimPdfManifestImages(protectedShell) {
|
| 2245 |
const images = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
|
|
|
|
| 2252 |
);
|
| 2253 |
for (const shell of images.slice(25)) {
|
| 2254 |
shell.querySelector("img")?.remove();
|
| 2255 |
+
const textLayer = shell.querySelector(".reader-pdf-text");
|
| 2256 |
+
if (textLayer) textLayer.replaceChildren();
|
| 2257 |
+
shell._bookmarkTextItems = [];
|
| 2258 |
+
shell.dataset.textReady = "0";
|
| 2259 |
shell.dataset.renderState = "idle";
|
| 2260 |
}
|
| 2261 |
}
|
|
|
|
| 2418 |
const totalPages = manifest.page_count;
|
| 2419 |
if (!source) throw new Error("PDF_MANIFEST_INVALID");
|
| 2420 |
pdfPageManifest = { ...source, pageCount: totalPages };
|
| 2421 |
+
if (ocrManifestUrl) loadPdfOcrManifest().catch(() => {});
|
| 2422 |
updateDocumentState({ pageCount: totalPages });
|
| 2423 |
pageInput.max = String(totalPages);
|
| 2424 |
document.querySelector("#page-total").textContent = `/ ${totalPages}`;
|
|
|
|
| 2434 |
const observer = new IntersectionObserver(
|
| 2435 |
(items) => {
|
| 2436 |
if (readerAbortController.signal.aborted) return;
|
| 2437 |
+
for (const entry of items) if (entry.isIntersecting) {
|
| 2438 |
+
entry.target.dataset.renderVisible = isPdfPageVisible(entry.target) ? "1" : "0";
|
| 2439 |
+
renderPdfInBackground(entry.target);
|
| 2440 |
+
}
|
| 2441 |
},
|
| 2442 |
{ root: viewport, rootMargin: "1200px 0px" }
|
| 2443 |
);
|
|
|
|
| 2481 |
(entries) =>
|
| 2482 |
entries.forEach((entry) => {
|
| 2483 |
if (readerAbortController.signal.aborted) return;
|
| 2484 |
+
entry.target.dataset.renderVisible = entry.isIntersecting && isPdfPageVisible(entry.target) ? "1" : "0";
|
| 2485 |
if (entry.isIntersecting) renderPdfInBackground(entry.target);
|
| 2486 |
}),
|
| 2487 |
{ root: document.querySelector("#viewport"), rootMargin: "1200px 0px" }
|
|
|
|
| 2557 |
assertReaderActive();
|
| 2558 |
}
|
| 2559 |
|
| 2560 |
+
function isPdfPageVisible(shell) {
|
| 2561 |
+
const pageRect = shell?.getBoundingClientRect?.();
|
| 2562 |
+
const viewportRect = viewport?.getBoundingClientRect?.();
|
| 2563 |
+
return !!pageRect && !!viewportRect && pageRect.bottom > viewportRect.top && pageRect.top < viewportRect.bottom;
|
| 2564 |
+
}
|
| 2565 |
+
function renderPdfInBackground(shell, force = false, priority = isPdfPageVisible(shell)) {
|
| 2566 |
if (readerAbortController.signal.aborted) return;
|
| 2567 |
if (shell._backgroundRender) {
|
| 2568 |
if (force) shell.dataset.pendingRerender = "1";
|
|
|
|
| 2570 |
}
|
| 2571 |
const task =
|
| 2572 |
capability.mode === "pdf-pages"
|
| 2573 |
+
? renderPdfManifestShell(shell, force, priority)
|
| 2574 |
+
: renderPdfShell(shell, force, priority);
|
| 2575 |
shell._backgroundRender = task;
|
| 2576 |
task
|
| 2577 |
.catch((error) => {
|
|
|
|
| 2611 |
image.decoding = "async";
|
| 2612 |
shell.appendChild(image);
|
| 2613 |
}
|
| 2614 |
+
const fallbackPath = entry.w || `${pdfPageManifest.root}/pages/page-${String(entry.page).padStart(6, "0")}.webp`;
|
| 2615 |
+
const targets = [...new Set([entry.j, fallbackPath].filter(Boolean))].map((path) =>
|
| 2616 |
+
pdfPageManifest.assetUrl(path)
|
| 2617 |
+
);
|
| 2618 |
+
await new Promise((resolve, reject) => {
|
| 2619 |
+
let settled = false,
|
| 2620 |
+
targetIndex = 0,
|
| 2621 |
timeout = 0,
|
| 2622 |
untrack = () => {};
|
| 2623 |
const finish = (error) => {
|
|
|
|
| 2637 |
() => finish(new Error("PDF page image timeout")),
|
| 2638 |
READER_PROXY_TIMEOUT_MS
|
| 2639 |
);
|
| 2640 |
+
image.onload = () => finish();
|
| 2641 |
+
image.onerror = () => {
|
| 2642 |
+
if (targetIndex + 1 < targets.length) {
|
| 2643 |
+
targetIndex += 1;
|
| 2644 |
+
image.src = targets[targetIndex];
|
| 2645 |
+
} else finish(new Error(`PDF page ${entry.page} image failed`));
|
| 2646 |
+
};
|
| 2647 |
+
if (!targets.length) return finish(new Error(`PDF page ${entry.page} image missing`));
|
| 2648 |
+
if (image.src !== targets[targetIndex]) image.src = targets[targetIndex];
|
| 2649 |
+
if (image.complete && image.naturalWidth) finish();
|
| 2650 |
});
|
| 2651 |
assertReaderActive();
|
| 2652 |
shell.style.aspectRatio = `${image.naturalWidth || 1} / ${image.naturalHeight || 1}`;
|
| 2653 |
+
const loadText = pdfOcrManifestPromise
|
| 2654 |
+
? pdfOcrManifestPromise.then(() => renderPdfOcrText(shell, pdfPageEntry(Number(shell.dataset.page))))
|
| 2655 |
+
: renderPdfOcrText(shell, pdfPageEntry(Number(shell.dataset.page)));
|
| 2656 |
+
loadText.catch(() => {});
|
| 2657 |
image.classList.add("ready");
|
| 2658 |
shell.dataset.renderState = "rendered";
|
| 2659 |
shell.dataset.renderUsedAt = String(Date.now());
|
|
|
|
| 2760 |
function highlightPdfText(shell, query = searchState.query) {
|
| 2761 |
if (!shell || readerAbortController.signal.aborted || !query) return;
|
| 2762 |
const pattern = new RegExp(fullSearchEscape(query), "iu");
|
| 2763 |
+
const spans = [...shell.querySelectorAll(".reader-pdf-text span")];
|
| 2764 |
+
let text = "";
|
| 2765 |
+
const ranges = spans.map((span) => {
|
| 2766 |
+
const start = text.length;
|
| 2767 |
+
text += `${span.textContent || ""} `;
|
| 2768 |
+
return { span, start, end: text.length };
|
| 2769 |
+
});
|
| 2770 |
+
const hit = text.search(pattern);
|
| 2771 |
+
for (const { span, start, end } of ranges)
|
| 2772 |
+
if (!span.classList.contains("full-search-highlight") &&
|
| 2773 |
+
((hit >= 0 && end > hit && start < hit + query.length) || pattern.test(span.textContent))) {
|
| 2774 |
span.classList.add("full-search-highlight");
|
| 2775 |
fullSearchActiveMarks.push(span);
|
| 2776 |
}
|
|
|
|
| 2801 |
const waiter = pdfRenderWaiters.shift();
|
| 2802 |
if (waiter) waiter.resolve();
|
| 2803 |
}
|
| 2804 |
+
function acquirePdfTextSlot(priority = false) {
|
| 2805 |
+
if (readerAbortController.signal.aborted) return Promise.reject(readerAbortError());
|
| 2806 |
+
const limit = matchMedia("(max-width: 700px)").matches ? 1 : 2;
|
| 2807 |
+
if (pdfActiveTextLoads < limit) {
|
| 2808 |
+
pdfActiveTextLoads++;
|
| 2809 |
+
return Promise.resolve();
|
| 2810 |
+
}
|
| 2811 |
+
return new Promise((resolve, reject) => {
|
| 2812 |
+
const waiter = {
|
| 2813 |
+
resolve: () => {
|
| 2814 |
+
pdfActiveTextLoads++;
|
| 2815 |
+
resolve();
|
| 2816 |
+
},
|
| 2817 |
+
reject
|
| 2818 |
+
};
|
| 2819 |
+
if (priority) pdfTextWaiters.unshift(waiter);
|
| 2820 |
+
else pdfTextWaiters.push(waiter);
|
| 2821 |
+
});
|
| 2822 |
+
}
|
| 2823 |
+
function releasePdfTextSlot() {
|
| 2824 |
+
pdfActiveTextLoads = Math.max(0, pdfActiveTextLoads - 1);
|
| 2825 |
+
if (readerAbortController.signal.aborted) return;
|
| 2826 |
+
const waiter = pdfTextWaiters.shift();
|
| 2827 |
+
if (waiter) waiter.resolve();
|
| 2828 |
+
}
|
| 2829 |
function trimPdfCanvases(protectedShell) {
|
| 2830 |
const limit = matchMedia("(max-width: 700px)").matches ? 7 : 11;
|
| 2831 |
const rendered = [...content.querySelectorAll('.reader-page[data-render-state="rendered"]')];
|
|
|
|
| 2857 |
canvas.width = 0;
|
| 2858 |
canvas.height = 0;
|
| 2859 |
canvas.classList.remove("ready");
|
| 2860 |
+
const textLayer = shell.querySelector(".reader-pdf-text");
|
| 2861 |
+
if (textLayer) textLayer.replaceChildren();
|
| 2862 |
+
shell._bookmarkTextItems = [];
|
| 2863 |
+
shell.dataset.textReady = "0";
|
| 2864 |
shell.dataset.renderState = "idle";
|
| 2865 |
}
|
| 2866 |
}
|
|
|
|
| 3536 |
if (["pdf", "pdf-pages"].includes(mode)) {
|
| 3537 |
pdfDocument = null;
|
| 3538 |
pdfPageManifest = null;
|
| 3539 |
+
pdfOcrManifest = null;
|
| 3540 |
+
pdfOcrManifestPromise = null;
|
| 3541 |
pdfFirstPagePreload?.removeAttribute("src");
|
| 3542 |
pdfFirstPagePreload = null;
|
| 3543 |
for (const canvas of content.querySelectorAll(".reader-page canvas")) {
|
|
|
|
| 5021 |
updateProgressTools();
|
| 5022 |
scheduleSave();
|
| 5023 |
}
|
| 5024 |
+
async function fullSearchPdfPageMatches(query, generation) {
|
| 5025 |
+
const manifest = await loadPdfOcrManifest();
|
| 5026 |
+
if (!manifest?.book_text?.path || !pdfPageManifest) return [];
|
| 5027 |
+
const book = await readPdfOcrJson(pdfPageManifest.assetUrl(manifest.book_text.path), VoiceOfMLReaderSecurity.LIMITS.chapterTotalBytes);
|
| 5028 |
+
const output = [], pattern = new RegExp(fullSearchEscape(query), "giu");
|
| 5029 |
+
for (const page of book.pages || []) {
|
| 5030 |
+
if (!isReaderGenerationCurrent("search", generation)) return [];
|
| 5031 |
+
const text = String(page.text || "");
|
| 5032 |
+
for (const match of text.matchAll(pattern)) {
|
| 5033 |
+
if (output.length >= 100) break;
|
| 5034 |
+
output.push({
|
| 5035 |
+
location: `第 ${page.page} 页`,
|
| 5036 |
+
snippet: fullSearchSnippet(text, match.index, match[0].length),
|
| 5037 |
+
activate: (navigationGeneration) => goToPage(page.page, navigationGeneration)
|
| 5038 |
+
});
|
| 5039 |
+
}
|
| 5040 |
+
if (output.length >= 100) break;
|
| 5041 |
+
}
|
| 5042 |
+
return output;
|
| 5043 |
+
}
|
| 5044 |
+
|
| 5045 |
+
async function fullSearchPdfApiMatches(query, generation) {
|
| 5046 |
+
if (!readerId) throw new Error("PDF_OCR_API_ID_MISSING");
|
| 5047 |
+
const url = `https://voiceofml-search.hf.space/api/reader-ocr-search?id=${encodeURIComponent(readerId)}&q=${encodeURIComponent(query)}&page=1&page_size=100`;
|
| 5048 |
+
const response = await fetchWithReaderTimeout(url);
|
| 5049 |
+
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
| 5050 |
+
const payload = await response.json();
|
| 5051 |
+
if (!payload || payload.version !== 1 || !Array.isArray(payload.results))
|
| 5052 |
+
throw new Error("PDF_OCR_API_INVALID");
|
| 5053 |
+
return payload.results.map((item) => {
|
| 5054 |
+
const text = String(item.snippet || "");
|
| 5055 |
+
const start = Math.max(0, Number(item.snippet_start) || 0);
|
| 5056 |
+
const length = Math.max(0, Number(item.length) || query.length);
|
| 5057 |
+
return {
|
| 5058 |
+
location: `第 ${item.page} 页`,
|
| 5059 |
+
snippet: fullSearchSnippet(text, start, length),
|
| 5060 |
+
activate: (navigationGeneration) => goToPage(item.page, navigationGeneration)
|
| 5061 |
+
};
|
| 5062 |
+
});
|
| 5063 |
+
}
|
| 5064 |
+
|
| 5065 |
async function fullSearchPdfMatches(query, generation) {
|
| 5066 |
const pdf = pdfDocument,
|
| 5067 |
output = [],
|
|
|
|
| 5131 |
}
|
| 5132 |
fullSearchStatus.textContent = "正在搜索正文…";
|
| 5133 |
try {
|
| 5134 |
+
if (capability.mode === "pdf-pages") {
|
| 5135 |
+
try {
|
| 5136 |
+
publishSearchResults(generation, await fullSearchPdfApiMatches(query, generation));
|
| 5137 |
+
} catch (error) {
|
| 5138 |
+
publishSearchResults(generation, await fullSearchPdfPageMatches(query, generation));
|
| 5139 |
+
}
|
| 5140 |
+
}
|
| 5141 |
+
else if (capability.mode === "pdf")
|
| 5142 |
publishSearchResults(generation, await fullSearchPdfMatches(query, generation));
|
| 5143 |
else if (capability.mode === "foliate")
|
| 5144 |
publishSearchResults(generation, await fullSearchFoliateMatches(query, generation));
|
tests/test_fastapi_endpoints.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import copy
|
|
|
|
| 4 |
import json
|
| 5 |
import tempfile
|
| 6 |
import time
|
|
@@ -41,6 +42,30 @@ class FastApiEndpointTests(unittest.TestCase):
|
|
| 41 |
self.assertRegex(response.headers["server-timing"], r"queue;dur=\d+\.\d+, search;dur=\d+\.\d+")
|
| 42 |
self.assertNotIn("readme", response.headers["server-timing"])
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
def test_search_endpoint_rejects_malformed_json(self):
|
| 45 |
self.assertEqual(self.client.post("/api/search", content="not-json", headers={"content-type": "application/json"}).status_code, 422)
|
| 46 |
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import copy
|
| 4 |
+
import gzip
|
| 5 |
import json
|
| 6 |
import tempfile
|
| 7 |
import time
|
|
|
|
| 42 |
self.assertRegex(response.headers["server-timing"], r"queue;dur=\d+\.\d+, search;dur=\d+\.\d+")
|
| 43 |
self.assertNotIn("readme", response.headers["server-timing"])
|
| 44 |
|
| 45 |
+
def test_reader_ocr_search_returns_exact_page_hits(self):
|
| 46 |
+
root = "objects/aa/" + "a" * 64 + "/" + "b" * 16
|
| 47 |
+
manifest_path = root + "/ocr-manifest.json"
|
| 48 |
+
book_path = root + "/ocr/book-text.json.gz"
|
| 49 |
+
manifest = {"version": 1, "kind": "pdf-ocr", "page_count": 2,
|
| 50 |
+
"pages": [{"p": 1, "o": root + "/ocr/page-000001.json.gz"},
|
| 51 |
+
{"p": 2, "o": root + "/ocr/page-000002.json.gz"}],
|
| 52 |
+
"book_text": {"path": book_path}}
|
| 53 |
+
book = gzip.compress(json.dumps({"version": 1, "kind": "pdf-ocr-book-text", "pages": [
|
| 54 |
+
{"page": 1, "text": "第一章 手机阅读"},
|
| 55 |
+
{"page": 2, "text": "第二章 手机搜索"},
|
| 56 |
+
]}).encode())
|
| 57 |
+
|
| 58 |
+
async def resource(path):
|
| 59 |
+
return json.dumps(manifest).encode() if path == manifest_path else book
|
| 60 |
+
|
| 61 |
+
resolved = app.JSONResponse({"ocr_manifest": "https://voiceofml-search.hf.space/api/reader-bucket-resource?path=" + manifest_path})
|
| 62 |
+
with mock.patch.object(app, "api_reader_resolve", mock.AsyncMock(return_value=resolved)), \
|
| 63 |
+
mock.patch.object(app, "get_reader_bucket_resource", side_effect=resource):
|
| 64 |
+
response = self.client.get("/api/reader-ocr-search", params={"id": "a" * 13, "q": "手机"})
|
| 65 |
+
self.assertEqual(response.status_code, 200)
|
| 66 |
+
self.assertEqual(response.json()["total"], 2)
|
| 67 |
+
self.assertEqual([item["page"] for item in response.json()["results"]], [1, 2])
|
| 68 |
+
|
| 69 |
def test_search_endpoint_rejects_malformed_json(self):
|
| 70 |
self.assertEqual(self.client.post("/api/search", content="not-json", headers={"content-type": "application/json"}).status_code, 422)
|
| 71 |
|
tests/test_reader_pdf_loading.py
CHANGED
|
@@ -12,7 +12,7 @@ from tests.test_reader_performance import (
|
|
| 12 |
PlaywrightError, StaticHandler, minimal_pdf, static_server, sync_playwright,
|
| 13 |
)
|
| 14 |
|
| 15 |
-
MODULE = '**/static/vendor/pdf.min.
|
| 16 |
SOURCE = 'https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/cold-retry.pdf'
|
| 17 |
|
| 18 |
|
|
|
|
| 12 |
PlaywrightError, StaticHandler, minimal_pdf, static_server, sync_playwright,
|
| 13 |
)
|
| 14 |
|
| 15 |
+
MODULE = '**/static/vendor/pdf.min.*.mjs*'
|
| 16 |
SOURCE = 'https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/cold-retry.pdf'
|
| 17 |
|
| 18 |
|
tests/test_reader_performance.py
CHANGED
|
@@ -7,6 +7,7 @@ import io
|
|
| 7 |
import json
|
| 8 |
import mimetypes
|
| 9 |
import pathlib
|
|
|
|
| 10 |
import threading
|
| 11 |
import time
|
| 12 |
import unittest
|
|
@@ -23,6 +24,43 @@ except ImportError:
|
|
| 23 |
|
| 24 |
|
| 25 |
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
SOURCE_URL = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/performance.pdf"
|
| 27 |
PDF_MODULE = """
|
| 28 |
export const GlobalWorkerOptions = {};
|
|
@@ -260,7 +298,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 260 |
self.page = self.context.new_page()
|
| 261 |
self.pdf_requested_at = None
|
| 262 |
self.page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 263 |
-
self.page.route("**/static/vendor/pdf.min.
|
| 264 |
self.page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
|
| 265 |
|
| 266 |
def tearDown(self):
|
|
@@ -474,7 +512,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 474 |
self.assertEqual(first_page.locator(".reader-pdf-text").text_content(), "Accessible PDF text")
|
| 475 |
|
| 476 |
def test_scanned_pdf_bookmark_has_empty_excerpt(self):
|
| 477 |
-
self.page.route('**/static/vendor/pdf.min.
|
| 478 |
content_type='text/javascript', body=PDF_MODULE.replace("[{ str: 'Accessible PDF text', hasEOL: false }]", '[]')))
|
| 479 |
self.open_pdf()
|
| 480 |
self.page.wait_for_function("document.querySelector('.reader-pdf-text')?.textContent.includes('此页没有可提取文本')")
|
|
@@ -583,7 +621,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 583 |
mobile_context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 584 |
mobile_page = mobile_context.new_page()
|
| 585 |
mobile_page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 586 |
-
mobile_page.route("**/static/vendor/pdf.min.
|
| 587 |
mobile_page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
|
| 588 |
query = urllib.parse.quote(SOURCE_URL, safe="")
|
| 589 |
mobile_page.goto(f"{self.origin}/static/reader.html?url={query}&ext=pdf&title=Performance", wait_until="domcontentloaded")
|
|
@@ -663,13 +701,13 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 663 |
context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 664 |
page = context.new_page()
|
| 665 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 666 |
-
page.route("**/static/vendor/pdf.min.
|
| 667 |
if extension == "md":
|
| 668 |
-
page.route("**/static/vendor/marked.min.
|
| 669 |
-
page.route("**/static/vendor/purify.min.
|
| 670 |
if extension == "docx":
|
| 671 |
-
page.route("**/static/vendor/jszip.min.
|
| 672 |
-
page.route("**/static/vendor/docx-preview.min.
|
| 673 |
page.route("**/api/reader-content**", lambda route, _request, content_type=content_type, body=body: route.fulfill(status=200, content_type=content_type, body=body))
|
| 674 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/matrix.{extension}"
|
| 675 |
if extension == "docx":
|
|
@@ -1126,8 +1164,8 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 1126 |
self.addCleanup(context.close)
|
| 1127 |
page = context.new_page()
|
| 1128 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1129 |
-
page.route("**/static/vendor/jszip.min.
|
| 1130 |
-
page.route("**/static/vendor/docx-preview.min.
|
| 1131 |
page.route("**/static/foliate-reader/view.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="customElements.define('foliate-view', class extends HTMLElement { open() { window.__archiveParserStarted=true; throw new Error('parser must not start'); } });"))
|
| 1132 |
page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/octet-stream", body=payload))
|
| 1133 |
source = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/bomb.epub" if extension == "epub" else f"https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/objects/aa/{'a' * 64}/docx-native-v1/document.docx"
|
|
@@ -1154,14 +1192,14 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 1154 |
context = self.browser.new_context(viewport={"width": 390, "height": 844}); page = context.new_page()
|
| 1155 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1156 |
if extension == "md":
|
| 1157 |
-
page.route("**/static/vendor/marked.min.
|
| 1158 |
page.route("**/api/reader-content**", lambda route, _request, body=body: route.fulfill(status=200, content_type="text/html" if extension == "html" else "text/markdown", body=body))
|
| 1159 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/toc.{extension}"; page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=TOC", wait_until="domcontentloaded"); page.locator("#history").click(); page.locator("#toc-list .panel-item-main").nth(1).click()
|
| 1160 |
page.wait_for_function("() => document.querySelector('iframe') ? document.querySelector('iframe').contentWindow.scrollY > 0 : document.querySelector('#viewport').scrollTop > 0"); context.close()
|
| 1161 |
|
| 1162 |
def test_pdf_outline_click_navigates_to_declared_page(self):
|
| 1163 |
self.page.add_init_script("window.__pdfOutlineEnabled = true")
|
| 1164 |
-
module = PDF_MODULE.replace("[{title: '第一章', dest: [{}], items: []}]", "[{title: '第三章', dest: [{}], items: []}]").replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: () => Promise.resolve(2)"); self.page.unroute("**/static/vendor/pdf.min.
|
| 1165 |
|
| 1166 |
def test_pdf_outline_marks_current_entry_when_page_changes(self):
|
| 1167 |
self.page.add_init_script("window.__pdfOutlineEnabled = true")
|
|
@@ -1169,8 +1207,8 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 1169 |
"[{title: '第一章', dest: [{}], items: []}]",
|
| 1170 |
"[{title: '第一章', dest: [{page: 0}], items: []}, {title: '第二章', dest: [{page: 2}], items: []}]",
|
| 1171 |
).replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: (ref) => Promise.resolve(ref.page || 0)")
|
| 1172 |
-
self.page.unroute("**/static/vendor/pdf.min.
|
| 1173 |
-
self.page.route("**/static/vendor/pdf.min.
|
| 1174 |
self.open_pdf()
|
| 1175 |
self.page.locator("#history").click()
|
| 1176 |
self.page.locator("#toc-list .toc-item").nth(1).wait_for()
|
|
@@ -1186,7 +1224,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 1186 |
def test_pdf_stale_navigation_and_search_keep_latest_state(self):
|
| 1187 |
errors = []
|
| 1188 |
self.page.on("pageerror", lambda error: errors.append(str(error)))
|
| 1189 |
-
self.page.route("**/static/vendor/pdf.min.
|
| 1190 |
status=200, content_type="text/javascript", body=PDF_TASK_MODULE,
|
| 1191 |
))
|
| 1192 |
self.open_pdf()
|
|
@@ -1249,7 +1287,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 1249 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1250 |
requests = []
|
| 1251 |
page.on("request", lambda request: requests.append(request.url) if "reader-content" in request.url or request.url == SOURCE_URL else None)
|
| 1252 |
-
page.route("**/static/vendor/pdf.min.
|
| 1253 |
page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(SOURCE_URL, safe='')}&ext=pdf", wait_until="domcontentloaded")
|
| 1254 |
page.wait_for_function("() => Boolean(window.__pdfProbe)")
|
| 1255 |
if phase == "rendering":
|
|
@@ -2007,8 +2045,8 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 2007 |
context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 2008 |
page = context.new_page()
|
| 2009 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 2010 |
-
page.route("**/static/vendor/marked.min.
|
| 2011 |
-
page.route("**/static/vendor/purify.min.
|
| 2012 |
page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="text/markdown", body=b"# Markdown readable"))
|
| 2013 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/readme.{extension}"
|
| 2014 |
page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=Markdown", wait_until="domcontentloaded")
|
|
@@ -2113,11 +2151,11 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 2113 |
return fulfill
|
| 2114 |
|
| 2115 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 2116 |
-
page.route("**/static/vendor/marked.min.
|
| 2117 |
-
page.route("**/static/vendor/purify.min.
|
| 2118 |
-
page.route("**/static/vendor/jszip.min.
|
| 2119 |
page.route("**/static/vendor/epub.min.06eae1574510.js", timed("epub", "text/javascript", EPUB_SCRIPT))
|
| 2120 |
-
page.route("**/static/vendor/docx-preview.min.
|
| 2121 |
content_type = "image/png" if extension == "png" else "application/octet-stream"
|
| 2122 |
body = PNG_BYTES if extension == "png" else minimal_docx() if extension == "docx" else minimal_epub() if extension == "epub" else b"Reader benchmark content"
|
| 2123 |
page.route("**/api/reader-content**", timed("content", content_type, body))
|
|
@@ -2151,9 +2189,9 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 2151 |
def test_cold_cache_first_read_with_real_format_engines(self):
|
| 2152 |
cases = [
|
| 2153 |
("txt", b"Cold TXT readable", "text/plain", "已加载", ".reader-text", []),
|
| 2154 |
-
("md", b"# Cold Markdown", "text/markdown", "已加载", ".reader-markdown", ["marked
|
| 2155 |
-
("pdf", minimal_pdf(), "application/pdf", "1 页", ".reader-page canvas.ready", ["pdf
|
| 2156 |
-
("docx", minimal_docx(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "1 页", ".docx-body", ["jszip
|
| 2157 |
("png", PNG_BYTES, "image/png", "图片", ".reader-image", []),
|
| 2158 |
]
|
| 2159 |
results = []
|
|
@@ -2164,6 +2202,7 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 2164 |
page = context.new_page(); session = context.new_cdp_session(page)
|
| 2165 |
session.send("Network.enable"); session.send("Network.setCacheDisabled", {"cacheDisabled": True})
|
| 2166 |
responses = []; page.on("response", lambda response: responses.append(response))
|
|
|
|
| 2167 |
|
| 2168 |
def serve_document(route):
|
| 2169 |
route.fulfill(status=200, content_type=content_type, body=document)
|
|
@@ -2192,7 +2231,11 @@ class ReaderPerformanceTest(unittest.TestCase):
|
|
| 2192 |
entry = next((item for item in entries if item["name"].endswith(engine)), None)
|
| 2193 |
self.assertIsNotNone(entry, engine)
|
| 2194 |
self.assertGreater(entry["transferSize"], 0, f"{engine} was not transferred on a cold load")
|
| 2195 |
-
byte_count = len(document) + sum(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2196 |
results.append((extension, elapsed, byte_count, len(responses)))
|
| 2197 |
finally:
|
| 2198 |
context.close()
|
|
|
|
| 7 |
import json
|
| 8 |
import mimetypes
|
| 9 |
import pathlib
|
| 10 |
+
import re
|
| 11 |
import threading
|
| 12 |
import time
|
| 13 |
import unittest
|
|
|
|
| 24 |
|
| 25 |
|
| 26 |
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def vendor_name(key, pattern):
|
| 30 |
+
resources = (ROOT / "static/reader-resources.js").read_text(encoding="utf-8")
|
| 31 |
+
match = re.search(rf"\b{re.escape(key)}:\s*\{{.*?sha256:\s*\"([0-9a-f]+)\"", resources, re.S)
|
| 32 |
+
if not match:
|
| 33 |
+
raise RuntimeError(f"missing vendor digest for {key!r}")
|
| 34 |
+
return pattern.replace("*", match.group(1)[:12], 1)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
VENDOR_FILES = {
|
| 38 |
+
"pdf": vendor_name("pdf", "pdf.min.*.mjs"),
|
| 39 |
+
"pdf_worker": vendor_name("pdfWorker", "pdf.worker.min.*.mjs"),
|
| 40 |
+
"marked": vendor_name("marked", "marked.min.*.js"),
|
| 41 |
+
"purify": vendor_name("purify", "purify.min.*.js"),
|
| 42 |
+
"jszip": vendor_name("jszip", "jszip.min.*.js"),
|
| 43 |
+
"docx": vendor_name("docx", "docx-preview.min.*.js"),
|
| 44 |
+
}
|
| 45 |
+
VENDOR_STEMS = {
|
| 46 |
+
"pdf": "pdf.min.*.mjs",
|
| 47 |
+
"pdf_worker": "pdf.worker.min.*.mjs",
|
| 48 |
+
"marked": "marked.min.*.js",
|
| 49 |
+
"purify": "purify.min.*.js",
|
| 50 |
+
"jszip": "jszip.min.*.js",
|
| 51 |
+
"docx": "docx-preview.min.*.js",
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def route_local_vendor_fallback(page):
|
| 56 |
+
for key, requested in VENDOR_FILES.items():
|
| 57 |
+
candidates = sorted((ROOT / "static/vendor").glob(VENDOR_STEMS[key]))
|
| 58 |
+
if not candidates:
|
| 59 |
+
continue
|
| 60 |
+
page.route(
|
| 61 |
+
f"**/static/vendor/{requested}",
|
| 62 |
+
lambda route, _request, path=str(candidates[-1]): route.fulfill(path=path),
|
| 63 |
+
)
|
| 64 |
SOURCE_URL = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/performance.pdf"
|
| 65 |
PDF_MODULE = """
|
| 66 |
export const GlobalWorkerOptions = {};
|
|
|
|
| 298 |
self.page = self.context.new_page()
|
| 299 |
self.pdf_requested_at = None
|
| 300 |
self.page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 301 |
+
self.page.route("**/static/vendor/pdf.min.*.mjs", self.route_pdf)
|
| 302 |
self.page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
|
| 303 |
|
| 304 |
def tearDown(self):
|
|
|
|
| 512 |
self.assertEqual(first_page.locator(".reader-pdf-text").text_content(), "Accessible PDF text")
|
| 513 |
|
| 514 |
def test_scanned_pdf_bookmark_has_empty_excerpt(self):
|
| 515 |
+
self.page.route('**/static/vendor/pdf.min.*.mjs', lambda route: route.fulfill(
|
| 516 |
content_type='text/javascript', body=PDF_MODULE.replace("[{ str: 'Accessible PDF text', hasEOL: false }]", '[]')))
|
| 517 |
self.open_pdf()
|
| 518 |
self.page.wait_for_function("document.querySelector('.reader-pdf-text')?.textContent.includes('此页没有可提取文本')")
|
|
|
|
| 621 |
mobile_context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 622 |
mobile_page = mobile_context.new_page()
|
| 623 |
mobile_page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 624 |
+
mobile_page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
|
| 625 |
mobile_page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/pdf", body=b"pdf"))
|
| 626 |
query = urllib.parse.quote(SOURCE_URL, safe="")
|
| 627 |
mobile_page.goto(f"{self.origin}/static/reader.html?url={query}&ext=pdf&title=Performance", wait_until="domcontentloaded")
|
|
|
|
| 701 |
context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 702 |
page = context.new_page()
|
| 703 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 704 |
+
page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_MODULE))
|
| 705 |
if extension == "md":
|
| 706 |
+
page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
|
| 707 |
+
page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
|
| 708 |
if extension == "docx":
|
| 709 |
+
page.route("**/static/vendor/jszip.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=JSZIP_SCRIPT))
|
| 710 |
+
page.route("**/static/vendor/docx-preview.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=DOCX_SCRIPT))
|
| 711 |
page.route("**/api/reader-content**", lambda route, _request, content_type=content_type, body=body: route.fulfill(status=200, content_type=content_type, body=body))
|
| 712 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/matrix.{extension}"
|
| 713 |
if extension == "docx":
|
|
|
|
| 1164 |
self.addCleanup(context.close)
|
| 1165 |
page = context.new_page()
|
| 1166 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1167 |
+
page.route("**/static/vendor/jszip.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.JSZip=function(){window.__archiveParserStarted=true};"))
|
| 1168 |
+
page.route("**/static/vendor/docx-preview.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.docx={renderAsync(){window.__archiveParserStarted=true}};"))
|
| 1169 |
page.route("**/static/foliate-reader/view.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="customElements.define('foliate-view', class extends HTMLElement { open() { window.__archiveParserStarted=true; throw new Error('parser must not start'); } });"))
|
| 1170 |
page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="application/octet-stream", body=payload))
|
| 1171 |
source = "https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/bomb.epub" if extension == "epub" else f"https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/objects/aa/{'a' * 64}/docx-native-v1/document.docx"
|
|
|
|
| 1192 |
context = self.browser.new_context(viewport={"width": 390, "height": 844}); page = context.new_page()
|
| 1193 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1194 |
if extension == "md":
|
| 1195 |
+
page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body="window.marked={parse:()=>'<h1>Markdown one</h1><p style=\"height:1000px\"></p><h2>Markdown two</h2>'};")); page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
|
| 1196 |
page.route("**/api/reader-content**", lambda route, _request, body=body: route.fulfill(status=200, content_type="text/html" if extension == "html" else "text/markdown", body=body))
|
| 1197 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/toc.{extension}"; page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=TOC", wait_until="domcontentloaded"); page.locator("#history").click(); page.locator("#toc-list .panel-item-main").nth(1).click()
|
| 1198 |
page.wait_for_function("() => document.querySelector('iframe') ? document.querySelector('iframe').contentWindow.scrollY > 0 : document.querySelector('#viewport').scrollTop > 0"); context.close()
|
| 1199 |
|
| 1200 |
def test_pdf_outline_click_navigates_to_declared_page(self):
|
| 1201 |
self.page.add_init_script("window.__pdfOutlineEnabled = true")
|
| 1202 |
+
module = PDF_MODULE.replace("[{title: '第一章', dest: [{}], items: []}]", "[{title: '第三章', dest: [{}], items: []}]").replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: () => Promise.resolve(2)"); self.page.unroute("**/static/vendor/pdf.min.*.mjs"); self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module)); self.open_pdf(); self.page.locator("#history").click(); self.page.locator("#toc-list .panel-item-main").click(); self.page.wait_for_function("() => document.querySelector('#page-number').value === '3'")
|
| 1203 |
|
| 1204 |
def test_pdf_outline_marks_current_entry_when_page_changes(self):
|
| 1205 |
self.page.add_init_script("window.__pdfOutlineEnabled = true")
|
|
|
|
| 1207 |
"[{title: '第一章', dest: [{}], items: []}]",
|
| 1208 |
"[{title: '第一章', dest: [{page: 0}], items: []}, {title: '第二章', dest: [{page: 2}], items: []}]",
|
| 1209 |
).replace("getPageIndex: () => Promise.resolve(0)", "getPageIndex: (ref) => Promise.resolve(ref.page || 0)")
|
| 1210 |
+
self.page.unroute("**/static/vendor/pdf.min.*.mjs")
|
| 1211 |
+
self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route, _request, module=module: route.fulfill(status=200, content_type="text/javascript", body=module))
|
| 1212 |
self.open_pdf()
|
| 1213 |
self.page.locator("#history").click()
|
| 1214 |
self.page.locator("#toc-list .toc-item").nth(1).wait_for()
|
|
|
|
| 1224 |
def test_pdf_stale_navigation_and_search_keep_latest_state(self):
|
| 1225 |
errors = []
|
| 1226 |
self.page.on("pageerror", lambda error: errors.append(str(error)))
|
| 1227 |
+
self.page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(
|
| 1228 |
status=200, content_type="text/javascript", body=PDF_TASK_MODULE,
|
| 1229 |
))
|
| 1230 |
self.open_pdf()
|
|
|
|
| 1287 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 1288 |
requests = []
|
| 1289 |
page.on("request", lambda request: requests.append(request.url) if "reader-content" in request.url or request.url == SOURCE_URL else None)
|
| 1290 |
+
page.route("**/static/vendor/pdf.min.*.mjs", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PDF_TASK_MODULE))
|
| 1291 |
page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(SOURCE_URL, safe='')}&ext=pdf", wait_until="domcontentloaded")
|
| 1292 |
page.wait_for_function("() => Boolean(window.__pdfProbe)")
|
| 1293 |
if phase == "rendering":
|
|
|
|
| 2045 |
context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 2046 |
page = context.new_page()
|
| 2047 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 2048 |
+
page.route("**/static/vendor/marked.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=MARKED_SCRIPT))
|
| 2049 |
+
page.route("**/static/vendor/purify.min.*.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=PURIFY_SCRIPT))
|
| 2050 |
page.route("**/api/reader-content**", lambda route: route.fulfill(status=200, content_type="text/markdown", body=b"# Markdown readable"))
|
| 2051 |
source = f"https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/readme.{extension}"
|
| 2052 |
page.goto(f"{self.origin}/static/reader.html?url={urllib.parse.quote(source, safe='')}&ext={extension}&title=Markdown", wait_until="domcontentloaded")
|
|
|
|
| 2151 |
return fulfill
|
| 2152 |
|
| 2153 |
page.route("**/static/reader-store.js", lambda route: route.fulfill(status=200, content_type="text/javascript", body=STORE_SCRIPT))
|
| 2154 |
+
page.route("**/static/vendor/marked.min.*.js", timed("marked", "text/javascript", MARKED_SCRIPT))
|
| 2155 |
+
page.route("**/static/vendor/purify.min.*.js", timed("purify", "text/javascript", PURIFY_SCRIPT))
|
| 2156 |
+
page.route("**/static/vendor/jszip.min.*.js", timed("jszip", "text/javascript", JSZIP_SCRIPT))
|
| 2157 |
page.route("**/static/vendor/epub.min.06eae1574510.js", timed("epub", "text/javascript", EPUB_SCRIPT))
|
| 2158 |
+
page.route("**/static/vendor/docx-preview.min.*.js", timed("docx", "text/javascript", DOCX_SCRIPT))
|
| 2159 |
content_type = "image/png" if extension == "png" else "application/octet-stream"
|
| 2160 |
body = PNG_BYTES if extension == "png" else minimal_docx() if extension == "docx" else minimal_epub() if extension == "epub" else b"Reader benchmark content"
|
| 2161 |
page.route("**/api/reader-content**", timed("content", content_type, body))
|
|
|
|
| 2189 |
def test_cold_cache_first_read_with_real_format_engines(self):
|
| 2190 |
cases = [
|
| 2191 |
("txt", b"Cold TXT readable", "text/plain", "已加载", ".reader-text", []),
|
| 2192 |
+
("md", b"# Cold Markdown", "text/markdown", "已加载", ".reader-markdown", [VENDOR_FILES["marked"], VENDOR_FILES["purify"]]),
|
| 2193 |
+
("pdf", minimal_pdf(), "application/pdf", "1 页", ".reader-page canvas.ready", [VENDOR_FILES["pdf"], VENDOR_FILES["pdf_worker"]]),
|
| 2194 |
+
("docx", minimal_docx(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "1 页", ".docx-body", [VENDOR_FILES["jszip"], VENDOR_FILES["docx"]]),
|
| 2195 |
("png", PNG_BYTES, "image/png", "图片", ".reader-image", []),
|
| 2196 |
]
|
| 2197 |
results = []
|
|
|
|
| 2202 |
page = context.new_page(); session = context.new_cdp_session(page)
|
| 2203 |
session.send("Network.enable"); session.send("Network.setCacheDisabled", {"cacheDisabled": True})
|
| 2204 |
responses = []; page.on("response", lambda response: responses.append(response))
|
| 2205 |
+
route_local_vendor_fallback(page)
|
| 2206 |
|
| 2207 |
def serve_document(route):
|
| 2208 |
route.fulfill(status=200, content_type=content_type, body=document)
|
|
|
|
| 2231 |
entry = next((item for item in entries if item["name"].endswith(engine)), None)
|
| 2232 |
self.assertIsNotNone(entry, engine)
|
| 2233 |
self.assertGreater(entry["transferSize"], 0, f"{engine} was not transferred on a cold load")
|
| 2234 |
+
byte_count = len(document) + sum(
|
| 2235 |
+
(ROOT / "static/vendor" / engine).stat().st_size
|
| 2236 |
+
if (ROOT / "static/vendor" / engine).exists() else 0
|
| 2237 |
+
for engine in engines
|
| 2238 |
+
)
|
| 2239 |
results.append((extension, elapsed, byte_count, len(responses)))
|
| 2240 |
finally:
|
| 2241 |
context.close()
|
upstream_http.py
CHANGED
|
@@ -9,7 +9,7 @@ ALLOWED_DOWNLOAD_HOSTS = {"huggingface.co", "hf-mirror.com", "hf.co"}
|
|
| 9 |
ALLOWED_DOWNLOAD_HOST_SUFFIXES = (".hf.co", ".huggingface.co", ".xethub.hf.co")
|
| 10 |
MAX_REDIRECTS = 5
|
| 11 |
READER_ASSET_PATH_RE = re.compile(
|
| 12 |
-
r"objects/[0-9a-f]{2}/[0-9a-f]{64}/(?:[0-9a-f]{16}/)?(?:page-manifest\.json|pages/page-[0-9]{6}\.webp|(?:[a-z0-9-]+/)?(?:linearized\.pdf|chapter-manifest\.json|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz)))"
|
| 13 |
)
|
| 14 |
READER_ASSET_SOURCE_RE = re.compile(
|
| 15 |
r"/datasets/vomebook/Reader-Assets/resolve/main/(?:pdf_manifest\.json|" + READER_ASSET_PATH_RE.pattern + r")"
|
|
|
|
| 9 |
ALLOWED_DOWNLOAD_HOST_SUFFIXES = (".hf.co", ".huggingface.co", ".xethub.hf.co")
|
| 10 |
MAX_REDIRECTS = 5
|
| 11 |
READER_ASSET_PATH_RE = re.compile(
|
| 12 |
+
r"objects/[0-9a-f]{2}/[0-9a-f]{64}/(?:[0-9a-f]{16}/)?(?:page-manifest\.json|ocr-manifest\.json|pages/page-[0-9]{6}\.(?:webp|jxl)|ocr/(?:page-[0-9]{6}\.json\.gz|book-text\.json\.gz)|(?:[a-z0-9-]+/)?(?:linearized\.pdf|chapter-manifest\.json|document\.(?:pdf|epub|mobi|azw|azw3|fb2|docx|html)|book\.epub|audio\.mp3|video\.mp4|epub-chapters/(?:chapter-manifest\.json|chapters/chapter-[0-9]{4}\.xhtml|resources/[A-Za-z0-9._~%+\-/]+|epub-search-index\.json\.gz)))"
|
| 13 |
)
|
| 14 |
READER_ASSET_SOURCE_RE = re.compile(
|
| 15 |
r"/datasets/vomebook/Reader-Assets/resolve/main/(?:pdf_manifest\.json|" + READER_ASSET_PATH_RE.pattern + r")"
|