test: publish project tests and ignore rules
Browse filesRestore root metadata, publish test sources and test requirements; exclude local guides, reports, caches, benchmarks and snapshots.
- .gitignore +16 -0
- ocr_release.py +266 -0
- requirements-test.txt +3 -0
- tests/__init__.py +0 -0
- tests/test_archive_compat.py +88 -0
- tests/test_benchmark.py +251 -0
- tests/test_browser.py +722 -0
- tests/test_integration.py +529 -0
- tests/test_live_smoke.py +129 -0
- tests/test_ocr_release.py +168 -0
- tests/test_prepare_fulltext_index.py +135 -0
- tests/test_search.py +1214 -0
- tests/test_search_oracle.py +229 -0
.gitignore
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
data/fulltext/
|
| 2 |
+
data/production-snapshot/
|
| 3 |
+
*.sqlite3
|
| 4 |
+
*.sqlite3.*
|
| 5 |
+
*.tmp
|
| 6 |
+
*.bak
|
| 7 |
+
*.before-rebuild
|
| 8 |
+
__pycache__/
|
| 9 |
+
*.pyc
|
| 10 |
+
.pytest_cache/
|
| 11 |
+
.mypy_cache/
|
| 12 |
+
.venv/
|
| 13 |
+
env/
|
| 14 |
+
tests/benchmark-results.json
|
| 15 |
+
AGENTS.md
|
| 16 |
+
TESTING.md
|
ocr_release.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict, startup-pinned OCR text/index releases. No network, pointers or deletion."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from collections import defaultdict
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path, PurePosixPath
|
| 9 |
+
import re
|
| 10 |
+
import shutil
|
| 11 |
+
import sqlite3
|
| 12 |
+
import tempfile
|
| 13 |
+
import unicodedata
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
BUCKET = "vomebook/ccrd-index"
|
| 17 |
+
TOKENIZER = "cjk-bigram-boundary-fts5-v6-snippet-anchors"
|
| 18 |
+
SOURCE = "OCR"
|
| 19 |
+
SOURCE_NAME = "OCR成果"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def canonical(value) -> bytes:
|
| 23 |
+
return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def digest(value) -> str:
|
| 27 |
+
return hashlib.sha256(value).hexdigest()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def sha(value) -> str:
|
| 31 |
+
if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value):
|
| 32 |
+
raise ValueError("invalid release SHA-256")
|
| 33 |
+
return value
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def safe_path(value: str) -> str:
|
| 37 |
+
if (not isinstance(value, str) or not value or value.startswith("/") or "\\" in value
|
| 38 |
+
or any(part in {"", ".", ".."} for part in value.split("/"))
|
| 39 |
+
or any(ord(char) < 32 for char in value)):
|
| 40 |
+
raise ValueError("invalid release relative path")
|
| 41 |
+
return value
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def read_resource(root: Path, ref: dict) -> bytes:
|
| 45 |
+
if not isinstance(ref, dict) or ref.get("bucket") != BUCKET:
|
| 46 |
+
raise ValueError("wrong OCR release bucket")
|
| 47 |
+
path = safe_path(ref.get("path"))
|
| 48 |
+
if not path.startswith(("text/objects/", "text/exports/", "indexes/ocr/")):
|
| 49 |
+
raise ValueError("unmanaged OCR resource path")
|
| 50 |
+
expected = sha(ref.get("sha256"))
|
| 51 |
+
size = ref.get("bytes")
|
| 52 |
+
if type(size) is not int or size < 1:
|
| 53 |
+
raise ValueError("invalid OCR resource size")
|
| 54 |
+
root = root.resolve()
|
| 55 |
+
target = (root / path).resolve()
|
| 56 |
+
if root not in target.parents or not target.is_file():
|
| 57 |
+
raise ValueError("missing OCR release resource or path escape")
|
| 58 |
+
raw = target.read_bytes()
|
| 59 |
+
if len(raw) != size or digest(raw) != expected:
|
| 60 |
+
raise ValueError("OCR release resource checksum mismatch")
|
| 61 |
+
return raw
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def export_record(root: Path, ref: dict) -> tuple[dict, dict]:
|
| 65 |
+
export = json.loads(read_resource(root, ref))
|
| 66 |
+
if (export.get("kind") != "ccrd-ocr-text-export" or export.get("version") != 1
|
| 67 |
+
or export.get("complete") is not True or export.get("recipe") != "ocr-plain-utf8-pages-v1"):
|
| 68 |
+
raise ValueError("unsupported OCR export schema")
|
| 69 |
+
if not re.fullmatch(r"OCR:[1-9][0-9]*", str(export.get("doc_id", ""))) or int(export["doc_id"].split(":")[1]) >= 1 << 63:
|
| 70 |
+
raise ValueError("invalid OCR document identity")
|
| 71 |
+
repo, original = export.get("origin_repo"), safe_path(export.get("origin_rel_path"))
|
| 72 |
+
if not isinstance(repo, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repo):
|
| 73 |
+
raise ValueError("invalid OCR origin repository")
|
| 74 |
+
if not re.fullmatch(r"[0-9a-f]{40}", str(export.get("origin_revision", ""))):
|
| 75 |
+
raise ValueError("OCR origin must be commit-pinned")
|
| 76 |
+
readable = safe_path(export.get("readable_rel_path"))
|
| 77 |
+
expected_path = repo + "/" + str(PurePosixPath(original).with_suffix(".txt"))
|
| 78 |
+
if readable != expected_path:
|
| 79 |
+
raise ValueError("OCR readable path does not preserve its origin")
|
| 80 |
+
raw = read_resource(root, export.get("text"))
|
| 81 |
+
body = raw.decode("utf-8", errors="strict")
|
| 82 |
+
if body.startswith("\ufeff"):
|
| 83 |
+
raise ValueError("plain OCR export must not hide a BOM in its indexed text")
|
| 84 |
+
identity = {key: export[key] for key in ("recipe", "origin_repo", "origin_rel_path", "origin_revision",
|
| 85 |
+
"source_sha256", "doc_id", "text_generation", "txt_sha256")}
|
| 86 |
+
for field in ("source_sha256", "text_generation", "txt_sha256", "generation", "book_id"):
|
| 87 |
+
sha(export.get(field))
|
| 88 |
+
generation = digest(canonical(identity).rstrip(b"\n"))
|
| 89 |
+
if (generation != export["generation"] or digest(raw) != export["txt_sha256"]
|
| 90 |
+
or export["book_id"] != digest((repo + "\0" + original).encode())):
|
| 91 |
+
raise ValueError("OCR export generation mismatch")
|
| 92 |
+
pages, count = export.get("pages"), export.get("page_count")
|
| 93 |
+
if type(count) is not int or count < 1 or not isinstance(pages, list) or len(pages) != count:
|
| 94 |
+
raise ValueError("OCR export is missing pages")
|
| 95 |
+
if export.get("offset_unit") != "unicode-codepoint" or export.get("page_separator") != "\n\n":
|
| 96 |
+
raise ValueError("unsupported page offset recipe")
|
| 97 |
+
cursor, reviewed = 0, 0
|
| 98 |
+
for number, page in enumerate(pages, 1):
|
| 99 |
+
start, end = page.get("start"), page.get("end")
|
| 100 |
+
if (page.get("page") != number or type(start) is not int or type(end) is not int
|
| 101 |
+
or start != cursor or not start <= end <= len(body)):
|
| 102 |
+
raise ValueError("OCR page order/offset mismatch")
|
| 103 |
+
sha(page.get("text_generation"))
|
| 104 |
+
if digest(body[start:end].encode()) != sha(page.get("text_sha256")):
|
| 105 |
+
raise ValueError("OCR page text digest mismatch")
|
| 106 |
+
if page.get("quality") not in {"unreviewed", "partially-reviewed", "accepted"}:
|
| 107 |
+
raise ValueError("invalid OCR page quality")
|
| 108 |
+
reviewed += page["quality"] != "unreviewed"
|
| 109 |
+
cursor = end
|
| 110 |
+
if number < count:
|
| 111 |
+
if body[cursor:cursor + 2] != "\n\n":
|
| 112 |
+
raise ValueError("OCR page separator mismatch")
|
| 113 |
+
cursor += 2
|
| 114 |
+
if cursor != len(body) or export.get("accepted_pages") != reviewed or export.get("quality") != ("partially-reviewed" if reviewed else "unreviewed"):
|
| 115 |
+
raise ValueError("OCR whole-book coverage/quality mismatch")
|
| 116 |
+
resources = export.get("retained_reader_resources")
|
| 117 |
+
if not isinstance(resources, list) or not resources:
|
| 118 |
+
raise ValueError("OCR provenance retention references are missing")
|
| 119 |
+
for resource in resources:
|
| 120 |
+
if resource.get("bucket") not in {"vomebook/reader-assets-v2", "vomebook/pdf-pages-v2", "melsm/pdf-archive-v2"}:
|
| 121 |
+
raise ValueError("unknown retained Reader bucket")
|
| 122 |
+
safe_path(resource.get("path")); sha(resource.get("sha256"))
|
| 123 |
+
if type(resource.get("bytes")) is not int or resource["bytes"] < 1:
|
| 124 |
+
raise ValueError("invalid retained Reader resource")
|
| 125 |
+
path = PurePosixPath(readable)
|
| 126 |
+
record = {"doc_id": export["doc_id"], "source": SOURCE, "source_name": SOURCE_NAME,
|
| 127 |
+
"display_name": path.stem, "display_dirs": list(path.parts[:-1]), "display_rel_path": readable,
|
| 128 |
+
"extension": "txt", "size": len(raw), "has_preview": True, "has_fulltext": True,
|
| 129 |
+
"storage_root": ".", "storage_rel_path": export["text"]["path"],
|
| 130 |
+
"text_generation": export["text_generation"], "text_quality": export["quality"]}
|
| 131 |
+
return record, export
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def directory_payloads(records: list[dict]) -> tuple[list[dict], dict]:
|
| 135 |
+
browser = defaultdict(lambda: {"folders": [], "files": []})
|
| 136 |
+
counts, direct = defaultdict(int), defaultdict(list)
|
| 137 |
+
children = defaultdict(set)
|
| 138 |
+
for record in records:
|
| 139 |
+
parts = record["display_dirs"]
|
| 140 |
+
for depth in range(len(parts) + 1):
|
| 141 |
+
path = "/".join(parts[:depth]); counts[path] += 1
|
| 142 |
+
if depth < len(parts): children[path].add(parts[depth])
|
| 143 |
+
direct["/".join(parts)].append(record)
|
| 144 |
+
for path in counts:
|
| 145 |
+
browser[path]["folders"] = [{"name": name, "path": (path + "/" if path else "") + name,
|
| 146 |
+
"count": counts[(path + "/" if path else "") + name]} for name in sorted(children[path])]
|
| 147 |
+
browser[path]["files"] = [{"name": item["display_name"], "ext": "txt", "size": item["size"],
|
| 148 |
+
"doc_id": item["doc_id"]} for item in sorted(direct[path], key=lambda item: (item["display_name"], item["doc_id"]))]
|
| 149 |
+
def node(name, path):
|
| 150 |
+
return {"name": name, "path": path, "count": counts[path], "hasDirectFiles": bool(direct[path]),
|
| 151 |
+
"hasChildren": bool(children[path]), "showSelfToggle": bool(path and direct[path] and children[path]),
|
| 152 |
+
"children": [node(child, (path + "/" if path else "") + child) for child in sorted(children[path])]}
|
| 153 |
+
return [node(SOURCE_NAME, "")], dict(browser)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def zip_member(record: dict) -> str:
|
| 157 |
+
"""Keep legal origin hierarchy; bound non-portable components deterministically."""
|
| 158 |
+
replacements = {":": ":", "*": "*", "?": "?", '"': """, "<": "<", ">": ">", "|": "|", "\\": "\"}
|
| 159 |
+
def component(original):
|
| 160 |
+
value = re.sub(r'[:*?"<>|\\\x00-\x1f]', lambda match: replacements.get(match[0], "_"), original).rstrip(" .") or "_"
|
| 161 |
+
if value != original or len(value.encode()) > 180:
|
| 162 |
+
suffix = "-" + digest(original.encode())[:12]
|
| 163 |
+
raw = value.encode()[:180 - len(suffix)]
|
| 164 |
+
value = raw.decode("utf-8", errors="ignore").rstrip(" .") + suffix
|
| 165 |
+
return value
|
| 166 |
+
parts = safe_path(record["display_rel_path"]).split("/")
|
| 167 |
+
parts[-1] = component(PurePosixPath(parts[-1]).stem) + ".txt"
|
| 168 |
+
return "/".join([SOURCE_NAME, *[component(part) for part in parts[:-1]], parts[-1]])
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def validate_database(path: Path, records: list[dict], text_root: Path) -> None:
|
| 172 |
+
with sqlite3.connect(f"file:{path.resolve()}?mode=ro&immutable=1", uri=True) as db:
|
| 173 |
+
if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
|
| 174 |
+
raise ValueError("invalid OCR SQLite integrity")
|
| 175 |
+
version = db.execute("SELECT value FROM metadata WHERE key='tokenizer_version'").fetchone()
|
| 176 |
+
if version != (TOKENIZER,):
|
| 177 |
+
raise ValueError("OCR tokenizer mismatch")
|
| 178 |
+
ids = {row[0] for row in db.execute("SELECT doc_id FROM documents")}
|
| 179 |
+
if ids != {record["doc_id"] for record in records}:
|
| 180 |
+
raise ValueError("OCR metadata/database ID set mismatch")
|
| 181 |
+
if db.execute("SELECT COUNT(*) FROM content_fts").fetchone()[0] != len(records):
|
| 182 |
+
raise ValueError("OCR content rows mismatch")
|
| 183 |
+
for record in records:
|
| 184 |
+
body = (text_root / record["storage_rel_path"]).read_text(encoding="utf-8", errors="strict")
|
| 185 |
+
normalized = unicodedata.normalize("NFKC", body).casefold().replace("\0", " ")
|
| 186 |
+
row = db.execute("SELECT content FROM content_fts WHERE rowid=?", (int(record["doc_id"].split(":")[1]),)).fetchone()
|
| 187 |
+
if row != (normalized,):
|
| 188 |
+
raise ValueError("OCR indexed content differs from readable TXT")
|
| 189 |
+
columns = {row[1] for row in db.execute("PRAGMA table_info(snippet_anchors)")}
|
| 190 |
+
if not {"token_hash", "doc_number", "char_offset"} <= columns:
|
| 191 |
+
raise ValueError("OCR snippet anchor schema missing")
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def load_release(bucket: Path, manifest_path: str, expected_sha256: str, base_metadata_sha256: str) -> dict:
|
| 195 |
+
safe_path(manifest_path); sha(expected_sha256); sha(base_metadata_sha256)
|
| 196 |
+
bucket = bucket.resolve()
|
| 197 |
+
target = (bucket / manifest_path).resolve()
|
| 198 |
+
if bucket not in target.parents or not manifest_path.startswith("releases/ocr-"):
|
| 199 |
+
raise ValueError("OCR release path outside immutable release namespace")
|
| 200 |
+
raw = target.read_bytes()
|
| 201 |
+
if digest(raw) != expected_sha256:
|
| 202 |
+
raise ValueError("OCR release manifest checksum mismatch")
|
| 203 |
+
manifest = json.loads(raw)
|
| 204 |
+
if (manifest.get("kind") != "ccrd-ocr-search-release" or manifest.get("version") != 1
|
| 205 |
+
or manifest.get("tokenizer_version") != TOKENIZER or manifest.get("base_metadata_sha256") != base_metadata_sha256):
|
| 206 |
+
raise ValueError("incompatible OCR release/base snapshot")
|
| 207 |
+
sha(manifest.get("generation")); sha(manifest.get("builder_sha256"))
|
| 208 |
+
identity = {key: manifest[key] for key in ("version", "kind", "base_metadata_sha256", "builder_sha256", "tokenizer_version", "exports")}
|
| 209 |
+
if (manifest["generation"] != digest(canonical(identity))
|
| 210 |
+
or manifest_path != f'releases/ocr-{manifest["generation"]}/manifest.json'):
|
| 211 |
+
raise ValueError("OCR release identity/path mismatch")
|
| 212 |
+
refs = manifest.get("exports")
|
| 213 |
+
if not isinstance(refs, list) or not refs:
|
| 214 |
+
raise ValueError("empty OCR release")
|
| 215 |
+
records, exports = [], []
|
| 216 |
+
ids, paths = set(), set()
|
| 217 |
+
for ref in refs:
|
| 218 |
+
record, export = export_record(bucket, ref)
|
| 219 |
+
if record["doc_id"] in ids or record["display_rel_path"] in paths:
|
| 220 |
+
raise ValueError("OCR ID/readable-path collision; no silent overwrite")
|
| 221 |
+
ids.add(record["doc_id"]); paths.add(record["display_rel_path"])
|
| 222 |
+
records.append(record); exports.append(export)
|
| 223 |
+
if manifest.get("documents") != len(records):
|
| 224 |
+
raise ValueError("OCR release count mismatch")
|
| 225 |
+
database = manifest.get("database")
|
| 226 |
+
read_resource(bucket, database)
|
| 227 |
+
db_path = (bucket / database["path"]).resolve()
|
| 228 |
+
if database["path"] != f'indexes/ocr/{manifest["generation"]}/OCR.sqlite3':
|
| 229 |
+
raise ValueError("OCR database must retain its source namespace")
|
| 230 |
+
validate_database(db_path, records, bucket)
|
| 231 |
+
tree, browser = directory_payloads(records)
|
| 232 |
+
return {"manifest": manifest, "records": records, "exports": exports, "database": db_path,
|
| 233 |
+
"text_paths": {record["doc_id"]: bucket / record["storage_rel_path"] for record in records},
|
| 234 |
+
"tree": tree, "browser": browser}
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def pin_release(bucket: Path, manifest_path: str, expected_sha256: str,
|
| 238 |
+
base_metadata_sha256: str, cache: Path) -> dict:
|
| 239 |
+
"""Verify the whole candidate before pinning a process-local immutable copy."""
|
| 240 |
+
candidate = load_release(bucket, manifest_path, expected_sha256, base_metadata_sha256)
|
| 241 |
+
cache.mkdir(parents=True, exist_ok=True)
|
| 242 |
+
destination = cache / candidate["manifest"]["generation"]
|
| 243 |
+
if destination.exists():
|
| 244 |
+
return load_release(destination, manifest_path, expected_sha256, base_metadata_sha256)
|
| 245 |
+
stage = Path(tempfile.mkdtemp(prefix="ocr-stage-", dir=cache))
|
| 246 |
+
try:
|
| 247 |
+
refs = [*candidate["manifest"]["exports"], candidate["manifest"]["database"],
|
| 248 |
+
*[export["text"] for export in candidate["exports"]]]
|
| 249 |
+
for ref in refs:
|
| 250 |
+
raw = read_resource(bucket, ref)
|
| 251 |
+
target = stage / ref["path"]
|
| 252 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 253 |
+
with target.open("wb") as output:
|
| 254 |
+
output.write(raw); output.flush(); os.fsync(output.fileno())
|
| 255 |
+
target = stage / manifest_path
|
| 256 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 257 |
+
manifest_raw = (bucket / manifest_path).read_bytes()
|
| 258 |
+
if digest(manifest_raw) != expected_sha256:
|
| 259 |
+
raise ValueError("OCR candidate changed while being copied")
|
| 260 |
+
target.write_bytes(manifest_raw)
|
| 261 |
+
load_release(stage, manifest_path, expected_sha256, base_metadata_sha256)
|
| 262 |
+
stage.rename(destination)
|
| 263 |
+
finally:
|
| 264 |
+
if stage.exists():
|
| 265 |
+
shutil.rmtree(stage)
|
| 266 |
+
return load_release(destination, manifest_path, expected_sha256, base_metadata_sha256)
|
requirements-test.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r requirements.txt
|
| 2 |
+
httpx==0.28.1
|
| 3 |
+
playwright==1.63.0
|
tests/__init__.py
ADDED
|
File without changes
|
tests/test_archive_compat.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import shutil
|
| 7 |
+
import subprocess
|
| 8 |
+
import tempfile
|
| 9 |
+
import unittest
|
| 10 |
+
import zipfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from unittest.mock import patch
|
| 13 |
+
|
| 14 |
+
import app
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
| 18 |
+
RUN_LARGE = os.environ.get("CCRD_LARGE_ARCHIVE_TEST") == "1"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ArchiveNameCompatibilityTests(unittest.TestCase):
|
| 22 |
+
@classmethod
|
| 23 |
+
def setUpClass(cls):
|
| 24 |
+
app.load_data()
|
| 25 |
+
|
| 26 |
+
def test_every_corpus_record_maps_to_unique_portable_member_name(self):
|
| 27 |
+
entries = app.zip_entries([record["doc_id"] for record in app.records])
|
| 28 |
+
names = [name for name, _path in entries]
|
| 29 |
+
self.assertEqual(len(names), 47250)
|
| 30 |
+
self.assertEqual(len(names), len(set(names)))
|
| 31 |
+
self.assertTrue(all(name.count("/") == 1 for name in names))
|
| 32 |
+
self.assertTrue(all(not any(character in name for character in ':*?"<>|\\') for name in names))
|
| 33 |
+
self.assertTrue(all(not any(part.endswith((" ", ".")) for part in name.split("/")) for name in names))
|
| 34 |
+
self.assertLessEqual(max(len(name.encode("utf-8")) for name in names), 220)
|
| 35 |
+
|
| 36 |
+
def test_multiple_independent_extractors_accept_adversarial_archive(self):
|
| 37 |
+
payload = json.loads(gzip.decompress((BASE_DIR / "data/search_data.json.gz").read_bytes()))
|
| 38 |
+
records = payload["records"]
|
| 39 |
+
selected = sorted(
|
| 40 |
+
records,
|
| 41 |
+
key=lambda record: (
|
| 42 |
+
any(character in record["display_name"] for character in ':*?"<>|\\'),
|
| 43 |
+
len(record["display_name"].encode("utf-8")),
|
| 44 |
+
),
|
| 45 |
+
reverse=True,
|
| 46 |
+
)[:100]
|
| 47 |
+
response = app.zip_response([record["doc_id"] for record in selected])
|
| 48 |
+
archive_path = Path(response.path)
|
| 49 |
+
try:
|
| 50 |
+
with zipfile.ZipFile(archive_path) as archive:
|
| 51 |
+
self.assertEqual(len(archive.namelist()), 100)
|
| 52 |
+
self.assertIsNone(archive.testzip())
|
| 53 |
+
commands = []
|
| 54 |
+
if shutil.which("unzip"):
|
| 55 |
+
commands.append(["unzip", "-t", str(archive_path)])
|
| 56 |
+
if shutil.which("7z"):
|
| 57 |
+
commands.append(["7z", "t", str(archive_path)])
|
| 58 |
+
if shutil.which("bsdtar"):
|
| 59 |
+
commands.append(["bsdtar", "-tf", str(archive_path)])
|
| 60 |
+
self.assertGreaterEqual(len(commands), 3)
|
| 61 |
+
for command in commands:
|
| 62 |
+
result = subprocess.run(command, capture_output=True, timeout=120)
|
| 63 |
+
self.assertEqual(result.returncode, 0, f"{command}: {result.stderr.decode(errors='ignore')}")
|
| 64 |
+
finally:
|
| 65 |
+
archive_path.unlink(missing_ok=True)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@unittest.skipUnless(RUN_LARGE, "set CCRD_LARGE_ARCHIVE_TEST=1 to test the 500 largest documents")
|
| 69 |
+
class MaximumArchiveCompatibilityTests(unittest.TestCase):
|
| 70 |
+
def test_500_largest_documents_open_with_multiple_extractors(self):
|
| 71 |
+
app.load_data()
|
| 72 |
+
selected = sorted(app.records, key=lambda record: record["size"], reverse=True)[:500]
|
| 73 |
+
response = app.zip_response([record["doc_id"] for record in selected])
|
| 74 |
+
archive_path = Path(response.path)
|
| 75 |
+
try:
|
| 76 |
+
self.assertGreater(archive_path.stat().st_size, 0)
|
| 77 |
+
with zipfile.ZipFile(archive_path) as archive:
|
| 78 |
+
self.assertEqual(len(archive.namelist()), 500)
|
| 79 |
+
self.assertIsNone(archive.testzip())
|
| 80 |
+
for command in (["unzip", "-t", str(archive_path)], ["7z", "t", str(archive_path)], ["bsdtar", "-tf", str(archive_path)]):
|
| 81 |
+
result = subprocess.run(command, capture_output=True, timeout=300)
|
| 82 |
+
self.assertEqual(result.returncode, 0, f"{command}: {result.stderr.decode(errors='ignore')}")
|
| 83 |
+
finally:
|
| 84 |
+
archive_path.unlink(missing_ok=True)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
if __name__ == "__main__":
|
| 88 |
+
unittest.main()
|
tests/test_benchmark.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import socket
|
| 6 |
+
import statistics
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
import time
|
| 10 |
+
import unittest
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
+
from urllib.request import Request, urlopen
|
| 13 |
+
|
| 14 |
+
RESULTS_PATH = os.path.join(os.path.dirname(__file__), "benchmark-results.json")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _git_sha():
|
| 18 |
+
try:
|
| 19 |
+
return subprocess.run(
|
| 20 |
+
["git", "log", "-1", "--format=%H"],
|
| 21 |
+
capture_output=True, text=True, timeout=5,
|
| 22 |
+
).stdout.strip()
|
| 23 |
+
except Exception:
|
| 24 |
+
return ""
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _save_results(project, queries):
|
| 28 |
+
existing = {"results": []}
|
| 29 |
+
if os.path.isfile(RESULTS_PATH):
|
| 30 |
+
try:
|
| 31 |
+
with open(RESULTS_PATH) as f:
|
| 32 |
+
existing = json.load(f)
|
| 33 |
+
except (json.JSONDecodeError, OSError):
|
| 34 |
+
pass
|
| 35 |
+
existing.setdefault("results", []).append({
|
| 36 |
+
"timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
| 37 |
+
"project": project,
|
| 38 |
+
"git_sha": _git_sha(),
|
| 39 |
+
"queries": queries,
|
| 40 |
+
})
|
| 41 |
+
existing["results"] = existing["results"][-50:]
|
| 42 |
+
with open(RESULTS_PATH, "w") as f:
|
| 43 |
+
json.dump(existing, f, indent=2, ensure_ascii=False)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _print_and_collect(results_list, label, iterations, times):
|
| 47 |
+
median = statistics.median(times)
|
| 48 |
+
mn = min(times)
|
| 49 |
+
mx = max(times)
|
| 50 |
+
print(f" {label:<46s} median={median:>8.3f}ms min={mn:>8.3f}ms max={mx:>8.3f}ms (n={iterations})")
|
| 51 |
+
results_list.append({
|
| 52 |
+
"label": label.strip(),
|
| 53 |
+
"iterations": iterations,
|
| 54 |
+
"median_ms": round(median, 3),
|
| 55 |
+
"min_ms": round(mn, 3),
|
| 56 |
+
"max_ms": round(mx, 3),
|
| 57 |
+
})
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def free_port():
|
| 61 |
+
with socket.socket() as sock:
|
| 62 |
+
sock.bind(("127.0.0.1", 0))
|
| 63 |
+
return sock.getsockname()[1]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# ---- shared server for API & browser tests ----
|
| 67 |
+
_server_proc = None
|
| 68 |
+
_server_port = None
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def start_server():
|
| 72 |
+
global _server_proc, _server_port
|
| 73 |
+
if _server_proc is not None:
|
| 74 |
+
return
|
| 75 |
+
_server_port = free_port()
|
| 76 |
+
_server_proc = subprocess.Popen(
|
| 77 |
+
[sys.executable, "-m", "uvicorn", "app:app",
|
| 78 |
+
"--host", "127.0.0.1", "--port", str(_server_port),
|
| 79 |
+
"--log-level", "error"],
|
| 80 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
| 81 |
+
)
|
| 82 |
+
for _ in range(120):
|
| 83 |
+
try:
|
| 84 |
+
with urlopen(f"http://127.0.0.1:{_server_port}/api/sources", timeout=2) as r:
|
| 85 |
+
if r.status == 200:
|
| 86 |
+
return
|
| 87 |
+
except Exception:
|
| 88 |
+
time.sleep(0.5)
|
| 89 |
+
stop_server()
|
| 90 |
+
raise RuntimeError("CCRD server did not start")
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def stop_server():
|
| 94 |
+
global _server_proc, _server_port
|
| 95 |
+
if _server_proc:
|
| 96 |
+
try:
|
| 97 |
+
_server_proc.terminate()
|
| 98 |
+
_server_proc.wait(timeout=10)
|
| 99 |
+
except Exception:
|
| 100 |
+
_server_proc.kill()
|
| 101 |
+
_server_proc = None
|
| 102 |
+
_server_port = None
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# ---- local benchmarks (pure function) ----
|
| 106 |
+
import app # noqa: E402
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class LocalSearchBenchmarkTests(unittest.TestCase):
|
| 110 |
+
_results: list = []
|
| 111 |
+
|
| 112 |
+
@classmethod
|
| 113 |
+
def setUpClass(cls):
|
| 114 |
+
if not app.source_counts:
|
| 115 |
+
app.load_data()
|
| 116 |
+
|
| 117 |
+
def _bench(self, label, fn, iterations=10):
|
| 118 |
+
times = []
|
| 119 |
+
for _ in range(iterations):
|
| 120 |
+
start = time.perf_counter()
|
| 121 |
+
fn()
|
| 122 |
+
elapsed = (time.perf_counter() - start) * 1000
|
| 123 |
+
times.append(elapsed)
|
| 124 |
+
_print_and_collect(self._results, label, iterations, times)
|
| 125 |
+
|
| 126 |
+
def test_benchmarks(self):
|
| 127 |
+
self._bench('"手机" normal', lambda: app.search("手机"))
|
| 128 |
+
self._bench('"手机" exact', lambda: app.search("手机", exact=True))
|
| 129 |
+
self._bench('"文化革命" exact', lambda: app.search("文化革命", exact=True))
|
| 130 |
+
self._bench('"文化 革命" normal', lambda: app.search("文化 革命"))
|
| 131 |
+
self._bench('"手。机" exact', lambda: app.search("手。机", exact=True))
|
| 132 |
+
self._bench('"手*机" exact', lambda: app.search("手*机", exact=True))
|
| 133 |
+
self._bench('"手?机" exact', lambda: app.search("手?机", exact=True))
|
| 134 |
+
self._bench('"文" exact', lambda: app.search("文", exact=True))
|
| 135 |
+
self._bench('"ABC" normal', lambda: app.search("ABC"))
|
| 136 |
+
self._bench('"文化革命" exact search_paths=False',
|
| 137 |
+
lambda: app.search("文化革命", exact=True, search_paths=False))
|
| 138 |
+
self._bench('"手机" exact min_size=100',
|
| 139 |
+
lambda: app.search("手机", exact=True, min_size=100))
|
| 140 |
+
self._bench('sort by name', lambda: app.search(page=1, page_size=100, sort="name"))
|
| 141 |
+
self._bench('sort by size', lambda: app.search(page=1, page_size=100, sort="size"))
|
| 142 |
+
|
| 143 |
+
@classmethod
|
| 144 |
+
def tearDownClass(cls):
|
| 145 |
+
if cls._results:
|
| 146 |
+
_save_results("CCRD-Search-local", cls._results)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# ---- API benchmarks (HTTP to local server) ----
|
| 150 |
+
class ApiSearchBenchmarkTests(unittest.TestCase):
|
| 151 |
+
_results: list = []
|
| 152 |
+
|
| 153 |
+
def _bench(self, label, path, payload=None, iterations=5):
|
| 154 |
+
body = json.dumps(payload).encode() if payload else None
|
| 155 |
+
req = Request(f"http://127.0.0.1:{_server_port}{path}", data=body,
|
| 156 |
+
headers={"Content-Type": "application/json"} if body else {})
|
| 157 |
+
times = []
|
| 158 |
+
for i in range(iterations + 1):
|
| 159 |
+
start = time.perf_counter()
|
| 160 |
+
with urlopen(req, timeout=60) as r:
|
| 161 |
+
r.read()
|
| 162 |
+
elapsed = (time.perf_counter() - start) * 1000
|
| 163 |
+
if i > 0:
|
| 164 |
+
times.append(elapsed)
|
| 165 |
+
_print_and_collect(self._results, label, iterations, times)
|
| 166 |
+
|
| 167 |
+
def test_search_api_benchmarks(self):
|
| 168 |
+
self._bench('"手机" normal', "/api/search", {"q": "手机"})
|
| 169 |
+
self._bench('"手机" exact', "/api/search", {"q": "手机", "exact": True})
|
| 170 |
+
self._bench('"文化革命" exact', "/api/search", {"q": "文化革命", "exact": True})
|
| 171 |
+
self._bench('"文化 革命" normal', "/api/search", {"q": "文化 革命"})
|
| 172 |
+
self._bench('"手。机" exact', "/api/search", {"q": "手。机", "exact": True})
|
| 173 |
+
self._bench('"手*机" exact', "/api/search", {"q": "手*机", "exact": True})
|
| 174 |
+
self._bench('"手?机" exact', "/api/search", {"q": "手?机", "exact": True})
|
| 175 |
+
self._bench('"文" exact', "/api/search", {"q": "文", "exact": True})
|
| 176 |
+
self._bench('"ABC" normal', "/api/search", {"q": "ABC"})
|
| 177 |
+
self._bench('sources', "/api/sources")
|
| 178 |
+
|
| 179 |
+
@classmethod
|
| 180 |
+
def tearDownClass(cls):
|
| 181 |
+
if cls._results:
|
| 182 |
+
_save_results("CCRD-Search-api", cls._results)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
# ---- browser benchmarks (Playwright) ----
|
| 186 |
+
try:
|
| 187 |
+
from playwright.sync_api import sync_playwright
|
| 188 |
+
except ImportError:
|
| 189 |
+
sync_playwright = None
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
@unittest.skipUnless(sync_playwright, "install requirements-test.txt, run: playwright install chromium")
|
| 193 |
+
class BrowserSearchBenchmarkTests(unittest.TestCase):
|
| 194 |
+
_results: list = []
|
| 195 |
+
|
| 196 |
+
@classmethod
|
| 197 |
+
def setUpClass(cls):
|
| 198 |
+
cls.playwright = sync_playwright().start()
|
| 199 |
+
cls.browser = cls.playwright.chromium.launch(headless=True, args=["--no-sandbox",
|
| 200 |
+
"--disable-gpu", "--disable-dev-shm-usage"])
|
| 201 |
+
|
| 202 |
+
@classmethod
|
| 203 |
+
def tearDownClass(cls):
|
| 204 |
+
if cls._results:
|
| 205 |
+
_save_results("CCRD-Search-browser", cls._results)
|
| 206 |
+
cls.browser.close()
|
| 207 |
+
cls.playwright.stop()
|
| 208 |
+
|
| 209 |
+
def _bench_search(self, label, query, iterations=3):
|
| 210 |
+
ctx = self.browser.new_context(viewport={"width": 1440, "height": 900})
|
| 211 |
+
page = ctx.new_page()
|
| 212 |
+
times = []
|
| 213 |
+
for i in range(iterations + 1):
|
| 214 |
+
page.goto(f"http://127.0.0.1:{_server_port}/")
|
| 215 |
+
page.wait_for_selector("#results-list .result-item", timeout=30000)
|
| 216 |
+
before_hash = page.evaluate("""() => {
|
| 217 |
+
const items = document.querySelectorAll('.result-item');
|
| 218 |
+
return Array.from(items).slice(0, 3).map(el => el.textContent.substring(0, 50)).join('|');
|
| 219 |
+
}""")
|
| 220 |
+
inp = page.locator("#search-input")
|
| 221 |
+
start = time.perf_counter()
|
| 222 |
+
inp.fill(query)
|
| 223 |
+
page.wait_for_function(f"""
|
| 224 |
+
() => {{
|
| 225 |
+
const items = document.querySelectorAll('.result-item');
|
| 226 |
+
if (items.length > 0) {{
|
| 227 |
+
const hash = Array.from(items).slice(0, 3).map(el => el.textContent.substring(0, 50)).join('|');
|
| 228 |
+
if (hash !== {json.dumps(before_hash)}) return true;
|
| 229 |
+
}}
|
| 230 |
+
const empty = document.querySelector('#empty-state');
|
| 231 |
+
return !!(empty && empty.style.display !== 'none');
|
| 232 |
+
}}
|
| 233 |
+
""", timeout=15000)
|
| 234 |
+
elapsed = (time.perf_counter() - start) * 1000
|
| 235 |
+
if i > 0:
|
| 236 |
+
times.append(elapsed)
|
| 237 |
+
ctx.close()
|
| 238 |
+
_print_and_collect(self._results, label, iterations, times)
|
| 239 |
+
|
| 240 |
+
def test_browser_search_benchmarks(self):
|
| 241 |
+
self._bench_search('"手机" normal', "手机")
|
| 242 |
+
self._bench_search('"文化革命" exact', "文化革命")
|
| 243 |
+
self._bench_search('"手*机" exact', "手*机")
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def setUpModule():
|
| 247 |
+
start_server()
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def tearDownModule():
|
| 251 |
+
stop_server()
|
tests/test_browser.py
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import io
|
| 4 |
+
import os
|
| 5 |
+
import socket
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
import time
|
| 9 |
+
import unittest
|
| 10 |
+
import zipfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from urllib.request import urlopen
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
from playwright.sync_api import sync_playwright
|
| 16 |
+
except ImportError:
|
| 17 |
+
sync_playwright = None
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def free_port() -> int:
|
| 24 |
+
with socket.socket() as sock:
|
| 25 |
+
sock.bind(("127.0.0.1", 0))
|
| 26 |
+
return int(sock.getsockname()[1])
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@unittest.skipUnless(sync_playwright, "install requirements-test.txt and run: playwright install chromium")
|
| 30 |
+
class BrowserInteractionTests(unittest.TestCase):
|
| 31 |
+
@classmethod
|
| 32 |
+
def setUpClass(cls):
|
| 33 |
+
cls.base_url = os.environ.get("CCRD_BROWSER_BASE_URL", "").rstrip("/")
|
| 34 |
+
cls.server = None
|
| 35 |
+
if not cls.base_url:
|
| 36 |
+
cls.port = free_port()
|
| 37 |
+
cls.base_url = f"http://127.0.0.1:{cls.port}"
|
| 38 |
+
cls.server = subprocess.Popen(
|
| 39 |
+
[
|
| 40 |
+
sys.executable, "-m", "uvicorn", "app:app", "--host", "127.0.0.1",
|
| 41 |
+
"--port", str(cls.port), "--log-level", "warning",
|
| 42 |
+
],
|
| 43 |
+
cwd=BASE_DIR,
|
| 44 |
+
stdout=subprocess.DEVNULL,
|
| 45 |
+
stderr=subprocess.DEVNULL,
|
| 46 |
+
)
|
| 47 |
+
for _ in range(120):
|
| 48 |
+
if cls.server.poll() is not None:
|
| 49 |
+
raise RuntimeError("Uvicorn exited before browser tests started")
|
| 50 |
+
try:
|
| 51 |
+
with urlopen(cls.base_url + "/api/sources", timeout=1) as response:
|
| 52 |
+
if response.status == 200:
|
| 53 |
+
break
|
| 54 |
+
except Exception:
|
| 55 |
+
time.sleep(0.25)
|
| 56 |
+
else:
|
| 57 |
+
cls.server.terminate()
|
| 58 |
+
raise RuntimeError("Uvicorn did not become ready")
|
| 59 |
+
cls.playwright = sync_playwright().start()
|
| 60 |
+
cls.browser = cls.playwright.chromium.launch(headless=True, args=["--no-sandbox"])
|
| 61 |
+
|
| 62 |
+
@classmethod
|
| 63 |
+
def tearDownClass(cls):
|
| 64 |
+
cls.browser.close()
|
| 65 |
+
cls.playwright.stop()
|
| 66 |
+
if cls.server is not None:
|
| 67 |
+
cls.server.terminate()
|
| 68 |
+
try:
|
| 69 |
+
cls.server.wait(timeout=10)
|
| 70 |
+
except subprocess.TimeoutExpired:
|
| 71 |
+
cls.server.kill()
|
| 72 |
+
|
| 73 |
+
def page(self, viewport=None):
|
| 74 |
+
context = self.browser.new_context(
|
| 75 |
+
viewport=viewport or {"width": 1440, "height": 900},
|
| 76 |
+
accept_downloads=True,
|
| 77 |
+
)
|
| 78 |
+
page = context.new_page()
|
| 79 |
+
errors = []
|
| 80 |
+
page.on("pageerror", lambda error: errors.append(str(error)))
|
| 81 |
+
page.goto(self.base_url, wait_until="domcontentloaded")
|
| 82 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 83 |
+
return context, page, errors
|
| 84 |
+
|
| 85 |
+
def search_request_pages(self, page):
|
| 86 |
+
pages = []
|
| 87 |
+
|
| 88 |
+
def handle_request(request):
|
| 89 |
+
if request.method != "POST":
|
| 90 |
+
return
|
| 91 |
+
if not request.url.endswith("/api/search") and "/api/search/" not in request.url:
|
| 92 |
+
return
|
| 93 |
+
try:
|
| 94 |
+
body = request.post_data_json
|
| 95 |
+
except Exception:
|
| 96 |
+
return
|
| 97 |
+
pages.append(body.get("page"))
|
| 98 |
+
|
| 99 |
+
page.on("request", handle_request)
|
| 100 |
+
return pages
|
| 101 |
+
|
| 102 |
+
def test_late_initial_payload_cannot_take_over_a_new_search(self):
|
| 103 |
+
context, page, errors = self.page()
|
| 104 |
+
try:
|
| 105 |
+
result = page.evaluate("""async () => {
|
| 106 |
+
abortSearchPrefetch();
|
| 107 |
+
let releaseInitial;
|
| 108 |
+
loadInitialPayload = () => new Promise(resolve => { releaseInitial = resolve; });
|
| 109 |
+
STATE.query = '';
|
| 110 |
+
STATE.page = 1;
|
| 111 |
+
DOM.sortSelect.value = 'relevance';
|
| 112 |
+
const oldSearch = doSearch();
|
| 113 |
+
STATE.query = 'new-query';
|
| 114 |
+
const requests = [];
|
| 115 |
+
API.search = (body, source, signal) => new Promise(resolve => {
|
| 116 |
+
requests.push({signal, resolve});
|
| 117 |
+
});
|
| 118 |
+
const newSearch = doSearch();
|
| 119 |
+
await Promise.resolve();
|
| 120 |
+
const owner = STATE.searchController;
|
| 121 |
+
releaseInitial(null);
|
| 122 |
+
for (let i = 0; i < 10; i++) await Promise.resolve();
|
| 123 |
+
const retainedOwner = STATE.searchController === owner;
|
| 124 |
+
for (const request of requests) request.resolve({results: [], total: 0});
|
| 125 |
+
await Promise.all([oldSearch, newSearch]);
|
| 126 |
+
return {retainedOwner, requests: requests.length, loading: STATE.isLoading};
|
| 127 |
+
}""")
|
| 128 |
+
self.assertEqual(result, {"retainedOwner": True, "requests": 1, "loading": False})
|
| 129 |
+
self.assertEqual(errors, [])
|
| 130 |
+
finally:
|
| 131 |
+
context.close()
|
| 132 |
+
|
| 133 |
+
def test_initial_json_body_timeout_falls_back_to_search(self):
|
| 134 |
+
context, page, errors = self.page()
|
| 135 |
+
try:
|
| 136 |
+
result = page.evaluate("""async () => {
|
| 137 |
+
abortSearchPrefetch();
|
| 138 |
+
initialPayloadCache.clear();
|
| 139 |
+
const originalFetch = window.fetch;
|
| 140 |
+
const originalTimeout = window.setTimeout;
|
| 141 |
+
let aborted = false;
|
| 142 |
+
window.setTimeout = (fn, ms, ...args) => originalTimeout(fn, ms === REQUEST_TIMEOUT_MS ? 30 : ms, ...args);
|
| 143 |
+
window.fetch = (url, options = {}) => Promise.resolve({ok: true,
|
| 144 |
+
json: () => new Promise((resolve, reject) => {
|
| 145 |
+
if (options.signal) options.signal.addEventListener('abort', () => {
|
| 146 |
+
aborted = true;
|
| 147 |
+
reject(new DOMException('aborted', 'AbortError'));
|
| 148 |
+
}, {once: true});
|
| 149 |
+
})});
|
| 150 |
+
let searched = false;
|
| 151 |
+
API.search = async () => { searched = true; return {results: [], total: 0}; };
|
| 152 |
+
STATE.query = '';
|
| 153 |
+
STATE.page = 1;
|
| 154 |
+
DOM.sortSelect.value = 'relevance';
|
| 155 |
+
const completed = await Promise.race([
|
| 156 |
+
doSearch().then(() => true),
|
| 157 |
+
new Promise(resolve => originalTimeout(() => resolve(false), 500))
|
| 158 |
+
]);
|
| 159 |
+
window.fetch = originalFetch;
|
| 160 |
+
window.setTimeout = originalTimeout;
|
| 161 |
+
return {completed, aborted, searched};
|
| 162 |
+
}""")
|
| 163 |
+
self.assertEqual(result, {"completed": True, "aborted": True, "searched": True})
|
| 164 |
+
self.assertEqual(errors, [])
|
| 165 |
+
finally:
|
| 166 |
+
context.close()
|
| 167 |
+
|
| 168 |
+
def test_stale_search_failure_does_not_change_current_page(self):
|
| 169 |
+
context, page, errors = self.page()
|
| 170 |
+
try:
|
| 171 |
+
result = page.evaluate("""async () => {
|
| 172 |
+
abortSearchPrefetch();
|
| 173 |
+
let failOld;
|
| 174 |
+
let finishNew;
|
| 175 |
+
const toasts = [];
|
| 176 |
+
showToast = message => toasts.push(message);
|
| 177 |
+
API.search = body => new Promise((resolve, reject) => {
|
| 178 |
+
if (body.q === 'old-query') failOld = reject;
|
| 179 |
+
else finishNew = resolve;
|
| 180 |
+
});
|
| 181 |
+
STATE.query = 'old-query';
|
| 182 |
+
const oldSearch = doSearch();
|
| 183 |
+
await Promise.resolve();
|
| 184 |
+
STATE.query = 'new-query';
|
| 185 |
+
STATE.page = 2;
|
| 186 |
+
const newSearch = doSearch();
|
| 187 |
+
await Promise.resolve();
|
| 188 |
+
failOld(new TypeError('late network failure'));
|
| 189 |
+
await oldSearch;
|
| 190 |
+
const currentPage = STATE.page;
|
| 191 |
+
finishNew({results: [], total: 0});
|
| 192 |
+
await newSearch;
|
| 193 |
+
return {currentPage, toasts};
|
| 194 |
+
}""")
|
| 195 |
+
self.assertEqual(result, {"currentPage": 2, "toasts": []})
|
| 196 |
+
self.assertEqual(errors, [])
|
| 197 |
+
finally:
|
| 198 |
+
context.close()
|
| 199 |
+
|
| 200 |
+
def test_failed_sidebar_payload_can_be_retried(self):
|
| 201 |
+
context, page, errors = self.page()
|
| 202 |
+
try:
|
| 203 |
+
result = page.evaluate("""async () => {
|
| 204 |
+
sidebarInitialCache.clear();
|
| 205 |
+
let attempts = 0;
|
| 206 |
+
window.fetch = async () => {
|
| 207 |
+
if (++attempts === 1) throw new TypeError('connection lost');
|
| 208 |
+
return {ok: true, json: async () => ({sources: []})};
|
| 209 |
+
};
|
| 210 |
+
await loadSidebarInitial(null).catch(() => null);
|
| 211 |
+
const data = await loadSidebarInitial(null).catch(() => null);
|
| 212 |
+
return {attempts, recovered: Array.isArray(data && data.sources)};
|
| 213 |
+
}""")
|
| 214 |
+
self.assertEqual(result, {"attempts": 2, "recovered": True})
|
| 215 |
+
self.assertEqual(errors, [])
|
| 216 |
+
finally:
|
| 217 |
+
context.close()
|
| 218 |
+
|
| 219 |
+
def test_late_root_sidebar_does_not_replace_open_folder(self):
|
| 220 |
+
for initial in (True, False):
|
| 221 |
+
with self.subTest(initial=initial):
|
| 222 |
+
context, page, errors = self.page()
|
| 223 |
+
try:
|
| 224 |
+
result = page.evaluate("""async initial => {
|
| 225 |
+
STATE.source = 'CCRD';
|
| 226 |
+
const route = ++routeRenderId;
|
| 227 |
+
let release;
|
| 228 |
+
const rootData = {source: 'CCRD', folders: [{name: 'root', path: 'root'}], files: []};
|
| 229 |
+
loadSidebarInitial = initial
|
| 230 |
+
? () => new Promise(resolve => { release = () => resolve(rootData); })
|
| 231 |
+
: async () => null;
|
| 232 |
+
API.getContents = (source, path) => path
|
| 233 |
+
? Promise.resolve({folders: [], files: [{doc_id: 'child', name: '当前目录文件'}]})
|
| 234 |
+
: new Promise(resolve => { release = () => resolve(rootData); });
|
| 235 |
+
const pending = renderSidebar(route);
|
| 236 |
+
for (let i = 0; i < 5; i++) await Promise.resolve();
|
| 237 |
+
await openFolder('child', route);
|
| 238 |
+
const html = DOM.sidebarContent.innerHTML;
|
| 239 |
+
release();
|
| 240 |
+
await pending;
|
| 241 |
+
return {preserved: DOM.sidebarContent.innerHTML === html, path: sidebarCurrentPath};
|
| 242 |
+
}""", initial)
|
| 243 |
+
self.assertEqual(result, {"preserved": True, "path": "child"})
|
| 244 |
+
self.assertEqual(errors, [])
|
| 245 |
+
finally:
|
| 246 |
+
context.close()
|
| 247 |
+
|
| 248 |
+
def test_old_sidebar_retry_cannot_reopen_previous_folder(self):
|
| 249 |
+
context, page, errors = self.page()
|
| 250 |
+
try:
|
| 251 |
+
result = page.evaluate("""async () => {
|
| 252 |
+
STATE.source = 'CCRD';
|
| 253 |
+
const route = ++routeRenderId;
|
| 254 |
+
const originalTimeout = window.setTimeout;
|
| 255 |
+
const retries = [];
|
| 256 |
+
window.setTimeout = (fn, ms, ...args) => ms === 1200
|
| 257 |
+
? (retries.push(fn), 0) : originalTimeout(fn, ms, ...args);
|
| 258 |
+
const paths = [];
|
| 259 |
+
API.getContents = async (source, path) => {
|
| 260 |
+
paths.push(path);
|
| 261 |
+
if (path === 'old') throw new TypeError('offline');
|
| 262 |
+
return {folders: [], files: [{doc_id: 'new', name: '新目录'}]};
|
| 263 |
+
};
|
| 264 |
+
await openFolder('old', route);
|
| 265 |
+
await openFolder('new', route);
|
| 266 |
+
for (const retry of retries.slice()) retry();
|
| 267 |
+
for (let i = 0; i < 5; i++) await Promise.resolve();
|
| 268 |
+
window.setTimeout = originalTimeout;
|
| 269 |
+
return {paths, path: sidebarCurrentPath, visible: DOM.sidebarContent.textContent.includes('新目录')};
|
| 270 |
+
}""")
|
| 271 |
+
self.assertEqual(result, {"paths": ["old", "new"], "path": "new", "visible": True})
|
| 272 |
+
self.assertEqual(errors, [])
|
| 273 |
+
finally:
|
| 274 |
+
context.close()
|
| 275 |
+
|
| 276 |
+
def test_folder_http_errors_are_not_treated_as_empty_directories(self):
|
| 277 |
+
context, page, errors = self.page()
|
| 278 |
+
try:
|
| 279 |
+
result = page.evaluate("""async () => {
|
| 280 |
+
folderTreePending.clear();
|
| 281 |
+
folderContentsPending.clear();
|
| 282 |
+
STATE.folderContentsCache.clear();
|
| 283 |
+
window.fetch = async () => ({ok: false, status: 503, json: async () => ({folders: [], files: []})});
|
| 284 |
+
const treeRejected = await API.getFolders('CCRD').then(() => false, () => true);
|
| 285 |
+
const contentsRejected = await API.getContents('CCRD', 'retry').then(() => false, () => true);
|
| 286 |
+
let requests = 0;
|
| 287 |
+
window.fetch = async () => {
|
| 288 |
+
requests++;
|
| 289 |
+
return {ok: true, json: async () => ({folders: [], files: [{doc_id: 'recovered'}]})};
|
| 290 |
+
};
|
| 291 |
+
const recovered = await API.getContents('CCRD', 'retry');
|
| 292 |
+
return {treeRejected, contentsRejected, requests, recovered: recovered.files[0]?.doc_id};
|
| 293 |
+
}""")
|
| 294 |
+
self.assertEqual(result, {"treeRejected": True, "contentsRejected": True, "requests": 1, "recovered": "recovered"})
|
| 295 |
+
self.assertEqual(errors, [])
|
| 296 |
+
finally:
|
| 297 |
+
context.close()
|
| 298 |
+
|
| 299 |
+
def test_old_folder_navigation_does_not_restart_search_after_going_home(self):
|
| 300 |
+
context, page, errors = self.page()
|
| 301 |
+
try:
|
| 302 |
+
result = page.evaluate("""async () => {
|
| 303 |
+
let release;
|
| 304 |
+
renderSidebar = () => STATE.source
|
| 305 |
+
? new Promise(resolve => { release = resolve; }) : Promise.resolve();
|
| 306 |
+
const searches = [];
|
| 307 |
+
const trees = [];
|
| 308 |
+
renderFolderTree = async () => trees.push(STATE.source);
|
| 309 |
+
doSearch = () => searches.push(STATE.source);
|
| 310 |
+
const oldNavigation = navigateToSourceFolder('CCRD', 'old');
|
| 311 |
+
navigateHome();
|
| 312 |
+
release();
|
| 313 |
+
await oldNavigation;
|
| 314 |
+
return {searches, trees, source: STATE.source};
|
| 315 |
+
}""")
|
| 316 |
+
self.assertEqual(result, {"searches": [None], "trees": [None], "source": None})
|
| 317 |
+
self.assertEqual(errors, [])
|
| 318 |
+
finally:
|
| 319 |
+
context.close()
|
| 320 |
+
|
| 321 |
+
def test_desktop_controls_search_preview_and_selection(self):
|
| 322 |
+
context, page, errors = self.page()
|
| 323 |
+
try:
|
| 324 |
+
static_controls = (
|
| 325 |
+
"hamburger-btn", "search-input", "settings-btn", "theme-btn", "mobile-toggle-btn",
|
| 326 |
+
"multi-select-toggle", "multi-select-all", "multi-deselect", "multi-batch-download",
|
| 327 |
+
"multi-zip-download", "empty-random-btn", "close-filters-btn", "fulltext-toggle",
|
| 328 |
+
"search-paths-toggle", "history-toggle", "exact-search-toggle", "folder-select-all",
|
| 329 |
+
"folder-deselect-all", "source-filter-cancel", "folder-filter-cancel",
|
| 330 |
+
"filter-min-size", "filter-min-unit", "filter-max-size",
|
| 331 |
+
"filter-max-unit", "random-book-btn", "sort-select",
|
| 332 |
+
)
|
| 333 |
+
for control_id in static_controls:
|
| 334 |
+
self.assertEqual(page.locator(f"#{control_id}").count(), 1, control_id)
|
| 335 |
+
|
| 336 |
+
original_theme = page.locator("body").get_attribute("class")
|
| 337 |
+
page.locator("#theme-btn").click()
|
| 338 |
+
self.assertNotEqual(page.locator("body").get_attribute("class"), original_theme)
|
| 339 |
+
self.assertEqual(page.locator("body.theme-transitioning").count(), 0)
|
| 340 |
+
self.assertGreater(page.locator("#theme-btn .theme-icon-svg").evaluate("node => node.getAnimations().length"), 0)
|
| 341 |
+
|
| 342 |
+
page.locator("#settings-btn").click()
|
| 343 |
+
self.assertTrue(page.evaluate("STATE.rightSidebarOpen"))
|
| 344 |
+
page.locator("#close-filters-btn").click()
|
| 345 |
+
self.assertFalse(page.evaluate("STATE.rightSidebarOpen"))
|
| 346 |
+
|
| 347 |
+
page.locator("#mobile-toggle-btn").click()
|
| 348 |
+
self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 349 |
+
page.locator("#mobile-toggle-btn").click()
|
| 350 |
+
self.assertFalse(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 351 |
+
|
| 352 |
+
page.locator("#settings-btn").click()
|
| 353 |
+
page.locator("label:has(#fulltext-toggle)").click()
|
| 354 |
+
self.assertEqual(page.locator("#search-input").get_attribute("placeholder"), "搜索 TXT 文件或路径...")
|
| 355 |
+
page.locator("label:has(#exact-search-toggle)").click()
|
| 356 |
+
page.locator("label:has(#search-paths-toggle)").click()
|
| 357 |
+
page.locator("label:has(#history-toggle)").click()
|
| 358 |
+
self.assertIn("fulltext=0", page.url)
|
| 359 |
+
self.assertIn("exact=0", page.url)
|
| 360 |
+
self.assertIn("search_paths=0", page.url)
|
| 361 |
+
self.assertIn("history=0", page.url)
|
| 362 |
+
page.locator("#close-filters-btn").click()
|
| 363 |
+
|
| 364 |
+
page.locator("#sort-select").select_option("name")
|
| 365 |
+
page.locator("#search-input").fill("文化革命")
|
| 366 |
+
page.wait_for_timeout(400)
|
| 367 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 368 |
+
self.assertIn("q=", page.url)
|
| 369 |
+
|
| 370 |
+
page.locator("#multi-select-toggle").check()
|
| 371 |
+
page.locator(".result-checkbox").first.check()
|
| 372 |
+
self.assertIn("已选 1 项", page.locator("#multi-selected-count").text_content())
|
| 373 |
+
page.locator("#multi-select-all").click()
|
| 374 |
+
self.assertGreater(page.locator(".result-checkbox:checked").count(), 0)
|
| 375 |
+
page.locator("#multi-select-all").click()
|
| 376 |
+
self.assertEqual(page.locator(".result-checkbox:checked").count(), 0)
|
| 377 |
+
page.locator("#multi-deselect").click()
|
| 378 |
+
self.assertEqual(page.locator(".result-checkbox:checked").count(), 0)
|
| 379 |
+
|
| 380 |
+
page.locator("[data-action='preview']").first.click()
|
| 381 |
+
page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
|
| 382 |
+
self.assertEqual(page.locator("[data-action='close-preview']").count(), 1)
|
| 383 |
+
page.locator("[data-action='close-preview']").click()
|
| 384 |
+
page.locator("#preview-panel").wait_for(state="hidden")
|
| 385 |
+
|
| 386 |
+
page.locator("#random-book-btn").click()
|
| 387 |
+
page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
|
| 388 |
+
page.keyboard.press("Escape")
|
| 389 |
+
page.locator("#preview-panel").wait_for(state="hidden")
|
| 390 |
+
self.assertEqual(errors, [])
|
| 391 |
+
finally:
|
| 392 |
+
context.close()
|
| 393 |
+
|
| 394 |
+
def test_section_filter_cancel_buttons_preserve_other_filters(self):
|
| 395 |
+
context, page, errors = self.page()
|
| 396 |
+
try:
|
| 397 |
+
page.locator("#settings-btn").click()
|
| 398 |
+
page.locator("#filter-min-size").fill("1")
|
| 399 |
+
page.locator("#filter-source-list input").first.check()
|
| 400 |
+
page.locator("#source-filter-cancel").wait_for(state="visible")
|
| 401 |
+
page.locator("#source-filter-cancel").click()
|
| 402 |
+
self.assertEqual(page.evaluate("STATE.selectedSources"), [])
|
| 403 |
+
self.assertEqual(page.locator("#filter-min-size").input_value(), "1")
|
| 404 |
+
page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
|
| 405 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 406 |
+
page.locator("#settings-btn").click()
|
| 407 |
+
page.locator("#folder-select-all").click()
|
| 408 |
+
page.locator("#folder-filter-cancel").wait_for(state="visible")
|
| 409 |
+
page.locator("#folder-filter-cancel").click()
|
| 410 |
+
self.assertEqual(page.evaluate("STATE.folderSelections"), [])
|
| 411 |
+
self.assertNotIn("folder=", page.url)
|
| 412 |
+
self.assertEqual(errors, [])
|
| 413 |
+
finally:
|
| 414 |
+
context.close()
|
| 415 |
+
|
| 416 |
+
def test_short_snippet_does_not_keep_three_line_height(self):
|
| 417 |
+
context, page, errors = self.page()
|
| 418 |
+
try:
|
| 419 |
+
lazy_rendering = page.locator(".result-item").first.evaluate(
|
| 420 |
+
"node => ({ visibility: getComputedStyle(node).contentVisibility, intrinsicSize: getComputedStyle(node).containIntrinsicSize })"
|
| 421 |
+
)
|
| 422 |
+
self.assertEqual(lazy_rendering["visibility"], "auto")
|
| 423 |
+
self.assertIn("112px", lazy_rendering["intrinsicSize"])
|
| 424 |
+
page.locator(".result-snippet").first.wait_for()
|
| 425 |
+
page.locator(".result-snippet").first.evaluate("node => node.textContent = '短摘要'")
|
| 426 |
+
height = page.locator(".result-snippet").first.bounding_box()["height"]
|
| 427 |
+
self.assertLess(height, 40)
|
| 428 |
+
self.assertEqual(errors, [])
|
| 429 |
+
finally:
|
| 430 |
+
context.close()
|
| 431 |
+
|
| 432 |
+
def test_theme_switch_does_not_animate_large_result_tree(self):
|
| 433 |
+
context, page, errors = self.page()
|
| 434 |
+
try:
|
| 435 |
+
page.locator("#results-list").evaluate(
|
| 436 |
+
"""list => {
|
| 437 |
+
const template = list.querySelector('.result-item');
|
| 438 |
+
while (list.children.length < 500) list.appendChild(template.cloneNode(true));
|
| 439 |
+
}"""
|
| 440 |
+
)
|
| 441 |
+
page.locator("#theme-btn").click()
|
| 442 |
+
result_animations = page.locator("#results-list").evaluate(
|
| 443 |
+
"list => list.getAnimations({subtree: true}).length"
|
| 444 |
+
)
|
| 445 |
+
icon_animations = page.locator("#theme-btn .theme-icon-svg").evaluate(
|
| 446 |
+
"node => node.getAnimations().length"
|
| 447 |
+
)
|
| 448 |
+
self.assertEqual(page.locator(".result-item").count(), 500)
|
| 449 |
+
self.assertEqual(result_animations, 0)
|
| 450 |
+
self.assertGreater(icon_animations, 0)
|
| 451 |
+
self.assertEqual(errors, [])
|
| 452 |
+
finally:
|
| 453 |
+
context.close()
|
| 454 |
+
|
| 455 |
+
def test_long_snippet_clamps_to_three_lines(self):
|
| 456 |
+
context, page, errors = self.page()
|
| 457 |
+
try:
|
| 458 |
+
page.locator(".result-snippet").first.wait_for()
|
| 459 |
+
page.locator(".result-snippet").first.evaluate("node => { node.textContent = ['第一行','第二行','第三行','第四行','第五行'].join('\\n'); }")
|
| 460 |
+
height = page.locator(".result-snippet").first.bounding_box()["height"]
|
| 461 |
+
self.assertLess(height, 90)
|
| 462 |
+
self.assertEqual(errors, [])
|
| 463 |
+
finally:
|
| 464 |
+
context.close()
|
| 465 |
+
|
| 466 |
+
def test_source_folder_controls_and_mobile_sidebars(self):
|
| 467 |
+
context = self.browser.new_context(viewport={"width": 390, "height": 844})
|
| 468 |
+
page = context.new_page()
|
| 469 |
+
errors = []
|
| 470 |
+
page.on("pageerror", lambda error: errors.append(str(error)))
|
| 471 |
+
try:
|
| 472 |
+
page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
|
| 473 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 474 |
+
self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 475 |
+
page.locator("#hamburger-btn").click()
|
| 476 |
+
self.assertTrue(page.evaluate("STATE.leftSidebarOpen"))
|
| 477 |
+
page.locator("#overlay").click(position={"x": 380, "y": 400})
|
| 478 |
+
self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
|
| 479 |
+
page.locator("#settings-btn").click()
|
| 480 |
+
self.assertTrue(page.evaluate("STATE.rightSidebarOpen"))
|
| 481 |
+
page.locator("#close-filters-btn").click()
|
| 482 |
+
self.assertFalse(page.evaluate("STATE.rightSidebarOpen"))
|
| 483 |
+
page.locator("#settings-btn").click()
|
| 484 |
+
page.locator("#folder-select-all").click()
|
| 485 |
+
self.assertGreaterEqual(page.locator("#filter-folder-tree input:checked").count(), 1)
|
| 486 |
+
page.evaluate("""() => {
|
| 487 |
+
const paths = [];
|
| 488 |
+
flattenNodes(STATE.folderTree, paths);
|
| 489 |
+
const leaf = paths.sort((left, right) => right.split('/').length - left.split('/').length)[0];
|
| 490 |
+
applyFolderSelections(deselectFolderPath(leaf));
|
| 491 |
+
applyFolderSelections(selectFolderPath(leaf));
|
| 492 |
+
}""")
|
| 493 |
+
page.locator("#folder-select-all").click()
|
| 494 |
+
self.assertEqual(page.evaluate("STATE.folderSelections"), [])
|
| 495 |
+
page.locator("#folder-select-all").click()
|
| 496 |
+
page.locator("#folder-deselect-all").click()
|
| 497 |
+
page.locator("#filter-min-size").fill("1")
|
| 498 |
+
page.locator("#filter-min-unit").select_option("KB")
|
| 499 |
+
page.locator("#close-filters-btn").click()
|
| 500 |
+
page.locator("#clear-filters-btn").click()
|
| 501 |
+
self.assertEqual(page.locator("#filter-min-size").input_value(), "")
|
| 502 |
+
page.locator("[data-action='preview']").first.click()
|
| 503 |
+
page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
|
| 504 |
+
page.locator("[data-action='toggle-preview-theme']").click()
|
| 505 |
+
page.locator("[data-action='close-preview']").click()
|
| 506 |
+
self.assertEqual(errors, [])
|
| 507 |
+
finally:
|
| 508 |
+
context.close()
|
| 509 |
+
|
| 510 |
+
def test_mobile_long_folder_uses_remaining_path_line(self):
|
| 511 |
+
context, page, errors = self.page({"width": 390, "height": 844})
|
| 512 |
+
try:
|
| 513 |
+
layout = page.evaluate("""async () => {
|
| 514 |
+
STATE.results = [{
|
| 515 |
+
Source: "CCRD",
|
| 516 |
+
SourceName: "CCRD",
|
| 517 |
+
File: "路径换行测试",
|
| 518 |
+
Folder: [
|
| 519 |
+
"马列之声ebook小组第七批成果集成(2018.8.17)",
|
| 520 |
+
"马列之声ebook小组第七批成果集成(2018.8.17)part 1a",
|
| 521 |
+
"2018.8 part 1",
|
| 522 |
+
],
|
| 523 |
+
Size: 1,
|
| 524 |
+
doc_id: "browser-path-test",
|
| 525 |
+
snippet: "",
|
| 526 |
+
snippet_complete: true,
|
| 527 |
+
}];
|
| 528 |
+
STATE.total = 1;
|
| 529 |
+
renderResults({ animate: false });
|
| 530 |
+
await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
|
| 531 |
+
const path = document.querySelector('.result-path');
|
| 532 |
+
const folders = [...path.querySelectorAll('.path-folder')];
|
| 533 |
+
const first = document.createRange();
|
| 534 |
+
first.selectNodeContents(folders[1]);
|
| 535 |
+
const source = folders[0].getBoundingClientRect();
|
| 536 |
+
const firstRect = first.getClientRects()[0];
|
| 537 |
+
return {
|
| 538 |
+
sourceTop: source.top,
|
| 539 |
+
firstTop: firstRect.top,
|
| 540 |
+
firstLeft: firstRect.left,
|
| 541 |
+
separatorsNested: [...path.querySelectorAll('.path-sep')].every(sep => sep.parentElement.classList.contains('path-folder')),
|
| 542 |
+
};
|
| 543 |
+
}""")
|
| 544 |
+
self.assertAlmostEqual(layout["firstTop"], layout["sourceTop"], delta=1)
|
| 545 |
+
self.assertGreater(layout["firstLeft"], 0)
|
| 546 |
+
self.assertTrue(layout["separatorsNested"])
|
| 547 |
+
self.assertEqual(errors, [])
|
| 548 |
+
finally:
|
| 549 |
+
context.close()
|
| 550 |
+
|
| 551 |
+
def test_individual_batch_and_zip_downloads_are_browser_valid(self):
|
| 552 |
+
context, page, errors = self.page()
|
| 553 |
+
try:
|
| 554 |
+
with page.expect_download(timeout=30000) as download_info:
|
| 555 |
+
page.locator(".result-action-btn.primary").first.click()
|
| 556 |
+
individual = download_info.value
|
| 557 |
+
self.assertTrue(individual.suggested_filename.endswith(".txt"))
|
| 558 |
+
self.assertGreater(Path(individual.path()).stat().st_size, 0)
|
| 559 |
+
|
| 560 |
+
page.locator("#multi-select-toggle").check()
|
| 561 |
+
page.locator(".result-checkbox").first.check()
|
| 562 |
+
with page.expect_download(timeout=30000) as download_info:
|
| 563 |
+
page.locator("#multi-batch-download").click()
|
| 564 |
+
batch = download_info.value
|
| 565 |
+
self.assertTrue(batch.suggested_filename.endswith(".txt"))
|
| 566 |
+
self.assertGreater(Path(batch.path()).stat().st_size, 0)
|
| 567 |
+
|
| 568 |
+
checkboxes = page.locator(".result-checkbox")
|
| 569 |
+
checkboxes.nth(1).check()
|
| 570 |
+
with page.expect_download(timeout=120000) as download_info:
|
| 571 |
+
page.locator("#multi-zip-download").click()
|
| 572 |
+
archive_download = download_info.value
|
| 573 |
+
archive_bytes = Path(archive_download.path()).read_bytes()
|
| 574 |
+
self.assertTrue(archive_bytes.startswith(b"PK"))
|
| 575 |
+
with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
|
| 576 |
+
self.assertEqual(len(archive.namelist()), 2)
|
| 577 |
+
self.assertIsNone(archive.testzip())
|
| 578 |
+
self.assertNotIn(b"PK\x06\x06", archive_bytes)
|
| 579 |
+
self.assertEqual(errors, [])
|
| 580 |
+
finally:
|
| 581 |
+
context.close()
|
| 582 |
+
|
| 583 |
+
def test_navigation_history_source_filters_and_keyboard(self):
|
| 584 |
+
context, page, errors = self.page()
|
| 585 |
+
try:
|
| 586 |
+
page.locator("#hamburger-btn").click()
|
| 587 |
+
self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
|
| 588 |
+
page.locator("#hamburger-btn").click()
|
| 589 |
+
self.assertTrue(page.evaluate("STATE.leftSidebarOpen"))
|
| 590 |
+
|
| 591 |
+
page.locator("#settings-btn").click()
|
| 592 |
+
source_inputs = page.locator("#filter-source-list input")
|
| 593 |
+
self.assertEqual(source_inputs.count(), 2)
|
| 594 |
+
page.locator("#filter-source-list label").first.click()
|
| 595 |
+
self.assertEqual(len(page.evaluate("STATE.selectedSources")), 1)
|
| 596 |
+
page.locator("#close-filters-btn").click()
|
| 597 |
+
page.locator("#clear-filters-btn").click()
|
| 598 |
+
self.assertEqual(page.evaluate("STATE.selectedSources"), [])
|
| 599 |
+
|
| 600 |
+
page.locator("#search-input").fill("文化")
|
| 601 |
+
page.wait_for_timeout(400)
|
| 602 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 603 |
+
page.locator("#search-input").fill("")
|
| 604 |
+
page.wait_for_timeout(400)
|
| 605 |
+
page.locator("#search-input").focus()
|
| 606 |
+
history_item = page.locator("#search-history-dropdown [data-query]").first
|
| 607 |
+
history_item.wait_for(state="visible", timeout=10000)
|
| 608 |
+
history_item.click()
|
| 609 |
+
self.assertEqual(page.locator("#search-input").input_value(), "文化")
|
| 610 |
+
|
| 611 |
+
page.locator("#search-input").press("Escape")
|
| 612 |
+
self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
|
| 613 |
+
page.keyboard.press("/")
|
| 614 |
+
self.assertTrue(page.locator("#search-input").evaluate("node => document.activeElement === node"))
|
| 615 |
+
|
| 616 |
+
page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
|
| 617 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 618 |
+
self.assertTrue(page.locator("#sidebar-expand-btn").is_visible())
|
| 619 |
+
page.locator("#sidebar-expand-btn").click()
|
| 620 |
+
self.assertTrue(page.locator("#left-sidebar").evaluate("node => node.classList.contains('expanded-wide')"))
|
| 621 |
+
page.locator("#sidebar-expand-btn").click()
|
| 622 |
+
self.assertFalse(page.locator("#left-sidebar").evaluate("node => node.classList.contains('expanded-wide')"))
|
| 623 |
+
back_home = page.locator("#back-home")
|
| 624 |
+
if back_home.count():
|
| 625 |
+
back_home.click()
|
| 626 |
+
self.assertEqual(page.url.rstrip("/"), self.base_url)
|
| 627 |
+
self.assertEqual(errors, [])
|
| 628 |
+
finally:
|
| 629 |
+
context.close()
|
| 630 |
+
|
| 631 |
+
def test_accessibility_keyboard_paths_sidebar_state_and_preview_focus(self):
|
| 632 |
+
context, page, errors = self.page()
|
| 633 |
+
try:
|
| 634 |
+
self.assertEqual(page.locator("#search-input").get_attribute("aria-label"), "搜索 TXT 文件和正文")
|
| 635 |
+
self.assertEqual(page.locator("#results-list").get_attribute("role"), "list")
|
| 636 |
+
path = page.locator(".path-folder[role='button']").first
|
| 637 |
+
self.assertEqual(path.get_attribute("tabindex"), "0")
|
| 638 |
+
self.assertTrue(path.get_attribute("aria-label"))
|
| 639 |
+
|
| 640 |
+
page.locator("#settings-btn").press("Enter")
|
| 641 |
+
self.assertEqual(page.locator("#settings-btn").get_attribute("aria-expanded"), "true")
|
| 642 |
+
self.assertFalse(page.locator("#right-sidebar").evaluate("node => node.inert"))
|
| 643 |
+
page.keyboard.press("Escape")
|
| 644 |
+
self.assertEqual(page.locator("#settings-btn").get_attribute("aria-expanded"), "false")
|
| 645 |
+
self.assertEqual(page.evaluate("document.activeElement.id"), "hamburger-btn")
|
| 646 |
+
|
| 647 |
+
page.locator("[data-action='preview']").first.click()
|
| 648 |
+
page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
|
| 649 |
+
self.assertEqual(page.locator("#preview-panel").get_attribute("role"), "dialog")
|
| 650 |
+
self.assertEqual(page.locator("#preview-panel").get_attribute("aria-modal"), "true")
|
| 651 |
+
self.assertTrue(page.locator("#header").evaluate("node => node.inert"))
|
| 652 |
+
self.assertEqual(page.evaluate("document.activeElement.getAttribute('data-action')"), "close-preview")
|
| 653 |
+
page.keyboard.press("Escape")
|
| 654 |
+
page.locator("#preview-panel").wait_for(state="hidden")
|
| 655 |
+
self.assertFalse(page.locator("#header").evaluate("node => node.inert"))
|
| 656 |
+
self.assertEqual(page.evaluate("document.activeElement.getAttribute('data-action')"), "preview")
|
| 657 |
+
self.assertEqual(errors, [])
|
| 658 |
+
finally:
|
| 659 |
+
context.close()
|
| 660 |
+
|
| 661 |
+
def test_search_prefetch_advances_to_third_page_after_second_page_loads(self):
|
| 662 |
+
context, page, errors = self.page()
|
| 663 |
+
try:
|
| 664 |
+
pages = self.search_request_pages(page)
|
| 665 |
+
page.locator("#search-input").fill("文化")
|
| 666 |
+
page.wait_for_timeout(400)
|
| 667 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 668 |
+
|
| 669 |
+
page.locator("#results-container").evaluate("el => { el.scrollTop = el.scrollHeight; }")
|
| 670 |
+
page.wait_for_timeout(1200)
|
| 671 |
+
page.wait_for_timeout(1200)
|
| 672 |
+
self.assertGreaterEqual(page.evaluate("STATE.results.length"), 200)
|
| 673 |
+
self.assertIn(1, pages)
|
| 674 |
+
self.assertIn(2, pages)
|
| 675 |
+
self.assertIn(3, pages)
|
| 676 |
+
self.assertEqual(pages.count(3), 1)
|
| 677 |
+
self.assertEqual(errors, [])
|
| 678 |
+
finally:
|
| 679 |
+
context.close()
|
| 680 |
+
|
| 681 |
+
def test_initial_search_prefetches_second_page_once(self):
|
| 682 |
+
context = self.browser.new_context(viewport={"width": 1440, "height": 900})
|
| 683 |
+
page = context.new_page()
|
| 684 |
+
errors = []
|
| 685 |
+
page.on("pageerror", lambda error: errors.append(str(error)))
|
| 686 |
+
pages = self.search_request_pages(page)
|
| 687 |
+
try:
|
| 688 |
+
page.goto(self.base_url, wait_until="domcontentloaded")
|
| 689 |
+
page.locator(".result-item").first.wait_for(timeout=30000)
|
| 690 |
+
page.wait_for_timeout(2500)
|
| 691 |
+
self.assertEqual(pages.count(2), 1)
|
| 692 |
+
self.assertEqual(errors, [])
|
| 693 |
+
finally:
|
| 694 |
+
context.close()
|
| 695 |
+
|
| 696 |
+
def test_initial_results_skip_entry_animation_but_search_results_animate(self):
|
| 697 |
+
context, page, errors = self.page()
|
| 698 |
+
try:
|
| 699 |
+
self.assertEqual(page.locator(".result-item.result-enter").count(), 0)
|
| 700 |
+
page.evaluate("""
|
| 701 |
+
window.__sawResultEnter = false;
|
| 702 |
+
new MutationObserver((mutations, observer) => {
|
| 703 |
+
if (!mutations.some((mutation) =>
|
| 704 |
+
mutation.target instanceof Element && mutation.target.classList.contains("result-enter")
|
| 705 |
+
)) return;
|
| 706 |
+
window.__sawResultEnter = true;
|
| 707 |
+
observer.disconnect();
|
| 708 |
+
}).observe(document.querySelector("#results-list"), {
|
| 709 |
+
subtree: true,
|
| 710 |
+
attributes: true,
|
| 711 |
+
attributeFilter: ["class"],
|
| 712 |
+
});
|
| 713 |
+
""")
|
| 714 |
+
page.locator("#search-input").fill("文化")
|
| 715 |
+
page.wait_for_function("window.__sawResultEnter", timeout=30000)
|
| 716 |
+
self.assertEqual(errors, [])
|
| 717 |
+
finally:
|
| 718 |
+
context.close()
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
if __name__ == "__main__":
|
| 722 |
+
unittest.main()
|
tests/test_integration.py
ADDED
|
@@ -0,0 +1,529 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import io
|
| 5 |
+
import json
|
| 6 |
+
import sqlite3
|
| 7 |
+
import tempfile
|
| 8 |
+
import threading
|
| 9 |
+
import unittest
|
| 10 |
+
import zipfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from unittest.mock import patch
|
| 13 |
+
from urllib.parse import urlencode
|
| 14 |
+
|
| 15 |
+
from fastapi.testclient import TestClient
|
| 16 |
+
|
| 17 |
+
import app
|
| 18 |
+
import build_fulltext_db
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def create_fts_database(path: Path, rows: list[tuple[int, str]]) -> None:
|
| 22 |
+
with sqlite3.connect(path) as connection:
|
| 23 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 24 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 25 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 26 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 27 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 28 |
+
connection.execute("CREATE TABLE snippet_anchors (token_hash BLOB NOT NULL, doc_number INTEGER NOT NULL, char_offset INTEGER NOT NULL, PRIMARY KEY (token_hash, doc_number)) WITHOUT ROWID")
|
| 29 |
+
for number, content in rows:
|
| 30 |
+
connection.execute("INSERT INTO documents VALUES (?, ?)", (f"CCRD:{number}", content[:700]))
|
| 31 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", (number, content))
|
| 32 |
+
connection.executemany(
|
| 33 |
+
"INSERT INTO snippet_anchors VALUES (?, ?, ?)",
|
| 34 |
+
((token_hash, number, char_offset) for token_hash, char_offset in build_fulltext_db.snippet_anchors(content).items()),
|
| 35 |
+
)
|
| 36 |
+
for token in build_fulltext_db.index_tokens(content) | build_fulltext_db.literal_tokens(content):
|
| 37 |
+
token_hash = __import__("hashlib").sha256(token.encode("utf-8")).digest()[:16]
|
| 38 |
+
existing = connection.execute("SELECT docs FROM postings WHERE token_hash = ?", (token_hash,)).fetchone()
|
| 39 |
+
numbers = [number] if existing is None else sorted(build_doc_numbers(existing[0]) | {number})
|
| 40 |
+
payload = build_fulltext_db.encode_doc_ids(numbers)
|
| 41 |
+
connection.execute(
|
| 42 |
+
"INSERT OR REPLACE INTO postings VALUES (?, ?, ?)",
|
| 43 |
+
(token_hash, payload, len(numbers)),
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def build_doc_numbers(payload: bytes) -> set[int]:
|
| 48 |
+
return app.decode_doc_numbers(payload)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class FulltextIntegrationTests(unittest.TestCase):
|
| 52 |
+
def test_match_cache_is_complete_isolated_mode_scoped_and_expiring(self):
|
| 53 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 54 |
+
root = Path(directory)
|
| 55 |
+
create_fts_database(root / "CCRD.sqlite3", [(1, "文化 革命"), (2, "文化革命")])
|
| 56 |
+
databases = app.FulltextDatabases(root)
|
| 57 |
+
original = databases._search_source_connection
|
| 58 |
+
try:
|
| 59 |
+
with patch.object(databases, "_search_source_connection", wraps=original) as search, \
|
| 60 |
+
patch.object(app.time, "monotonic", return_value=100) as clock:
|
| 61 |
+
first = databases.search_source("CCRD", "文化 革命")
|
| 62 |
+
first.clear()
|
| 63 |
+
self.assertEqual(databases.search_source("CCRD", "文化 革命"), {1, 2})
|
| 64 |
+
self.assertEqual(search.call_count, 1)
|
| 65 |
+
self.assertEqual(databases.search_source("CCRD", "文化 革命", True), {1})
|
| 66 |
+
self.assertEqual(search.call_count, 2)
|
| 67 |
+
clock.return_value = 401
|
| 68 |
+
self.assertEqual(databases.search_source("CCRD", "文化 革命"), {1, 2})
|
| 69 |
+
self.assertEqual(search.call_count, 3)
|
| 70 |
+
self.assertEqual(databases.match_inflight, {})
|
| 71 |
+
finally:
|
| 72 |
+
databases.close()
|
| 73 |
+
|
| 74 |
+
def test_match_cache_obeys_entry_and_total_id_budgets_without_truncation(self):
|
| 75 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 76 |
+
root = Path(directory)
|
| 77 |
+
create_fts_database(root / "CCRD.sqlite3", [(1, "正文")])
|
| 78 |
+
databases = app.FulltextDatabases(root)
|
| 79 |
+
try:
|
| 80 |
+
with patch.object(app, "FULLTEXT_MATCH_CACHE_MAX_ENTRIES", 2), \
|
| 81 |
+
patch.object(app, "FULLTEXT_MATCH_CACHE_MAX_TOTAL_IDS", 3), \
|
| 82 |
+
patch.object(databases, "_search_source_connection", return_value={1, 2}):
|
| 83 |
+
for query in ("甲", "乙", "丙"):
|
| 84 |
+
self.assertEqual(databases.search_source("CCRD", query), {1, 2})
|
| 85 |
+
self.assertEqual(len(databases.match_cache), 1)
|
| 86 |
+
self.assertLessEqual(sum(len(value[1]) for value in databases.match_cache.values()), 3)
|
| 87 |
+
databases.match_cache.clear()
|
| 88 |
+
with patch.object(databases, "_search_source_connection", return_value={1}) as search:
|
| 89 |
+
for query in ("甲", "乙", "丙"):
|
| 90 |
+
self.assertEqual(databases.search_source("CCRD", query), {1})
|
| 91 |
+
self.assertEqual(len(databases.match_cache), 2)
|
| 92 |
+
databases.match_cache.clear()
|
| 93 |
+
with patch.object(databases, "_search_source_connection", return_value={1, 2, 3, 4}) as search:
|
| 94 |
+
for _ in range(2):
|
| 95 |
+
self.assertEqual(databases.search_source("CCRD", "oversized"), {1, 2, 3, 4})
|
| 96 |
+
self.assertEqual(search.call_count, 2)
|
| 97 |
+
self.assertEqual(databases.match_cache, {})
|
| 98 |
+
self.assertEqual(databases.match_inflight, {})
|
| 99 |
+
finally:
|
| 100 |
+
databases.close()
|
| 101 |
+
|
| 102 |
+
def test_concurrent_source_queries_share_complete_work_and_failures_release(self):
|
| 103 |
+
import concurrent.futures
|
| 104 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 105 |
+
root = Path(directory)
|
| 106 |
+
create_fts_database(root / "CCRD.sqlite3", [(1, "正文")])
|
| 107 |
+
databases = app.FulltextDatabases(root)
|
| 108 |
+
entered = threading.Event()
|
| 109 |
+
joined = threading.Event()
|
| 110 |
+
release = threading.Event()
|
| 111 |
+
calls = []
|
| 112 |
+
def execute(*_args):
|
| 113 |
+
calls.append(1)
|
| 114 |
+
entered.set()
|
| 115 |
+
self.assertTrue(release.wait(5))
|
| 116 |
+
return {1, 2, 3}
|
| 117 |
+
try:
|
| 118 |
+
with patch.object(databases, "_search_source_connection", side_effect=execute):
|
| 119 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
| 120 |
+
first = executor.submit(databases.search_source, "CCRD", "query")
|
| 121 |
+
try:
|
| 122 |
+
self.assertTrue(entered.wait(5))
|
| 123 |
+
pending = next(iter(databases.match_inflight.values()))
|
| 124 |
+
original_wait = pending.wait
|
| 125 |
+
def wait(timeout=None):
|
| 126 |
+
joined.set()
|
| 127 |
+
return original_wait(timeout)
|
| 128 |
+
with patch.object(pending, "wait", side_effect=wait):
|
| 129 |
+
second = executor.submit(databases.search_source, "CCRD", "query")
|
| 130 |
+
self.assertTrue(joined.wait(5))
|
| 131 |
+
self.assertEqual(len(calls), 1)
|
| 132 |
+
finally:
|
| 133 |
+
release.set()
|
| 134 |
+
self.assertEqual(first.result(timeout=5), {1, 2, 3})
|
| 135 |
+
self.assertEqual(second.result(timeout=5), {1, 2, 3})
|
| 136 |
+
self.assertEqual(len(calls), 1)
|
| 137 |
+
self.assertEqual(databases.match_inflight, {})
|
| 138 |
+
with patch.object(databases, "_search_source_connection", side_effect=[RuntimeError("failed"), {9}]) as search:
|
| 139 |
+
with self.assertRaises(RuntimeError):
|
| 140 |
+
databases.search_source("CCRD", "failure")
|
| 141 |
+
self.assertEqual(databases.match_inflight, {})
|
| 142 |
+
self.assertEqual(databases.search_source("CCRD", "failure"), {9})
|
| 143 |
+
self.assertEqual(search.call_count, 2)
|
| 144 |
+
finally:
|
| 145 |
+
databases.close()
|
| 146 |
+
def test_fulltext_databases_can_load_in_memory(self):
|
| 147 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 148 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 149 |
+
create_fts_database(path, [(1, "开头 文化革命 正文")])
|
| 150 |
+
databases = app.FulltextDatabases(Path(directory), in_memory=True)
|
| 151 |
+
try:
|
| 152 |
+
self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), {1})
|
| 153 |
+
self.assertEqual(databases.search_source("CCRD", "文化 革命", exact=False), {1})
|
| 154 |
+
finally:
|
| 155 |
+
databases.close()
|
| 156 |
+
|
| 157 |
+
def test_short_latin_exact_and_latin_wildcards_do_not_miss_substrings(self):
|
| 158 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 159 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 160 |
+
create_fts_database(path, [(1, "smartphone abc123xyz"), (2, "alpha beta"), (3, "unrelated")])
|
| 161 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 162 |
+
try:
|
| 163 |
+
self.assertEqual(databases.search_source("CCRD", "abc", exact=True), {1})
|
| 164 |
+
self.assertEqual(databases.search_source("CCRD", "phone", exact=True), {1})
|
| 165 |
+
self.assertEqual(databases.search_source("CCRD", "abc*xyz", exact=True), {1})
|
| 166 |
+
self.assertEqual(databases.search_source("CCRD", "a?c*xyz", exact=True), {1})
|
| 167 |
+
finally:
|
| 168 |
+
databases.close()
|
| 169 |
+
|
| 170 |
+
def test_builder_creates_complete_reusable_database(self):
|
| 171 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 172 |
+
root = Path(directory)
|
| 173 |
+
corpus = root / "corpus"
|
| 174 |
+
corpus.mkdir()
|
| 175 |
+
(corpus / "one.txt").write_text("开头 文化革命 正文", encoding="utf-8")
|
| 176 |
+
(corpus / "two.txt").write_text("全角ABC和手机", encoding="utf-8")
|
| 177 |
+
records = [
|
| 178 |
+
{"doc_id": "TEST:1", "storage_root": "corpus", "storage_rel_path": "one.txt"},
|
| 179 |
+
{"doc_id": "TEST:2", "storage_root": "corpus", "storage_rel_path": "two.txt"},
|
| 180 |
+
]
|
| 181 |
+
with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
|
| 182 |
+
build_fulltext_db.FULLTEXT_DIR.mkdir()
|
| 183 |
+
build_fulltext_db.build_source_db("TEST", records)
|
| 184 |
+
database = build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3"
|
| 185 |
+
first_bytes = database.read_bytes()
|
| 186 |
+
self.assertTrue(build_fulltext_db.database_ready(database, 2))
|
| 187 |
+
with sqlite3.connect(database) as connection:
|
| 188 |
+
self.assertEqual(connection.execute("SELECT COUNT(*) FROM documents").fetchone()[0], 2)
|
| 189 |
+
self.assertEqual(connection.execute("SELECT COUNT(*) FROM content_fts").fetchone()[0], 2)
|
| 190 |
+
self.assertGreater(connection.execute("SELECT COUNT(*) FROM snippet_anchors").fetchone()[0], 0)
|
| 191 |
+
self.assertEqual(connection.execute("SELECT value FROM metadata WHERE key='tokenizer_version'").fetchone()[0], app.TOKENIZER_VERSION)
|
| 192 |
+
self.assertGreater(connection.execute("SELECT COUNT(*) FROM postings").fetchone()[0], 0)
|
| 193 |
+
build_fulltext_db.build_source_db("TEST", records)
|
| 194 |
+
self.assertEqual(database.read_bytes(), first_bytes)
|
| 195 |
+
|
| 196 |
+
def test_builder_rejects_invalid_document_ids_without_publishing_database(self):
|
| 197 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 198 |
+
root = Path(directory)
|
| 199 |
+
corpus = root / "corpus"
|
| 200 |
+
corpus.mkdir()
|
| 201 |
+
(corpus / "one.txt").write_text("正文", encoding="utf-8")
|
| 202 |
+
record = {"doc_id": "invalid", "storage_root": "corpus", "storage_rel_path": "one.txt"}
|
| 203 |
+
with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
|
| 204 |
+
build_fulltext_db.FULLTEXT_DIR.mkdir()
|
| 205 |
+
with self.assertRaises(ValueError):
|
| 206 |
+
build_fulltext_db.build_source_db("TEST", [record])
|
| 207 |
+
self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3").exists())
|
| 208 |
+
|
| 209 |
+
def test_builder_does_not_publish_database_with_missing_files(self):
|
| 210 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 211 |
+
root = Path(directory)
|
| 212 |
+
record = {"doc_id": "TEST:1", "storage_root": "corpus", "storage_rel_path": "missing.txt"}
|
| 213 |
+
with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
|
| 214 |
+
build_fulltext_db.FULLTEXT_DIR.mkdir()
|
| 215 |
+
with self.assertRaises(RuntimeError):
|
| 216 |
+
build_fulltext_db.build_source_db("TEST", [record])
|
| 217 |
+
self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3").exists())
|
| 218 |
+
self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3.tmp").exists())
|
| 219 |
+
|
| 220 |
+
def test_atomic_database_replacement_reopens_and_increments_generation(self):
|
| 221 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 222 |
+
root = Path(directory)
|
| 223 |
+
path = root / "CCRD.sqlite3"
|
| 224 |
+
create_fts_database(path, [(1, "旧数据库文化")])
|
| 225 |
+
databases = app.FulltextDatabases(root)
|
| 226 |
+
first_generation = databases.generation
|
| 227 |
+
replacement = root / "replacement.sqlite3"
|
| 228 |
+
create_fts_database(replacement, [(2, "新数据库革命资料")])
|
| 229 |
+
replacement.replace(path)
|
| 230 |
+
databases.refresh()
|
| 231 |
+
try:
|
| 232 |
+
self.assertGreater(databases.generation, first_generation)
|
| 233 |
+
self.assertEqual(databases.search_source("CCRD", "革命"), {2})
|
| 234 |
+
self.assertEqual(databases.search_source("CCRD", "文化"), set())
|
| 235 |
+
finally:
|
| 236 |
+
databases.close()
|
| 237 |
+
|
| 238 |
+
def test_refresh_builds_replacement_without_blocking_existing_queries(self):
|
| 239 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 240 |
+
root = Path(directory)
|
| 241 |
+
path = root / "CCRD.sqlite3"
|
| 242 |
+
create_fts_database(path, [(1, "旧数据库文化")])
|
| 243 |
+
databases = app.FulltextDatabases(root, in_memory=True)
|
| 244 |
+
replacement = root / "replacement.sqlite3"
|
| 245 |
+
create_fts_database(replacement, [(2, "新数据库革命资料")])
|
| 246 |
+
replacement.replace(path)
|
| 247 |
+
opening = threading.Event()
|
| 248 |
+
release = threading.Event()
|
| 249 |
+
original_open = databases._open_candidate
|
| 250 |
+
|
| 251 |
+
def blocked_open(candidate_path):
|
| 252 |
+
opening.set()
|
| 253 |
+
self.assertTrue(release.wait(2))
|
| 254 |
+
return original_open(candidate_path)
|
| 255 |
+
|
| 256 |
+
thread = threading.Thread(target=databases.refresh)
|
| 257 |
+
try:
|
| 258 |
+
with patch.object(databases, "_open_candidate", side_effect=blocked_open):
|
| 259 |
+
thread.start()
|
| 260 |
+
self.assertTrue(opening.wait(2))
|
| 261 |
+
self.assertEqual(databases.search_source("CCRD", "文化"), {1})
|
| 262 |
+
release.set()
|
| 263 |
+
thread.join(2)
|
| 264 |
+
self.assertFalse(thread.is_alive())
|
| 265 |
+
self.assertEqual(databases.search_source("CCRD", "革命"), {2})
|
| 266 |
+
finally:
|
| 267 |
+
release.set()
|
| 268 |
+
thread.join(2)
|
| 269 |
+
databases.close()
|
| 270 |
+
|
| 271 |
+
def test_active_query_keeps_retired_connection_alive_until_release(self):
|
| 272 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 273 |
+
root = Path(directory)
|
| 274 |
+
path = root / "CCRD.sqlite3"
|
| 275 |
+
create_fts_database(path, [(1, "旧数据库文化")])
|
| 276 |
+
databases = app.FulltextDatabases(root, in_memory=True)
|
| 277 |
+
entered = threading.Event()
|
| 278 |
+
release = threading.Event()
|
| 279 |
+
result = []
|
| 280 |
+
original_search = databases._search_source_connection
|
| 281 |
+
|
| 282 |
+
def blocked_search(connection, query, exact, state):
|
| 283 |
+
entered.set()
|
| 284 |
+
self.assertTrue(release.wait(2))
|
| 285 |
+
return original_search(connection, query, exact, state)
|
| 286 |
+
|
| 287 |
+
thread = threading.Thread(target=lambda: result.append(databases.search_source("CCRD", "文化")))
|
| 288 |
+
try:
|
| 289 |
+
with patch.object(databases, "_search_source_connection", side_effect=blocked_search):
|
| 290 |
+
thread.start()
|
| 291 |
+
self.assertTrue(entered.wait(2))
|
| 292 |
+
replacement = root / "replacement.sqlite3"
|
| 293 |
+
create_fts_database(replacement, [(2, "新数据库革命资料")])
|
| 294 |
+
replacement.replace(path)
|
| 295 |
+
databases.refresh()
|
| 296 |
+
self.assertTrue(databases.retired_connections)
|
| 297 |
+
release.set()
|
| 298 |
+
thread.join(2)
|
| 299 |
+
self.assertFalse(thread.is_alive())
|
| 300 |
+
self.assertEqual(result, [{1}])
|
| 301 |
+
self.assertFalse(databases.retired_connections)
|
| 302 |
+
self.assertEqual(databases.match_cache, {})
|
| 303 |
+
self.assertEqual(databases.search_source("CCRD", "革命"), {2})
|
| 304 |
+
finally:
|
| 305 |
+
release.set()
|
| 306 |
+
thread.join(2)
|
| 307 |
+
databases.close()
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
class HttpIntegrationTests(unittest.TestCase):
|
| 311 |
+
def test_fulltext_in_memory_defaults_enabled(self):
|
| 312 |
+
self.assertTrue(app.FULLTEXT_IN_MEMORY)
|
| 313 |
+
|
| 314 |
+
@classmethod
|
| 315 |
+
def setUpClass(cls):
|
| 316 |
+
app.load_data()
|
| 317 |
+
cls.original_databases = app.fulltext_databases
|
| 318 |
+
app.fulltext_databases = app.FulltextDatabases(Path("/tmp/ccrd-test-no-index"))
|
| 319 |
+
cls.client = TestClient(app.app)
|
| 320 |
+
|
| 321 |
+
@classmethod
|
| 322 |
+
def tearDownClass(cls):
|
| 323 |
+
app.fulltext_databases.close()
|
| 324 |
+
app.fulltext_databases = cls.original_databases
|
| 325 |
+
|
| 326 |
+
def test_search_http_validation_and_option_isolation(self):
|
| 327 |
+
valid = self.client.post("/api/search", json={"q": "文化 革命", "fulltext": False, "page_size": 1})
|
| 328 |
+
self.assertEqual(valid.status_code, 200)
|
| 329 |
+
self.assertEqual(valid.json()["page_size"], 1)
|
| 330 |
+
self.assertEqual(self.client.post("/api/search", json={"q": "x" * 257}).status_code, 422)
|
| 331 |
+
self.assertEqual(self.client.post("/api/search", json={"page": 0}).status_code, 422)
|
| 332 |
+
self.assertEqual(self.client.post("/api/search", json={"page_size": 201}).status_code, 422)
|
| 333 |
+
self.assertEqual(self.client.post("/api/search", json={"min_size": -1}).status_code, 422)
|
| 334 |
+
self.assertEqual(self.client.post("/api/search/missing", json={}).status_code, 404)
|
| 335 |
+
|
| 336 |
+
def test_preview_snippet_and_download_http_contracts(self):
|
| 337 |
+
doc_id, record = next(iter(app.record_map.items()))
|
| 338 |
+
preview = self.client.get(f"/api/preview/{doc_id}")
|
| 339 |
+
self.assertEqual(preview.status_code, 200)
|
| 340 |
+
self.assertEqual(preview.json()["doc_id"], doc_id)
|
| 341 |
+
snippet = self.client.get(f"/api/snippet/{doc_id}", params={"q": "文化", "exact": "false"})
|
| 342 |
+
self.assertEqual(snippet.status_code, 200)
|
| 343 |
+
self.assertIn("snippet", snippet.json())
|
| 344 |
+
download = self.client.get(f"/api/download/{doc_id}")
|
| 345 |
+
self.assertEqual(download.status_code, 200)
|
| 346 |
+
self.assertTrue(download.headers["content-type"].startswith("text/plain"))
|
| 347 |
+
self.assertIn("attachment", download.headers["content-disposition"])
|
| 348 |
+
self.assertEqual(download.content, app.get_doc_storage_path(record).read_bytes())
|
| 349 |
+
self.assertEqual(self.client.get("/api/preview/missing:1").status_code, 404)
|
| 350 |
+
self.assertEqual(self.client.get("/api/snippet/missing:1").status_code, 404)
|
| 351 |
+
self.assertEqual(self.client.get("/api/download/missing:1").status_code, 404)
|
| 352 |
+
self.assertEqual(self.client.get(f"/api/snippet/{doc_id}", params={"q": "x" * 257}).status_code, 422)
|
| 353 |
+
|
| 354 |
+
def test_json_and_form_zip_are_valid_and_enforce_limits(self):
|
| 355 |
+
selected = sorted(app.records, key=lambda record: record["size"])[:2]
|
| 356 |
+
ids = [record["doc_id"] for record in selected]
|
| 357 |
+
for response in (
|
| 358 |
+
self.client.post("/api/zip", json={"doc_ids": ids}),
|
| 359 |
+
self.client.post(
|
| 360 |
+
"/api/zip-download",
|
| 361 |
+
content=urlencode([("doc_id", ids[0]), ("doc_id", ids[1]), ("doc_id", ids[0])]),
|
| 362 |
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
| 363 |
+
),
|
| 364 |
+
):
|
| 365 |
+
self.assertEqual(response.status_code, 200)
|
| 366 |
+
self.assertEqual(response.headers["content-type"], "application/zip")
|
| 367 |
+
self.assertIn("attachment", response.headers["content-disposition"])
|
| 368 |
+
self.assertEqual(response.headers["cache-control"], "no-store")
|
| 369 |
+
self.assertEqual(response.headers["content-encoding"], "identity")
|
| 370 |
+
self.assertEqual(int(response.headers["content-length"]), len(response.content))
|
| 371 |
+
self.assertTrue(response.content.startswith(b"PK"))
|
| 372 |
+
self.assertNotIn(b"PK\x06\x06", response.content)
|
| 373 |
+
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
| 374 |
+
self.assertEqual(len(archive.namelist()), 2)
|
| 375 |
+
self.assertEqual(archive.testzip(), None)
|
| 376 |
+
with self.assertRaises(zipfile.BadZipFile):
|
| 377 |
+
with zipfile.ZipFile(io.BytesIO(response.content[:-22])) as truncated:
|
| 378 |
+
truncated.testzip()
|
| 379 |
+
self.assertEqual(self.client.post("/api/zip", json={"doc_ids": []}).status_code, 400)
|
| 380 |
+
self.assertEqual(self.client.post("/api/zip", json={"doc_ids": ["x"] * 501}).status_code, 422)
|
| 381 |
+
too_many = [("doc_id", f"CCRD:{index}") for index in range(501)]
|
| 382 |
+
self.assertEqual(self.client.post(
|
| 383 |
+
"/api/zip-download",
|
| 384 |
+
content=urlencode(too_many),
|
| 385 |
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
| 386 |
+
).status_code, 400)
|
| 387 |
+
self.assertEqual(self.client.post("/api/zip-download", json={"doc_id": ids}).status_code, 415)
|
| 388 |
+
self.assertEqual(self.client.get("/api/zip-download").status_code, 400)
|
| 389 |
+
legacy = self.client.post(
|
| 390 |
+
"/api/zip-download",
|
| 391 |
+
content=urlencode([("doc_id", ids[0]), ("doc_id", ids[1])]),
|
| 392 |
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
| 393 |
+
follow_redirects=False,
|
| 394 |
+
)
|
| 395 |
+
self.assertEqual(legacy.status_code, 303)
|
| 396 |
+
self.assertTrue(legacy.headers["location"].startswith("/api/zip-download/"))
|
| 397 |
+
legacy_download = self.client.get(legacy.headers["location"])
|
| 398 |
+
self.assertTrue(legacy_download.content.startswith(b"PK"))
|
| 399 |
+
prepared = self.client.post("/api/zip-prepare", json={"doc_ids": ids})
|
| 400 |
+
self.assertEqual(prepared.status_code, 200)
|
| 401 |
+
self.assertEqual(prepared.json()["file_count"], 2)
|
| 402 |
+
download_url = prepared.json()["download_url"]
|
| 403 |
+
for _attempt in range(2):
|
| 404 |
+
downloaded = self.client.get(download_url, headers={"Accept-Encoding": "gzip"})
|
| 405 |
+
self.assertEqual(downloaded.status_code, 200)
|
| 406 |
+
self.assertEqual(downloaded.headers["content-encoding"], "identity")
|
| 407 |
+
self.assertTrue(downloaded.content.startswith(b"PK"))
|
| 408 |
+
with zipfile.ZipFile(io.BytesIO(downloaded.content)) as archive:
|
| 409 |
+
self.assertEqual(len(archive.namelist()), 2)
|
| 410 |
+
self.assertIsNone(archive.testzip())
|
| 411 |
+
self.assertEqual(self.client.get("/api/zip-download/invalid-token").status_code, 404)
|
| 412 |
+
|
| 413 |
+
def test_zip_disambiguates_duplicate_display_paths_and_deduplicates_ids(self):
|
| 414 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 415 |
+
root = Path(directory)
|
| 416 |
+
first = root / "first.txt"
|
| 417 |
+
second = root / "second.txt"
|
| 418 |
+
first.write_text("first", encoding="utf-8")
|
| 419 |
+
second.write_text("second", encoding="utf-8")
|
| 420 |
+
original_map = app.record_map
|
| 421 |
+
app.record_map = {
|
| 422 |
+
"CCRD:1": {"source": "CCRD", "display_name": 'same:name*?"<>|', "display_rel_path": "same/name.txt"},
|
| 423 |
+
"CCRD:2": {"source": "CCRD", "display_name": 'same:name*?"<>|', "display_rel_path": "same/name.txt"},
|
| 424 |
+
}
|
| 425 |
+
paths = {"CCRD:1": first, "CCRD:2": second}
|
| 426 |
+
try:
|
| 427 |
+
with patch.object(app, "get_doc_storage_path", side_effect=lambda record: paths[next(key for key, value in app.record_map.items() if value is record)]):
|
| 428 |
+
response = app.zip_response(["CCRD:1", "CCRD:1", "CCRD:2"])
|
| 429 |
+
self.assertEqual(response.status_code, 200)
|
| 430 |
+
with open(response.path, "rb") as archive_file:
|
| 431 |
+
payload = archive_file.read()
|
| 432 |
+
self.assertNotIn(b"PK\x06\x06", payload)
|
| 433 |
+
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
| 434 |
+
self.assertEqual(len(set(archive.namelist())), 2)
|
| 435 |
+
self.assertTrue(all(not any(character in name for character in ':*?"<>|\\') for name in archive.namelist()))
|
| 436 |
+
self.assertTrue(all(len(name.encode("utf-8")) <= 220 for name in archive.namelist()))
|
| 437 |
+
self.assertEqual(sorted(archive.read(name).decode() for name in archive.namelist()), ["first", "second"])
|
| 438 |
+
finally:
|
| 439 |
+
app.record_map = original_map
|
| 440 |
+
|
| 441 |
+
def test_static_status_and_cors_contracts(self):
|
| 442 |
+
self.assertEqual(self.client.get("/manifest.json").status_code, 200)
|
| 443 |
+
self.assertIn("javascript", self.client.get("/sw.js").headers["content-type"])
|
| 444 |
+
status = self.client.get("/api/index-status")
|
| 445 |
+
self.assertEqual(status.status_code, 200)
|
| 446 |
+
self.assertEqual(status.json()["loaded_sources"], [])
|
| 447 |
+
cors = self.client.options(
|
| 448 |
+
"/api/search",
|
| 449 |
+
headers={"Origin": "https://example.com", "Access-Control-Request-Method": "POST"},
|
| 450 |
+
)
|
| 451 |
+
self.assertEqual(cors.status_code, 200)
|
| 452 |
+
self.assertEqual(cors.headers["access-control-allow-origin"], "*")
|
| 453 |
+
self.assertEqual(self.client.get("/data/fulltext/CCRD.sqlite3").status_code, 404)
|
| 454 |
+
self.assertEqual(self.client.get("/data/fulltext/CCRD.sqlite3.tmp").status_code, 404)
|
| 455 |
+
|
| 456 |
+
def test_ping_is_empty_and_not_cached(self):
|
| 457 |
+
response = self.client.get("/api/ping")
|
| 458 |
+
self.assertEqual(response.status_code, 204)
|
| 459 |
+
self.assertEqual(response.content, b"")
|
| 460 |
+
self.assertEqual(response.headers["cache-control"], "no-store")
|
| 461 |
+
|
| 462 |
+
def test_sources_extensions_folders_random_and_spa_routes(self):
|
| 463 |
+
sources = self.client.get("/api/sources")
|
| 464 |
+
self.assertEqual(sources.status_code, 200)
|
| 465 |
+
self.assertEqual({item["slug"] for item in sources.json()}, {"CCRD", "CW"})
|
| 466 |
+
self.assertEqual(self.client.get("/api/extensions").json(), [{"name": "txt", "count": 47250}])
|
| 467 |
+
self.assertEqual(self.client.get("/api/extensions", params={"source": "CCRD"}).json(), [{"name": "txt", "count": 35586}])
|
| 468 |
+
folders = self.client.get("/api/folders/CCRD")
|
| 469 |
+
self.assertEqual(folders.status_code, 200)
|
| 470 |
+
self.assertIsInstance(folders.json(), list)
|
| 471 |
+
contents = self.client.get("/api/folders/CCRD/contents")
|
| 472 |
+
self.assertEqual(contents.status_code, 200)
|
| 473 |
+
self.assertEqual(contents.json()["current_path"], "")
|
| 474 |
+
random_doc = self.client.get("/api/random", params={"source": "CW"})
|
| 475 |
+
self.assertEqual(random_doc.status_code, 200)
|
| 476 |
+
self.assertEqual(random_doc.json()["Source"], "CW")
|
| 477 |
+
self.assertEqual(self.client.get("/api/random", params={"source": "missing"}).status_code, 404)
|
| 478 |
+
for route in ("/", "/CCRD", "/CW"):
|
| 479 |
+
page = self.client.get(route)
|
| 480 |
+
self.assertEqual(page.status_code, 200)
|
| 481 |
+
self.assertIn("text/html", page.headers["content-type"])
|
| 482 |
+
self.assertIn("initial-sidebar-data", page.text)
|
| 483 |
+
|
| 484 |
+
def test_search_cache_ttl_is_longer_than_burst_idle_gap(self):
|
| 485 |
+
self.assertGreaterEqual(app.SEARCH_CACHE_TTL_SECONDS, 300)
|
| 486 |
+
|
| 487 |
+
def test_search_cache_returns_identical_payload(self):
|
| 488 |
+
app.search_response_cache.clear()
|
| 489 |
+
body = {"q": "server-timing-unique-query", "fulltext": False, "page_size": 1}
|
| 490 |
+
first = self.client.post("/api/search", json=body)
|
| 491 |
+
second = self.client.post("/api/search", json=body)
|
| 492 |
+
self.assertEqual(first.status_code, 200)
|
| 493 |
+
self.assertEqual(first.json(), second.json())
|
| 494 |
+
|
| 495 |
+
def test_missing_physical_files_return_stable_errors(self):
|
| 496 |
+
original_map = app.record_map
|
| 497 |
+
app.record_map = {
|
| 498 |
+
"CCRD:missing": {
|
| 499 |
+
"display_name": "missing", "display_rel_path": "missing.txt", "source_name": "CCRD",
|
| 500 |
+
"storage_root": "CCRD", "storage_rel_path": "does-not-exist.txt",
|
| 501 |
+
}
|
| 502 |
+
}
|
| 503 |
+
try:
|
| 504 |
+
for endpoint in ("preview", "snippet", "download"):
|
| 505 |
+
response = self.client.get(f"/api/{endpoint}/CCRD:missing")
|
| 506 |
+
self.assertEqual(response.status_code, 404)
|
| 507 |
+
self.assertIn("missing file", response.json()["error"])
|
| 508 |
+
finally:
|
| 509 |
+
app.record_map = original_map
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
class CorpusIntegrityTests(unittest.TestCase):
|
| 513 |
+
def test_metadata_counts_and_every_storage_file_exists(self):
|
| 514 |
+
payload = json.loads(gzip.decompress((Path(app.BASE_DIR) / "data/search_data.json.gz").read_bytes()))
|
| 515 |
+
records = payload["records"]
|
| 516 |
+
self.assertEqual(len(records), 47250)
|
| 517 |
+
counts = {source["slug"]: source["count"] for source in payload["sources"]}
|
| 518 |
+
self.assertEqual(counts, {"CCRD": 35586, "CW": 11664})
|
| 519 |
+
doc_ids = {record["doc_id"] for record in records}
|
| 520 |
+
self.assertEqual(len(doc_ids), len(records))
|
| 521 |
+
missing = [
|
| 522 |
+
record["doc_id"] for record in records
|
| 523 |
+
if not (Path(app.BASE_DIR) / record["storage_root"] / record["storage_rel_path"]).is_file()
|
| 524 |
+
]
|
| 525 |
+
self.assertEqual(missing, [])
|
| 526 |
+
self.assertEqual(sum(record["source"] == "CCRD" for record in records), 35586)
|
| 527 |
+
self.assertEqual(sum(record["source"] == "CW" for record in records), 11664)
|
| 528 |
+
archive_keys = [(record["source"], record["display_rel_path"], record["doc_id"]) for record in records]
|
| 529 |
+
self.assertEqual(len(archive_keys), len(set(archive_keys)))
|
tests/test_live_smoke.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import io
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
import unittest
|
| 8 |
+
import zipfile
|
| 9 |
+
from http.client import RemoteDisconnected
|
| 10 |
+
from urllib.error import HTTPError, URLError
|
| 11 |
+
from urllib.request import Request, urlopen
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
BASE_URL = os.environ.get("CCRD_LIVE_BASE_URL", "").rstrip("/")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@unittest.skipUnless(BASE_URL, "set CCRD_LIVE_BASE_URL to run deployment smoke tests")
|
| 18 |
+
class LiveSmokeTests(unittest.TestCase):
|
| 19 |
+
def request_json(self, path: str, payload: dict | None = None):
|
| 20 |
+
data = json.dumps(payload).encode() if payload is not None else None
|
| 21 |
+
for attempt in range(3):
|
| 22 |
+
request = Request(
|
| 23 |
+
BASE_URL + path,
|
| 24 |
+
data=data,
|
| 25 |
+
headers={"Content-Type": "application/json"} if data else {},
|
| 26 |
+
)
|
| 27 |
+
try:
|
| 28 |
+
with urlopen(request, timeout=45) as response:
|
| 29 |
+
self.assertEqual(response.status, 200)
|
| 30 |
+
return json.load(response)
|
| 31 |
+
except HTTPError as exc:
|
| 32 |
+
if exc.code < 500 or attempt == 2:
|
| 33 |
+
raise
|
| 34 |
+
except (RemoteDisconnected, TimeoutError, URLError):
|
| 35 |
+
if attempt == 2:
|
| 36 |
+
raise
|
| 37 |
+
time.sleep(attempt + 1)
|
| 38 |
+
self.fail("request retries exhausted")
|
| 39 |
+
|
| 40 |
+
def request_text(self, path: str):
|
| 41 |
+
with urlopen(BASE_URL + path, timeout=45) as response:
|
| 42 |
+
self.assertEqual(response.status, 200)
|
| 43 |
+
return response.read().decode("utf-8")
|
| 44 |
+
|
| 45 |
+
def test_sources_and_extensions_match_corpus(self):
|
| 46 |
+
sources = self.request_json("/api/sources")
|
| 47 |
+
self.assertEqual({item["slug"] for item in sources}, {"CCRD", "CW"})
|
| 48 |
+
self.assertEqual(sum(item["count"] for item in sources), 47250)
|
| 49 |
+
extensions = self.request_json("/api/extensions")
|
| 50 |
+
self.assertEqual(extensions, [{"name": "txt", "count": 47250}])
|
| 51 |
+
|
| 52 |
+
def test_exact_fulltext_results_contain_contiguous_query(self):
|
| 53 |
+
status = self.request_json("/api/index-status")
|
| 54 |
+
if not status.get("ready"):
|
| 55 |
+
self.skipTest(f'fulltext indexes still building: {status.get("building_sources", [])}')
|
| 56 |
+
payload = self.request_json("/api/search", {
|
| 57 |
+
"q": "手机",
|
| 58 |
+
"page": 1,
|
| 59 |
+
"page_size": 5,
|
| 60 |
+
"exact": True,
|
| 61 |
+
"fulltext": True,
|
| 62 |
+
})
|
| 63 |
+
self.assertGreater(payload["total"], 0)
|
| 64 |
+
self.assertLessEqual(len(payload["results"]), 5)
|
| 65 |
+
for item in payload["results"]:
|
| 66 |
+
preview = self.request_json(f'/api/preview/{item["doc_id"]}')
|
| 67 |
+
searchable = " ".join([item.get("DisplayPath", ""), preview.get("text", "")])
|
| 68 |
+
self.assertIn("手机", searchable)
|
| 69 |
+
|
| 70 |
+
def test_source_filter_and_pagination_are_stable(self):
|
| 71 |
+
request = {
|
| 72 |
+
"q": "文化 革命",
|
| 73 |
+
"page": 1,
|
| 74 |
+
"page_size": 5,
|
| 75 |
+
"exact": False,
|
| 76 |
+
"fulltext": True,
|
| 77 |
+
}
|
| 78 |
+
first = self.request_json("/api/search/CCRD", request)
|
| 79 |
+
request["page"] = 2
|
| 80 |
+
second = self.request_json("/api/search/CCRD", request)
|
| 81 |
+
first_ids = {item["doc_id"] for item in first["results"]}
|
| 82 |
+
second_ids = {item["doc_id"] for item in second["results"]}
|
| 83 |
+
self.assertTrue(first_ids)
|
| 84 |
+
self.assertTrue(second_ids)
|
| 85 |
+
self.assertTrue(first_ids.isdisjoint(second_ids))
|
| 86 |
+
self.assertTrue(all(item["Source"] == "CCRD" for item in first["results"] + second["results"]))
|
| 87 |
+
|
| 88 |
+
def test_deployed_ui_contains_controls_and_matching_javascript(self):
|
| 89 |
+
html = self.request_text("/")
|
| 90 |
+
javascript = self.request_text("/static/app.js")
|
| 91 |
+
css = self.request_text("/static/style.css")
|
| 92 |
+
for control_id in (
|
| 93 |
+
"hamburger-btn", "search-input", "settings-btn", "theme-btn", "mobile-toggle-btn",
|
| 94 |
+
"multi-select-toggle", "multi-select-all", "multi-deselect", "multi-batch-download",
|
| 95 |
+
"multi-zip-download", "fulltext-toggle", "search-paths-toggle", "history-toggle",
|
| 96 |
+
"exact-search-toggle", "folder-select-all", "folder-deselect-all", "source-filter-cancel",
|
| 97 |
+
"folder-filter-cancel", "random-book-btn",
|
| 98 |
+
):
|
| 99 |
+
self.assertIn(f'id="{control_id}"', html)
|
| 100 |
+
for binding in (
|
| 101 |
+
"DOM.hamburgerBtn.addEventListener", "DOM.searchInput.addEventListener",
|
| 102 |
+
"DOM.multiBatchDownload.addEventListener", "DOM.multiZipDownload.addEventListener",
|
| 103 |
+
"DOM.resultsList.addEventListener", "DOM.previewPanel.addEventListener",
|
| 104 |
+
):
|
| 105 |
+
self.assertIn(binding, javascript)
|
| 106 |
+
self.assertIn(".result-snippet", css)
|
| 107 |
+
|
| 108 |
+
def test_deployed_zip_is_not_http_gzipped_and_opens(self):
|
| 109 |
+
prepared = self.request_json("/api/zip-prepare", {"doc_ids": ["CCRD:0", "CW:0"]})
|
| 110 |
+
request = Request(
|
| 111 |
+
BASE_URL + prepared["download_url"],
|
| 112 |
+
headers={
|
| 113 |
+
"Accept-Encoding": "gzip",
|
| 114 |
+
},
|
| 115 |
+
)
|
| 116 |
+
with urlopen(request, timeout=120) as response:
|
| 117 |
+
payload = response.read()
|
| 118 |
+
self.assertEqual(response.status, 200)
|
| 119 |
+
self.assertEqual(response.headers.get("Content-Encoding"), "identity")
|
| 120 |
+
self.assertEqual(int(response.headers["Content-Length"]), len(payload))
|
| 121 |
+
self.assertTrue(payload.startswith(b"PK"))
|
| 122 |
+
self.assertNotIn(b"PK\x06\x06", payload)
|
| 123 |
+
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
| 124 |
+
self.assertEqual(len(archive.namelist()), 2)
|
| 125 |
+
self.assertIsNone(archive.testzip())
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
if __name__ == "__main__":
|
| 129 |
+
unittest.main()
|
tests/test_ocr_release.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny real SQLite/text candidates; never open/rebuild production databases."""
|
| 2 |
+
import copy
|
| 3 |
+
import gzip
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
import tempfile
|
| 8 |
+
import unittest
|
| 9 |
+
from unittest.mock import patch
|
| 10 |
+
|
| 11 |
+
import app
|
| 12 |
+
import build_fulltext_db as builder
|
| 13 |
+
import ocr_release as release
|
| 14 |
+
from fastapi.testclient import TestClient
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class OcrReleaseTests(unittest.TestCase):
|
| 18 |
+
def setUp(self):
|
| 19 |
+
temporary = tempfile.TemporaryDirectory()
|
| 20 |
+
self.addCleanup(temporary.cleanup)
|
| 21 |
+
self.root = Path(temporary.name)
|
| 22 |
+
self.bucket = self.root / "bucket"
|
| 23 |
+
self.bucket.mkdir()
|
| 24 |
+
self.base = self.root / "base.json.gz"
|
| 25 |
+
base_record = {"doc_id": "CCRD:1", "source": "CCRD", "source_name": "CCRD",
|
| 26 |
+
"storage_root": "CCRD", "storage_rel_path": "old.txt", "display_name": "旧文件",
|
| 27 |
+
"display_dirs": [], "display_rel_path": "旧文件.txt", "size": 7, "extension": "txt"}
|
| 28 |
+
self.base.write_bytes(gzip.compress(json.dumps({"records": [base_record], "sources": [
|
| 29 |
+
{"slug": "CCRD", "name": "CCRD", "count": 1}]}).encode(), mtime=0))
|
| 30 |
+
self.base_sha = hashlib.sha256(self.base.read_bytes()).hexdigest()
|
| 31 |
+
self.content = "手机文化\n\n\n\n末页手。机 ABC"
|
| 32 |
+
texts = ["手机文化", "", "末页手。机 ABC"]
|
| 33 |
+
body = self.content.encode()
|
| 34 |
+
txt_sha = release.digest(body)
|
| 35 |
+
text_ref = {"bucket": release.BUCKET, "path": f"text/objects/{txt_sha[:2]}/{txt_sha}.txt",
|
| 36 |
+
"sha256": txt_sha, "bytes": len(body), "role": "runtime"}
|
| 37 |
+
self.write(text_ref["path"], body)
|
| 38 |
+
original = "原目录/book.pdf"
|
| 39 |
+
identity = {"recipe": "ocr-plain-utf8-pages-v1", "origin_repo": "VoiceOfML/Test", "origin_rel_path": original,
|
| 40 |
+
"origin_revision": "b" * 40, "source_sha256": "a" * 64, "doc_id": "OCR:1",
|
| 41 |
+
"text_generation": "c" * 64, "txt_sha256": txt_sha}
|
| 42 |
+
generation = release.digest(release.canonical(identity).rstrip(b"\n"))
|
| 43 |
+
book_id = release.digest(("VoiceOfML/Test\0" + original).encode())
|
| 44 |
+
pages, cursor = [], 0
|
| 45 |
+
for number, text in enumerate(texts, 1):
|
| 46 |
+
pages.append({"page": number, "start": cursor, "end": cursor + len(text),
|
| 47 |
+
"text_sha256": release.digest(text.encode()), "text_generation": str(number) * 64,
|
| 48 |
+
"quality": "partially-reviewed" if number == 1 else "unreviewed"})
|
| 49 |
+
cursor += len(text) + 2
|
| 50 |
+
self.export = {"version": 1, "kind": "ccrd-ocr-text-export", "complete": True, **identity,
|
| 51 |
+
"generation": generation, "book_id": book_id, "title": "book", "page_count": 3,
|
| 52 |
+
"readable_rel_path": "VoiceOfML/Test/原目录/book.txt", "accepted_pages": 1,
|
| 53 |
+
"quality": "partially-reviewed", "offset_unit": "unicode-codepoint", "page_separator": "\n\n",
|
| 54 |
+
"pages": pages, "text": text_ref,
|
| 55 |
+
"retained_reader_resources": [{"bucket": "vomebook/pdf-pages-v2", "path": "objects/aa/reading-manifest.json",
|
| 56 |
+
"sha256": "a" * 64, "bytes": 1, "role": "provenance"}]}
|
| 57 |
+
self.export_path = f"text/exports/{book_id}/{generation}/manifest.json"
|
| 58 |
+
self.export_ref = self.ref(self.export_path, release.canonical(self.export))
|
| 59 |
+
core = {"version": 1, "kind": "ccrd-ocr-search-release", "base_metadata_sha256": self.base_sha,
|
| 60 |
+
"builder_sha256": release.digest(Path(builder.__file__).read_bytes()), "tokenizer_version": release.TOKENIZER,
|
| 61 |
+
"exports": [self.export_ref]}
|
| 62 |
+
generation = release.digest(release.canonical(core))
|
| 63 |
+
records = [release.export_record(self.bucket, self.export_ref)[0]]
|
| 64 |
+
db_relative = f"indexes/ocr/{generation}/OCR.sqlite3"
|
| 65 |
+
directory = self.bucket / Path(db_relative).parent
|
| 66 |
+
directory.mkdir(parents=True)
|
| 67 |
+
with patch.object(builder, "BASE_DIR", self.bucket), patch.object(builder, "FULLTEXT_DIR", directory):
|
| 68 |
+
builder.build_source_db("OCR", records)
|
| 69 |
+
self.manifest = {**core, "generation": generation, "documents": 1,
|
| 70 |
+
"database": self.ref(db_relative, (self.bucket / db_relative).read_bytes())}
|
| 71 |
+
self.manifest_path = f"releases/ocr-{generation}/manifest.json"
|
| 72 |
+
raw = release.canonical(self.manifest)
|
| 73 |
+
self.write(self.manifest_path, raw)
|
| 74 |
+
self.manifest_sha = release.digest(raw)
|
| 75 |
+
|
| 76 |
+
def write(self, path, body):
|
| 77 |
+
target = self.bucket / path
|
| 78 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 79 |
+
target.write_bytes(body)
|
| 80 |
+
|
| 81 |
+
def ref(self, path, body):
|
| 82 |
+
self.write(path, body)
|
| 83 |
+
return {"bucket": release.BUCKET, "path": path, "sha256": release.digest(body), "bytes": len(body), "role": "runtime"}
|
| 84 |
+
|
| 85 |
+
def load(self):
|
| 86 |
+
return release.load_release(self.bucket, self.manifest_path, self.manifest_sha, self.base_sha)
|
| 87 |
+
|
| 88 |
+
def test_whole_release_is_verified_and_origin_hierarchy_is_preserved(self):
|
| 89 |
+
candidate = self.load()
|
| 90 |
+
self.assertEqual(candidate["records"][0]["display_rel_path"], "VoiceOfML/Test/原目录/book.txt")
|
| 91 |
+
self.assertEqual(candidate["text_paths"]["OCR:1"].read_text(), self.content)
|
| 92 |
+
self.assertEqual(candidate["tree"][0]["count"], 1)
|
| 93 |
+
self.assertEqual(candidate["browser"]["VoiceOfML/Test/原目录"]["files"][0]["doc_id"], "OCR:1")
|
| 94 |
+
|
| 95 |
+
def test_corrupt_text_or_database_blocks_the_release(self):
|
| 96 |
+
for ref in (self.export["text"], self.manifest["database"]):
|
| 97 |
+
target = self.bucket / ref["path"]
|
| 98 |
+
original = target.read_bytes()
|
| 99 |
+
try:
|
| 100 |
+
target.write_bytes(original + b"damage")
|
| 101 |
+
with self.assertRaisesRegex(ValueError, "checksum"):
|
| 102 |
+
self.load()
|
| 103 |
+
finally:
|
| 104 |
+
target.write_bytes(original)
|
| 105 |
+
|
| 106 |
+
def test_wrong_manifest_or_base_identity_cannot_activate(self):
|
| 107 |
+
for manifest_sha, base_sha in (("d" * 64, self.base_sha), (self.manifest_sha, "e" * 64)):
|
| 108 |
+
with self.subTest(manifest_sha=manifest_sha), self.assertRaises(ValueError):
|
| 109 |
+
release.load_release(self.bucket, self.manifest_path, manifest_sha, base_sha)
|
| 110 |
+
|
| 111 |
+
def test_export_page_offsets_and_missing_provenance_are_rejected(self):
|
| 112 |
+
for change in ({"pages": []}, {"retained_reader_resources": []}, {"readable_rel_path": "wrong/book.txt"},
|
| 113 |
+
{"accepted_pages": 3}, {"origin_repo": "../bad"}):
|
| 114 |
+
value = {**copy.deepcopy(self.export), **change}
|
| 115 |
+
ref = self.ref(self.export_path, release.canonical(value))
|
| 116 |
+
with self.subTest(change=list(change)), self.assertRaises(ValueError):
|
| 117 |
+
release.export_record(self.bucket, ref)
|
| 118 |
+
|
| 119 |
+
def test_pin_copies_verified_data_and_is_independent_of_the_mount(self):
|
| 120 |
+
pinned = release.pin_release(self.bucket, self.manifest_path, self.manifest_sha, self.base_sha, self.root / "cache")
|
| 121 |
+
self.assertNotIn(self.bucket, pinned["database"].parents)
|
| 122 |
+
self.assertEqual(pinned["text_paths"]["OCR:1"].read_text(), self.content)
|
| 123 |
+
self.write(self.export["text"]["path"], b"changed remote mount")
|
| 124 |
+
self.assertEqual(pinned["text_paths"]["OCR:1"].read_text(), self.content)
|
| 125 |
+
|
| 126 |
+
def test_small_index_uses_existing_in_memory_search_contract(self):
|
| 127 |
+
candidate = self.load()
|
| 128 |
+
databases = app.FulltextDatabases(self.root / "empty-db", in_memory=True, extra_paths=[candidate["database"]])
|
| 129 |
+
try:
|
| 130 |
+
self.assertEqual(databases.search_source("OCR", "手机", exact=False), {1})
|
| 131 |
+
self.assertEqual(databases.search_source("OCR", "手。机", exact=True), {1})
|
| 132 |
+
self.assertEqual(databases.search_source("OCR", "文化 ABC", exact=False), {1})
|
| 133 |
+
self.assertEqual(databases.search_source("OCR", "手机 ABC", exact=True), set())
|
| 134 |
+
self.assertEqual(databases.connections["OCR"].execute("PRAGMA database_list").fetchone()[2], "")
|
| 135 |
+
finally:
|
| 136 |
+
databases.close()
|
| 137 |
+
|
| 138 |
+
def test_startup_loads_one_pinned_release_and_serves_plain_txt(self):
|
| 139 |
+
tree, browser = self.root / "tree.gz", self.root / "browser.gz"
|
| 140 |
+
tree.write_bytes(gzip.compress(b'{"CCRD":[]}')); browser.write_bytes(gzip.compress(b'{"CCRD":{}}'))
|
| 141 |
+
saved = {key: getattr(app, key) for key in ("records", "record_map", "record_map_index", "sources", "source_counts",
|
| 142 |
+
"folder_tree_data", "folder_browser_data", "active_ocr_release")}
|
| 143 |
+
try:
|
| 144 |
+
with patch.dict("os.environ", {"CCRD_INDEX_BUCKET_DIR": str(self.bucket), "CCRD_OCR_RELEASE_PATH": self.manifest_path,
|
| 145 |
+
"CCRD_OCR_RELEASE_SHA256": self.manifest_sha}), \
|
| 146 |
+
patch.object(app, "DATA_PATH", self.base), patch.object(app, "FOLDER_TREE_PATH", tree), \
|
| 147 |
+
patch.object(app, "FOLDER_BROWSER_PATH", browser), patch.object(app, "BASE_DIR", self.root):
|
| 148 |
+
app.load_data()
|
| 149 |
+
self.assertEqual(app.source_counts, {"CCRD": 1, "OCR": 1})
|
| 150 |
+
self.assertEqual(app.record_map["CCRD:1"]["display_name"], "旧文件")
|
| 151 |
+
client = TestClient(app.app)
|
| 152 |
+
response = client.get("/txt/OCR/VoiceOfML/Test/原目录/book.txt")
|
| 153 |
+
self.assertEqual(response.status_code, 200)
|
| 154 |
+
self.assertEqual(response.text, self.content)
|
| 155 |
+
self.assertIn("inline", response.headers["content-disposition"])
|
| 156 |
+
self.assertEqual(client.get("/txt/OCR/missing.txt").status_code, 404)
|
| 157 |
+
self.assertEqual(client.get("/api/preview/OCR:1").json()["text"], self.content)
|
| 158 |
+
self.assertEqual(json.loads(app.sidebar_payload_for_path(""))["sources"][-1]["slug"], "OCR")
|
| 159 |
+
self.assertEqual(app.zip_entries(["OCR:1"])[0][0], "OCR成果/VoiceOfML/Test/原目录/book.txt")
|
| 160 |
+
self.assertEqual(client.get("/data/sidebar/global.json").json()["sources"][-1]["slug"], "OCR")
|
| 161 |
+
self.assertEqual(client.get("/data/initial/sources/OCR.json").json()["total"], 1)
|
| 162 |
+
finally:
|
| 163 |
+
for key, value in saved.items(): setattr(app, key, value)
|
| 164 |
+
app.build_indexes()
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
if __name__ == "__main__":
|
| 168 |
+
unittest.main()
|
tests/test_prepare_fulltext_index.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import sqlite3
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import tempfile
|
| 8 |
+
import unittest
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
+
PREPARE = ROOT / "prepare_fulltext_index.py"
|
| 14 |
+
TOKENIZER_VERSION = "cjk-bigram-boundary-fts5-v6-snippet-anchors"
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def create_database(path: Path, doc_id: str) -> str:
|
| 18 |
+
with sqlite3.connect(path) as connection:
|
| 19 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 20 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (TOKENIZER_VERSION,))
|
| 21 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 22 |
+
connection.execute("INSERT INTO documents VALUES (?, 'summary')", (doc_id,))
|
| 23 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 24 |
+
connection.execute("INSERT INTO content_fts (rowid, content) VALUES (1, 'content')")
|
| 25 |
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class PrepareFulltextIndexTests(unittest.TestCase):
|
| 29 |
+
def setUp(self):
|
| 30 |
+
self.temporary = tempfile.TemporaryDirectory()
|
| 31 |
+
self.root = Path(self.temporary.name)
|
| 32 |
+
self.bucket = self.root / "bucket"
|
| 33 |
+
self.generation = self.bucket / "generations/test"
|
| 34 |
+
self.generation.mkdir(parents=True)
|
| 35 |
+
self.fulltext = self.root / "fulltext"
|
| 36 |
+
|
| 37 |
+
def tearDown(self):
|
| 38 |
+
self.temporary.cleanup()
|
| 39 |
+
|
| 40 |
+
def write_manifest(self, hashes):
|
| 41 |
+
self.bucket.joinpath("current.json").write_text(json.dumps({
|
| 42 |
+
"format": 1,
|
| 43 |
+
"tokenizer_version": TOKENIZER_VERSION,
|
| 44 |
+
"databases": {
|
| 45 |
+
source: {
|
| 46 |
+
"path": f"generations/test/{source}.sqlite3",
|
| 47 |
+
"sha256": digest,
|
| 48 |
+
"bytes": (self.generation / f"{source}.sqlite3").stat().st_size,
|
| 49 |
+
"documents": 1,
|
| 50 |
+
"fts_rows": 1,
|
| 51 |
+
}
|
| 52 |
+
for source, digest in hashes.items()
|
| 53 |
+
},
|
| 54 |
+
}), encoding="utf-8")
|
| 55 |
+
|
| 56 |
+
def run_prepare(self):
|
| 57 |
+
environment = os.environ | {
|
| 58 |
+
"CCRD_INDEX_BUCKET_DIR": str(self.bucket),
|
| 59 |
+
"CCRD_FULLTEXT_DIR": str(self.fulltext),
|
| 60 |
+
}
|
| 61 |
+
return subprocess.run([sys.executable, str(PREPARE)], env=environment, capture_output=True, text=True)
|
| 62 |
+
|
| 63 |
+
def test_activates_verified_bucket_generation(self):
|
| 64 |
+
hashes = {
|
| 65 |
+
source: create_database(self.generation / f"{source}.sqlite3", f"{source}:1")
|
| 66 |
+
for source in ("CCRD", "CW")
|
| 67 |
+
}
|
| 68 |
+
self.write_manifest(hashes)
|
| 69 |
+
newer = self.bucket / "generations/ccrd-newer"
|
| 70 |
+
newer.mkdir(parents=True)
|
| 71 |
+
for source in ("CCRD", "CW"):
|
| 72 |
+
create_database(newer / f"{source}.sqlite3", f"{source}:newer")
|
| 73 |
+
|
| 74 |
+
result = self.run_prepare()
|
| 75 |
+
|
| 76 |
+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
| 77 |
+
status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
|
| 78 |
+
self.assertEqual(status['state'], 'ready')
|
| 79 |
+
for source in hashes:
|
| 80 |
+
target = self.fulltext / f"{source}.sqlite3"
|
| 81 |
+
self.assertTrue(target.is_file())
|
| 82 |
+
self.assertFalse(target.is_symlink())
|
| 83 |
+
self.assertEqual(target.read_bytes(), (self.generation / f"{source}.sqlite3").read_bytes())
|
| 84 |
+
|
| 85 |
+
def test_rejects_invalid_hash_format(self):
|
| 86 |
+
hashes = {
|
| 87 |
+
source: create_database(self.generation / f"{source}.sqlite3", f"{source}:1")
|
| 88 |
+
for source in ("CCRD", "CW")
|
| 89 |
+
}
|
| 90 |
+
hashes["CW"] = "not-a-sha256"
|
| 91 |
+
self.write_manifest(hashes)
|
| 92 |
+
|
| 93 |
+
result = self.run_prepare()
|
| 94 |
+
|
| 95 |
+
self.assertNotEqual(result.returncode, 0)
|
| 96 |
+
status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
|
| 97 |
+
self.assertEqual(status['state'], 'unavailable')
|
| 98 |
+
self.assertEqual(status['error'], 'ValueError')
|
| 99 |
+
self.assertFalse((self.fulltext / "CCRD.sqlite3").exists())
|
| 100 |
+
|
| 101 |
+
def test_recovers_from_stale_pointer_with_latest_complete_generation(self):
|
| 102 |
+
latest = self.bucket / "generations/ccrd-new"
|
| 103 |
+
latest.mkdir(parents=True)
|
| 104 |
+
for source in ("CCRD", "CW"):
|
| 105 |
+
create_database(latest / f"{source}.sqlite3", f"{source}:latest")
|
| 106 |
+
self.bucket.joinpath("current.json").write_text(json.dumps({
|
| 107 |
+
"format": 1,
|
| 108 |
+
"tokenizer_version": TOKENIZER_VERSION,
|
| 109 |
+
"databases": {
|
| 110 |
+
source: {
|
| 111 |
+
"path": f"generations/ccrd-old/{source}.sqlite3",
|
| 112 |
+
"sha256": "0" * 64,
|
| 113 |
+
"bytes": 1,
|
| 114 |
+
"documents": 1,
|
| 115 |
+
"fts_rows": 1,
|
| 116 |
+
}
|
| 117 |
+
for source in ("CCRD", "CW")
|
| 118 |
+
},
|
| 119 |
+
}), encoding="utf-8")
|
| 120 |
+
|
| 121 |
+
result = self.run_prepare()
|
| 122 |
+
|
| 123 |
+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
| 124 |
+
status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
|
| 125 |
+
self.assertEqual(status["state"], "ready")
|
| 126 |
+
self.assertEqual(status["pointer_error"]["error"], "ValueError")
|
| 127 |
+
for source in ("CCRD", "CW"):
|
| 128 |
+
self.assertEqual(
|
| 129 |
+
(self.fulltext / f"{source}.sqlite3").read_bytes(),
|
| 130 |
+
(latest / f"{source}.sqlite3").read_bytes(),
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
if __name__ == "__main__":
|
| 135 |
+
unittest.main()
|
tests/test_search.py
ADDED
|
@@ -0,0 +1,1214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import sqlite3
|
| 5 |
+
import tempfile
|
| 6 |
+
import unittest
|
| 7 |
+
from html.parser import HTMLParser
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from unittest.mock import MagicMock, patch
|
| 10 |
+
|
| 11 |
+
import app
|
| 12 |
+
import build_fulltext_db
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class InteractiveIdParser(HTMLParser):
|
| 16 |
+
def __init__(self):
|
| 17 |
+
super().__init__()
|
| 18 |
+
self.ids = set()
|
| 19 |
+
|
| 20 |
+
def handle_starttag(self, tag, attrs):
|
| 21 |
+
values = dict(attrs)
|
| 22 |
+
if tag in {"button", "input", "select", "a"} and values.get("id"):
|
| 23 |
+
self.ids.add(values["id"])
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def encode_doc_numbers(numbers: list[int]) -> bytes:
|
| 27 |
+
return build_fulltext_db.encode_doc_ids(numbers)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class TokenizationTests(unittest.TestCase):
|
| 31 |
+
def test_normalized_snippet_payload_matches_current_path(self):
|
| 32 |
+
text = build_fulltext_db.fts_content("前文\nABC 与 文化 革命,后文。")
|
| 33 |
+
self.assertEqual(
|
| 34 |
+
app.build_normalized_snippet_payload(text, "文化 革命"),
|
| 35 |
+
app.build_snippet_payload(text, "文化 革命"),
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
def test_normalized_exact_snippet_payload_matches_current_path(self):
|
| 39 |
+
text = build_fulltext_db.fts_content("前文\nABC 与 文化革命,后文。")
|
| 40 |
+
self.assertEqual(
|
| 41 |
+
app.build_normalized_exact_snippet_payload(text, "文化革命"),
|
| 42 |
+
app.build_snippet_payload(text, "文化革命", exact=True),
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
def test_normalizes_latin_and_indexes_chinese_unigrams_and_bigrams(self):
|
| 46 |
+
self.assertEqual(app.query_terms("ABC 手机"), ["abc", "手机"])
|
| 47 |
+
self.assertEqual(app.query_tokens("文化革命"), ["文化", "化革", "革命"])
|
| 48 |
+
self.assertEqual(app.index_tokens("手机"), {"手", "机", "手机"})
|
| 49 |
+
|
| 50 |
+
def test_builder_and_runtime_tokenizers_stay_in_sync(self):
|
| 51 |
+
text = "文化革命 ABC 手机"
|
| 52 |
+
self.assertEqual(app.index_tokens(text), build_fulltext_db.index_tokens(text))
|
| 53 |
+
expected_hashes = {
|
| 54 |
+
hashlib.sha256(token.encode("utf-8")).digest()[:16]
|
| 55 |
+
for token in build_fulltext_db.index_tokens(text) | build_fulltext_db.literal_tokens(text)
|
| 56 |
+
}
|
| 57 |
+
self.assertEqual(build_fulltext_db.token_hashes(text), expected_hashes)
|
| 58 |
+
|
| 59 |
+
def test_literal_tokens_preserve_punctuation_boundaries(self):
|
| 60 |
+
self.assertEqual(app.literal_query_tokens("手。机"), ["\0literal:手。机"])
|
| 61 |
+
self.assertEqual(
|
| 62 |
+
set(app.literal_query_tokens("手。机")),
|
| 63 |
+
build_fulltext_db.literal_tokens("手。机"),
|
| 64 |
+
)
|
| 65 |
+
self.assertEqual(app.literal_query_tokens("文化革命"), [])
|
| 66 |
+
self.assertEqual(app.literal_query_tokens("手。机文化"), ["\0literal:手。机"])
|
| 67 |
+
|
| 68 |
+
def test_snippet_anchors_use_normalized_character_offsets_for_all_token_types(self):
|
| 69 |
+
anchors = build_fulltext_db.snippet_anchors("ABC 手机 手。机")
|
| 70 |
+
offset_for = lambda token: anchors[hashlib.sha256(token.encode("utf-8")).digest()[:16]]
|
| 71 |
+
self.assertEqual(offset_for("abc"), 0)
|
| 72 |
+
self.assertEqual(offset_for("手机"), 4)
|
| 73 |
+
self.assertEqual(offset_for("手"), 4)
|
| 74 |
+
self.assertEqual(offset_for("\0literal:手。机"), 7)
|
| 75 |
+
|
| 76 |
+
def test_exact_wildcards_are_non_greedy_and_cross_newlines(self):
|
| 77 |
+
self.assertTrue(app.matches_exact_query("手\n持电话机", "手*机"))
|
| 78 |
+
self.assertTrue(app.matches_exact_query("手机", "手?"))
|
| 79 |
+
self.assertFalse(app.matches_exact_query("手持电话机", "手?机"))
|
| 80 |
+
|
| 81 |
+
def test_wildcard_required_tokens_keep_only_cjk_fixed_parts(self):
|
| 82 |
+
self.assertEqual(app.wildcard_required_tokens("手*机"), ["手", "机"])
|
| 83 |
+
self.assertEqual(app.wildcard_required_tokens("文化*资料"), ["文化", "资料"])
|
| 84 |
+
self.assertEqual(app.wildcard_required_tokens("ABC*文化"), ["文化"])
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class MetadataSearchTests(unittest.TestCase):
|
| 88 |
+
def setUp(self):
|
| 89 |
+
self.original_records = app.records
|
| 90 |
+
self.original_sources = app.source_counts
|
| 91 |
+
app.records = [
|
| 92 |
+
self.record("CCRD:0", "文化革命资料", ["历史"]),
|
| 93 |
+
self.record("CCRD:1", "文化资料", ["革命文献"]),
|
| 94 |
+
self.record("CW:2", "革命资料", ["其他"]),
|
| 95 |
+
]
|
| 96 |
+
app.source_counts = {"CCRD": 2, "CW": 1}
|
| 97 |
+
app.build_indexes()
|
| 98 |
+
|
| 99 |
+
def tearDown(self):
|
| 100 |
+
app.records = self.original_records
|
| 101 |
+
app.source_counts = self.original_sources
|
| 102 |
+
app.build_indexes()
|
| 103 |
+
|
| 104 |
+
@staticmethod
|
| 105 |
+
def record(doc_id: str, name: str, dirs: list[str]) -> dict:
|
| 106 |
+
source = doc_id.split(":", 1)[0]
|
| 107 |
+
path = "/".join([*dirs, name + ".txt"])
|
| 108 |
+
return {
|
| 109 |
+
"doc_id": doc_id,
|
| 110 |
+
"source": source,
|
| 111 |
+
"source_name": source,
|
| 112 |
+
"display_name": name,
|
| 113 |
+
"display_dirs": dirs,
|
| 114 |
+
"display_rel_path": path,
|
| 115 |
+
"extension": ".txt",
|
| 116 |
+
"size": 100,
|
| 117 |
+
"storage_root": source,
|
| 118 |
+
"storage_rel_path": path,
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
def test_multiple_terms_are_and_not_or(self):
|
| 122 |
+
result = app.search("文化 革命")
|
| 123 |
+
self.assertEqual(result["total"], 2)
|
| 124 |
+
self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0", "CCRD:1"})
|
| 125 |
+
|
| 126 |
+
def test_metadata_prefilter_keeps_shared_postings_and_matches_direct_intersection(self):
|
| 127 |
+
snapshot = {token: set(indices) for token, indices in app.word_index.items()}
|
| 128 |
+
for terms in (["文化", "革命"], ["革命", "文化"], ["文化革命"], ["不存在"], ["文化", "文化"], ["。"], []):
|
| 129 |
+
expected = None
|
| 130 |
+
for term in terms:
|
| 131 |
+
for token in app.query_tokens(term):
|
| 132 |
+
posting = snapshot.get(token, set())
|
| 133 |
+
expected = set(posting) if expected is None else expected & posting
|
| 134 |
+
with self.subTest(terms=terms):
|
| 135 |
+
self.assertEqual(app.metadata_prefilter(terms, True), expected)
|
| 136 |
+
self.assertEqual(app.word_index, snapshot)
|
| 137 |
+
|
| 138 |
+
def test_mixed_latin_metadata_candidates_keep_path_option_and_substrings(self):
|
| 139 |
+
app.records = [
|
| 140 |
+
self.record("CCRD:0", "文化 ABC123", []),
|
| 141 |
+
self.record("CCRD:1", "文化", ["ABC123"]),
|
| 142 |
+
self.record("CW:2", "革命 ABC123", []),
|
| 143 |
+
]
|
| 144 |
+
app.build_indexes()
|
| 145 |
+
for paths, expected in ((True, {0, 1}), (False, {0})):
|
| 146 |
+
for terms in (["文化", "abc"], ["abc", "文化"]):
|
| 147 |
+
self.assertEqual(app.metadata_prefilter(terms, paths), expected)
|
| 148 |
+
|
| 149 |
+
def test_search_paths_false_excludes_path_only_match(self):
|
| 150 |
+
result = app.search("革命", search_paths=False)
|
| 151 |
+
self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0", "CW:2"})
|
| 152 |
+
|
| 153 |
+
def test_filters_sorting_and_pagination_are_stable(self):
|
| 154 |
+
app.records[0]["size"] = 50
|
| 155 |
+
app.records[1]["size"] = 200
|
| 156 |
+
app.build_indexes()
|
| 157 |
+
result = app.search("文化", sources_filter=["CCRD"], min_size=100, sort="size", page_size=1)
|
| 158 |
+
self.assertEqual(result["total"], 1)
|
| 159 |
+
self.assertEqual(result["results"][0]["doc_id"], "CCRD:1")
|
| 160 |
+
|
| 161 |
+
def test_directory_filter_preserves_boundaries_and_root_semantics(self):
|
| 162 |
+
app.records = [
|
| 163 |
+
self.record("CCRD:0", "one", []),
|
| 164 |
+
self.record("CCRD:1", "two", ["a"]),
|
| 165 |
+
self.record("CCRD:2", "three", ["a", "b"]),
|
| 166 |
+
self.record("CW:3", "four", ["ab"]),
|
| 167 |
+
self.record("CW:4", "five", ["x", "y"]),
|
| 168 |
+
self.record("CW:5", "six", ["", "a"]),
|
| 169 |
+
self.record("CW:6", "seven", ["a", "", "b"]),
|
| 170 |
+
]
|
| 171 |
+
for folders in ([], [""], ["a"], ["a", "x"], ["", "a"], ["a/", "x/y"], ["missing", "ab"], ["a", "a"]):
|
| 172 |
+
for sources in (None, [], ["CCRD"], ["CCRD", "CW"]):
|
| 173 |
+
with self.subTest(folders=folders, sources=sources):
|
| 174 |
+
expected = [
|
| 175 |
+
idx for idx, rec in enumerate(app.records)
|
| 176 |
+
if (sources is None or rec["source"] in sources)
|
| 177 |
+
and (not folders or any(
|
| 178 |
+
"/".join(rec["display_dirs"]) == folder
|
| 179 |
+
or "/".join(rec["display_dirs"]).startswith(folder + "/")
|
| 180 |
+
for folder in folders
|
| 181 |
+
))
|
| 182 |
+
]
|
| 183 |
+
self.assertEqual(app.apply_filters(range(len(app.records)), sources, folders), expected)
|
| 184 |
+
|
| 185 |
+
def test_unfiltered_candidates_do_not_read_record_fields(self):
|
| 186 |
+
with patch.object(app, "records", [None, None, None]):
|
| 187 |
+
self.assertEqual(app.apply_filters(iter([2, 0, 1])), [2, 0, 1])
|
| 188 |
+
|
| 189 |
+
def test_precomputed_name_and_size_orders_match_previous_sort_keys(self):
|
| 190 |
+
app.records[0]["size"] = 50
|
| 191 |
+
app.records[1]["size"] = 200
|
| 192 |
+
app.records[2]["size"] = 200
|
| 193 |
+
app.build_indexes()
|
| 194 |
+
expected_name = sorted(
|
| 195 |
+
range(len(app.records)),
|
| 196 |
+
key=lambda idx: (app.records[idx]["display_rel_path"].lower(), app.records[idx]["doc_id"]),
|
| 197 |
+
)
|
| 198 |
+
expected_size = sorted(
|
| 199 |
+
range(len(app.records)),
|
| 200 |
+
key=lambda idx: (-app.records[idx]["size"], app.records[idx]["display_rel_path"].lower(), app.records[idx]["doc_id"]),
|
| 201 |
+
)
|
| 202 |
+
self.assertEqual(app.name_order, expected_name)
|
| 203 |
+
self.assertEqual(app.size_order, expected_size)
|
| 204 |
+
self.assertEqual(app.source_name_order["CCRD"], [idx for idx in expected_name if app.records[idx]["source"] == "CCRD"])
|
| 205 |
+
self.assertEqual(app.source_size_order["CCRD"], [idx for idx in expected_size if app.records[idx]["source"] == "CCRD"])
|
| 206 |
+
self.assertEqual(app.ordered_page(set(range(len(app.records))), app.size_order, 1, 2), expected_size[1:3])
|
| 207 |
+
|
| 208 |
+
def test_sparse_ordered_pages_match_global_order_with_ties(self):
|
| 209 |
+
app.records = [
|
| 210 |
+
self.record(f"CCRD:{number}", "Name" if number % 2 else "name", [str(number % 3)])
|
| 211 |
+
for number in range(40)
|
| 212 |
+
]
|
| 213 |
+
app.build_indexes()
|
| 214 |
+
for ordering in (app.name_order, app.size_order, list(reversed(app.name_order))):
|
| 215 |
+
for candidates in ([39, 2, 10, 1], list(range(40)), []):
|
| 216 |
+
expected = [idx for idx in ordering if idx in set(candidates)]
|
| 217 |
+
for start, page_size in ((0, 1), (0, 10), (1, 10), (4, 10), (30, 10)):
|
| 218 |
+
with self.subTest(candidates=len(candidates), start=start, page_size=page_size):
|
| 219 |
+
self.assertEqual(
|
| 220 |
+
app.ordered_page(candidates, ordering, start, page_size),
|
| 221 |
+
expected[start:start + page_size],
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
def test_empty_global_and_single_source_queries_bypass_filtering_and_order_scan(self):
|
| 225 |
+
with patch.object(app, "apply_filters", side_effect=AssertionError("fast path scanned records")), \
|
| 226 |
+
patch.object(app, "ordered_page", side_effect=AssertionError("fast path scanned order")):
|
| 227 |
+
global_name = app.search("", sort="name", page=2, page_size=1)
|
| 228 |
+
source_size = app.search("", sources_filter=["CCRD"], sort="size", page_size=2)
|
| 229 |
+
self.assertEqual(global_name["results"][0]["doc_id"], app.records[app.name_order[1]]["doc_id"])
|
| 230 |
+
self.assertEqual(
|
| 231 |
+
[item["doc_id"] for item in source_size["results"]],
|
| 232 |
+
[app.records[idx]["doc_id"] for idx in app.source_size_order["CCRD"][:2]],
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
def test_empty_query_with_record_filters_keeps_general_path(self):
|
| 236 |
+
with patch.object(app, "apply_filters", wraps=app.apply_filters) as filtered:
|
| 237 |
+
result = app.search("", sources_filter=["CCRD"], min_size=100, sort="size")
|
| 238 |
+
filtered.assert_called_once()
|
| 239 |
+
self.assertEqual(result["total"], 2)
|
| 240 |
+
|
| 241 |
+
def test_bounded_relevance_page_matches_full_sort(self):
|
| 242 |
+
indices = list(range(len(app.records)))
|
| 243 |
+
tokens = app.query_tokens("文化 革命")
|
| 244 |
+
for search_paths in (False, True):
|
| 245 |
+
expected = sorted(
|
| 246 |
+
indices,
|
| 247 |
+
key=lambda idx: (
|
| 248 |
+
-app.score_record(idx, tokens, search_paths),
|
| 249 |
+
app.records[idx]["display_rel_path"].lower(),
|
| 250 |
+
app.records[idx]["doc_id"],
|
| 251 |
+
),
|
| 252 |
+
)
|
| 253 |
+
for start, page_size in ((0, 1), (1, 2), (2, 2)):
|
| 254 |
+
self.assertEqual(
|
| 255 |
+
app.relevance_page(indices, tokens, search_paths, start, page_size),
|
| 256 |
+
expected[start:start + page_size],
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
def test_heavy_relevance_pages_preserve_full_order_and_duplicates(self):
|
| 260 |
+
app.records = [self.record(f"CCRD:{number:05d}", ("文化" if number % 3 else "革命"), [str(number % 7)])
|
| 261 |
+
for number in range(10020)]
|
| 262 |
+
app.build_indexes()
|
| 263 |
+
indices = list(range(len(app.records)))
|
| 264 |
+
for paths in (True, False):
|
| 265 |
+
tokens = app.query_tokens("文化 革命")
|
| 266 |
+
expected = sorted(indices, key=lambda idx: (
|
| 267 |
+
-app.score_record(idx, tokens, paths), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
|
| 268 |
+
))
|
| 269 |
+
for start in (0, 100, 999, 1000, 5000, 9900, 10020):
|
| 270 |
+
self.assertEqual(app.relevance_page(indices, tokens, paths, start, 200), expected[start:start + 200])
|
| 271 |
+
duplicate_indices = indices + [0]
|
| 272 |
+
expected = sorted(duplicate_indices, key=lambda idx: (
|
| 273 |
+
-app.score_record(idx, tokens, True), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
|
| 274 |
+
))
|
| 275 |
+
self.assertEqual(app.relevance_page(duplicate_indices, tokens, True, 1000, 200), expected[1000:1200])
|
| 276 |
+
self.assertEqual(app.relevance_page(duplicate_indices, tokens, True, 0, 200), expected[:200])
|
| 277 |
+
|
| 278 |
+
def test_shallow_relevance_stops_only_after_enough_maximum_score_hits(self):
|
| 279 |
+
app.records = [self.record(f"CCRD:{number}", "文化革命", ["same"])
|
| 280 |
+
for number in range(10020)]
|
| 281 |
+
app.build_indexes()
|
| 282 |
+
candidates = list(range(len(app.records)))
|
| 283 |
+
for paths in (True, False):
|
| 284 |
+
with self.subTest(paths=paths), patch.object(app, "score_record", wraps=app.score_record) as scored:
|
| 285 |
+
self.assertEqual(app.relevance_page(candidates, ["文化", "革命"], paths, 100, 100), app.name_order[100:200])
|
| 286 |
+
self.assertEqual(scored.call_count, 200)
|
| 287 |
+
# A file-only source-name match cannot attain the path-inclusive maximum.
|
| 288 |
+
with patch.object(app, "score_record", wraps=app.score_record) as scored:
|
| 289 |
+
self.assertEqual(app.relevance_page(candidates, ["ccrd"], True, 0, 100), app.name_order[:100])
|
| 290 |
+
self.assertEqual(scored.call_count, len(candidates))
|
| 291 |
+
|
| 292 |
+
def test_numeric_fulltext_mapping_preserves_source_identity_and_missing_ids(self):
|
| 293 |
+
app.records = [
|
| 294 |
+
self.record("CW:0", "first", []),
|
| 295 |
+
self.record("CCRD:0", "second", []),
|
| 296 |
+
self.record("CCRD:7", "third", []),
|
| 297 |
+
self.record("CCRD:007", "padded", []),
|
| 298 |
+
self.record("CCRD:invalid", "invalid", []),
|
| 299 |
+
]
|
| 300 |
+
app.build_indexes()
|
| 301 |
+
fake = MagicMock()
|
| 302 |
+
fake.search_source.side_effect = lambda source, q, exact: {0, 7, 99}
|
| 303 |
+
with patch.object(app, "fulltext_databases", fake), \
|
| 304 |
+
patch.object(app, "add_summaries", side_effect=lambda items, *args, **kwargs: items):
|
| 305 |
+
result = app.fulltext_search("missing", sort="name")
|
| 306 |
+
self.assertEqual(result["total"], 3)
|
| 307 |
+
self.assertEqual({item["doc_id"] for item in result["results"]}, {"CW:0", "CCRD:0", "CCRD:7"})
|
| 308 |
+
cw = app.fulltext_search("missing", sources_filter=["CW"])
|
| 309 |
+
self.assertEqual([item["doc_id"] for item in cw["results"]], ["CW:0"])
|
| 310 |
+
self.assertEqual(app.fulltext_search("missing", sources_filter=["unknown"])["total"], 0)
|
| 311 |
+
app.records = [self.record("CCRD:99", "replacement", [])]
|
| 312 |
+
app.build_indexes()
|
| 313 |
+
self.assertEqual(app.source_doc_number_index, {"CCRD": {99: 0}})
|
| 314 |
+
|
| 315 |
+
def test_shallow_relevance_falls_back_when_maximum_score_hits_are_insufficient(self):
|
| 316 |
+
app.records = [self.record(f"CCRD:{number}", "文化革命" if number < 3 else "文化", ["same"])
|
| 317 |
+
for number in range(10020)]
|
| 318 |
+
for record in app.records:
|
| 319 |
+
record["source_name"] = "文化 革命"
|
| 320 |
+
app.build_indexes()
|
| 321 |
+
candidates = list(reversed(range(len(app.records))))
|
| 322 |
+
tokens = ["文化", "革命"]
|
| 323 |
+
expected = sorted(candidates, key=lambda idx: (
|
| 324 |
+
-app.score_record(idx, tokens, True), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
|
| 325 |
+
))
|
| 326 |
+
for start in (0, 100):
|
| 327 |
+
self.assertEqual(app.relevance_page(candidates, tokens, True, start, 100), expected[start:start + 100])
|
| 328 |
+
|
| 329 |
+
def test_fulltext_relevance_respects_search_paths_false(self):
|
| 330 |
+
original_databases = app.fulltext_databases
|
| 331 |
+
fake_databases = MagicMock()
|
| 332 |
+
fake_databases.search_source.return_value = set()
|
| 333 |
+
fake_databases.has_content_fts = {"CCRD": False, "CW": False}
|
| 334 |
+
fake_databases.summaries.return_value = {}
|
| 335 |
+
fake_databases.matched_snippets.return_value = {}
|
| 336 |
+
app.fulltext_databases = fake_databases
|
| 337 |
+
try:
|
| 338 |
+
result = app.fulltext_search("文化", search_paths=False)
|
| 339 |
+
expected = sorted(
|
| 340 |
+
app.metadata_matches("文化", False, False),
|
| 341 |
+
key=lambda idx: (-app.score_record(idx, app.query_tokens("文化"), False), app.records[idx]["display_rel_path"].lower()),
|
| 342 |
+
)
|
| 343 |
+
self.assertEqual([item["doc_id"] for item in result["results"]], [app.records[idx]["doc_id"] for idx in expected])
|
| 344 |
+
finally:
|
| 345 |
+
app.fulltext_databases = original_databases
|
| 346 |
+
|
| 347 |
+
def test_exact_metadata_index_matches_full_scan(self):
|
| 348 |
+
for query in ("文化革命", "革命资料", "不存在"):
|
| 349 |
+
indexed = app.metadata_matches(query, True, True)
|
| 350 |
+
scanned = {
|
| 351 |
+
idx for idx, record in enumerate(app.records)
|
| 352 |
+
if app.matches_exact_query(record["_search_text"], query)
|
| 353 |
+
}
|
| 354 |
+
self.assertEqual(indexed, scanned)
|
| 355 |
+
|
| 356 |
+
def test_exact_search_uses_metadata_candidate_path(self):
|
| 357 |
+
with patch.object(app, "metadata_matches", wraps=app.metadata_matches) as metadata_matches:
|
| 358 |
+
result = app.search("文化革命", exact=True)
|
| 359 |
+
metadata_matches.assert_called_once_with("文化革命", True, True)
|
| 360 |
+
self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0"})
|
| 361 |
+
|
| 362 |
+
def test_metadata_match_cache_is_option_scoped_bounded_and_cleared_on_rebuild(self):
|
| 363 |
+
app.metadata_match_cache.clear()
|
| 364 |
+
with patch.object(app, "_metadata_matches_uncached", wraps=app._metadata_matches_uncached) as uncached:
|
| 365 |
+
first = app.metadata_matches("文化革命", True, True)
|
| 366 |
+
second = app.metadata_matches("文化革命", True, True)
|
| 367 |
+
self.assertEqual(first, second)
|
| 368 |
+
first.clear()
|
| 369 |
+
self.assertEqual(app.metadata_matches("文化革命", True, True), second)
|
| 370 |
+
self.assertEqual(uncached.call_count, 1)
|
| 371 |
+
app.metadata_matches("文化革命", True, False)
|
| 372 |
+
app.metadata_matches("文化 革命", False, True)
|
| 373 |
+
self.assertEqual(uncached.call_count, 3)
|
| 374 |
+
with patch.object(app, "METADATA_MATCH_CACHE_MAX_ENTRIES", 2):
|
| 375 |
+
app.metadata_matches("甲", False, True)
|
| 376 |
+
app.metadata_matches("乙", False, True)
|
| 377 |
+
app.metadata_matches("丙", False, True)
|
| 378 |
+
self.assertLessEqual(len(app.metadata_match_cache), 2)
|
| 379 |
+
app.build_indexes()
|
| 380 |
+
self.assertEqual(app.metadata_match_cache, {})
|
| 381 |
+
|
| 382 |
+
def test_metadata_cache_id_budget_never_truncates_results(self):
|
| 383 |
+
app.clear_metadata_match_cache()
|
| 384 |
+
with patch.object(app, "METADATA_MATCH_CACHE_MAX_TOTAL_IDS", 3), \
|
| 385 |
+
patch.object(app, "_metadata_matches_uncached", return_value={0, 1}):
|
| 386 |
+
for query in ("甲", "乙", "丙"):
|
| 387 |
+
self.assertEqual(app.metadata_matches(query, False, True), {0, 1})
|
| 388 |
+
self.assertEqual(sum(map(len, app.metadata_match_cache.values())), 2)
|
| 389 |
+
app.clear_metadata_match_cache()
|
| 390 |
+
with patch.object(app, "_metadata_matches_uncached", return_value={0, 1, 2, 3}) as search:
|
| 391 |
+
for _ in range(2):
|
| 392 |
+
self.assertEqual(app.metadata_matches("oversized", False, True), {0, 1, 2, 3})
|
| 393 |
+
self.assertEqual(search.call_count, 2)
|
| 394 |
+
self.assertEqual(app.metadata_match_cache, {})
|
| 395 |
+
|
| 396 |
+
def test_old_metadata_computation_cannot_repopulate_after_invalidation(self):
|
| 397 |
+
app.clear_metadata_match_cache()
|
| 398 |
+
def old_result(*_args):
|
| 399 |
+
app.clear_metadata_match_cache()
|
| 400 |
+
return {99}
|
| 401 |
+
with patch.object(app, "_metadata_matches_uncached", side_effect=old_result):
|
| 402 |
+
self.assertEqual(app.metadata_matches("文化", False, True), {99})
|
| 403 |
+
self.assertEqual(app.metadata_match_cache, {})
|
| 404 |
+
with patch.object(app, "_metadata_matches_uncached", side_effect=RuntimeError("failure")):
|
| 405 |
+
with self.assertRaises(RuntimeError):
|
| 406 |
+
app.metadata_matches("文化", False, True)
|
| 407 |
+
self.assertEqual(app.metadata_match_cache, {})
|
| 408 |
+
|
| 409 |
+
def test_exact_metadata_wildcard_preserves_full_scan_semantics(self):
|
| 410 |
+
for query in ("文化*资料", "手?机", "*文化*"):
|
| 411 |
+
indexed = app.metadata_matches(query, True, True)
|
| 412 |
+
scanned = {
|
| 413 |
+
idx for idx, record in enumerate(app.records)
|
| 414 |
+
if app.matches_exact_query(record["_search_text"], query)
|
| 415 |
+
}
|
| 416 |
+
self.assertEqual(indexed, scanned)
|
| 417 |
+
|
| 418 |
+
def test_exact_and_normal_latin_metadata_substrings_are_not_lost(self):
|
| 419 |
+
app.records[0]["display_name"] = "smartphone"
|
| 420 |
+
app.records[0]["display_rel_path"] = "folder/smartphone.txt"
|
| 421 |
+
app.build_indexes()
|
| 422 |
+
self.assertIn(0, app.metadata_matches("phone", True, False))
|
| 423 |
+
self.assertIn(0, app.metadata_matches("phone", False, False))
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
class FulltextDatabaseTests(unittest.TestCase):
|
| 427 |
+
def test_uses_rarest_posting_first_and_intersects_all_tokens(self):
|
| 428 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 429 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 430 |
+
with sqlite3.connect(path) as connection:
|
| 431 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 432 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 433 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 434 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 435 |
+
for token, numbers in (("文化", [1, 2, 3]), ("化革", [2]), ("革命", [2, 3])):
|
| 436 |
+
token_hash = hashlib.sha256(token.encode()).digest()[:16]
|
| 437 |
+
connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
|
| 438 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 439 |
+
try:
|
| 440 |
+
self.assertEqual(databases.search_source("CCRD", "文化革命"), {2})
|
| 441 |
+
finally:
|
| 442 |
+
databases.close()
|
| 443 |
+
|
| 444 |
+
def test_exact_phrase_uses_literal_postings(self):
|
| 445 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 446 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 447 |
+
with sqlite3.connect(path) as connection:
|
| 448 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 449 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 450 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 451 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 452 |
+
for token in [*app.query_tokens("手。机"), *app.literal_query_tokens("手。机")]:
|
| 453 |
+
token_hash = hashlib.sha256(token.encode()).digest()[:16]
|
| 454 |
+
connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers([7]), 1))
|
| 455 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 456 |
+
try:
|
| 457 |
+
self.assertEqual(databases.search_source("CCRD", "手。机", exact=True), {7})
|
| 458 |
+
self.assertEqual(databases.search_source("CCRD", "手机", exact=True), set())
|
| 459 |
+
finally:
|
| 460 |
+
databases.close()
|
| 461 |
+
|
| 462 |
+
def test_short_exact_query_uses_postings_before_verification(self):
|
| 463 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 464 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 465 |
+
with sqlite3.connect(path) as connection:
|
| 466 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 467 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 468 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 469 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 470 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 471 |
+
for token in [*app.query_tokens("手机"), *app.literal_query_tokens("手机")]:
|
| 472 |
+
token_hash = hashlib.sha256(token.encode()).digest()[:16]
|
| 473 |
+
connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers([7]), 1))
|
| 474 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", (7, "这是手机测试"))
|
| 475 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 476 |
+
try:
|
| 477 |
+
self.assertEqual(databases.search_source("CCRD", "手机", exact=True), {7})
|
| 478 |
+
finally:
|
| 479 |
+
databases.close()
|
| 480 |
+
|
| 481 |
+
def test_fts_content_strictly_verifies_wildcard_candidates(self):
|
| 482 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 483 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 484 |
+
with sqlite3.connect(path) as connection:
|
| 485 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 486 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 487 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 488 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 489 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 490 |
+
connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
|
| 491 |
+
(1, "手持电话机"),
|
| 492 |
+
(2, "手工劳动和机器"),
|
| 493 |
+
(3, "只有手持机"),
|
| 494 |
+
(4, "字面[手持机"),
|
| 495 |
+
])
|
| 496 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 497 |
+
try:
|
| 498 |
+
connection = databases.connections["CCRD"]
|
| 499 |
+
self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "手?机"), {3, 4})
|
| 500 |
+
self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "手*机"), {1, 2, 3, 4})
|
| 501 |
+
self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "[手?机"), {4})
|
| 502 |
+
finally:
|
| 503 |
+
databases.close()
|
| 504 |
+
|
| 505 |
+
def test_large_wildcard_candidates_use_one_fts_scan_without_widening_results(self):
|
| 506 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 507 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 508 |
+
with sqlite3.connect(path) as connection:
|
| 509 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 510 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 511 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 512 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 513 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 514 |
+
connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)",
|
| 515 |
+
[(number, "文化资料" if number % 2 else "其他") for number in range(1, 10001)])
|
| 516 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 517 |
+
try:
|
| 518 |
+
connection = databases.connections["CCRD"]
|
| 519 |
+
with patch.object(databases, "verify_wildcard_content", wraps=databases.verify_wildcard_content) as verifier:
|
| 520 |
+
result = databases.verify_wildcard_content(connection, set(range(1, 10001)), "文化*资料", fts_filtered=True)
|
| 521 |
+
self.assertEqual(result, set(range(1, 10001, 2)))
|
| 522 |
+
verifier.assert_called_once()
|
| 523 |
+
finally:
|
| 524 |
+
databases.close()
|
| 525 |
+
|
| 526 |
+
def test_short_wildcard_uses_required_postings_before_fts_verification(self):
|
| 527 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 528 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 529 |
+
with sqlite3.connect(path) as connection:
|
| 530 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 531 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 532 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 533 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 534 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 535 |
+
for token, numbers in (("手", [1, 2]), ("机", [1, 3])):
|
| 536 |
+
token_hash = hashlib.sha256(token.encode()).digest()[:16]
|
| 537 |
+
connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
|
| 538 |
+
connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
|
| 539 |
+
(1, "手持机"), (2, "手持电话"), (3, "机器"), (4, "手机"),
|
| 540 |
+
])
|
| 541 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 542 |
+
try:
|
| 543 |
+
self.assertEqual(databases.search_source("CCRD", "手*机", exact=True), {1})
|
| 544 |
+
finally:
|
| 545 |
+
databases.close()
|
| 546 |
+
|
| 547 |
+
def test_content_fts_returns_contiguous_cjk_phrase(self):
|
| 548 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 549 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 550 |
+
with sqlite3.connect(path) as connection:
|
| 551 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 552 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 553 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 554 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 555 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 556 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", ("这是文化革命资料",))
|
| 557 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (8, ?)", ("这是文化和革命资料",))
|
| 558 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 559 |
+
try:
|
| 560 |
+
self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), {7})
|
| 561 |
+
finally:
|
| 562 |
+
databases.close()
|
| 563 |
+
|
| 564 |
+
def test_exact_short_non_cjk_literals_verify_against_fts_content(self):
|
| 565 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 566 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 567 |
+
with sqlite3.connect(path) as connection:
|
| 568 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 569 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 570 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 571 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 572 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 573 |
+
connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
|
| 574 |
+
(1, "smartphone abc123xyz"),
|
| 575 |
+
(2, "手 单独"),
|
| 576 |
+
(3, "手。句号"),
|
| 577 |
+
(4, "手机 没有句号"),
|
| 578 |
+
])
|
| 579 |
+
for token, numbers in (("手", [2, 3, 4]), ("机", [4])):
|
| 580 |
+
token_hash = hashlib.sha256(token.encode()).digest()[:16]
|
| 581 |
+
connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
|
| 582 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 583 |
+
try:
|
| 584 |
+
# Latin substrings shorter than three characters must not be lost.
|
| 585 |
+
self.assertEqual(databases.search_source("CCRD", "ab", exact=True), {1})
|
| 586 |
+
self.assertEqual(databases.search_source("CCRD", "a", exact=True), {1})
|
| 587 |
+
self.assertEqual(databases.search_source("CCRD", "12", exact=True), {1})
|
| 588 |
+
# Short punctuation-separated CJK must not collapse to the bare character.
|
| 589 |
+
self.assertEqual(databases.search_source("CCRD", "手。", exact=True), {3})
|
| 590 |
+
# Pure CJK single characters still resolve through postings.
|
| 591 |
+
self.assertEqual(databases.search_source("CCRD", "手", exact=True), {2, 3, 4})
|
| 592 |
+
finally:
|
| 593 |
+
databases.close()
|
| 594 |
+
|
| 595 |
+
def test_builder_sanitizes_nul_before_fts_indexing(self):
|
| 596 |
+
self.assertEqual(build_fulltext_db.fts_content("ABC\0文化革命"), "abc 文化革命")
|
| 597 |
+
|
| 598 |
+
def test_metadata_only_result_does_not_read_original_text_for_snippet(self):
|
| 599 |
+
original_records = app.records
|
| 600 |
+
original_map = app.record_map
|
| 601 |
+
original_indices = app.record_map_index
|
| 602 |
+
original_databases = app.fulltext_databases
|
| 603 |
+
app.records = [{"doc_id": "CCRD:0", "storage_root": "unused", "storage_rel_path": "unused"}]
|
| 604 |
+
app.record_map = {"CCRD:0": app.records[0]}
|
| 605 |
+
app.record_map_index = {"CCRD:0": 0}
|
| 606 |
+
try:
|
| 607 |
+
item = {"doc_id": "CCRD:0"}
|
| 608 |
+
fake_databases = MagicMock()
|
| 609 |
+
fake_databases.summaries.return_value = {"CCRD:0": "摘要没有查询词"}
|
| 610 |
+
with patch.object(app, "get_doc_storage_path", side_effect=AssertionError("metadata hit read source text")):
|
| 611 |
+
app.fulltext_databases = fake_databases
|
| 612 |
+
self.assertEqual(app.add_summaries([item], "手机", True, set()), [item])
|
| 613 |
+
finally:
|
| 614 |
+
app.records = original_records
|
| 615 |
+
app.record_map = original_map
|
| 616 |
+
app.record_map_index = original_indices
|
| 617 |
+
app.fulltext_databases = original_databases
|
| 618 |
+
|
| 619 |
+
def test_fulltext_result_list_uses_stored_summaries_without_source_reads(self):
|
| 620 |
+
original_databases = app.fulltext_databases
|
| 621 |
+
fake_databases = MagicMock()
|
| 622 |
+
fake_databases.summaries.return_value = {"CCRD:0": "文档摘要"}
|
| 623 |
+
fake_databases.matched_snippets.return_value = {}
|
| 624 |
+
app.fulltext_databases = fake_databases
|
| 625 |
+
try:
|
| 626 |
+
item = {"doc_id": "CCRD:0"}
|
| 627 |
+
with patch.object(app, "get_doc_storage_path", side_effect=AssertionError("list summary read source text")):
|
| 628 |
+
result = app.add_summaries([item], "手机", matched_snippets=False, content_indices={0})
|
| 629 |
+
self.assertEqual(result[0]["snippet"], "文档摘要")
|
| 630 |
+
finally:
|
| 631 |
+
app.fulltext_databases = original_databases
|
| 632 |
+
|
| 633 |
+
def test_current_page_snippets_are_bounded_and_read_from_fts(self):
|
| 634 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 635 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 636 |
+
with sqlite3.connect(path) as connection:
|
| 637 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 638 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 639 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 640 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 641 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 642 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", ("甲" * 1000 + "文化革命" + "乙" * 1000,))
|
| 643 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 644 |
+
try:
|
| 645 |
+
calls = []
|
| 646 |
+
def counted_instr(content, needle):
|
| 647 |
+
calls.append(needle)
|
| 648 |
+
return content.find(needle) + 1
|
| 649 |
+
databases.connections["CCRD"].create_function("instr", 2, counted_instr)
|
| 650 |
+
payload = databases.matched_snippets(["CCRD:7"], "文化革命", exact=True)["CCRD:7"]
|
| 651 |
+
self.assertIn("文化革命", payload["snippet"])
|
| 652 |
+
self.assertIn("文化革命", payload["highlights"])
|
| 653 |
+
self.assertLessEqual(len(payload["snippet"]), 230)
|
| 654 |
+
self.assertEqual(calls, ["文化革命"])
|
| 655 |
+
finally:
|
| 656 |
+
databases.close()
|
| 657 |
+
|
| 658 |
+
def test_normal_snippets_seek_token_anchors_without_instr_scanning(self):
|
| 659 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 660 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 661 |
+
content = "甲" * 1000 + "文化革命" + "乙" * 1000
|
| 662 |
+
with sqlite3.connect(path) as connection:
|
| 663 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 664 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 665 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 666 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 667 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 668 |
+
connection.execute("CREATE TABLE snippet_anchors (token_hash BLOB NOT NULL, doc_number INTEGER NOT NULL, char_offset INTEGER NOT NULL, PRIMARY KEY (token_hash, doc_number)) WITHOUT ROWID")
|
| 669 |
+
connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", (content,))
|
| 670 |
+
for token_hash, char_offset in build_fulltext_db.snippet_anchors(content).items():
|
| 671 |
+
connection.execute("INSERT INTO snippet_anchors VALUES (?, 7, ?)", (token_hash, char_offset))
|
| 672 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 673 |
+
statements = []
|
| 674 |
+
databases.connections["CCRD"].set_trace_callback(statements.append)
|
| 675 |
+
try:
|
| 676 |
+
payload = databases.matched_snippets(["CCRD:7"], "文化革命")["CCRD:7"]
|
| 677 |
+
self.assertIn("文化", payload["highlights"])
|
| 678 |
+
self.assertIn("snippet_anchors", "\n".join(statements))
|
| 679 |
+
self.assertNotIn("instr(content", "\n".join(statements).lower())
|
| 680 |
+
finally:
|
| 681 |
+
databases.close()
|
| 682 |
+
|
| 683 |
+
def test_fulltext_results_use_synchronous_fts_snippets(self):
|
| 684 |
+
original_records = app.records
|
| 685 |
+
original_map = app.record_map
|
| 686 |
+
original_indices = app.record_map_index
|
| 687 |
+
original_counts = app.source_counts
|
| 688 |
+
original_databases = app.fulltext_databases
|
| 689 |
+
record = {
|
| 690 |
+
"doc_id": "CCRD:7", "source": "CCRD", "source_name": "CCRD",
|
| 691 |
+
"display_name": "文档", "extension": "txt", "display_dirs": [],
|
| 692 |
+
"display_rel_path": "文档.txt", "size": 10,
|
| 693 |
+
"_file_search_text": "文档 ccrd", "_path_search_text": "文档.txt",
|
| 694 |
+
"_search_text": "文档 ccrd 文档.txt",
|
| 695 |
+
}
|
| 696 |
+
fake_databases = MagicMock()
|
| 697 |
+
fake_databases.search_source.return_value = {7}
|
| 698 |
+
fake_databases.summaries.return_value = {"CCRD:7": "开头摘要"}
|
| 699 |
+
fake_databases.matched_snippets.return_value = {
|
| 700 |
+
"CCRD:7": {"snippet": "...命中文化革命正文...", "highlights": ["文化革命"]}
|
| 701 |
+
}
|
| 702 |
+
app.records = [record]
|
| 703 |
+
app.record_map = {"CCRD:7": record}
|
| 704 |
+
app.record_map_index = {"CCRD:7": 0}
|
| 705 |
+
app.source_counts = {"CCRD": 1}
|
| 706 |
+
app.build_indexes()
|
| 707 |
+
app.fulltext_databases = fake_databases
|
| 708 |
+
try:
|
| 709 |
+
result = app.fulltext_search("文化革命", exact=True)
|
| 710 |
+
self.assertEqual(result["results"][0]["snippet"], "...命中文化革命正文...")
|
| 711 |
+
self.assertEqual(result["results"][0]["snippet_highlights"], ["文化革命"])
|
| 712 |
+
self.assertTrue(result["results"][0]["snippet_complete"])
|
| 713 |
+
fake_databases.matched_snippets.assert_called_once_with(["CCRD:7"], "文化革命", exact=True, timings=None)
|
| 714 |
+
finally:
|
| 715 |
+
app.records = original_records
|
| 716 |
+
app.record_map = original_map
|
| 717 |
+
app.record_map_index = original_indices
|
| 718 |
+
app.source_counts = original_counts
|
| 719 |
+
app.build_indexes()
|
| 720 |
+
app.fulltext_databases = original_databases
|
| 721 |
+
|
| 722 |
+
def test_normal_snippet_uses_actual_bigram_match(self):
|
| 723 |
+
payload = app.build_snippet_payload("前文很多内容,文化在这里,后面另有革命。", "文化革命")
|
| 724 |
+
self.assertIn("文化", payload["snippet"])
|
| 725 |
+
self.assertIn("文化", payload["highlights"])
|
| 726 |
+
|
| 727 |
+
def test_exact_snippet_requires_contiguous_query(self):
|
| 728 |
+
text = "手。机只是干扰,后文才出现手机。"
|
| 729 |
+
payload = app.build_snippet_payload(text, "手机", exact=True)
|
| 730 |
+
self.assertIn("后文才出现手机", payload["snippet"])
|
| 731 |
+
self.assertEqual(payload["highlights"], ["手机"])
|
| 732 |
+
|
| 733 |
+
def test_snippet_returns_original_text_for_normalized_highlight(self):
|
| 734 |
+
payload = app.build_snippet_payload("正文使用全角ABC字符。", "ABC", exact=True)
|
| 735 |
+
self.assertEqual(payload["highlights"], ["ABC"])
|
| 736 |
+
|
| 737 |
+
def test_wildcard_snippet_highlights_fixed_original_parts(self):
|
| 738 |
+
payload = app.build_snippet_payload("这是手持电话机测试。", "手*机", exact=True)
|
| 739 |
+
self.assertEqual(payload["highlights"], ["手", "机"])
|
| 740 |
+
|
| 741 |
+
def test_missing_token_returns_no_matches(self):
|
| 742 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 743 |
+
path = Path(directory) / "CCRD.sqlite3"
|
| 744 |
+
with sqlite3.connect(path) as connection:
|
| 745 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 746 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 747 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 748 |
+
try:
|
| 749 |
+
self.assertEqual(databases.search_source("CCRD", "不存在"), set())
|
| 750 |
+
finally:
|
| 751 |
+
databases.close()
|
| 752 |
+
|
| 753 |
+
def test_missing_runtime_database_returns_no_matches_until_hot_loaded(self):
|
| 754 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 755 |
+
databases = app.FulltextDatabases(Path(directory))
|
| 756 |
+
try:
|
| 757 |
+
self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), set())
|
| 758 |
+
finally:
|
| 759 |
+
databases.close()
|
| 760 |
+
|
| 761 |
+
def test_hot_loading_increments_cache_generation(self):
|
| 762 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 763 |
+
root = Path(directory)
|
| 764 |
+
databases = app.FulltextDatabases(root)
|
| 765 |
+
self.assertEqual(databases.generation, 0)
|
| 766 |
+
path = root / "CCRD.sqlite3"
|
| 767 |
+
with sqlite3.connect(path) as connection:
|
| 768 |
+
connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
|
| 769 |
+
connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
|
| 770 |
+
connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
|
| 771 |
+
connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
|
| 772 |
+
connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
|
| 773 |
+
databases.refresh()
|
| 774 |
+
try:
|
| 775 |
+
self.assertEqual(databases.generation, 1)
|
| 776 |
+
finally:
|
| 777 |
+
databases.close()
|
| 778 |
+
|
| 779 |
+
|
| 780 |
+
class RequestValidationTests(unittest.TestCase):
|
| 781 |
+
def test_search_request_limits_expensive_inputs(self):
|
| 782 |
+
with self.assertRaises(ValueError):
|
| 783 |
+
app.SearchRequest(q="x" * 257)
|
| 784 |
+
with self.assertRaises(ValueError):
|
| 785 |
+
app.SearchRequest(page=0)
|
| 786 |
+
with self.assertRaises(ValueError):
|
| 787 |
+
app.SearchRequest(page_size=201)
|
| 788 |
+
with self.assertRaises(ValueError):
|
| 789 |
+
app.SearchRequest(sort="unknown")
|
| 790 |
+
|
| 791 |
+
def test_zip_request_has_independent_default_list(self):
|
| 792 |
+
first = app.ZipRequest()
|
| 793 |
+
second = app.ZipRequest()
|
| 794 |
+
first.doc_ids.append("CCRD:1")
|
| 795 |
+
self.assertEqual(second.doc_ids, [])
|
| 796 |
+
|
| 797 |
+
def test_zip_entries_deduplicate_ids_and_preserve_safe_paths(self):
|
| 798 |
+
original_map = app.record_map
|
| 799 |
+
try:
|
| 800 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 801 |
+
path = Path(directory) / "article.txt"
|
| 802 |
+
path.write_bytes(b"x" * 70000)
|
| 803 |
+
record = {
|
| 804 |
+
"source": "CCRD", "display_rel_path": "folder/article.txt",
|
| 805 |
+
"storage_root": "unused", "storage_rel_path": "unused",
|
| 806 |
+
}
|
| 807 |
+
app.record_map = {"CCRD:1": record}
|
| 808 |
+
with patch.object(app, "get_doc_storage_path", return_value=path):
|
| 809 |
+
entries = app.zip_entries(["CCRD:1", "CCRD:1"])
|
| 810 |
+
self.assertEqual(entries, [("CCRD/article [CCRD_1].txt", path)])
|
| 811 |
+
finally:
|
| 812 |
+
app.record_map = original_map
|
| 813 |
+
|
| 814 |
+
def test_unknown_source_returns_404_shape(self):
|
| 815 |
+
original_counts = app.source_counts
|
| 816 |
+
app.source_counts = {"CCRD": 1}
|
| 817 |
+
try:
|
| 818 |
+
response = app.api_search_source("missing", app.SearchRequest())
|
| 819 |
+
self.assertEqual(response.status_code, 404)
|
| 820 |
+
self.assertIn(b'"total":0', response.body)
|
| 821 |
+
finally:
|
| 822 |
+
app.source_counts = original_counts
|
| 823 |
+
|
| 824 |
+
def test_index_status_reports_background_build_progress(self):
|
| 825 |
+
original_databases = app.fulltext_databases
|
| 826 |
+
original_counts = app.source_counts
|
| 827 |
+
fake_databases = MagicMock()
|
| 828 |
+
fake_databases.connections = {"CCRD": MagicMock()}
|
| 829 |
+
fake_databases.tokenizer_versions = {"CCRD": app.TOKENIZER_VERSION}
|
| 830 |
+
fake_databases.has_content_fts = {"CCRD": True}
|
| 831 |
+
fake_databases.document_counts = {"CCRD": 1}
|
| 832 |
+
fake_databases.fts_counts = {"CCRD": 1}
|
| 833 |
+
app.fulltext_databases = fake_databases
|
| 834 |
+
app.source_counts = {"CCRD": 1, "CW": 1}
|
| 835 |
+
try:
|
| 836 |
+
response = app.api_index_status()
|
| 837 |
+
payload = __import__("json").loads(response.body)
|
| 838 |
+
self.assertFalse(payload["ready"])
|
| 839 |
+
self.assertEqual(payload["loaded_sources"], ["CCRD"])
|
| 840 |
+
self.assertEqual(payload["building_sources"], ["CW"])
|
| 841 |
+
finally:
|
| 842 |
+
app.fulltext_databases = original_databases
|
| 843 |
+
app.source_counts = original_counts
|
| 844 |
+
|
| 845 |
+
def test_index_status_rejects_wrong_version_or_incomplete_database(self):
|
| 846 |
+
original_databases = app.fulltext_databases
|
| 847 |
+
original_counts = app.source_counts
|
| 848 |
+
fake = MagicMock()
|
| 849 |
+
fake.connections = {"CCRD": MagicMock()}
|
| 850 |
+
fake.tokenizer_versions = {"CCRD": "wrong-version"}
|
| 851 |
+
fake.has_content_fts = {"CCRD": True}
|
| 852 |
+
fake.document_counts = {"CCRD": 1}
|
| 853 |
+
fake.fts_counts = {"CCRD": 1}
|
| 854 |
+
app.fulltext_databases = fake
|
| 855 |
+
app.source_counts = {"CCRD": 1}
|
| 856 |
+
try:
|
| 857 |
+
payload = __import__("json").loads(app.api_index_status().body)
|
| 858 |
+
self.assertFalse(payload["ready"])
|
| 859 |
+
self.assertEqual(payload["loaded_sources"], [])
|
| 860 |
+
self.assertEqual(payload["building_sources"], ["CCRD"])
|
| 861 |
+
finally:
|
| 862 |
+
app.fulltext_databases = original_databases
|
| 863 |
+
app.source_counts = original_counts
|
| 864 |
+
|
| 865 |
+
def test_index_status_rejects_database_without_snippet_anchors(self):
|
| 866 |
+
original_databases = app.fulltext_databases
|
| 867 |
+
original_counts = app.source_counts
|
| 868 |
+
fake = MagicMock()
|
| 869 |
+
fake.connections = {"CCRD": MagicMock()}
|
| 870 |
+
fake.tokenizer_versions = {"CCRD": app.TOKENIZER_VERSION}
|
| 871 |
+
fake.has_content_fts = {"CCRD": True}
|
| 872 |
+
fake.has_snippet_anchors = {"CCRD": False}
|
| 873 |
+
fake.document_counts = {"CCRD": 1}
|
| 874 |
+
fake.fts_counts = {"CCRD": 1}
|
| 875 |
+
app.fulltext_databases = fake
|
| 876 |
+
app.source_counts = {"CCRD": 1}
|
| 877 |
+
try:
|
| 878 |
+
payload = __import__("json").loads(app.api_index_status().body)
|
| 879 |
+
self.assertFalse(payload["ready"])
|
| 880 |
+
self.assertEqual(payload["building_sources"], ["CCRD"])
|
| 881 |
+
finally:
|
| 882 |
+
app.fulltext_databases = original_databases
|
| 883 |
+
app.source_counts = original_counts
|
| 884 |
+
|
| 885 |
+
def test_identical_search_requests_use_bounded_response_cache(self):
|
| 886 |
+
original_records = app.records
|
| 887 |
+
original_sources = app.source_counts
|
| 888 |
+
original_cache = app.search_response_cache
|
| 889 |
+
app.records = []
|
| 890 |
+
app.source_counts = {}
|
| 891 |
+
app.search_response_cache = {}
|
| 892 |
+
try:
|
| 893 |
+
with patch.object(app, "search", wraps=app.search) as search_call:
|
| 894 |
+
first = app.run_search(app.SearchRequest(q="手机"))
|
| 895 |
+
second = app.run_search(app.SearchRequest(q="手机"))
|
| 896 |
+
self.assertEqual(first, second)
|
| 897 |
+
self.assertEqual(search_call.call_count, 1)
|
| 898 |
+
self.assertTrue(any(key[0] == "search" for key in app.search_response_cache))
|
| 899 |
+
finally:
|
| 900 |
+
app.search_response_cache = original_cache
|
| 901 |
+
app.records = original_records
|
| 902 |
+
app.source_counts = original_sources
|
| 903 |
+
|
| 904 |
+
def test_search_cache_key_isolates_every_result_option(self):
|
| 905 |
+
original_cache = app.search_response_cache
|
| 906 |
+
app.search_response_cache = {}
|
| 907 |
+
try:
|
| 908 |
+
base = app.SearchRequest(q="手机")
|
| 909 |
+
variants = [
|
| 910 |
+
app.SearchRequest(q="手机", page=2),
|
| 911 |
+
app.SearchRequest(q="手机", page_size=1),
|
| 912 |
+
app.SearchRequest(q="手机", exact=True),
|
| 913 |
+
app.SearchRequest(q="手机", fulltext=True),
|
| 914 |
+
app.SearchRequest(q="手机", search_paths=False),
|
| 915 |
+
app.SearchRequest(q="手机", sort="name"),
|
| 916 |
+
app.SearchRequest(q="手机", sources=["CCRD"]),
|
| 917 |
+
app.SearchRequest(q="手机", folders=["历史"]),
|
| 918 |
+
app.SearchRequest(q="手机", min_size=1),
|
| 919 |
+
app.SearchRequest(q="手机", max_size=100),
|
| 920 |
+
]
|
| 921 |
+
app.run_search(base)
|
| 922 |
+
for variant in variants:
|
| 923 |
+
app.run_search(variant)
|
| 924 |
+
keys = [key for key in app.search_response_cache if key[0] == "search"]
|
| 925 |
+
self.assertEqual(len(keys), 1 + len(variants))
|
| 926 |
+
finally:
|
| 927 |
+
app.search_response_cache = original_cache
|
| 928 |
+
|
| 929 |
+
def test_api_cache_expires_and_evicts_oldest_entry(self):
|
| 930 |
+
original_cache = app.api_response_cache
|
| 931 |
+
app.api_response_cache = {}
|
| 932 |
+
try:
|
| 933 |
+
with patch.object(app.time, "monotonic", side_effect=[0, 0, 0, app.API_CACHE_TTL_SECONDS + 1, app.API_CACHE_TTL_SECONDS + 1]):
|
| 934 |
+
first = app.cached_payload(("sources",), lambda: object())
|
| 935 |
+
cached = app.cached_payload(("sources",), lambda: object())
|
| 936 |
+
expired = app.cached_payload(("sources",), lambda: object())
|
| 937 |
+
self.assertIs(first, cached)
|
| 938 |
+
self.assertIsNot(first, expired)
|
| 939 |
+
app.api_response_cache = {
|
| 940 |
+
("entry", index): (float(index), index) for index in range(501)
|
| 941 |
+
}
|
| 942 |
+
with patch.object(app.time, "monotonic", return_value=1000):
|
| 943 |
+
app.cached_payload(("new",), lambda: "value")
|
| 944 |
+
self.assertLessEqual(len(app.api_response_cache), app.API_CACHE_MAX_ENTRIES)
|
| 945 |
+
self.assertNotIn(("entry", 0), app.api_response_cache)
|
| 946 |
+
finally:
|
| 947 |
+
app.api_response_cache = original_cache
|
| 948 |
+
|
| 949 |
+
def test_search_and_api_caches_evict_independently(self):
|
| 950 |
+
original_api_cache = app.api_response_cache
|
| 951 |
+
original_search_cache = app.search_response_cache
|
| 952 |
+
app.api_response_cache = {("sources",): (1.0, "sources")}
|
| 953 |
+
app.search_response_cache = {("search", 1): (1.0, "results")}
|
| 954 |
+
try:
|
| 955 |
+
with patch.object(app, "API_CACHE_MAX_ENTRIES", 1):
|
| 956 |
+
app.cached_payload(("folders", "CCRD"), lambda: [])
|
| 957 |
+
self.assertEqual(app.search_response_cache[("search", 1)][1], "results")
|
| 958 |
+
with patch.object(app, "SEARCH_CACHE_MAX_ENTRIES", 1):
|
| 959 |
+
app.cached_search_payload(("search", 2), lambda: {"results": []})
|
| 960 |
+
self.assertIn(("folders", "CCRD"), app.api_response_cache)
|
| 961 |
+
finally:
|
| 962 |
+
app.api_response_cache = original_api_cache
|
| 963 |
+
app.search_response_cache = original_search_cache
|
| 964 |
+
|
| 965 |
+
|
| 966 |
+
class FrontendContractTests(unittest.TestCase):
|
| 967 |
+
def test_every_static_control_exists_is_cached_and_has_behavior(self):
|
| 968 |
+
html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
|
| 969 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 970 |
+
parser = InteractiveIdParser()
|
| 971 |
+
parser.feed(html)
|
| 972 |
+
expected = {
|
| 973 |
+
"hamburger-btn", "header-logo", "search-input", "settings-btn", "mobile-toggle-btn",
|
| 974 |
+
"theme-btn", "sidebar-expand-btn", "multi-select-toggle", "clear-filters-btn", "sort-select",
|
| 975 |
+
"multi-select-all", "multi-deselect", "multi-batch-download", "multi-zip-download",
|
| 976 |
+
"empty-random-btn", "close-filters-btn", "fulltext-toggle", "search-paths-toggle",
|
| 977 |
+
"history-toggle", "exact-search-toggle", "folder-select-all", "folder-deselect-all",
|
| 978 |
+
"source-filter-cancel", "folder-filter-cancel",
|
| 979 |
+
"filter-min-size", "filter-min-unit", "filter-max-size", "filter-max-unit", "random-book-btn",
|
| 980 |
+
}
|
| 981 |
+
self.assertEqual(parser.ids, expected)
|
| 982 |
+
selector_exceptions = {"header-logo", "search-input"}
|
| 983 |
+
for control_id in expected - selector_exceptions:
|
| 984 |
+
self.assertIn(f'$("#{control_id}")', source, control_id)
|
| 985 |
+
self.assertIn('DOM.headerLogo = $("#header-logo")', source)
|
| 986 |
+
self.assertIn('DOM.searchInput = $("#search-input")', source)
|
| 987 |
+
event_objects = {
|
| 988 |
+
"hamburger-btn": "DOM.hamburgerBtn", "header-logo": "DOM.headerLogo",
|
| 989 |
+
"search-input": "DOM.searchInput", "settings-btn": "DOM.settingsBtn",
|
| 990 |
+
"mobile-toggle-btn": "DOM.mobileToggleBtn", "theme-btn": "DOM.themeBtn",
|
| 991 |
+
"sidebar-expand-btn": "DOM.sidebarExpandBtn", "multi-select-toggle": "DOM.multiSelectToggle",
|
| 992 |
+
"clear-filters-btn": "DOM.clearFiltersBtn", "sort-select": "DOM.sortSelect",
|
| 993 |
+
"multi-select-all": "DOM.multiSelectAll", "multi-deselect": "DOM.multiDeselect",
|
| 994 |
+
"multi-batch-download": "DOM.multiBatchDownload", "multi-zip-download": "DOM.multiZipDownload",
|
| 995 |
+
"empty-random-btn": "DOM.emptyRandomBtn", "close-filters-btn": "DOM.closeFiltersBtn",
|
| 996 |
+
"fulltext-toggle": "DOM.fulltextToggle", "search-paths-toggle": "DOM.searchPathsToggle",
|
| 997 |
+
"history-toggle": "DOM.historyToggle", "exact-search-toggle": "DOM.exactToggle",
|
| 998 |
+
"folder-select-all": "DOM.folderSelectAll", "folder-deselect-all": "DOM.folderDeselectAll",
|
| 999 |
+
"source-filter-cancel": "DOM.sourceFilterCancel", "folder-filter-cancel": "DOM.folderFilterCancel",
|
| 1000 |
+
"filter-min-size": "DOM.filterMinSize", "filter-min-unit": "DOM.filterMinUnit",
|
| 1001 |
+
"filter-max-size": "DOM.filterMaxSize", "filter-max-unit": "DOM.filterMaxUnit",
|
| 1002 |
+
"random-book-btn": "DOM.randomBookBtn",
|
| 1003 |
+
}
|
| 1004 |
+
for control_id, object_name in event_objects.items():
|
| 1005 |
+
self.assertIn(f'{object_name}.addEventListener(', source, control_id)
|
| 1006 |
+
|
| 1007 |
+
def test_section_filter_cancel_buttons_clear_only_their_selected_state(self):
|
| 1008 |
+
html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
|
| 1009 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1010 |
+
css = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
|
| 1011 |
+
for control_id in ("source-filter-cancel", "folder-filter-cancel"):
|
| 1012 |
+
self.assertRegex(html, rf'id="{control_id}"[^>]*hidden>取消<')
|
| 1013 |
+
self.assertIn("DOM.sourceFilterCancel.hidden = STATE.selectedSources.length === 0", source)
|
| 1014 |
+
self.assertIn("DOM.folderFilterCancel.hidden = STATE.folderSelections.length === 0", source)
|
| 1015 |
+
self.assertIn('DOM.folderFilterCancel.addEventListener("click", () => applyFolderSelections([]))', source)
|
| 1016 |
+
self.assertIn(".filter-title-leading", css)
|
| 1017 |
+
|
| 1018 |
+
def test_dynamic_controls_have_delegated_event_roots(self):
|
| 1019 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1020 |
+
self.assertIn('DOM.resultsList.addEventListener("click"', source)
|
| 1021 |
+
self.assertIn('DOM.sidebarContent.addEventListener("click"', source)
|
| 1022 |
+
self.assertIn('DOM.previewPanel.addEventListener("click"', source)
|
| 1023 |
+
self.assertIn('DOM.searchHistoryDropdown.addEventListener("click"', source)
|
| 1024 |
+
for selector in (
|
| 1025 |
+
".result-checkbox", "[data-action='preview']", ".path-folder[data-folder]",
|
| 1026 |
+
"[data-source]", "#back-home", "[data-folder-open]", "[data-folder-nav]",
|
| 1027 |
+
"[data-doc-id]", "[data-action='close-preview']", "[data-action='toggle-preview-theme']", "[data-query]",
|
| 1028 |
+
):
|
| 1029 |
+
self.assertIn(selector, source)
|
| 1030 |
+
|
| 1031 |
+
def test_mobile_result_paths_keep_separators_with_their_folder(self):
|
| 1032 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1033 |
+
css = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
|
| 1034 |
+
self.assertIn("${highlightText(part, STATE.query)}${separator}</span>", source)
|
| 1035 |
+
self.assertNotIn('<span class="path-sep">/</span><span class="path-folder"', source)
|
| 1036 |
+
self.assertIn("body.mobile .result-path", css)
|
| 1037 |
+
self.assertIn("word-break: break-all", css)
|
| 1038 |
+
self.assertIn("overflow-wrap: normal", css)
|
| 1039 |
+
|
| 1040 |
+
def test_highlighting_is_bounded_for_common_single_character_queries(self):
|
| 1041 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1042 |
+
self.assertIn("const MAX_HIGHLIGHTS_PER_TEXT = 16", source)
|
| 1043 |
+
self.assertIn("if (!remaining) return match", source)
|
| 1044 |
+
|
| 1045 |
+
def test_search_debounce_allows_ime_input_to_settle(self):
|
| 1046 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1047 |
+
self.assertIn("const SEARCH_DEBOUNCE_MS = 120", source)
|
| 1048 |
+
self.assertIn("}, SEARCH_DEBOUNCE_MS);", source)
|
| 1049 |
+
|
| 1050 |
+
def test_visible_page_keeps_the_search_connection_warm(self):
|
| 1051 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1052 |
+
html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
|
| 1053 |
+
self.assertIn("const KEEPALIVE_INTERVAL_MS = 45 * 1000", source)
|
| 1054 |
+
self.assertIn('fetch("/api/ping"', source)
|
| 1055 |
+
self.assertIn('fetch("/api/ping", { cache: "no-store" }).catch(() => {})', html)
|
| 1056 |
+
self.assertLess(html.index('fetch("/api/ping"'), html.index('<link rel="stylesheet"'))
|
| 1057 |
+
self.assertIn("if (document.hidden || !navigator.onLine || keepalivePending)", source)
|
| 1058 |
+
self.assertIn('window.addEventListener("online", () => warmConnection(true))', source)
|
| 1059 |
+
self.assertIn('document.addEventListener("visibilitychange"', source)
|
| 1060 |
+
self.assertIn("window.setInterval(() => warmConnection(), KEEPALIVE_INTERVAL_MS)", source)
|
| 1061 |
+
|
| 1062 |
+
def test_visible_results_lazily_load_exact_snippets(self):
|
| 1063 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1064 |
+
self.assertIn("const SNIPPET_CONCURRENCY = 4", source)
|
| 1065 |
+
self.assertIn('priority: "low"', source)
|
| 1066 |
+
self.assertIn("function pauseSnippetLoadingForPreview()", source)
|
| 1067 |
+
self.assertIn("pauseSnippetLoadingForPreview();", source)
|
| 1068 |
+
self.assertIn('priority: "high"', source)
|
| 1069 |
+
append_results = source[source.index("function appendResults(items)") : source.index("function renderResults({ animate = false } = {})")]
|
| 1070 |
+
self.assertNotIn("animateResultRows", append_results)
|
| 1071 |
+
self.assertNotIn("preview-loading", source)
|
| 1072 |
+
self.assertIn("new IntersectionObserver", source)
|
| 1073 |
+
self.assertIn("API.snippet(task.docId, task.query, task.exact, controller.signal)", source)
|
| 1074 |
+
self.assertIn("for (const controller of snippetControllers) controller.abort()", source)
|
| 1075 |
+
self.assertIn("task.sequence !== STATE.searchSequence", source)
|
| 1076 |
+
self.assertIn("data.highlights || []", source)
|
| 1077 |
+
self.assertIn('if (node.dataset.snippetPending !== "1") return', source)
|
| 1078 |
+
self.assertIn('node.querySelector(".snippet-loading")', source)
|
| 1079 |
+
self.assertIn("正在提取摘要...", source)
|
| 1080 |
+
self.assertIn("function renderResults({ animate = false } = {}) {\n resetSnippetObserver();", source)
|
| 1081 |
+
|
| 1082 |
+
def test_zip_download_uses_native_form_instead_of_buffering_blob(self):
|
| 1083 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1084 |
+
self.assertIn('fetch("/api/zip-prepare"', source)
|
| 1085 |
+
self.assertIn("window.location.assign(data.download_url)", source)
|
| 1086 |
+
self.assertNotIn('const blob = await resp.blob()', source)
|
| 1087 |
+
self.assertIn("合并下载最多支持 500 个文件", source)
|
| 1088 |
+
self.assertNotIn("zip-download-frame", source)
|
| 1089 |
+
|
| 1090 |
+
def test_results_reserve_snippet_height_and_clear_stale_selection(self):
|
| 1091 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1092 |
+
style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
|
| 1093 |
+
html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
|
| 1094 |
+
self.assertIn("if (STATE.page === 1 && STATE.selectedIds.size)", source)
|
| 1095 |
+
self.assertIn("min-height: 1.5em", style)
|
| 1096 |
+
self.assertIn("-webkit-line-clamp: 3", style)
|
| 1097 |
+
self.assertIn("content-visibility: auto", style)
|
| 1098 |
+
self.assertIn("contain-intrinsic-size: auto 112px", style)
|
| 1099 |
+
self.assertIn('id="multi-select-all" class="multi-action-btn">全选</button>', html)
|
| 1100 |
+
self.assertNotIn("全选已加载", html)
|
| 1101 |
+
|
| 1102 |
+
def test_frontend_motion_preserves_loading_and_accessibility_contracts(self):
|
| 1103 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1104 |
+
style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
|
| 1105 |
+
html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
|
| 1106 |
+
self.assertIn('matchMedia("(prefers-reduced-motion: reduce)")', source)
|
| 1107 |
+
self.assertIn('setSearchMotion(true, STATE.page === 1 && STATE.results.length > 0)', source)
|
| 1108 |
+
self.assertIn('DOM.resultsContainer.setAttribute("aria-busy", loading ? "true" : "false")', source)
|
| 1109 |
+
self.assertIn("function renderResults({ animate = false } = {})", source)
|
| 1110 |
+
self.assertIn("if (animate) animateResultRows", source)
|
| 1111 |
+
self.assertIn("renderResults({ animate: true })", source)
|
| 1112 |
+
self.assertIn("Math.round(24 * Math.sqrt(index))", source)
|
| 1113 |
+
self.assertIn('classList.add("preview-enter")', source)
|
| 1114 |
+
self.assertIn(".result-item.result-enter", style)
|
| 1115 |
+
self.assertIn("animation: result-enter 320ms cubic-bezier(0.2, 0, 0, 1) both", style)
|
| 1116 |
+
self.assertIn(".results-container.is-refreshing .results-list", style)
|
| 1117 |
+
self.assertIn("--motion-emphasized-easing: cubic-bezier(0.05, 0.7, 0.1, 1)", style)
|
| 1118 |
+
self.assertIn("--motion-spring-easing: cubic-bezier(0.18, 1.32, 0.32, 1)", style)
|
| 1119 |
+
self.assertIn(".preview-panel.preview-enter .preview-body", style)
|
| 1120 |
+
self.assertIn("transition: width 150ms ease, opacity 150ms ease", style)
|
| 1121 |
+
self.assertIn('document.querySelectorAll(".theme-icon-svg")', source)
|
| 1122 |
+
self.assertIn("icon.getAnimations().forEach", source)
|
| 1123 |
+
self.assertIn('DOM.resultsList.classList.add("theme-static")', source)
|
| 1124 |
+
self.assertIn(".results-list.theme-static *", style)
|
| 1125 |
+
self.assertNotIn("document.body.offsetWidth", source)
|
| 1126 |
+
self.assertNotIn("theme-transitioning", source + style)
|
| 1127 |
+
self.assertNotIn("sidebar-opening", source + style)
|
| 1128 |
+
self.assertIn("@media (prefers-reduced-motion: reduce)", style)
|
| 1129 |
+
self.assertIn('id="results-container" class="results-container" aria-busy="false"', html)
|
| 1130 |
+
self.assertIn('id="toast" class="toast" role="status" aria-live="polite"', html)
|
| 1131 |
+
|
| 1132 |
+
def test_preview_open_limits_text_selection_to_the_preview_panel(self):
|
| 1133 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1134 |
+
style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
|
| 1135 |
+
do_search = source[source.index("async function doSearch()") : source.index("function resultPathHtml")]
|
| 1136 |
+
self.assertNotIn('DOM.previewPanel.style.display = "none"', do_search)
|
| 1137 |
+
self.assertNotIn('document.body.classList.remove("preview-open")', do_search)
|
| 1138 |
+
self.assertIn('document.body.classList.add("preview-open")', source)
|
| 1139 |
+
self.assertIn('document.body.classList.remove("preview-open")', source)
|
| 1140 |
+
self.assertIn("body.preview-open {\n user-select: none;", style)
|
| 1141 |
+
self.assertIn("body.preview-open #preview-panel {\n user-select: text;", style)
|
| 1142 |
+
self.assertIn("body.preview-open #preview-panel .preview-header {\n user-select: none;", style)
|
| 1143 |
+
|
| 1144 |
+
def test_frontend_guards_initial_search_races_and_popup_blocking(self):
|
| 1145 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1146 |
+
self.assertIn("async function applyInitialSearch(sequence)", source)
|
| 1147 |
+
self.assertIn("sequence !== STATE.searchSequence", source)
|
| 1148 |
+
self.assertIn("if (await applyInitialSearch(sequence)) return", source)
|
| 1149 |
+
self.assertIn('frame.src = `/api/download/${encodeURIComponent(id)}`', source)
|
| 1150 |
+
self.assertNotIn("window.open(`/api/download/", source)
|
| 1151 |
+
self.assertIn('if (event.key === "Escape")', source)
|
| 1152 |
+
self.assertIn('showToast("随机文章加载失败")', source)
|
| 1153 |
+
self.assertIn('DOM.searchInput.placeholder = STATE.fulltext ?', source)
|
| 1154 |
+
self.assertIn('if (!resp.ok) throw new Error(`HTTP ${resp.status}`)', source)
|
| 1155 |
+
self.assertIn('if (value && value.indexing) return', source)
|
| 1156 |
+
self.assertIn('if (STATE.page > 1) STATE.page -= 1', source)
|
| 1157 |
+
self.assertIn('showToast(timedOut ? "搜索超时,请重试" : "搜索失败")', source)
|
| 1158 |
+
self.assertIn("text = highlightText(rawText, keyword)", source)
|
| 1159 |
+
|
| 1160 |
+
def test_sidebar_uses_inline_payload_before_network(self):
|
| 1161 |
+
source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
|
| 1162 |
+
self.assertIn('readInlineJson("initial-sidebar-data")', source)
|
| 1163 |
+
self.assertIn('readInlineJson("initial-sidebar-sources")', source)
|
| 1164 |
+
|
| 1165 |
+
def test_server_embeds_route_sidebar_payload(self):
|
| 1166 |
+
html = app.render_index_html("CCRD")
|
| 1167 |
+
self.assertIn('id="initial-sidebar-data"', html)
|
| 1168 |
+
self.assertIn('id="initial-sidebar-sources"', html)
|
| 1169 |
+
self.assertIn('"source":"CCRD"', html)
|
| 1170 |
+
|
| 1171 |
+
def test_docker_build_generates_indexes_in_image(self):
|
| 1172 |
+
dockerfile = (Path(app.BASE_DIR) / "Dockerfile").read_text(encoding="utf-8")
|
| 1173 |
+
self.assertIn("FROM node:26.11.1-bookworm-slim AS static-assets", dockerfile)
|
| 1174 |
+
self.assertIn("ARG ESBUILD_VERSION=0.28.2", dockerfile)
|
| 1175 |
+
self.assertIn("esbuild static/app.js --minify --target=es2019 --charset=utf8 --legal-comments=none", dockerfile)
|
| 1176 |
+
self.assertIn("esbuild static/style.css --minify", dockerfile)
|
| 1177 |
+
self.assertIn("esbuild static/sw.js --minify --target=es2019 --charset=utf8 --legal-comments=none", dockerfile)
|
| 1178 |
+
self.assertIn("COPY --from=static-assets /out/ static/", dockerfile)
|
| 1179 |
+
self.assertEqual(dockerfile.count("FROM node:"), 1)
|
| 1180 |
+
self.assertEqual(dockerfile.count("FROM python:"), 1)
|
| 1181 |
+
self.assertIn("COPY CCRD/ CCRD/", dockerfile)
|
| 1182 |
+
self.assertIn("COPY CW/ CW/", dockerfile)
|
| 1183 |
+
self.assertIn("COPY app.py build_fulltext_db.py prepare_fulltext_index.py start.sh ./", dockerfile)
|
| 1184 |
+
self.assertIn('CMD ["sh", "/app/start.sh"]', dockerfile)
|
| 1185 |
+
self.assertNotIn("RUN python build_fulltext_db.py", dockerfile)
|
| 1186 |
+
start_script = (Path(app.BASE_DIR) / "start.sh").read_text(encoding="utf-8")
|
| 1187 |
+
self.assertIn("python /app/prepare_fulltext_index.py", start_script)
|
| 1188 |
+
self.assertIn("python /app/build_fulltext_db.py", start_script)
|
| 1189 |
+
self.assertIn(") &", start_script)
|
| 1190 |
+
self.assertIn("exec uvicorn", start_script)
|
| 1191 |
+
|
| 1192 |
+
def test_generated_indexes_are_excluded_from_build_context(self):
|
| 1193 |
+
dockerignore = (Path(app.BASE_DIR) / ".dockerignore").read_text(encoding="utf-8")
|
| 1194 |
+
gitattributes = (Path(app.BASE_DIR) / ".gitattributes").read_text(encoding="utf-8")
|
| 1195 |
+
self.assertIn("data/fulltext/", dockerignore)
|
| 1196 |
+
self.assertNotIn("data/fulltext/CCRD.sqlite3", gitattributes)
|
| 1197 |
+
self.assertNotIn("data/fulltext/CW.sqlite3", gitattributes)
|
| 1198 |
+
|
| 1199 |
+
def test_service_worker_only_caches_explicit_static_assets(self):
|
| 1200 |
+
source = (Path(app.BASE_DIR) / "static" / "sw.js").read_text(encoding="utf-8")
|
| 1201 |
+
precache = source.split("];", 1)[0]
|
| 1202 |
+
self.assertNotIn('"/",', precache)
|
| 1203 |
+
self.assertNotIn("/data/initial/manifest.json", precache)
|
| 1204 |
+
self.assertNotIn("/data/sidebar/manifest.json", precache)
|
| 1205 |
+
self.assertNotIn("cacheManifestUrls", source)
|
| 1206 |
+
self.assertIn('event.request.method !== "GET"', source)
|
| 1207 |
+
self.assertIn('url.pathname.startsWith("/static/")', source)
|
| 1208 |
+
self.assertIn('url.pathname.startsWith("/data/initial/")', source)
|
| 1209 |
+
self.assertIn('url.pathname.startsWith("/data/sidebar/")', source)
|
| 1210 |
+
self.assertIn("if (!cacheable)", source)
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
if __name__ == "__main__":
|
| 1214 |
+
unittest.main()
|
tests/test_search_oracle.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import re
|
| 7 |
+
import time
|
| 8 |
+
import unicodedata
|
| 9 |
+
import unittest
|
| 10 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from urllib.error import HTTPError, URLError
|
| 13 |
+
from urllib.request import Request, urlopen
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
| 17 |
+
BASE_URL = os.environ.get("CCRD_ORACLE_BASE_URL", "").rstrip("/")
|
| 18 |
+
TOKEN_RE = re.compile(r"[a-z0-9]+|[\u4e00-\u9fff\u3400-\u4dbf]+")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def normalize(value: str) -> str:
|
| 22 |
+
return unicodedata.normalize("NFKC", value or "").casefold()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def unique(values):
|
| 26 |
+
return list(dict.fromkeys(values))
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def parts(query: str) -> list[str]:
|
| 30 |
+
return unique(TOKEN_RE.findall(normalize(query)))
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def query_tokens(query: str) -> list[str]:
|
| 34 |
+
result = []
|
| 35 |
+
for part in parts(query):
|
| 36 |
+
if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", part) and len(part) > 1:
|
| 37 |
+
result.extend(part[index:index + 2] for index in range(len(part) - 1))
|
| 38 |
+
else:
|
| 39 |
+
result.append(part)
|
| 40 |
+
return unique(result)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def body_tokens(text: str) -> set[str]:
|
| 44 |
+
result = set()
|
| 45 |
+
for part in TOKEN_RE.findall(text):
|
| 46 |
+
if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", part):
|
| 47 |
+
result.update(part)
|
| 48 |
+
result.update(part[index:index + 2] for index in range(len(part) - 1))
|
| 49 |
+
else:
|
| 50 |
+
result.add(part)
|
| 51 |
+
return result
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def wildcard_regex(query: str):
|
| 55 |
+
pattern = []
|
| 56 |
+
for character in normalize(query):
|
| 57 |
+
pattern.append(".*?" if character == "*" else "." if character == "?" else re.escape(character))
|
| 58 |
+
return re.compile("".join(pattern), re.DOTALL)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def metadata_text(record: dict, search_paths: bool) -> str:
|
| 62 |
+
fields = [record["display_name"], record["source_name"]]
|
| 63 |
+
if search_paths:
|
| 64 |
+
fields.append(record["display_rel_path"])
|
| 65 |
+
return normalize(" ".join(fields))
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def metadata_match(record: dict, query: str, exact: bool, search_paths: bool) -> bool:
|
| 69 |
+
text = metadata_text(record, search_paths)
|
| 70 |
+
if exact:
|
| 71 |
+
return bool(wildcard_regex(query).search(text)) if "*" in query or "?" in query else normalize(query) in text
|
| 72 |
+
terms = parts(query)
|
| 73 |
+
return bool(terms) and all(term in text for term in terms)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@unittest.skipUnless(BASE_URL, "set CCRD_ORACLE_BASE_URL to run brute-force corpus oracle")
|
| 77 |
+
class ProductionSearchOracleTests(unittest.TestCase):
|
| 78 |
+
@classmethod
|
| 79 |
+
def setUpClass(cls):
|
| 80 |
+
payload = json.loads(gzip.decompress((BASE_DIR / "data/search_data.json.gz").read_bytes()))
|
| 81 |
+
cls.records = payload["records"]
|
| 82 |
+
cls.by_id = {record["doc_id"]: record for record in cls.records}
|
| 83 |
+
status = cls.request_json("/api/index-status")
|
| 84 |
+
if not status.get("ready"):
|
| 85 |
+
raise unittest.SkipTest(f'fulltext indexes are not ready: {status.get("building_sources", [])}')
|
| 86 |
+
cls.matrix = [
|
| 87 |
+
("手机", True), ("手。机", True), ("文化革命", True),
|
| 88 |
+
("手*机", True), ("手?机", True), ("文化 革命", False),
|
| 89 |
+
("ABC", True), ("ABC", True), ("hair", True), ("hair", False),
|
| 90 |
+
]
|
| 91 |
+
cls.body_matches = {(query, exact): set() for query, exact in cls.matrix}
|
| 92 |
+
started = time.monotonic()
|
| 93 |
+
for record in cls.records:
|
| 94 |
+
text = normalize((BASE_DIR / record["storage_root"] / record["storage_rel_path"]).read_text(encoding="utf-8", errors="ignore"))
|
| 95 |
+
tokens = None
|
| 96 |
+
for query, exact in cls.matrix:
|
| 97 |
+
if exact:
|
| 98 |
+
matched = bool(wildcard_regex(query).search(text)) if "*" in query or "?" in query else normalize(query) in text
|
| 99 |
+
else:
|
| 100 |
+
if tokens is None:
|
| 101 |
+
tokens = body_tokens(text)
|
| 102 |
+
matched = all(token in tokens for token in query_tokens(query))
|
| 103 |
+
if matched:
|
| 104 |
+
cls.body_matches[(query, exact)].add(record["doc_id"])
|
| 105 |
+
print(f"oracle_scanned={len(cls.records)} elapsed_seconds={time.monotonic() - started:.2f}")
|
| 106 |
+
|
| 107 |
+
@classmethod
|
| 108 |
+
def request_json(cls, path: str, payload: dict | None = None):
|
| 109 |
+
data = json.dumps(payload).encode() if payload is not None else None
|
| 110 |
+
for attempt in range(3):
|
| 111 |
+
try:
|
| 112 |
+
with urlopen(Request(BASE_URL + path, data=data, headers={"Content-Type": "application/json"} if data else {}), timeout=120) as response:
|
| 113 |
+
return json.load(response)
|
| 114 |
+
except (HTTPError, URLError, TimeoutError):
|
| 115 |
+
if attempt == 2:
|
| 116 |
+
raise
|
| 117 |
+
time.sleep(attempt + 1)
|
| 118 |
+
|
| 119 |
+
@classmethod
|
| 120 |
+
def api_ids(cls, body: dict, path: str = "/api/search") -> tuple[set[str], int]:
|
| 121 |
+
first = cls.request_json(path, {**body, "page": 1, "page_size": 200})
|
| 122 |
+
total = first["total"]
|
| 123 |
+
page_count = (total + 199) // 200
|
| 124 |
+
ids_by_page = {1: [item["doc_id"] for item in first["results"]]}
|
| 125 |
+
|
| 126 |
+
def fetch(page: int):
|
| 127 |
+
payload = cls.request_json(path, {**body, "page": page, "page_size": 200})
|
| 128 |
+
if payload["total"] != total:
|
| 129 |
+
raise AssertionError(f"total changed during pagination for {body}: {payload['total']} != {total}")
|
| 130 |
+
return page, [item["doc_id"] for item in payload["results"]]
|
| 131 |
+
|
| 132 |
+
with ThreadPoolExecutor(max_workers=6) as executor:
|
| 133 |
+
for page, page_ids in executor.map(fetch, range(2, page_count + 1)):
|
| 134 |
+
ids_by_page[page] = page_ids
|
| 135 |
+
ids = [doc_id for page in range(1, page_count + 1) for doc_id in ids_by_page[page]]
|
| 136 |
+
if len(ids) != len(set(ids)):
|
| 137 |
+
raise AssertionError(f"duplicate IDs for {body}")
|
| 138 |
+
if len(ids) != total:
|
| 139 |
+
raise AssertionError(f"pagination did not cover total for {body}: {len(ids)} != {total}")
|
| 140 |
+
return set(ids), total
|
| 141 |
+
|
| 142 |
+
def expected_ids(self, query: str, exact: bool, fulltext: bool = True, search_paths: bool = True, sources=None, folders=None, min_size=None, max_size=None):
|
| 143 |
+
body = self.body_matches[(query, exact)] if fulltext else set()
|
| 144 |
+
result = set()
|
| 145 |
+
for record in self.records:
|
| 146 |
+
if record["doc_id"] not in body and not metadata_match(record, query, exact, search_paths):
|
| 147 |
+
continue
|
| 148 |
+
if sources and record["source"] not in sources:
|
| 149 |
+
continue
|
| 150 |
+
rel_dir = "/".join(record["display_dirs"])
|
| 151 |
+
if folders and not any(rel_dir == folder or rel_dir.startswith(folder + "/") for folder in folders):
|
| 152 |
+
continue
|
| 153 |
+
if min_size is not None and record["size"] < min_size:
|
| 154 |
+
continue
|
| 155 |
+
if max_size is not None and record["size"] > max_size:
|
| 156 |
+
continue
|
| 157 |
+
result.add(record["doc_id"])
|
| 158 |
+
return result
|
| 159 |
+
|
| 160 |
+
def test_full_result_sets_and_totals_match_independent_oracle(self):
|
| 161 |
+
for query, exact in unique(self.matrix):
|
| 162 |
+
with self.subTest(query=query, exact=exact):
|
| 163 |
+
body = {"q": query, "exact": exact, "fulltext": True, "search_paths": True, "sort": "name"}
|
| 164 |
+
actual, total = self.api_ids(body)
|
| 165 |
+
expected = self.expected_ids(query, exact)
|
| 166 |
+
self.assertEqual(total, len(expected))
|
| 167 |
+
self.assertEqual(actual, expected)
|
| 168 |
+
|
| 169 |
+
def test_filters_modes_and_normalization_match_oracle(self):
|
| 170 |
+
query, exact = "文化 革命", False
|
| 171 |
+
folder = self.records[0]["display_dirs"][0]
|
| 172 |
+
variants = [
|
| 173 |
+
{"fulltext": False, "search_paths": True},
|
| 174 |
+
{"fulltext": False, "search_paths": False},
|
| 175 |
+
{"fulltext": True, "search_paths": False, "sources": ["CCRD"]},
|
| 176 |
+
{"fulltext": True, "search_paths": True, "sources": ["CW"]},
|
| 177 |
+
{"fulltext": True, "search_paths": True, "sources": ["CCRD"], "folders": [folder]},
|
| 178 |
+
{"fulltext": True, "search_paths": True, "min_size": 1000, "max_size": 10000},
|
| 179 |
+
]
|
| 180 |
+
for options in variants:
|
| 181 |
+
with self.subTest(options=options):
|
| 182 |
+
body = {"q": query, "exact": exact, "sort": "name", **options}
|
| 183 |
+
actual, total = self.api_ids(body)
|
| 184 |
+
expected = self.expected_ids(query, exact, **options)
|
| 185 |
+
self.assertEqual(total, len(expected))
|
| 186 |
+
self.assertEqual(actual, expected)
|
| 187 |
+
self.assertEqual(self.body_matches[("ABC", True)], self.body_matches[("ABC", True)])
|
| 188 |
+
|
| 189 |
+
def test_sorting_page_sizes_source_route_and_exact_snippets(self):
|
| 190 |
+
base = {"q": "手机", "exact": True, "fulltext": True, "search_paths": True, "sources": ["CW"]}
|
| 191 |
+
expected = self.expected_ids("手机", True, sources=["CW"])
|
| 192 |
+
reconstructed = []
|
| 193 |
+
for page_size in (17, 100, 200):
|
| 194 |
+
page = 1
|
| 195 |
+
ids = []
|
| 196 |
+
while len(ids) < len(expected):
|
| 197 |
+
payload = self.request_json("/api/search", {**base, "sort": "size", "page": page, "page_size": page_size})
|
| 198 |
+
batch = payload["results"]
|
| 199 |
+
if not batch:
|
| 200 |
+
break
|
| 201 |
+
ids.extend(item["doc_id"] for item in batch)
|
| 202 |
+
sizes = [item["Size"] for item in batch]
|
| 203 |
+
self.assertEqual(sizes, sorted(sizes, reverse=True))
|
| 204 |
+
page += 1
|
| 205 |
+
self.assertEqual(set(ids), expected)
|
| 206 |
+
self.assertEqual(len(ids), len(expected))
|
| 207 |
+
reconstructed.append(set(ids))
|
| 208 |
+
self.assertTrue(all(result == reconstructed[0] for result in reconstructed))
|
| 209 |
+
|
| 210 |
+
route_ids, route_total = self.api_ids({**base, "sort": "name", "sources": ["CW"]}, "/api/search/CCRD")
|
| 211 |
+
expected_ccrd = self.expected_ids("手机", True, sources=["CCRD"])
|
| 212 |
+
self.assertEqual(route_total, len(expected_ccrd))
|
| 213 |
+
self.assertEqual(route_ids, expected_ccrd)
|
| 214 |
+
|
| 215 |
+
for query in ("手机", "文化革命", "手*机", "手?机"):
|
| 216 |
+
payload = self.request_json("/api/search", {
|
| 217 |
+
"q": query, "exact": True, "fulltext": True, "search_paths": True,
|
| 218 |
+
"page": 1, "page_size": 20,
|
| 219 |
+
})
|
| 220 |
+
pattern = wildcard_regex(query) if "*" in query or "?" in query else None
|
| 221 |
+
for item in payload["results"]:
|
| 222 |
+
record = self.by_id[item["doc_id"]]
|
| 223 |
+
content = normalize((BASE_DIR / record["storage_root"] / record["storage_rel_path"]).read_text(encoding="utf-8", errors="ignore"))
|
| 224 |
+
metadata = metadata_text(record, True)
|
| 225 |
+
self.assertTrue(bool(pattern.search(content) or pattern.search(metadata)) if pattern else normalize(query) in content or normalize(query) in metadata)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
if __name__ == "__main__":
|
| 229 |
+
unittest.main()
|