vomebook commited on
Commit
b8b5b2c
·
verified ·
1 Parent(s): fc53af8

test: publish project tests and ignore rules

Browse files

Restore root metadata, publish test sources and test requirements; exclude local guides, reports, caches, benchmarks and snapshots.

.gitignore ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data/fulltext/
2
+ data/production-snapshot/
3
+ *.sqlite3
4
+ *.sqlite3.*
5
+ *.tmp
6
+ *.bak
7
+ *.before-rebuild
8
+ __pycache__/
9
+ *.pyc
10
+ .pytest_cache/
11
+ .mypy_cache/
12
+ .venv/
13
+ env/
14
+ tests/benchmark-results.json
15
+ AGENTS.md
16
+ TESTING.md
ocr_release.py ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Strict, startup-pinned OCR text/index releases. No network, pointers or deletion."""
2
+ from __future__ import annotations
3
+
4
+ from collections import defaultdict
5
+ import hashlib
6
+ import json
7
+ import os
8
+ from pathlib import Path, PurePosixPath
9
+ import re
10
+ import shutil
11
+ import sqlite3
12
+ import tempfile
13
+ import unicodedata
14
+
15
+
16
+ BUCKET = "vomebook/ccrd-index"
17
+ TOKENIZER = "cjk-bigram-boundary-fts5-v6-snippet-anchors"
18
+ SOURCE = "OCR"
19
+ SOURCE_NAME = "OCR成果"
20
+
21
+
22
+ def canonical(value) -> bytes:
23
+ return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode()
24
+
25
+
26
+ def digest(value) -> str:
27
+ return hashlib.sha256(value).hexdigest()
28
+
29
+
30
+ def sha(value) -> str:
31
+ if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value):
32
+ raise ValueError("invalid release SHA-256")
33
+ return value
34
+
35
+
36
+ def safe_path(value: str) -> str:
37
+ if (not isinstance(value, str) or not value or value.startswith("/") or "\\" in value
38
+ or any(part in {"", ".", ".."} for part in value.split("/"))
39
+ or any(ord(char) < 32 for char in value)):
40
+ raise ValueError("invalid release relative path")
41
+ return value
42
+
43
+
44
+ def read_resource(root: Path, ref: dict) -> bytes:
45
+ if not isinstance(ref, dict) or ref.get("bucket") != BUCKET:
46
+ raise ValueError("wrong OCR release bucket")
47
+ path = safe_path(ref.get("path"))
48
+ if not path.startswith(("text/objects/", "text/exports/", "indexes/ocr/")):
49
+ raise ValueError("unmanaged OCR resource path")
50
+ expected = sha(ref.get("sha256"))
51
+ size = ref.get("bytes")
52
+ if type(size) is not int or size < 1:
53
+ raise ValueError("invalid OCR resource size")
54
+ root = root.resolve()
55
+ target = (root / path).resolve()
56
+ if root not in target.parents or not target.is_file():
57
+ raise ValueError("missing OCR release resource or path escape")
58
+ raw = target.read_bytes()
59
+ if len(raw) != size or digest(raw) != expected:
60
+ raise ValueError("OCR release resource checksum mismatch")
61
+ return raw
62
+
63
+
64
+ def export_record(root: Path, ref: dict) -> tuple[dict, dict]:
65
+ export = json.loads(read_resource(root, ref))
66
+ if (export.get("kind") != "ccrd-ocr-text-export" or export.get("version") != 1
67
+ or export.get("complete") is not True or export.get("recipe") != "ocr-plain-utf8-pages-v1"):
68
+ raise ValueError("unsupported OCR export schema")
69
+ if not re.fullmatch(r"OCR:[1-9][0-9]*", str(export.get("doc_id", ""))) or int(export["doc_id"].split(":")[1]) >= 1 << 63:
70
+ raise ValueError("invalid OCR document identity")
71
+ repo, original = export.get("origin_repo"), safe_path(export.get("origin_rel_path"))
72
+ if not isinstance(repo, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repo):
73
+ raise ValueError("invalid OCR origin repository")
74
+ if not re.fullmatch(r"[0-9a-f]{40}", str(export.get("origin_revision", ""))):
75
+ raise ValueError("OCR origin must be commit-pinned")
76
+ readable = safe_path(export.get("readable_rel_path"))
77
+ expected_path = repo + "/" + str(PurePosixPath(original).with_suffix(".txt"))
78
+ if readable != expected_path:
79
+ raise ValueError("OCR readable path does not preserve its origin")
80
+ raw = read_resource(root, export.get("text"))
81
+ body = raw.decode("utf-8", errors="strict")
82
+ if body.startswith("\ufeff"):
83
+ raise ValueError("plain OCR export must not hide a BOM in its indexed text")
84
+ identity = {key: export[key] for key in ("recipe", "origin_repo", "origin_rel_path", "origin_revision",
85
+ "source_sha256", "doc_id", "text_generation", "txt_sha256")}
86
+ for field in ("source_sha256", "text_generation", "txt_sha256", "generation", "book_id"):
87
+ sha(export.get(field))
88
+ generation = digest(canonical(identity).rstrip(b"\n"))
89
+ if (generation != export["generation"] or digest(raw) != export["txt_sha256"]
90
+ or export["book_id"] != digest((repo + "\0" + original).encode())):
91
+ raise ValueError("OCR export generation mismatch")
92
+ pages, count = export.get("pages"), export.get("page_count")
93
+ if type(count) is not int or count < 1 or not isinstance(pages, list) or len(pages) != count:
94
+ raise ValueError("OCR export is missing pages")
95
+ if export.get("offset_unit") != "unicode-codepoint" or export.get("page_separator") != "\n\n":
96
+ raise ValueError("unsupported page offset recipe")
97
+ cursor, reviewed = 0, 0
98
+ for number, page in enumerate(pages, 1):
99
+ start, end = page.get("start"), page.get("end")
100
+ if (page.get("page") != number or type(start) is not int or type(end) is not int
101
+ or start != cursor or not start <= end <= len(body)):
102
+ raise ValueError("OCR page order/offset mismatch")
103
+ sha(page.get("text_generation"))
104
+ if digest(body[start:end].encode()) != sha(page.get("text_sha256")):
105
+ raise ValueError("OCR page text digest mismatch")
106
+ if page.get("quality") not in {"unreviewed", "partially-reviewed", "accepted"}:
107
+ raise ValueError("invalid OCR page quality")
108
+ reviewed += page["quality"] != "unreviewed"
109
+ cursor = end
110
+ if number < count:
111
+ if body[cursor:cursor + 2] != "\n\n":
112
+ raise ValueError("OCR page separator mismatch")
113
+ cursor += 2
114
+ if cursor != len(body) or export.get("accepted_pages") != reviewed or export.get("quality") != ("partially-reviewed" if reviewed else "unreviewed"):
115
+ raise ValueError("OCR whole-book coverage/quality mismatch")
116
+ resources = export.get("retained_reader_resources")
117
+ if not isinstance(resources, list) or not resources:
118
+ raise ValueError("OCR provenance retention references are missing")
119
+ for resource in resources:
120
+ if resource.get("bucket") not in {"vomebook/reader-assets-v2", "vomebook/pdf-pages-v2", "melsm/pdf-archive-v2"}:
121
+ raise ValueError("unknown retained Reader bucket")
122
+ safe_path(resource.get("path")); sha(resource.get("sha256"))
123
+ if type(resource.get("bytes")) is not int or resource["bytes"] < 1:
124
+ raise ValueError("invalid retained Reader resource")
125
+ path = PurePosixPath(readable)
126
+ record = {"doc_id": export["doc_id"], "source": SOURCE, "source_name": SOURCE_NAME,
127
+ "display_name": path.stem, "display_dirs": list(path.parts[:-1]), "display_rel_path": readable,
128
+ "extension": "txt", "size": len(raw), "has_preview": True, "has_fulltext": True,
129
+ "storage_root": ".", "storage_rel_path": export["text"]["path"],
130
+ "text_generation": export["text_generation"], "text_quality": export["quality"]}
131
+ return record, export
132
+
133
+
134
+ def directory_payloads(records: list[dict]) -> tuple[list[dict], dict]:
135
+ browser = defaultdict(lambda: {"folders": [], "files": []})
136
+ counts, direct = defaultdict(int), defaultdict(list)
137
+ children = defaultdict(set)
138
+ for record in records:
139
+ parts = record["display_dirs"]
140
+ for depth in range(len(parts) + 1):
141
+ path = "/".join(parts[:depth]); counts[path] += 1
142
+ if depth < len(parts): children[path].add(parts[depth])
143
+ direct["/".join(parts)].append(record)
144
+ for path in counts:
145
+ browser[path]["folders"] = [{"name": name, "path": (path + "/" if path else "") + name,
146
+ "count": counts[(path + "/" if path else "") + name]} for name in sorted(children[path])]
147
+ browser[path]["files"] = [{"name": item["display_name"], "ext": "txt", "size": item["size"],
148
+ "doc_id": item["doc_id"]} for item in sorted(direct[path], key=lambda item: (item["display_name"], item["doc_id"]))]
149
+ def node(name, path):
150
+ return {"name": name, "path": path, "count": counts[path], "hasDirectFiles": bool(direct[path]),
151
+ "hasChildren": bool(children[path]), "showSelfToggle": bool(path and direct[path] and children[path]),
152
+ "children": [node(child, (path + "/" if path else "") + child) for child in sorted(children[path])]}
153
+ return [node(SOURCE_NAME, "")], dict(browser)
154
+
155
+
156
+ def zip_member(record: dict) -> str:
157
+ """Keep legal origin hierarchy; bound non-portable components deterministically."""
158
+ replacements = {":": ":", "*": "*", "?": "?", '"': """, "<": "<", ">": ">", "|": "|", "\\": "\"}
159
+ def component(original):
160
+ value = re.sub(r'[:*?"<>|\\\x00-\x1f]', lambda match: replacements.get(match[0], "_"), original).rstrip(" .") or "_"
161
+ if value != original or len(value.encode()) > 180:
162
+ suffix = "-" + digest(original.encode())[:12]
163
+ raw = value.encode()[:180 - len(suffix)]
164
+ value = raw.decode("utf-8", errors="ignore").rstrip(" .") + suffix
165
+ return value
166
+ parts = safe_path(record["display_rel_path"]).split("/")
167
+ parts[-1] = component(PurePosixPath(parts[-1]).stem) + ".txt"
168
+ return "/".join([SOURCE_NAME, *[component(part) for part in parts[:-1]], parts[-1]])
169
+
170
+
171
+ def validate_database(path: Path, records: list[dict], text_root: Path) -> None:
172
+ with sqlite3.connect(f"file:{path.resolve()}?mode=ro&immutable=1", uri=True) as db:
173
+ if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
174
+ raise ValueError("invalid OCR SQLite integrity")
175
+ version = db.execute("SELECT value FROM metadata WHERE key='tokenizer_version'").fetchone()
176
+ if version != (TOKENIZER,):
177
+ raise ValueError("OCR tokenizer mismatch")
178
+ ids = {row[0] for row in db.execute("SELECT doc_id FROM documents")}
179
+ if ids != {record["doc_id"] for record in records}:
180
+ raise ValueError("OCR metadata/database ID set mismatch")
181
+ if db.execute("SELECT COUNT(*) FROM content_fts").fetchone()[0] != len(records):
182
+ raise ValueError("OCR content rows mismatch")
183
+ for record in records:
184
+ body = (text_root / record["storage_rel_path"]).read_text(encoding="utf-8", errors="strict")
185
+ normalized = unicodedata.normalize("NFKC", body).casefold().replace("\0", " ")
186
+ row = db.execute("SELECT content FROM content_fts WHERE rowid=?", (int(record["doc_id"].split(":")[1]),)).fetchone()
187
+ if row != (normalized,):
188
+ raise ValueError("OCR indexed content differs from readable TXT")
189
+ columns = {row[1] for row in db.execute("PRAGMA table_info(snippet_anchors)")}
190
+ if not {"token_hash", "doc_number", "char_offset"} <= columns:
191
+ raise ValueError("OCR snippet anchor schema missing")
192
+
193
+
194
+ def load_release(bucket: Path, manifest_path: str, expected_sha256: str, base_metadata_sha256: str) -> dict:
195
+ safe_path(manifest_path); sha(expected_sha256); sha(base_metadata_sha256)
196
+ bucket = bucket.resolve()
197
+ target = (bucket / manifest_path).resolve()
198
+ if bucket not in target.parents or not manifest_path.startswith("releases/ocr-"):
199
+ raise ValueError("OCR release path outside immutable release namespace")
200
+ raw = target.read_bytes()
201
+ if digest(raw) != expected_sha256:
202
+ raise ValueError("OCR release manifest checksum mismatch")
203
+ manifest = json.loads(raw)
204
+ if (manifest.get("kind") != "ccrd-ocr-search-release" or manifest.get("version") != 1
205
+ or manifest.get("tokenizer_version") != TOKENIZER or manifest.get("base_metadata_sha256") != base_metadata_sha256):
206
+ raise ValueError("incompatible OCR release/base snapshot")
207
+ sha(manifest.get("generation")); sha(manifest.get("builder_sha256"))
208
+ identity = {key: manifest[key] for key in ("version", "kind", "base_metadata_sha256", "builder_sha256", "tokenizer_version", "exports")}
209
+ if (manifest["generation"] != digest(canonical(identity))
210
+ or manifest_path != f'releases/ocr-{manifest["generation"]}/manifest.json'):
211
+ raise ValueError("OCR release identity/path mismatch")
212
+ refs = manifest.get("exports")
213
+ if not isinstance(refs, list) or not refs:
214
+ raise ValueError("empty OCR release")
215
+ records, exports = [], []
216
+ ids, paths = set(), set()
217
+ for ref in refs:
218
+ record, export = export_record(bucket, ref)
219
+ if record["doc_id"] in ids or record["display_rel_path"] in paths:
220
+ raise ValueError("OCR ID/readable-path collision; no silent overwrite")
221
+ ids.add(record["doc_id"]); paths.add(record["display_rel_path"])
222
+ records.append(record); exports.append(export)
223
+ if manifest.get("documents") != len(records):
224
+ raise ValueError("OCR release count mismatch")
225
+ database = manifest.get("database")
226
+ read_resource(bucket, database)
227
+ db_path = (bucket / database["path"]).resolve()
228
+ if database["path"] != f'indexes/ocr/{manifest["generation"]}/OCR.sqlite3':
229
+ raise ValueError("OCR database must retain its source namespace")
230
+ validate_database(db_path, records, bucket)
231
+ tree, browser = directory_payloads(records)
232
+ return {"manifest": manifest, "records": records, "exports": exports, "database": db_path,
233
+ "text_paths": {record["doc_id"]: bucket / record["storage_rel_path"] for record in records},
234
+ "tree": tree, "browser": browser}
235
+
236
+
237
+ def pin_release(bucket: Path, manifest_path: str, expected_sha256: str,
238
+ base_metadata_sha256: str, cache: Path) -> dict:
239
+ """Verify the whole candidate before pinning a process-local immutable copy."""
240
+ candidate = load_release(bucket, manifest_path, expected_sha256, base_metadata_sha256)
241
+ cache.mkdir(parents=True, exist_ok=True)
242
+ destination = cache / candidate["manifest"]["generation"]
243
+ if destination.exists():
244
+ return load_release(destination, manifest_path, expected_sha256, base_metadata_sha256)
245
+ stage = Path(tempfile.mkdtemp(prefix="ocr-stage-", dir=cache))
246
+ try:
247
+ refs = [*candidate["manifest"]["exports"], candidate["manifest"]["database"],
248
+ *[export["text"] for export in candidate["exports"]]]
249
+ for ref in refs:
250
+ raw = read_resource(bucket, ref)
251
+ target = stage / ref["path"]
252
+ target.parent.mkdir(parents=True, exist_ok=True)
253
+ with target.open("wb") as output:
254
+ output.write(raw); output.flush(); os.fsync(output.fileno())
255
+ target = stage / manifest_path
256
+ target.parent.mkdir(parents=True, exist_ok=True)
257
+ manifest_raw = (bucket / manifest_path).read_bytes()
258
+ if digest(manifest_raw) != expected_sha256:
259
+ raise ValueError("OCR candidate changed while being copied")
260
+ target.write_bytes(manifest_raw)
261
+ load_release(stage, manifest_path, expected_sha256, base_metadata_sha256)
262
+ stage.rename(destination)
263
+ finally:
264
+ if stage.exists():
265
+ shutil.rmtree(stage)
266
+ return load_release(destination, manifest_path, expected_sha256, base_metadata_sha256)
requirements-test.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ -r requirements.txt
2
+ httpx==0.28.1
3
+ playwright==1.63.0
tests/__init__.py ADDED
File without changes
tests/test_archive_compat.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import gzip
4
+ import json
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+ import tempfile
9
+ import unittest
10
+ import zipfile
11
+ from pathlib import Path
12
+ from unittest.mock import patch
13
+
14
+ import app
15
+
16
+
17
+ BASE_DIR = Path(__file__).resolve().parents[1]
18
+ RUN_LARGE = os.environ.get("CCRD_LARGE_ARCHIVE_TEST") == "1"
19
+
20
+
21
+ class ArchiveNameCompatibilityTests(unittest.TestCase):
22
+ @classmethod
23
+ def setUpClass(cls):
24
+ app.load_data()
25
+
26
+ def test_every_corpus_record_maps_to_unique_portable_member_name(self):
27
+ entries = app.zip_entries([record["doc_id"] for record in app.records])
28
+ names = [name for name, _path in entries]
29
+ self.assertEqual(len(names), 47250)
30
+ self.assertEqual(len(names), len(set(names)))
31
+ self.assertTrue(all(name.count("/") == 1 for name in names))
32
+ self.assertTrue(all(not any(character in name for character in ':*?"<>|\\') for name in names))
33
+ self.assertTrue(all(not any(part.endswith((" ", ".")) for part in name.split("/")) for name in names))
34
+ self.assertLessEqual(max(len(name.encode("utf-8")) for name in names), 220)
35
+
36
+ def test_multiple_independent_extractors_accept_adversarial_archive(self):
37
+ payload = json.loads(gzip.decompress((BASE_DIR / "data/search_data.json.gz").read_bytes()))
38
+ records = payload["records"]
39
+ selected = sorted(
40
+ records,
41
+ key=lambda record: (
42
+ any(character in record["display_name"] for character in ':*?"<>|\\'),
43
+ len(record["display_name"].encode("utf-8")),
44
+ ),
45
+ reverse=True,
46
+ )[:100]
47
+ response = app.zip_response([record["doc_id"] for record in selected])
48
+ archive_path = Path(response.path)
49
+ try:
50
+ with zipfile.ZipFile(archive_path) as archive:
51
+ self.assertEqual(len(archive.namelist()), 100)
52
+ self.assertIsNone(archive.testzip())
53
+ commands = []
54
+ if shutil.which("unzip"):
55
+ commands.append(["unzip", "-t", str(archive_path)])
56
+ if shutil.which("7z"):
57
+ commands.append(["7z", "t", str(archive_path)])
58
+ if shutil.which("bsdtar"):
59
+ commands.append(["bsdtar", "-tf", str(archive_path)])
60
+ self.assertGreaterEqual(len(commands), 3)
61
+ for command in commands:
62
+ result = subprocess.run(command, capture_output=True, timeout=120)
63
+ self.assertEqual(result.returncode, 0, f"{command}: {result.stderr.decode(errors='ignore')}")
64
+ finally:
65
+ archive_path.unlink(missing_ok=True)
66
+
67
+
68
+ @unittest.skipUnless(RUN_LARGE, "set CCRD_LARGE_ARCHIVE_TEST=1 to test the 500 largest documents")
69
+ class MaximumArchiveCompatibilityTests(unittest.TestCase):
70
+ def test_500_largest_documents_open_with_multiple_extractors(self):
71
+ app.load_data()
72
+ selected = sorted(app.records, key=lambda record: record["size"], reverse=True)[:500]
73
+ response = app.zip_response([record["doc_id"] for record in selected])
74
+ archive_path = Path(response.path)
75
+ try:
76
+ self.assertGreater(archive_path.stat().st_size, 0)
77
+ with zipfile.ZipFile(archive_path) as archive:
78
+ self.assertEqual(len(archive.namelist()), 500)
79
+ self.assertIsNone(archive.testzip())
80
+ for command in (["unzip", "-t", str(archive_path)], ["7z", "t", str(archive_path)], ["bsdtar", "-tf", str(archive_path)]):
81
+ result = subprocess.run(command, capture_output=True, timeout=300)
82
+ self.assertEqual(result.returncode, 0, f"{command}: {result.stderr.decode(errors='ignore')}")
83
+ finally:
84
+ archive_path.unlink(missing_ok=True)
85
+
86
+
87
+ if __name__ == "__main__":
88
+ unittest.main()
tests/test_benchmark.py ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import socket
6
+ import statistics
7
+ import subprocess
8
+ import sys
9
+ import time
10
+ import unittest
11
+ from datetime import datetime, timezone
12
+ from urllib.request import Request, urlopen
13
+
14
+ RESULTS_PATH = os.path.join(os.path.dirname(__file__), "benchmark-results.json")
15
+
16
+
17
+ def _git_sha():
18
+ try:
19
+ return subprocess.run(
20
+ ["git", "log", "-1", "--format=%H"],
21
+ capture_output=True, text=True, timeout=5,
22
+ ).stdout.strip()
23
+ except Exception:
24
+ return ""
25
+
26
+
27
+ def _save_results(project, queries):
28
+ existing = {"results": []}
29
+ if os.path.isfile(RESULTS_PATH):
30
+ try:
31
+ with open(RESULTS_PATH) as f:
32
+ existing = json.load(f)
33
+ except (json.JSONDecodeError, OSError):
34
+ pass
35
+ existing.setdefault("results", []).append({
36
+ "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
37
+ "project": project,
38
+ "git_sha": _git_sha(),
39
+ "queries": queries,
40
+ })
41
+ existing["results"] = existing["results"][-50:]
42
+ with open(RESULTS_PATH, "w") as f:
43
+ json.dump(existing, f, indent=2, ensure_ascii=False)
44
+
45
+
46
+ def _print_and_collect(results_list, label, iterations, times):
47
+ median = statistics.median(times)
48
+ mn = min(times)
49
+ mx = max(times)
50
+ print(f" {label:<46s} median={median:>8.3f}ms min={mn:>8.3f}ms max={mx:>8.3f}ms (n={iterations})")
51
+ results_list.append({
52
+ "label": label.strip(),
53
+ "iterations": iterations,
54
+ "median_ms": round(median, 3),
55
+ "min_ms": round(mn, 3),
56
+ "max_ms": round(mx, 3),
57
+ })
58
+
59
+
60
+ def free_port():
61
+ with socket.socket() as sock:
62
+ sock.bind(("127.0.0.1", 0))
63
+ return sock.getsockname()[1]
64
+
65
+
66
+ # ---- shared server for API & browser tests ----
67
+ _server_proc = None
68
+ _server_port = None
69
+
70
+
71
+ def start_server():
72
+ global _server_proc, _server_port
73
+ if _server_proc is not None:
74
+ return
75
+ _server_port = free_port()
76
+ _server_proc = subprocess.Popen(
77
+ [sys.executable, "-m", "uvicorn", "app:app",
78
+ "--host", "127.0.0.1", "--port", str(_server_port),
79
+ "--log-level", "error"],
80
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
81
+ )
82
+ for _ in range(120):
83
+ try:
84
+ with urlopen(f"http://127.0.0.1:{_server_port}/api/sources", timeout=2) as r:
85
+ if r.status == 200:
86
+ return
87
+ except Exception:
88
+ time.sleep(0.5)
89
+ stop_server()
90
+ raise RuntimeError("CCRD server did not start")
91
+
92
+
93
+ def stop_server():
94
+ global _server_proc, _server_port
95
+ if _server_proc:
96
+ try:
97
+ _server_proc.terminate()
98
+ _server_proc.wait(timeout=10)
99
+ except Exception:
100
+ _server_proc.kill()
101
+ _server_proc = None
102
+ _server_port = None
103
+
104
+
105
+ # ---- local benchmarks (pure function) ----
106
+ import app # noqa: E402
107
+
108
+
109
+ class LocalSearchBenchmarkTests(unittest.TestCase):
110
+ _results: list = []
111
+
112
+ @classmethod
113
+ def setUpClass(cls):
114
+ if not app.source_counts:
115
+ app.load_data()
116
+
117
+ def _bench(self, label, fn, iterations=10):
118
+ times = []
119
+ for _ in range(iterations):
120
+ start = time.perf_counter()
121
+ fn()
122
+ elapsed = (time.perf_counter() - start) * 1000
123
+ times.append(elapsed)
124
+ _print_and_collect(self._results, label, iterations, times)
125
+
126
+ def test_benchmarks(self):
127
+ self._bench('"手机" normal', lambda: app.search("手机"))
128
+ self._bench('"手机" exact', lambda: app.search("手机", exact=True))
129
+ self._bench('"文化革命" exact', lambda: app.search("文化革命", exact=True))
130
+ self._bench('"文化 革命" normal', lambda: app.search("文化 革命"))
131
+ self._bench('"手。机" exact', lambda: app.search("手。机", exact=True))
132
+ self._bench('"手*机" exact', lambda: app.search("手*机", exact=True))
133
+ self._bench('"手?机" exact', lambda: app.search("手?机", exact=True))
134
+ self._bench('"文" exact', lambda: app.search("文", exact=True))
135
+ self._bench('"ABC" normal', lambda: app.search("ABC"))
136
+ self._bench('"文化革命" exact search_paths=False',
137
+ lambda: app.search("文化革命", exact=True, search_paths=False))
138
+ self._bench('"手机" exact min_size=100',
139
+ lambda: app.search("手机", exact=True, min_size=100))
140
+ self._bench('sort by name', lambda: app.search(page=1, page_size=100, sort="name"))
141
+ self._bench('sort by size', lambda: app.search(page=1, page_size=100, sort="size"))
142
+
143
+ @classmethod
144
+ def tearDownClass(cls):
145
+ if cls._results:
146
+ _save_results("CCRD-Search-local", cls._results)
147
+
148
+
149
+ # ---- API benchmarks (HTTP to local server) ----
150
+ class ApiSearchBenchmarkTests(unittest.TestCase):
151
+ _results: list = []
152
+
153
+ def _bench(self, label, path, payload=None, iterations=5):
154
+ body = json.dumps(payload).encode() if payload else None
155
+ req = Request(f"http://127.0.0.1:{_server_port}{path}", data=body,
156
+ headers={"Content-Type": "application/json"} if body else {})
157
+ times = []
158
+ for i in range(iterations + 1):
159
+ start = time.perf_counter()
160
+ with urlopen(req, timeout=60) as r:
161
+ r.read()
162
+ elapsed = (time.perf_counter() - start) * 1000
163
+ if i > 0:
164
+ times.append(elapsed)
165
+ _print_and_collect(self._results, label, iterations, times)
166
+
167
+ def test_search_api_benchmarks(self):
168
+ self._bench('"手机" normal', "/api/search", {"q": "手机"})
169
+ self._bench('"手机" exact', "/api/search", {"q": "手机", "exact": True})
170
+ self._bench('"文化革命" exact', "/api/search", {"q": "文化革命", "exact": True})
171
+ self._bench('"文化 革命" normal', "/api/search", {"q": "文化 革命"})
172
+ self._bench('"手。机" exact', "/api/search", {"q": "手。机", "exact": True})
173
+ self._bench('"手*机" exact', "/api/search", {"q": "手*机", "exact": True})
174
+ self._bench('"手?机" exact', "/api/search", {"q": "手?机", "exact": True})
175
+ self._bench('"文" exact', "/api/search", {"q": "文", "exact": True})
176
+ self._bench('"ABC" normal', "/api/search", {"q": "ABC"})
177
+ self._bench('sources', "/api/sources")
178
+
179
+ @classmethod
180
+ def tearDownClass(cls):
181
+ if cls._results:
182
+ _save_results("CCRD-Search-api", cls._results)
183
+
184
+
185
+ # ---- browser benchmarks (Playwright) ----
186
+ try:
187
+ from playwright.sync_api import sync_playwright
188
+ except ImportError:
189
+ sync_playwright = None
190
+
191
+
192
+ @unittest.skipUnless(sync_playwright, "install requirements-test.txt, run: playwright install chromium")
193
+ class BrowserSearchBenchmarkTests(unittest.TestCase):
194
+ _results: list = []
195
+
196
+ @classmethod
197
+ def setUpClass(cls):
198
+ cls.playwright = sync_playwright().start()
199
+ cls.browser = cls.playwright.chromium.launch(headless=True, args=["--no-sandbox",
200
+ "--disable-gpu", "--disable-dev-shm-usage"])
201
+
202
+ @classmethod
203
+ def tearDownClass(cls):
204
+ if cls._results:
205
+ _save_results("CCRD-Search-browser", cls._results)
206
+ cls.browser.close()
207
+ cls.playwright.stop()
208
+
209
+ def _bench_search(self, label, query, iterations=3):
210
+ ctx = self.browser.new_context(viewport={"width": 1440, "height": 900})
211
+ page = ctx.new_page()
212
+ times = []
213
+ for i in range(iterations + 1):
214
+ page.goto(f"http://127.0.0.1:{_server_port}/")
215
+ page.wait_for_selector("#results-list .result-item", timeout=30000)
216
+ before_hash = page.evaluate("""() => {
217
+ const items = document.querySelectorAll('.result-item');
218
+ return Array.from(items).slice(0, 3).map(el => el.textContent.substring(0, 50)).join('|');
219
+ }""")
220
+ inp = page.locator("#search-input")
221
+ start = time.perf_counter()
222
+ inp.fill(query)
223
+ page.wait_for_function(f"""
224
+ () => {{
225
+ const items = document.querySelectorAll('.result-item');
226
+ if (items.length > 0) {{
227
+ const hash = Array.from(items).slice(0, 3).map(el => el.textContent.substring(0, 50)).join('|');
228
+ if (hash !== {json.dumps(before_hash)}) return true;
229
+ }}
230
+ const empty = document.querySelector('#empty-state');
231
+ return !!(empty && empty.style.display !== 'none');
232
+ }}
233
+ """, timeout=15000)
234
+ elapsed = (time.perf_counter() - start) * 1000
235
+ if i > 0:
236
+ times.append(elapsed)
237
+ ctx.close()
238
+ _print_and_collect(self._results, label, iterations, times)
239
+
240
+ def test_browser_search_benchmarks(self):
241
+ self._bench_search('"手机" normal', "手机")
242
+ self._bench_search('"文化革命" exact', "文化革命")
243
+ self._bench_search('"手*机" exact', "手*机")
244
+
245
+
246
+ def setUpModule():
247
+ start_server()
248
+
249
+
250
+ def tearDownModule():
251
+ stop_server()
tests/test_browser.py ADDED
@@ -0,0 +1,722 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import io
4
+ import os
5
+ import socket
6
+ import subprocess
7
+ import sys
8
+ import time
9
+ import unittest
10
+ import zipfile
11
+ from pathlib import Path
12
+ from urllib.request import urlopen
13
+
14
+ try:
15
+ from playwright.sync_api import sync_playwright
16
+ except ImportError:
17
+ sync_playwright = None
18
+
19
+
20
+ BASE_DIR = Path(__file__).resolve().parents[1]
21
+
22
+
23
+ def free_port() -> int:
24
+ with socket.socket() as sock:
25
+ sock.bind(("127.0.0.1", 0))
26
+ return int(sock.getsockname()[1])
27
+
28
+
29
+ @unittest.skipUnless(sync_playwright, "install requirements-test.txt and run: playwright install chromium")
30
+ class BrowserInteractionTests(unittest.TestCase):
31
+ @classmethod
32
+ def setUpClass(cls):
33
+ cls.base_url = os.environ.get("CCRD_BROWSER_BASE_URL", "").rstrip("/")
34
+ cls.server = None
35
+ if not cls.base_url:
36
+ cls.port = free_port()
37
+ cls.base_url = f"http://127.0.0.1:{cls.port}"
38
+ cls.server = subprocess.Popen(
39
+ [
40
+ sys.executable, "-m", "uvicorn", "app:app", "--host", "127.0.0.1",
41
+ "--port", str(cls.port), "--log-level", "warning",
42
+ ],
43
+ cwd=BASE_DIR,
44
+ stdout=subprocess.DEVNULL,
45
+ stderr=subprocess.DEVNULL,
46
+ )
47
+ for _ in range(120):
48
+ if cls.server.poll() is not None:
49
+ raise RuntimeError("Uvicorn exited before browser tests started")
50
+ try:
51
+ with urlopen(cls.base_url + "/api/sources", timeout=1) as response:
52
+ if response.status == 200:
53
+ break
54
+ except Exception:
55
+ time.sleep(0.25)
56
+ else:
57
+ cls.server.terminate()
58
+ raise RuntimeError("Uvicorn did not become ready")
59
+ cls.playwright = sync_playwright().start()
60
+ cls.browser = cls.playwright.chromium.launch(headless=True, args=["--no-sandbox"])
61
+
62
+ @classmethod
63
+ def tearDownClass(cls):
64
+ cls.browser.close()
65
+ cls.playwright.stop()
66
+ if cls.server is not None:
67
+ cls.server.terminate()
68
+ try:
69
+ cls.server.wait(timeout=10)
70
+ except subprocess.TimeoutExpired:
71
+ cls.server.kill()
72
+
73
+ def page(self, viewport=None):
74
+ context = self.browser.new_context(
75
+ viewport=viewport or {"width": 1440, "height": 900},
76
+ accept_downloads=True,
77
+ )
78
+ page = context.new_page()
79
+ errors = []
80
+ page.on("pageerror", lambda error: errors.append(str(error)))
81
+ page.goto(self.base_url, wait_until="domcontentloaded")
82
+ page.locator(".result-item").first.wait_for(timeout=30000)
83
+ return context, page, errors
84
+
85
+ def search_request_pages(self, page):
86
+ pages = []
87
+
88
+ def handle_request(request):
89
+ if request.method != "POST":
90
+ return
91
+ if not request.url.endswith("/api/search") and "/api/search/" not in request.url:
92
+ return
93
+ try:
94
+ body = request.post_data_json
95
+ except Exception:
96
+ return
97
+ pages.append(body.get("page"))
98
+
99
+ page.on("request", handle_request)
100
+ return pages
101
+
102
+ def test_late_initial_payload_cannot_take_over_a_new_search(self):
103
+ context, page, errors = self.page()
104
+ try:
105
+ result = page.evaluate("""async () => {
106
+ abortSearchPrefetch();
107
+ let releaseInitial;
108
+ loadInitialPayload = () => new Promise(resolve => { releaseInitial = resolve; });
109
+ STATE.query = '';
110
+ STATE.page = 1;
111
+ DOM.sortSelect.value = 'relevance';
112
+ const oldSearch = doSearch();
113
+ STATE.query = 'new-query';
114
+ const requests = [];
115
+ API.search = (body, source, signal) => new Promise(resolve => {
116
+ requests.push({signal, resolve});
117
+ });
118
+ const newSearch = doSearch();
119
+ await Promise.resolve();
120
+ const owner = STATE.searchController;
121
+ releaseInitial(null);
122
+ for (let i = 0; i < 10; i++) await Promise.resolve();
123
+ const retainedOwner = STATE.searchController === owner;
124
+ for (const request of requests) request.resolve({results: [], total: 0});
125
+ await Promise.all([oldSearch, newSearch]);
126
+ return {retainedOwner, requests: requests.length, loading: STATE.isLoading};
127
+ }""")
128
+ self.assertEqual(result, {"retainedOwner": True, "requests": 1, "loading": False})
129
+ self.assertEqual(errors, [])
130
+ finally:
131
+ context.close()
132
+
133
+ def test_initial_json_body_timeout_falls_back_to_search(self):
134
+ context, page, errors = self.page()
135
+ try:
136
+ result = page.evaluate("""async () => {
137
+ abortSearchPrefetch();
138
+ initialPayloadCache.clear();
139
+ const originalFetch = window.fetch;
140
+ const originalTimeout = window.setTimeout;
141
+ let aborted = false;
142
+ window.setTimeout = (fn, ms, ...args) => originalTimeout(fn, ms === REQUEST_TIMEOUT_MS ? 30 : ms, ...args);
143
+ window.fetch = (url, options = {}) => Promise.resolve({ok: true,
144
+ json: () => new Promise((resolve, reject) => {
145
+ if (options.signal) options.signal.addEventListener('abort', () => {
146
+ aborted = true;
147
+ reject(new DOMException('aborted', 'AbortError'));
148
+ }, {once: true});
149
+ })});
150
+ let searched = false;
151
+ API.search = async () => { searched = true; return {results: [], total: 0}; };
152
+ STATE.query = '';
153
+ STATE.page = 1;
154
+ DOM.sortSelect.value = 'relevance';
155
+ const completed = await Promise.race([
156
+ doSearch().then(() => true),
157
+ new Promise(resolve => originalTimeout(() => resolve(false), 500))
158
+ ]);
159
+ window.fetch = originalFetch;
160
+ window.setTimeout = originalTimeout;
161
+ return {completed, aborted, searched};
162
+ }""")
163
+ self.assertEqual(result, {"completed": True, "aborted": True, "searched": True})
164
+ self.assertEqual(errors, [])
165
+ finally:
166
+ context.close()
167
+
168
+ def test_stale_search_failure_does_not_change_current_page(self):
169
+ context, page, errors = self.page()
170
+ try:
171
+ result = page.evaluate("""async () => {
172
+ abortSearchPrefetch();
173
+ let failOld;
174
+ let finishNew;
175
+ const toasts = [];
176
+ showToast = message => toasts.push(message);
177
+ API.search = body => new Promise((resolve, reject) => {
178
+ if (body.q === 'old-query') failOld = reject;
179
+ else finishNew = resolve;
180
+ });
181
+ STATE.query = 'old-query';
182
+ const oldSearch = doSearch();
183
+ await Promise.resolve();
184
+ STATE.query = 'new-query';
185
+ STATE.page = 2;
186
+ const newSearch = doSearch();
187
+ await Promise.resolve();
188
+ failOld(new TypeError('late network failure'));
189
+ await oldSearch;
190
+ const currentPage = STATE.page;
191
+ finishNew({results: [], total: 0});
192
+ await newSearch;
193
+ return {currentPage, toasts};
194
+ }""")
195
+ self.assertEqual(result, {"currentPage": 2, "toasts": []})
196
+ self.assertEqual(errors, [])
197
+ finally:
198
+ context.close()
199
+
200
+ def test_failed_sidebar_payload_can_be_retried(self):
201
+ context, page, errors = self.page()
202
+ try:
203
+ result = page.evaluate("""async () => {
204
+ sidebarInitialCache.clear();
205
+ let attempts = 0;
206
+ window.fetch = async () => {
207
+ if (++attempts === 1) throw new TypeError('connection lost');
208
+ return {ok: true, json: async () => ({sources: []})};
209
+ };
210
+ await loadSidebarInitial(null).catch(() => null);
211
+ const data = await loadSidebarInitial(null).catch(() => null);
212
+ return {attempts, recovered: Array.isArray(data && data.sources)};
213
+ }""")
214
+ self.assertEqual(result, {"attempts": 2, "recovered": True})
215
+ self.assertEqual(errors, [])
216
+ finally:
217
+ context.close()
218
+
219
+ def test_late_root_sidebar_does_not_replace_open_folder(self):
220
+ for initial in (True, False):
221
+ with self.subTest(initial=initial):
222
+ context, page, errors = self.page()
223
+ try:
224
+ result = page.evaluate("""async initial => {
225
+ STATE.source = 'CCRD';
226
+ const route = ++routeRenderId;
227
+ let release;
228
+ const rootData = {source: 'CCRD', folders: [{name: 'root', path: 'root'}], files: []};
229
+ loadSidebarInitial = initial
230
+ ? () => new Promise(resolve => { release = () => resolve(rootData); })
231
+ : async () => null;
232
+ API.getContents = (source, path) => path
233
+ ? Promise.resolve({folders: [], files: [{doc_id: 'child', name: '当前目录文件'}]})
234
+ : new Promise(resolve => { release = () => resolve(rootData); });
235
+ const pending = renderSidebar(route);
236
+ for (let i = 0; i < 5; i++) await Promise.resolve();
237
+ await openFolder('child', route);
238
+ const html = DOM.sidebarContent.innerHTML;
239
+ release();
240
+ await pending;
241
+ return {preserved: DOM.sidebarContent.innerHTML === html, path: sidebarCurrentPath};
242
+ }""", initial)
243
+ self.assertEqual(result, {"preserved": True, "path": "child"})
244
+ self.assertEqual(errors, [])
245
+ finally:
246
+ context.close()
247
+
248
+ def test_old_sidebar_retry_cannot_reopen_previous_folder(self):
249
+ context, page, errors = self.page()
250
+ try:
251
+ result = page.evaluate("""async () => {
252
+ STATE.source = 'CCRD';
253
+ const route = ++routeRenderId;
254
+ const originalTimeout = window.setTimeout;
255
+ const retries = [];
256
+ window.setTimeout = (fn, ms, ...args) => ms === 1200
257
+ ? (retries.push(fn), 0) : originalTimeout(fn, ms, ...args);
258
+ const paths = [];
259
+ API.getContents = async (source, path) => {
260
+ paths.push(path);
261
+ if (path === 'old') throw new TypeError('offline');
262
+ return {folders: [], files: [{doc_id: 'new', name: '新目录'}]};
263
+ };
264
+ await openFolder('old', route);
265
+ await openFolder('new', route);
266
+ for (const retry of retries.slice()) retry();
267
+ for (let i = 0; i < 5; i++) await Promise.resolve();
268
+ window.setTimeout = originalTimeout;
269
+ return {paths, path: sidebarCurrentPath, visible: DOM.sidebarContent.textContent.includes('新目录')};
270
+ }""")
271
+ self.assertEqual(result, {"paths": ["old", "new"], "path": "new", "visible": True})
272
+ self.assertEqual(errors, [])
273
+ finally:
274
+ context.close()
275
+
276
+ def test_folder_http_errors_are_not_treated_as_empty_directories(self):
277
+ context, page, errors = self.page()
278
+ try:
279
+ result = page.evaluate("""async () => {
280
+ folderTreePending.clear();
281
+ folderContentsPending.clear();
282
+ STATE.folderContentsCache.clear();
283
+ window.fetch = async () => ({ok: false, status: 503, json: async () => ({folders: [], files: []})});
284
+ const treeRejected = await API.getFolders('CCRD').then(() => false, () => true);
285
+ const contentsRejected = await API.getContents('CCRD', 'retry').then(() => false, () => true);
286
+ let requests = 0;
287
+ window.fetch = async () => {
288
+ requests++;
289
+ return {ok: true, json: async () => ({folders: [], files: [{doc_id: 'recovered'}]})};
290
+ };
291
+ const recovered = await API.getContents('CCRD', 'retry');
292
+ return {treeRejected, contentsRejected, requests, recovered: recovered.files[0]?.doc_id};
293
+ }""")
294
+ self.assertEqual(result, {"treeRejected": True, "contentsRejected": True, "requests": 1, "recovered": "recovered"})
295
+ self.assertEqual(errors, [])
296
+ finally:
297
+ context.close()
298
+
299
+ def test_old_folder_navigation_does_not_restart_search_after_going_home(self):
300
+ context, page, errors = self.page()
301
+ try:
302
+ result = page.evaluate("""async () => {
303
+ let release;
304
+ renderSidebar = () => STATE.source
305
+ ? new Promise(resolve => { release = resolve; }) : Promise.resolve();
306
+ const searches = [];
307
+ const trees = [];
308
+ renderFolderTree = async () => trees.push(STATE.source);
309
+ doSearch = () => searches.push(STATE.source);
310
+ const oldNavigation = navigateToSourceFolder('CCRD', 'old');
311
+ navigateHome();
312
+ release();
313
+ await oldNavigation;
314
+ return {searches, trees, source: STATE.source};
315
+ }""")
316
+ self.assertEqual(result, {"searches": [None], "trees": [None], "source": None})
317
+ self.assertEqual(errors, [])
318
+ finally:
319
+ context.close()
320
+
321
+ def test_desktop_controls_search_preview_and_selection(self):
322
+ context, page, errors = self.page()
323
+ try:
324
+ static_controls = (
325
+ "hamburger-btn", "search-input", "settings-btn", "theme-btn", "mobile-toggle-btn",
326
+ "multi-select-toggle", "multi-select-all", "multi-deselect", "multi-batch-download",
327
+ "multi-zip-download", "empty-random-btn", "close-filters-btn", "fulltext-toggle",
328
+ "search-paths-toggle", "history-toggle", "exact-search-toggle", "folder-select-all",
329
+ "folder-deselect-all", "source-filter-cancel", "folder-filter-cancel",
330
+ "filter-min-size", "filter-min-unit", "filter-max-size",
331
+ "filter-max-unit", "random-book-btn", "sort-select",
332
+ )
333
+ for control_id in static_controls:
334
+ self.assertEqual(page.locator(f"#{control_id}").count(), 1, control_id)
335
+
336
+ original_theme = page.locator("body").get_attribute("class")
337
+ page.locator("#theme-btn").click()
338
+ self.assertNotEqual(page.locator("body").get_attribute("class"), original_theme)
339
+ self.assertEqual(page.locator("body.theme-transitioning").count(), 0)
340
+ self.assertGreater(page.locator("#theme-btn .theme-icon-svg").evaluate("node => node.getAnimations().length"), 0)
341
+
342
+ page.locator("#settings-btn").click()
343
+ self.assertTrue(page.evaluate("STATE.rightSidebarOpen"))
344
+ page.locator("#close-filters-btn").click()
345
+ self.assertFalse(page.evaluate("STATE.rightSidebarOpen"))
346
+
347
+ page.locator("#mobile-toggle-btn").click()
348
+ self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
349
+ page.locator("#mobile-toggle-btn").click()
350
+ self.assertFalse(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
351
+
352
+ page.locator("#settings-btn").click()
353
+ page.locator("label:has(#fulltext-toggle)").click()
354
+ self.assertEqual(page.locator("#search-input").get_attribute("placeholder"), "搜索 TXT 文件或路径...")
355
+ page.locator("label:has(#exact-search-toggle)").click()
356
+ page.locator("label:has(#search-paths-toggle)").click()
357
+ page.locator("label:has(#history-toggle)").click()
358
+ self.assertIn("fulltext=0", page.url)
359
+ self.assertIn("exact=0", page.url)
360
+ self.assertIn("search_paths=0", page.url)
361
+ self.assertIn("history=0", page.url)
362
+ page.locator("#close-filters-btn").click()
363
+
364
+ page.locator("#sort-select").select_option("name")
365
+ page.locator("#search-input").fill("文化革命")
366
+ page.wait_for_timeout(400)
367
+ page.locator(".result-item").first.wait_for(timeout=30000)
368
+ self.assertIn("q=", page.url)
369
+
370
+ page.locator("#multi-select-toggle").check()
371
+ page.locator(".result-checkbox").first.check()
372
+ self.assertIn("已选 1 项", page.locator("#multi-selected-count").text_content())
373
+ page.locator("#multi-select-all").click()
374
+ self.assertGreater(page.locator(".result-checkbox:checked").count(), 0)
375
+ page.locator("#multi-select-all").click()
376
+ self.assertEqual(page.locator(".result-checkbox:checked").count(), 0)
377
+ page.locator("#multi-deselect").click()
378
+ self.assertEqual(page.locator(".result-checkbox:checked").count(), 0)
379
+
380
+ page.locator("[data-action='preview']").first.click()
381
+ page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
382
+ self.assertEqual(page.locator("[data-action='close-preview']").count(), 1)
383
+ page.locator("[data-action='close-preview']").click()
384
+ page.locator("#preview-panel").wait_for(state="hidden")
385
+
386
+ page.locator("#random-book-btn").click()
387
+ page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
388
+ page.keyboard.press("Escape")
389
+ page.locator("#preview-panel").wait_for(state="hidden")
390
+ self.assertEqual(errors, [])
391
+ finally:
392
+ context.close()
393
+
394
+ def test_section_filter_cancel_buttons_preserve_other_filters(self):
395
+ context, page, errors = self.page()
396
+ try:
397
+ page.locator("#settings-btn").click()
398
+ page.locator("#filter-min-size").fill("1")
399
+ page.locator("#filter-source-list input").first.check()
400
+ page.locator("#source-filter-cancel").wait_for(state="visible")
401
+ page.locator("#source-filter-cancel").click()
402
+ self.assertEqual(page.evaluate("STATE.selectedSources"), [])
403
+ self.assertEqual(page.locator("#filter-min-size").input_value(), "1")
404
+ page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
405
+ page.locator(".result-item").first.wait_for(timeout=30000)
406
+ page.locator("#settings-btn").click()
407
+ page.locator("#folder-select-all").click()
408
+ page.locator("#folder-filter-cancel").wait_for(state="visible")
409
+ page.locator("#folder-filter-cancel").click()
410
+ self.assertEqual(page.evaluate("STATE.folderSelections"), [])
411
+ self.assertNotIn("folder=", page.url)
412
+ self.assertEqual(errors, [])
413
+ finally:
414
+ context.close()
415
+
416
+ def test_short_snippet_does_not_keep_three_line_height(self):
417
+ context, page, errors = self.page()
418
+ try:
419
+ lazy_rendering = page.locator(".result-item").first.evaluate(
420
+ "node => ({ visibility: getComputedStyle(node).contentVisibility, intrinsicSize: getComputedStyle(node).containIntrinsicSize })"
421
+ )
422
+ self.assertEqual(lazy_rendering["visibility"], "auto")
423
+ self.assertIn("112px", lazy_rendering["intrinsicSize"])
424
+ page.locator(".result-snippet").first.wait_for()
425
+ page.locator(".result-snippet").first.evaluate("node => node.textContent = '短摘要'")
426
+ height = page.locator(".result-snippet").first.bounding_box()["height"]
427
+ self.assertLess(height, 40)
428
+ self.assertEqual(errors, [])
429
+ finally:
430
+ context.close()
431
+
432
+ def test_theme_switch_does_not_animate_large_result_tree(self):
433
+ context, page, errors = self.page()
434
+ try:
435
+ page.locator("#results-list").evaluate(
436
+ """list => {
437
+ const template = list.querySelector('.result-item');
438
+ while (list.children.length < 500) list.appendChild(template.cloneNode(true));
439
+ }"""
440
+ )
441
+ page.locator("#theme-btn").click()
442
+ result_animations = page.locator("#results-list").evaluate(
443
+ "list => list.getAnimations({subtree: true}).length"
444
+ )
445
+ icon_animations = page.locator("#theme-btn .theme-icon-svg").evaluate(
446
+ "node => node.getAnimations().length"
447
+ )
448
+ self.assertEqual(page.locator(".result-item").count(), 500)
449
+ self.assertEqual(result_animations, 0)
450
+ self.assertGreater(icon_animations, 0)
451
+ self.assertEqual(errors, [])
452
+ finally:
453
+ context.close()
454
+
455
+ def test_long_snippet_clamps_to_three_lines(self):
456
+ context, page, errors = self.page()
457
+ try:
458
+ page.locator(".result-snippet").first.wait_for()
459
+ page.locator(".result-snippet").first.evaluate("node => { node.textContent = ['第一行','第二行','第三行','第四行','第五行'].join('\\n'); }")
460
+ height = page.locator(".result-snippet").first.bounding_box()["height"]
461
+ self.assertLess(height, 90)
462
+ self.assertEqual(errors, [])
463
+ finally:
464
+ context.close()
465
+
466
+ def test_source_folder_controls_and_mobile_sidebars(self):
467
+ context = self.browser.new_context(viewport={"width": 390, "height": 844})
468
+ page = context.new_page()
469
+ errors = []
470
+ page.on("pageerror", lambda error: errors.append(str(error)))
471
+ try:
472
+ page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
473
+ page.locator(".result-item").first.wait_for(timeout=30000)
474
+ self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
475
+ page.locator("#hamburger-btn").click()
476
+ self.assertTrue(page.evaluate("STATE.leftSidebarOpen"))
477
+ page.locator("#overlay").click(position={"x": 380, "y": 400})
478
+ self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
479
+ page.locator("#settings-btn").click()
480
+ self.assertTrue(page.evaluate("STATE.rightSidebarOpen"))
481
+ page.locator("#close-filters-btn").click()
482
+ self.assertFalse(page.evaluate("STATE.rightSidebarOpen"))
483
+ page.locator("#settings-btn").click()
484
+ page.locator("#folder-select-all").click()
485
+ self.assertGreaterEqual(page.locator("#filter-folder-tree input:checked").count(), 1)
486
+ page.evaluate("""() => {
487
+ const paths = [];
488
+ flattenNodes(STATE.folderTree, paths);
489
+ const leaf = paths.sort((left, right) => right.split('/').length - left.split('/').length)[0];
490
+ applyFolderSelections(deselectFolderPath(leaf));
491
+ applyFolderSelections(selectFolderPath(leaf));
492
+ }""")
493
+ page.locator("#folder-select-all").click()
494
+ self.assertEqual(page.evaluate("STATE.folderSelections"), [])
495
+ page.locator("#folder-select-all").click()
496
+ page.locator("#folder-deselect-all").click()
497
+ page.locator("#filter-min-size").fill("1")
498
+ page.locator("#filter-min-unit").select_option("KB")
499
+ page.locator("#close-filters-btn").click()
500
+ page.locator("#clear-filters-btn").click()
501
+ self.assertEqual(page.locator("#filter-min-size").input_value(), "")
502
+ page.locator("[data-action='preview']").first.click()
503
+ page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
504
+ page.locator("[data-action='toggle-preview-theme']").click()
505
+ page.locator("[data-action='close-preview']").click()
506
+ self.assertEqual(errors, [])
507
+ finally:
508
+ context.close()
509
+
510
+ def test_mobile_long_folder_uses_remaining_path_line(self):
511
+ context, page, errors = self.page({"width": 390, "height": 844})
512
+ try:
513
+ layout = page.evaluate("""async () => {
514
+ STATE.results = [{
515
+ Source: "CCRD",
516
+ SourceName: "CCRD",
517
+ File: "路径换行测试",
518
+ Folder: [
519
+ "马列之声ebook小组第七批成果集成(2018.8.17)",
520
+ "马列之声ebook小组第七批成果集成(2018.8.17)part 1a",
521
+ "2018.8 part 1",
522
+ ],
523
+ Size: 1,
524
+ doc_id: "browser-path-test",
525
+ snippet: "",
526
+ snippet_complete: true,
527
+ }];
528
+ STATE.total = 1;
529
+ renderResults({ animate: false });
530
+ await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
531
+ const path = document.querySelector('.result-path');
532
+ const folders = [...path.querySelectorAll('.path-folder')];
533
+ const first = document.createRange();
534
+ first.selectNodeContents(folders[1]);
535
+ const source = folders[0].getBoundingClientRect();
536
+ const firstRect = first.getClientRects()[0];
537
+ return {
538
+ sourceTop: source.top,
539
+ firstTop: firstRect.top,
540
+ firstLeft: firstRect.left,
541
+ separatorsNested: [...path.querySelectorAll('.path-sep')].every(sep => sep.parentElement.classList.contains('path-folder')),
542
+ };
543
+ }""")
544
+ self.assertAlmostEqual(layout["firstTop"], layout["sourceTop"], delta=1)
545
+ self.assertGreater(layout["firstLeft"], 0)
546
+ self.assertTrue(layout["separatorsNested"])
547
+ self.assertEqual(errors, [])
548
+ finally:
549
+ context.close()
550
+
551
+ def test_individual_batch_and_zip_downloads_are_browser_valid(self):
552
+ context, page, errors = self.page()
553
+ try:
554
+ with page.expect_download(timeout=30000) as download_info:
555
+ page.locator(".result-action-btn.primary").first.click()
556
+ individual = download_info.value
557
+ self.assertTrue(individual.suggested_filename.endswith(".txt"))
558
+ self.assertGreater(Path(individual.path()).stat().st_size, 0)
559
+
560
+ page.locator("#multi-select-toggle").check()
561
+ page.locator(".result-checkbox").first.check()
562
+ with page.expect_download(timeout=30000) as download_info:
563
+ page.locator("#multi-batch-download").click()
564
+ batch = download_info.value
565
+ self.assertTrue(batch.suggested_filename.endswith(".txt"))
566
+ self.assertGreater(Path(batch.path()).stat().st_size, 0)
567
+
568
+ checkboxes = page.locator(".result-checkbox")
569
+ checkboxes.nth(1).check()
570
+ with page.expect_download(timeout=120000) as download_info:
571
+ page.locator("#multi-zip-download").click()
572
+ archive_download = download_info.value
573
+ archive_bytes = Path(archive_download.path()).read_bytes()
574
+ self.assertTrue(archive_bytes.startswith(b"PK"))
575
+ with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
576
+ self.assertEqual(len(archive.namelist()), 2)
577
+ self.assertIsNone(archive.testzip())
578
+ self.assertNotIn(b"PK\x06\x06", archive_bytes)
579
+ self.assertEqual(errors, [])
580
+ finally:
581
+ context.close()
582
+
583
+ def test_navigation_history_source_filters_and_keyboard(self):
584
+ context, page, errors = self.page()
585
+ try:
586
+ page.locator("#hamburger-btn").click()
587
+ self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
588
+ page.locator("#hamburger-btn").click()
589
+ self.assertTrue(page.evaluate("STATE.leftSidebarOpen"))
590
+
591
+ page.locator("#settings-btn").click()
592
+ source_inputs = page.locator("#filter-source-list input")
593
+ self.assertEqual(source_inputs.count(), 2)
594
+ page.locator("#filter-source-list label").first.click()
595
+ self.assertEqual(len(page.evaluate("STATE.selectedSources")), 1)
596
+ page.locator("#close-filters-btn").click()
597
+ page.locator("#clear-filters-btn").click()
598
+ self.assertEqual(page.evaluate("STATE.selectedSources"), [])
599
+
600
+ page.locator("#search-input").fill("文化")
601
+ page.wait_for_timeout(400)
602
+ page.locator(".result-item").first.wait_for(timeout=30000)
603
+ page.locator("#search-input").fill("")
604
+ page.wait_for_timeout(400)
605
+ page.locator("#search-input").focus()
606
+ history_item = page.locator("#search-history-dropdown [data-query]").first
607
+ history_item.wait_for(state="visible", timeout=10000)
608
+ history_item.click()
609
+ self.assertEqual(page.locator("#search-input").input_value(), "文化")
610
+
611
+ page.locator("#search-input").press("Escape")
612
+ self.assertFalse(page.evaluate("STATE.leftSidebarOpen"))
613
+ page.keyboard.press("/")
614
+ self.assertTrue(page.locator("#search-input").evaluate("node => document.activeElement === node"))
615
+
616
+ page.goto(self.base_url + "/CCRD", wait_until="domcontentloaded")
617
+ page.locator(".result-item").first.wait_for(timeout=30000)
618
+ self.assertTrue(page.locator("#sidebar-expand-btn").is_visible())
619
+ page.locator("#sidebar-expand-btn").click()
620
+ self.assertTrue(page.locator("#left-sidebar").evaluate("node => node.classList.contains('expanded-wide')"))
621
+ page.locator("#sidebar-expand-btn").click()
622
+ self.assertFalse(page.locator("#left-sidebar").evaluate("node => node.classList.contains('expanded-wide')"))
623
+ back_home = page.locator("#back-home")
624
+ if back_home.count():
625
+ back_home.click()
626
+ self.assertEqual(page.url.rstrip("/"), self.base_url)
627
+ self.assertEqual(errors, [])
628
+ finally:
629
+ context.close()
630
+
631
+ def test_accessibility_keyboard_paths_sidebar_state_and_preview_focus(self):
632
+ context, page, errors = self.page()
633
+ try:
634
+ self.assertEqual(page.locator("#search-input").get_attribute("aria-label"), "搜索 TXT 文件和正文")
635
+ self.assertEqual(page.locator("#results-list").get_attribute("role"), "list")
636
+ path = page.locator(".path-folder[role='button']").first
637
+ self.assertEqual(path.get_attribute("tabindex"), "0")
638
+ self.assertTrue(path.get_attribute("aria-label"))
639
+
640
+ page.locator("#settings-btn").press("Enter")
641
+ self.assertEqual(page.locator("#settings-btn").get_attribute("aria-expanded"), "true")
642
+ self.assertFalse(page.locator("#right-sidebar").evaluate("node => node.inert"))
643
+ page.keyboard.press("Escape")
644
+ self.assertEqual(page.locator("#settings-btn").get_attribute("aria-expanded"), "false")
645
+ self.assertEqual(page.evaluate("document.activeElement.id"), "hamburger-btn")
646
+
647
+ page.locator("[data-action='preview']").first.click()
648
+ page.locator("#preview-panel").wait_for(state="visible", timeout=30000)
649
+ self.assertEqual(page.locator("#preview-panel").get_attribute("role"), "dialog")
650
+ self.assertEqual(page.locator("#preview-panel").get_attribute("aria-modal"), "true")
651
+ self.assertTrue(page.locator("#header").evaluate("node => node.inert"))
652
+ self.assertEqual(page.evaluate("document.activeElement.getAttribute('data-action')"), "close-preview")
653
+ page.keyboard.press("Escape")
654
+ page.locator("#preview-panel").wait_for(state="hidden")
655
+ self.assertFalse(page.locator("#header").evaluate("node => node.inert"))
656
+ self.assertEqual(page.evaluate("document.activeElement.getAttribute('data-action')"), "preview")
657
+ self.assertEqual(errors, [])
658
+ finally:
659
+ context.close()
660
+
661
+ def test_search_prefetch_advances_to_third_page_after_second_page_loads(self):
662
+ context, page, errors = self.page()
663
+ try:
664
+ pages = self.search_request_pages(page)
665
+ page.locator("#search-input").fill("文化")
666
+ page.wait_for_timeout(400)
667
+ page.locator(".result-item").first.wait_for(timeout=30000)
668
+
669
+ page.locator("#results-container").evaluate("el => { el.scrollTop = el.scrollHeight; }")
670
+ page.wait_for_timeout(1200)
671
+ page.wait_for_timeout(1200)
672
+ self.assertGreaterEqual(page.evaluate("STATE.results.length"), 200)
673
+ self.assertIn(1, pages)
674
+ self.assertIn(2, pages)
675
+ self.assertIn(3, pages)
676
+ self.assertEqual(pages.count(3), 1)
677
+ self.assertEqual(errors, [])
678
+ finally:
679
+ context.close()
680
+
681
+ def test_initial_search_prefetches_second_page_once(self):
682
+ context = self.browser.new_context(viewport={"width": 1440, "height": 900})
683
+ page = context.new_page()
684
+ errors = []
685
+ page.on("pageerror", lambda error: errors.append(str(error)))
686
+ pages = self.search_request_pages(page)
687
+ try:
688
+ page.goto(self.base_url, wait_until="domcontentloaded")
689
+ page.locator(".result-item").first.wait_for(timeout=30000)
690
+ page.wait_for_timeout(2500)
691
+ self.assertEqual(pages.count(2), 1)
692
+ self.assertEqual(errors, [])
693
+ finally:
694
+ context.close()
695
+
696
+ def test_initial_results_skip_entry_animation_but_search_results_animate(self):
697
+ context, page, errors = self.page()
698
+ try:
699
+ self.assertEqual(page.locator(".result-item.result-enter").count(), 0)
700
+ page.evaluate("""
701
+ window.__sawResultEnter = false;
702
+ new MutationObserver((mutations, observer) => {
703
+ if (!mutations.some((mutation) =>
704
+ mutation.target instanceof Element && mutation.target.classList.contains("result-enter")
705
+ )) return;
706
+ window.__sawResultEnter = true;
707
+ observer.disconnect();
708
+ }).observe(document.querySelector("#results-list"), {
709
+ subtree: true,
710
+ attributes: true,
711
+ attributeFilter: ["class"],
712
+ });
713
+ """)
714
+ page.locator("#search-input").fill("文化")
715
+ page.wait_for_function("window.__sawResultEnter", timeout=30000)
716
+ self.assertEqual(errors, [])
717
+ finally:
718
+ context.close()
719
+
720
+
721
+ if __name__ == "__main__":
722
+ unittest.main()
tests/test_integration.py ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import gzip
4
+ import io
5
+ import json
6
+ import sqlite3
7
+ import tempfile
8
+ import threading
9
+ import unittest
10
+ import zipfile
11
+ from pathlib import Path
12
+ from unittest.mock import patch
13
+ from urllib.parse import urlencode
14
+
15
+ from fastapi.testclient import TestClient
16
+
17
+ import app
18
+ import build_fulltext_db
19
+
20
+
21
+ def create_fts_database(path: Path, rows: list[tuple[int, str]]) -> None:
22
+ with sqlite3.connect(path) as connection:
23
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
24
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
25
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
26
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
27
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
28
+ connection.execute("CREATE TABLE snippet_anchors (token_hash BLOB NOT NULL, doc_number INTEGER NOT NULL, char_offset INTEGER NOT NULL, PRIMARY KEY (token_hash, doc_number)) WITHOUT ROWID")
29
+ for number, content in rows:
30
+ connection.execute("INSERT INTO documents VALUES (?, ?)", (f"CCRD:{number}", content[:700]))
31
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", (number, content))
32
+ connection.executemany(
33
+ "INSERT INTO snippet_anchors VALUES (?, ?, ?)",
34
+ ((token_hash, number, char_offset) for token_hash, char_offset in build_fulltext_db.snippet_anchors(content).items()),
35
+ )
36
+ for token in build_fulltext_db.index_tokens(content) | build_fulltext_db.literal_tokens(content):
37
+ token_hash = __import__("hashlib").sha256(token.encode("utf-8")).digest()[:16]
38
+ existing = connection.execute("SELECT docs FROM postings WHERE token_hash = ?", (token_hash,)).fetchone()
39
+ numbers = [number] if existing is None else sorted(build_doc_numbers(existing[0]) | {number})
40
+ payload = build_fulltext_db.encode_doc_ids(numbers)
41
+ connection.execute(
42
+ "INSERT OR REPLACE INTO postings VALUES (?, ?, ?)",
43
+ (token_hash, payload, len(numbers)),
44
+ )
45
+
46
+
47
+ def build_doc_numbers(payload: bytes) -> set[int]:
48
+ return app.decode_doc_numbers(payload)
49
+
50
+
51
+ class FulltextIntegrationTests(unittest.TestCase):
52
+ def test_match_cache_is_complete_isolated_mode_scoped_and_expiring(self):
53
+ with tempfile.TemporaryDirectory() as directory:
54
+ root = Path(directory)
55
+ create_fts_database(root / "CCRD.sqlite3", [(1, "文化 革命"), (2, "文化革命")])
56
+ databases = app.FulltextDatabases(root)
57
+ original = databases._search_source_connection
58
+ try:
59
+ with patch.object(databases, "_search_source_connection", wraps=original) as search, \
60
+ patch.object(app.time, "monotonic", return_value=100) as clock:
61
+ first = databases.search_source("CCRD", "文化 革命")
62
+ first.clear()
63
+ self.assertEqual(databases.search_source("CCRD", "文化 革命"), {1, 2})
64
+ self.assertEqual(search.call_count, 1)
65
+ self.assertEqual(databases.search_source("CCRD", "文化 革命", True), {1})
66
+ self.assertEqual(search.call_count, 2)
67
+ clock.return_value = 401
68
+ self.assertEqual(databases.search_source("CCRD", "文化 革命"), {1, 2})
69
+ self.assertEqual(search.call_count, 3)
70
+ self.assertEqual(databases.match_inflight, {})
71
+ finally:
72
+ databases.close()
73
+
74
+ def test_match_cache_obeys_entry_and_total_id_budgets_without_truncation(self):
75
+ with tempfile.TemporaryDirectory() as directory:
76
+ root = Path(directory)
77
+ create_fts_database(root / "CCRD.sqlite3", [(1, "正文")])
78
+ databases = app.FulltextDatabases(root)
79
+ try:
80
+ with patch.object(app, "FULLTEXT_MATCH_CACHE_MAX_ENTRIES", 2), \
81
+ patch.object(app, "FULLTEXT_MATCH_CACHE_MAX_TOTAL_IDS", 3), \
82
+ patch.object(databases, "_search_source_connection", return_value={1, 2}):
83
+ for query in ("甲", "乙", "丙"):
84
+ self.assertEqual(databases.search_source("CCRD", query), {1, 2})
85
+ self.assertEqual(len(databases.match_cache), 1)
86
+ self.assertLessEqual(sum(len(value[1]) for value in databases.match_cache.values()), 3)
87
+ databases.match_cache.clear()
88
+ with patch.object(databases, "_search_source_connection", return_value={1}) as search:
89
+ for query in ("甲", "乙", "丙"):
90
+ self.assertEqual(databases.search_source("CCRD", query), {1})
91
+ self.assertEqual(len(databases.match_cache), 2)
92
+ databases.match_cache.clear()
93
+ with patch.object(databases, "_search_source_connection", return_value={1, 2, 3, 4}) as search:
94
+ for _ in range(2):
95
+ self.assertEqual(databases.search_source("CCRD", "oversized"), {1, 2, 3, 4})
96
+ self.assertEqual(search.call_count, 2)
97
+ self.assertEqual(databases.match_cache, {})
98
+ self.assertEqual(databases.match_inflight, {})
99
+ finally:
100
+ databases.close()
101
+
102
+ def test_concurrent_source_queries_share_complete_work_and_failures_release(self):
103
+ import concurrent.futures
104
+ with tempfile.TemporaryDirectory() as directory:
105
+ root = Path(directory)
106
+ create_fts_database(root / "CCRD.sqlite3", [(1, "正文")])
107
+ databases = app.FulltextDatabases(root)
108
+ entered = threading.Event()
109
+ joined = threading.Event()
110
+ release = threading.Event()
111
+ calls = []
112
+ def execute(*_args):
113
+ calls.append(1)
114
+ entered.set()
115
+ self.assertTrue(release.wait(5))
116
+ return {1, 2, 3}
117
+ try:
118
+ with patch.object(databases, "_search_source_connection", side_effect=execute):
119
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
120
+ first = executor.submit(databases.search_source, "CCRD", "query")
121
+ try:
122
+ self.assertTrue(entered.wait(5))
123
+ pending = next(iter(databases.match_inflight.values()))
124
+ original_wait = pending.wait
125
+ def wait(timeout=None):
126
+ joined.set()
127
+ return original_wait(timeout)
128
+ with patch.object(pending, "wait", side_effect=wait):
129
+ second = executor.submit(databases.search_source, "CCRD", "query")
130
+ self.assertTrue(joined.wait(5))
131
+ self.assertEqual(len(calls), 1)
132
+ finally:
133
+ release.set()
134
+ self.assertEqual(first.result(timeout=5), {1, 2, 3})
135
+ self.assertEqual(second.result(timeout=5), {1, 2, 3})
136
+ self.assertEqual(len(calls), 1)
137
+ self.assertEqual(databases.match_inflight, {})
138
+ with patch.object(databases, "_search_source_connection", side_effect=[RuntimeError("failed"), {9}]) as search:
139
+ with self.assertRaises(RuntimeError):
140
+ databases.search_source("CCRD", "failure")
141
+ self.assertEqual(databases.match_inflight, {})
142
+ self.assertEqual(databases.search_source("CCRD", "failure"), {9})
143
+ self.assertEqual(search.call_count, 2)
144
+ finally:
145
+ databases.close()
146
+ def test_fulltext_databases_can_load_in_memory(self):
147
+ with tempfile.TemporaryDirectory() as directory:
148
+ path = Path(directory) / "CCRD.sqlite3"
149
+ create_fts_database(path, [(1, "开头 文化革命 正文")])
150
+ databases = app.FulltextDatabases(Path(directory), in_memory=True)
151
+ try:
152
+ self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), {1})
153
+ self.assertEqual(databases.search_source("CCRD", "文化 革命", exact=False), {1})
154
+ finally:
155
+ databases.close()
156
+
157
+ def test_short_latin_exact_and_latin_wildcards_do_not_miss_substrings(self):
158
+ with tempfile.TemporaryDirectory() as directory:
159
+ path = Path(directory) / "CCRD.sqlite3"
160
+ create_fts_database(path, [(1, "smartphone abc123xyz"), (2, "alpha beta"), (3, "unrelated")])
161
+ databases = app.FulltextDatabases(Path(directory))
162
+ try:
163
+ self.assertEqual(databases.search_source("CCRD", "abc", exact=True), {1})
164
+ self.assertEqual(databases.search_source("CCRD", "phone", exact=True), {1})
165
+ self.assertEqual(databases.search_source("CCRD", "abc*xyz", exact=True), {1})
166
+ self.assertEqual(databases.search_source("CCRD", "a?c*xyz", exact=True), {1})
167
+ finally:
168
+ databases.close()
169
+
170
+ def test_builder_creates_complete_reusable_database(self):
171
+ with tempfile.TemporaryDirectory() as directory:
172
+ root = Path(directory)
173
+ corpus = root / "corpus"
174
+ corpus.mkdir()
175
+ (corpus / "one.txt").write_text("开头 文化革命 正文", encoding="utf-8")
176
+ (corpus / "two.txt").write_text("全角ABC和手机", encoding="utf-8")
177
+ records = [
178
+ {"doc_id": "TEST:1", "storage_root": "corpus", "storage_rel_path": "one.txt"},
179
+ {"doc_id": "TEST:2", "storage_root": "corpus", "storage_rel_path": "two.txt"},
180
+ ]
181
+ with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
182
+ build_fulltext_db.FULLTEXT_DIR.mkdir()
183
+ build_fulltext_db.build_source_db("TEST", records)
184
+ database = build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3"
185
+ first_bytes = database.read_bytes()
186
+ self.assertTrue(build_fulltext_db.database_ready(database, 2))
187
+ with sqlite3.connect(database) as connection:
188
+ self.assertEqual(connection.execute("SELECT COUNT(*) FROM documents").fetchone()[0], 2)
189
+ self.assertEqual(connection.execute("SELECT COUNT(*) FROM content_fts").fetchone()[0], 2)
190
+ self.assertGreater(connection.execute("SELECT COUNT(*) FROM snippet_anchors").fetchone()[0], 0)
191
+ self.assertEqual(connection.execute("SELECT value FROM metadata WHERE key='tokenizer_version'").fetchone()[0], app.TOKENIZER_VERSION)
192
+ self.assertGreater(connection.execute("SELECT COUNT(*) FROM postings").fetchone()[0], 0)
193
+ build_fulltext_db.build_source_db("TEST", records)
194
+ self.assertEqual(database.read_bytes(), first_bytes)
195
+
196
+ def test_builder_rejects_invalid_document_ids_without_publishing_database(self):
197
+ with tempfile.TemporaryDirectory() as directory:
198
+ root = Path(directory)
199
+ corpus = root / "corpus"
200
+ corpus.mkdir()
201
+ (corpus / "one.txt").write_text("正文", encoding="utf-8")
202
+ record = {"doc_id": "invalid", "storage_root": "corpus", "storage_rel_path": "one.txt"}
203
+ with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
204
+ build_fulltext_db.FULLTEXT_DIR.mkdir()
205
+ with self.assertRaises(ValueError):
206
+ build_fulltext_db.build_source_db("TEST", [record])
207
+ self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3").exists())
208
+
209
+ def test_builder_does_not_publish_database_with_missing_files(self):
210
+ with tempfile.TemporaryDirectory() as directory:
211
+ root = Path(directory)
212
+ record = {"doc_id": "TEST:1", "storage_root": "corpus", "storage_rel_path": "missing.txt"}
213
+ with patch.object(build_fulltext_db, "BASE_DIR", root), patch.object(build_fulltext_db, "FULLTEXT_DIR", root / "fulltext"):
214
+ build_fulltext_db.FULLTEXT_DIR.mkdir()
215
+ with self.assertRaises(RuntimeError):
216
+ build_fulltext_db.build_source_db("TEST", [record])
217
+ self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3").exists())
218
+ self.assertFalse((build_fulltext_db.FULLTEXT_DIR / "TEST.sqlite3.tmp").exists())
219
+
220
+ def test_atomic_database_replacement_reopens_and_increments_generation(self):
221
+ with tempfile.TemporaryDirectory() as directory:
222
+ root = Path(directory)
223
+ path = root / "CCRD.sqlite3"
224
+ create_fts_database(path, [(1, "旧数据库文化")])
225
+ databases = app.FulltextDatabases(root)
226
+ first_generation = databases.generation
227
+ replacement = root / "replacement.sqlite3"
228
+ create_fts_database(replacement, [(2, "新数据库革命资料")])
229
+ replacement.replace(path)
230
+ databases.refresh()
231
+ try:
232
+ self.assertGreater(databases.generation, first_generation)
233
+ self.assertEqual(databases.search_source("CCRD", "革命"), {2})
234
+ self.assertEqual(databases.search_source("CCRD", "文化"), set())
235
+ finally:
236
+ databases.close()
237
+
238
+ def test_refresh_builds_replacement_without_blocking_existing_queries(self):
239
+ with tempfile.TemporaryDirectory() as directory:
240
+ root = Path(directory)
241
+ path = root / "CCRD.sqlite3"
242
+ create_fts_database(path, [(1, "旧数据库文化")])
243
+ databases = app.FulltextDatabases(root, in_memory=True)
244
+ replacement = root / "replacement.sqlite3"
245
+ create_fts_database(replacement, [(2, "新数据库革命资料")])
246
+ replacement.replace(path)
247
+ opening = threading.Event()
248
+ release = threading.Event()
249
+ original_open = databases._open_candidate
250
+
251
+ def blocked_open(candidate_path):
252
+ opening.set()
253
+ self.assertTrue(release.wait(2))
254
+ return original_open(candidate_path)
255
+
256
+ thread = threading.Thread(target=databases.refresh)
257
+ try:
258
+ with patch.object(databases, "_open_candidate", side_effect=blocked_open):
259
+ thread.start()
260
+ self.assertTrue(opening.wait(2))
261
+ self.assertEqual(databases.search_source("CCRD", "文化"), {1})
262
+ release.set()
263
+ thread.join(2)
264
+ self.assertFalse(thread.is_alive())
265
+ self.assertEqual(databases.search_source("CCRD", "革命"), {2})
266
+ finally:
267
+ release.set()
268
+ thread.join(2)
269
+ databases.close()
270
+
271
+ def test_active_query_keeps_retired_connection_alive_until_release(self):
272
+ with tempfile.TemporaryDirectory() as directory:
273
+ root = Path(directory)
274
+ path = root / "CCRD.sqlite3"
275
+ create_fts_database(path, [(1, "旧数据库文化")])
276
+ databases = app.FulltextDatabases(root, in_memory=True)
277
+ entered = threading.Event()
278
+ release = threading.Event()
279
+ result = []
280
+ original_search = databases._search_source_connection
281
+
282
+ def blocked_search(connection, query, exact, state):
283
+ entered.set()
284
+ self.assertTrue(release.wait(2))
285
+ return original_search(connection, query, exact, state)
286
+
287
+ thread = threading.Thread(target=lambda: result.append(databases.search_source("CCRD", "文化")))
288
+ try:
289
+ with patch.object(databases, "_search_source_connection", side_effect=blocked_search):
290
+ thread.start()
291
+ self.assertTrue(entered.wait(2))
292
+ replacement = root / "replacement.sqlite3"
293
+ create_fts_database(replacement, [(2, "新数据库革命资料")])
294
+ replacement.replace(path)
295
+ databases.refresh()
296
+ self.assertTrue(databases.retired_connections)
297
+ release.set()
298
+ thread.join(2)
299
+ self.assertFalse(thread.is_alive())
300
+ self.assertEqual(result, [{1}])
301
+ self.assertFalse(databases.retired_connections)
302
+ self.assertEqual(databases.match_cache, {})
303
+ self.assertEqual(databases.search_source("CCRD", "革命"), {2})
304
+ finally:
305
+ release.set()
306
+ thread.join(2)
307
+ databases.close()
308
+
309
+
310
+ class HttpIntegrationTests(unittest.TestCase):
311
+ def test_fulltext_in_memory_defaults_enabled(self):
312
+ self.assertTrue(app.FULLTEXT_IN_MEMORY)
313
+
314
+ @classmethod
315
+ def setUpClass(cls):
316
+ app.load_data()
317
+ cls.original_databases = app.fulltext_databases
318
+ app.fulltext_databases = app.FulltextDatabases(Path("/tmp/ccrd-test-no-index"))
319
+ cls.client = TestClient(app.app)
320
+
321
+ @classmethod
322
+ def tearDownClass(cls):
323
+ app.fulltext_databases.close()
324
+ app.fulltext_databases = cls.original_databases
325
+
326
+ def test_search_http_validation_and_option_isolation(self):
327
+ valid = self.client.post("/api/search", json={"q": "文化 革命", "fulltext": False, "page_size": 1})
328
+ self.assertEqual(valid.status_code, 200)
329
+ self.assertEqual(valid.json()["page_size"], 1)
330
+ self.assertEqual(self.client.post("/api/search", json={"q": "x" * 257}).status_code, 422)
331
+ self.assertEqual(self.client.post("/api/search", json={"page": 0}).status_code, 422)
332
+ self.assertEqual(self.client.post("/api/search", json={"page_size": 201}).status_code, 422)
333
+ self.assertEqual(self.client.post("/api/search", json={"min_size": -1}).status_code, 422)
334
+ self.assertEqual(self.client.post("/api/search/missing", json={}).status_code, 404)
335
+
336
+ def test_preview_snippet_and_download_http_contracts(self):
337
+ doc_id, record = next(iter(app.record_map.items()))
338
+ preview = self.client.get(f"/api/preview/{doc_id}")
339
+ self.assertEqual(preview.status_code, 200)
340
+ self.assertEqual(preview.json()["doc_id"], doc_id)
341
+ snippet = self.client.get(f"/api/snippet/{doc_id}", params={"q": "文化", "exact": "false"})
342
+ self.assertEqual(snippet.status_code, 200)
343
+ self.assertIn("snippet", snippet.json())
344
+ download = self.client.get(f"/api/download/{doc_id}")
345
+ self.assertEqual(download.status_code, 200)
346
+ self.assertTrue(download.headers["content-type"].startswith("text/plain"))
347
+ self.assertIn("attachment", download.headers["content-disposition"])
348
+ self.assertEqual(download.content, app.get_doc_storage_path(record).read_bytes())
349
+ self.assertEqual(self.client.get("/api/preview/missing:1").status_code, 404)
350
+ self.assertEqual(self.client.get("/api/snippet/missing:1").status_code, 404)
351
+ self.assertEqual(self.client.get("/api/download/missing:1").status_code, 404)
352
+ self.assertEqual(self.client.get(f"/api/snippet/{doc_id}", params={"q": "x" * 257}).status_code, 422)
353
+
354
+ def test_json_and_form_zip_are_valid_and_enforce_limits(self):
355
+ selected = sorted(app.records, key=lambda record: record["size"])[:2]
356
+ ids = [record["doc_id"] for record in selected]
357
+ for response in (
358
+ self.client.post("/api/zip", json={"doc_ids": ids}),
359
+ self.client.post(
360
+ "/api/zip-download",
361
+ content=urlencode([("doc_id", ids[0]), ("doc_id", ids[1]), ("doc_id", ids[0])]),
362
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
363
+ ),
364
+ ):
365
+ self.assertEqual(response.status_code, 200)
366
+ self.assertEqual(response.headers["content-type"], "application/zip")
367
+ self.assertIn("attachment", response.headers["content-disposition"])
368
+ self.assertEqual(response.headers["cache-control"], "no-store")
369
+ self.assertEqual(response.headers["content-encoding"], "identity")
370
+ self.assertEqual(int(response.headers["content-length"]), len(response.content))
371
+ self.assertTrue(response.content.startswith(b"PK"))
372
+ self.assertNotIn(b"PK\x06\x06", response.content)
373
+ with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
374
+ self.assertEqual(len(archive.namelist()), 2)
375
+ self.assertEqual(archive.testzip(), None)
376
+ with self.assertRaises(zipfile.BadZipFile):
377
+ with zipfile.ZipFile(io.BytesIO(response.content[:-22])) as truncated:
378
+ truncated.testzip()
379
+ self.assertEqual(self.client.post("/api/zip", json={"doc_ids": []}).status_code, 400)
380
+ self.assertEqual(self.client.post("/api/zip", json={"doc_ids": ["x"] * 501}).status_code, 422)
381
+ too_many = [("doc_id", f"CCRD:{index}") for index in range(501)]
382
+ self.assertEqual(self.client.post(
383
+ "/api/zip-download",
384
+ content=urlencode(too_many),
385
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
386
+ ).status_code, 400)
387
+ self.assertEqual(self.client.post("/api/zip-download", json={"doc_id": ids}).status_code, 415)
388
+ self.assertEqual(self.client.get("/api/zip-download").status_code, 400)
389
+ legacy = self.client.post(
390
+ "/api/zip-download",
391
+ content=urlencode([("doc_id", ids[0]), ("doc_id", ids[1])]),
392
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
393
+ follow_redirects=False,
394
+ )
395
+ self.assertEqual(legacy.status_code, 303)
396
+ self.assertTrue(legacy.headers["location"].startswith("/api/zip-download/"))
397
+ legacy_download = self.client.get(legacy.headers["location"])
398
+ self.assertTrue(legacy_download.content.startswith(b"PK"))
399
+ prepared = self.client.post("/api/zip-prepare", json={"doc_ids": ids})
400
+ self.assertEqual(prepared.status_code, 200)
401
+ self.assertEqual(prepared.json()["file_count"], 2)
402
+ download_url = prepared.json()["download_url"]
403
+ for _attempt in range(2):
404
+ downloaded = self.client.get(download_url, headers={"Accept-Encoding": "gzip"})
405
+ self.assertEqual(downloaded.status_code, 200)
406
+ self.assertEqual(downloaded.headers["content-encoding"], "identity")
407
+ self.assertTrue(downloaded.content.startswith(b"PK"))
408
+ with zipfile.ZipFile(io.BytesIO(downloaded.content)) as archive:
409
+ self.assertEqual(len(archive.namelist()), 2)
410
+ self.assertIsNone(archive.testzip())
411
+ self.assertEqual(self.client.get("/api/zip-download/invalid-token").status_code, 404)
412
+
413
+ def test_zip_disambiguates_duplicate_display_paths_and_deduplicates_ids(self):
414
+ with tempfile.TemporaryDirectory() as directory:
415
+ root = Path(directory)
416
+ first = root / "first.txt"
417
+ second = root / "second.txt"
418
+ first.write_text("first", encoding="utf-8")
419
+ second.write_text("second", encoding="utf-8")
420
+ original_map = app.record_map
421
+ app.record_map = {
422
+ "CCRD:1": {"source": "CCRD", "display_name": 'same:name*?"<>|', "display_rel_path": "same/name.txt"},
423
+ "CCRD:2": {"source": "CCRD", "display_name": 'same:name*?"<>|', "display_rel_path": "same/name.txt"},
424
+ }
425
+ paths = {"CCRD:1": first, "CCRD:2": second}
426
+ try:
427
+ with patch.object(app, "get_doc_storage_path", side_effect=lambda record: paths[next(key for key, value in app.record_map.items() if value is record)]):
428
+ response = app.zip_response(["CCRD:1", "CCRD:1", "CCRD:2"])
429
+ self.assertEqual(response.status_code, 200)
430
+ with open(response.path, "rb") as archive_file:
431
+ payload = archive_file.read()
432
+ self.assertNotIn(b"PK\x06\x06", payload)
433
+ with zipfile.ZipFile(io.BytesIO(payload)) as archive:
434
+ self.assertEqual(len(set(archive.namelist())), 2)
435
+ self.assertTrue(all(not any(character in name for character in ':*?"<>|\\') for name in archive.namelist()))
436
+ self.assertTrue(all(len(name.encode("utf-8")) <= 220 for name in archive.namelist()))
437
+ self.assertEqual(sorted(archive.read(name).decode() for name in archive.namelist()), ["first", "second"])
438
+ finally:
439
+ app.record_map = original_map
440
+
441
+ def test_static_status_and_cors_contracts(self):
442
+ self.assertEqual(self.client.get("/manifest.json").status_code, 200)
443
+ self.assertIn("javascript", self.client.get("/sw.js").headers["content-type"])
444
+ status = self.client.get("/api/index-status")
445
+ self.assertEqual(status.status_code, 200)
446
+ self.assertEqual(status.json()["loaded_sources"], [])
447
+ cors = self.client.options(
448
+ "/api/search",
449
+ headers={"Origin": "https://example.com", "Access-Control-Request-Method": "POST"},
450
+ )
451
+ self.assertEqual(cors.status_code, 200)
452
+ self.assertEqual(cors.headers["access-control-allow-origin"], "*")
453
+ self.assertEqual(self.client.get("/data/fulltext/CCRD.sqlite3").status_code, 404)
454
+ self.assertEqual(self.client.get("/data/fulltext/CCRD.sqlite3.tmp").status_code, 404)
455
+
456
+ def test_ping_is_empty_and_not_cached(self):
457
+ response = self.client.get("/api/ping")
458
+ self.assertEqual(response.status_code, 204)
459
+ self.assertEqual(response.content, b"")
460
+ self.assertEqual(response.headers["cache-control"], "no-store")
461
+
462
+ def test_sources_extensions_folders_random_and_spa_routes(self):
463
+ sources = self.client.get("/api/sources")
464
+ self.assertEqual(sources.status_code, 200)
465
+ self.assertEqual({item["slug"] for item in sources.json()}, {"CCRD", "CW"})
466
+ self.assertEqual(self.client.get("/api/extensions").json(), [{"name": "txt", "count": 47250}])
467
+ self.assertEqual(self.client.get("/api/extensions", params={"source": "CCRD"}).json(), [{"name": "txt", "count": 35586}])
468
+ folders = self.client.get("/api/folders/CCRD")
469
+ self.assertEqual(folders.status_code, 200)
470
+ self.assertIsInstance(folders.json(), list)
471
+ contents = self.client.get("/api/folders/CCRD/contents")
472
+ self.assertEqual(contents.status_code, 200)
473
+ self.assertEqual(contents.json()["current_path"], "")
474
+ random_doc = self.client.get("/api/random", params={"source": "CW"})
475
+ self.assertEqual(random_doc.status_code, 200)
476
+ self.assertEqual(random_doc.json()["Source"], "CW")
477
+ self.assertEqual(self.client.get("/api/random", params={"source": "missing"}).status_code, 404)
478
+ for route in ("/", "/CCRD", "/CW"):
479
+ page = self.client.get(route)
480
+ self.assertEqual(page.status_code, 200)
481
+ self.assertIn("text/html", page.headers["content-type"])
482
+ self.assertIn("initial-sidebar-data", page.text)
483
+
484
+ def test_search_cache_ttl_is_longer_than_burst_idle_gap(self):
485
+ self.assertGreaterEqual(app.SEARCH_CACHE_TTL_SECONDS, 300)
486
+
487
+ def test_search_cache_returns_identical_payload(self):
488
+ app.search_response_cache.clear()
489
+ body = {"q": "server-timing-unique-query", "fulltext": False, "page_size": 1}
490
+ first = self.client.post("/api/search", json=body)
491
+ second = self.client.post("/api/search", json=body)
492
+ self.assertEqual(first.status_code, 200)
493
+ self.assertEqual(first.json(), second.json())
494
+
495
+ def test_missing_physical_files_return_stable_errors(self):
496
+ original_map = app.record_map
497
+ app.record_map = {
498
+ "CCRD:missing": {
499
+ "display_name": "missing", "display_rel_path": "missing.txt", "source_name": "CCRD",
500
+ "storage_root": "CCRD", "storage_rel_path": "does-not-exist.txt",
501
+ }
502
+ }
503
+ try:
504
+ for endpoint in ("preview", "snippet", "download"):
505
+ response = self.client.get(f"/api/{endpoint}/CCRD:missing")
506
+ self.assertEqual(response.status_code, 404)
507
+ self.assertIn("missing file", response.json()["error"])
508
+ finally:
509
+ app.record_map = original_map
510
+
511
+
512
+ class CorpusIntegrityTests(unittest.TestCase):
513
+ def test_metadata_counts_and_every_storage_file_exists(self):
514
+ payload = json.loads(gzip.decompress((Path(app.BASE_DIR) / "data/search_data.json.gz").read_bytes()))
515
+ records = payload["records"]
516
+ self.assertEqual(len(records), 47250)
517
+ counts = {source["slug"]: source["count"] for source in payload["sources"]}
518
+ self.assertEqual(counts, {"CCRD": 35586, "CW": 11664})
519
+ doc_ids = {record["doc_id"] for record in records}
520
+ self.assertEqual(len(doc_ids), len(records))
521
+ missing = [
522
+ record["doc_id"] for record in records
523
+ if not (Path(app.BASE_DIR) / record["storage_root"] / record["storage_rel_path"]).is_file()
524
+ ]
525
+ self.assertEqual(missing, [])
526
+ self.assertEqual(sum(record["source"] == "CCRD" for record in records), 35586)
527
+ self.assertEqual(sum(record["source"] == "CW" for record in records), 11664)
528
+ archive_keys = [(record["source"], record["display_rel_path"], record["doc_id"]) for record in records]
529
+ self.assertEqual(len(archive_keys), len(set(archive_keys)))
tests/test_live_smoke.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import io
5
+ import os
6
+ import time
7
+ import unittest
8
+ import zipfile
9
+ from http.client import RemoteDisconnected
10
+ from urllib.error import HTTPError, URLError
11
+ from urllib.request import Request, urlopen
12
+
13
+
14
+ BASE_URL = os.environ.get("CCRD_LIVE_BASE_URL", "").rstrip("/")
15
+
16
+
17
+ @unittest.skipUnless(BASE_URL, "set CCRD_LIVE_BASE_URL to run deployment smoke tests")
18
+ class LiveSmokeTests(unittest.TestCase):
19
+ def request_json(self, path: str, payload: dict | None = None):
20
+ data = json.dumps(payload).encode() if payload is not None else None
21
+ for attempt in range(3):
22
+ request = Request(
23
+ BASE_URL + path,
24
+ data=data,
25
+ headers={"Content-Type": "application/json"} if data else {},
26
+ )
27
+ try:
28
+ with urlopen(request, timeout=45) as response:
29
+ self.assertEqual(response.status, 200)
30
+ return json.load(response)
31
+ except HTTPError as exc:
32
+ if exc.code < 500 or attempt == 2:
33
+ raise
34
+ except (RemoteDisconnected, TimeoutError, URLError):
35
+ if attempt == 2:
36
+ raise
37
+ time.sleep(attempt + 1)
38
+ self.fail("request retries exhausted")
39
+
40
+ def request_text(self, path: str):
41
+ with urlopen(BASE_URL + path, timeout=45) as response:
42
+ self.assertEqual(response.status, 200)
43
+ return response.read().decode("utf-8")
44
+
45
+ def test_sources_and_extensions_match_corpus(self):
46
+ sources = self.request_json("/api/sources")
47
+ self.assertEqual({item["slug"] for item in sources}, {"CCRD", "CW"})
48
+ self.assertEqual(sum(item["count"] for item in sources), 47250)
49
+ extensions = self.request_json("/api/extensions")
50
+ self.assertEqual(extensions, [{"name": "txt", "count": 47250}])
51
+
52
+ def test_exact_fulltext_results_contain_contiguous_query(self):
53
+ status = self.request_json("/api/index-status")
54
+ if not status.get("ready"):
55
+ self.skipTest(f'fulltext indexes still building: {status.get("building_sources", [])}')
56
+ payload = self.request_json("/api/search", {
57
+ "q": "手机",
58
+ "page": 1,
59
+ "page_size": 5,
60
+ "exact": True,
61
+ "fulltext": True,
62
+ })
63
+ self.assertGreater(payload["total"], 0)
64
+ self.assertLessEqual(len(payload["results"]), 5)
65
+ for item in payload["results"]:
66
+ preview = self.request_json(f'/api/preview/{item["doc_id"]}')
67
+ searchable = " ".join([item.get("DisplayPath", ""), preview.get("text", "")])
68
+ self.assertIn("手机", searchable)
69
+
70
+ def test_source_filter_and_pagination_are_stable(self):
71
+ request = {
72
+ "q": "文化 革命",
73
+ "page": 1,
74
+ "page_size": 5,
75
+ "exact": False,
76
+ "fulltext": True,
77
+ }
78
+ first = self.request_json("/api/search/CCRD", request)
79
+ request["page"] = 2
80
+ second = self.request_json("/api/search/CCRD", request)
81
+ first_ids = {item["doc_id"] for item in first["results"]}
82
+ second_ids = {item["doc_id"] for item in second["results"]}
83
+ self.assertTrue(first_ids)
84
+ self.assertTrue(second_ids)
85
+ self.assertTrue(first_ids.isdisjoint(second_ids))
86
+ self.assertTrue(all(item["Source"] == "CCRD" for item in first["results"] + second["results"]))
87
+
88
+ def test_deployed_ui_contains_controls_and_matching_javascript(self):
89
+ html = self.request_text("/")
90
+ javascript = self.request_text("/static/app.js")
91
+ css = self.request_text("/static/style.css")
92
+ for control_id in (
93
+ "hamburger-btn", "search-input", "settings-btn", "theme-btn", "mobile-toggle-btn",
94
+ "multi-select-toggle", "multi-select-all", "multi-deselect", "multi-batch-download",
95
+ "multi-zip-download", "fulltext-toggle", "search-paths-toggle", "history-toggle",
96
+ "exact-search-toggle", "folder-select-all", "folder-deselect-all", "source-filter-cancel",
97
+ "folder-filter-cancel", "random-book-btn",
98
+ ):
99
+ self.assertIn(f'id="{control_id}"', html)
100
+ for binding in (
101
+ "DOM.hamburgerBtn.addEventListener", "DOM.searchInput.addEventListener",
102
+ "DOM.multiBatchDownload.addEventListener", "DOM.multiZipDownload.addEventListener",
103
+ "DOM.resultsList.addEventListener", "DOM.previewPanel.addEventListener",
104
+ ):
105
+ self.assertIn(binding, javascript)
106
+ self.assertIn(".result-snippet", css)
107
+
108
+ def test_deployed_zip_is_not_http_gzipped_and_opens(self):
109
+ prepared = self.request_json("/api/zip-prepare", {"doc_ids": ["CCRD:0", "CW:0"]})
110
+ request = Request(
111
+ BASE_URL + prepared["download_url"],
112
+ headers={
113
+ "Accept-Encoding": "gzip",
114
+ },
115
+ )
116
+ with urlopen(request, timeout=120) as response:
117
+ payload = response.read()
118
+ self.assertEqual(response.status, 200)
119
+ self.assertEqual(response.headers.get("Content-Encoding"), "identity")
120
+ self.assertEqual(int(response.headers["Content-Length"]), len(payload))
121
+ self.assertTrue(payload.startswith(b"PK"))
122
+ self.assertNotIn(b"PK\x06\x06", payload)
123
+ with zipfile.ZipFile(io.BytesIO(payload)) as archive:
124
+ self.assertEqual(len(archive.namelist()), 2)
125
+ self.assertIsNone(archive.testzip())
126
+
127
+
128
+ if __name__ == "__main__":
129
+ unittest.main()
tests/test_ocr_release.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tiny real SQLite/text candidates; never open/rebuild production databases."""
2
+ import copy
3
+ import gzip
4
+ import hashlib
5
+ import json
6
+ from pathlib import Path
7
+ import tempfile
8
+ import unittest
9
+ from unittest.mock import patch
10
+
11
+ import app
12
+ import build_fulltext_db as builder
13
+ import ocr_release as release
14
+ from fastapi.testclient import TestClient
15
+
16
+
17
+ class OcrReleaseTests(unittest.TestCase):
18
+ def setUp(self):
19
+ temporary = tempfile.TemporaryDirectory()
20
+ self.addCleanup(temporary.cleanup)
21
+ self.root = Path(temporary.name)
22
+ self.bucket = self.root / "bucket"
23
+ self.bucket.mkdir()
24
+ self.base = self.root / "base.json.gz"
25
+ base_record = {"doc_id": "CCRD:1", "source": "CCRD", "source_name": "CCRD",
26
+ "storage_root": "CCRD", "storage_rel_path": "old.txt", "display_name": "旧文件",
27
+ "display_dirs": [], "display_rel_path": "旧文件.txt", "size": 7, "extension": "txt"}
28
+ self.base.write_bytes(gzip.compress(json.dumps({"records": [base_record], "sources": [
29
+ {"slug": "CCRD", "name": "CCRD", "count": 1}]}).encode(), mtime=0))
30
+ self.base_sha = hashlib.sha256(self.base.read_bytes()).hexdigest()
31
+ self.content = "手机文化\n\n\n\n末页手。机 ABC"
32
+ texts = ["手机文化", "", "末页手。机 ABC"]
33
+ body = self.content.encode()
34
+ txt_sha = release.digest(body)
35
+ text_ref = {"bucket": release.BUCKET, "path": f"text/objects/{txt_sha[:2]}/{txt_sha}.txt",
36
+ "sha256": txt_sha, "bytes": len(body), "role": "runtime"}
37
+ self.write(text_ref["path"], body)
38
+ original = "原目录/book.pdf"
39
+ identity = {"recipe": "ocr-plain-utf8-pages-v1", "origin_repo": "VoiceOfML/Test", "origin_rel_path": original,
40
+ "origin_revision": "b" * 40, "source_sha256": "a" * 64, "doc_id": "OCR:1",
41
+ "text_generation": "c" * 64, "txt_sha256": txt_sha}
42
+ generation = release.digest(release.canonical(identity).rstrip(b"\n"))
43
+ book_id = release.digest(("VoiceOfML/Test\0" + original).encode())
44
+ pages, cursor = [], 0
45
+ for number, text in enumerate(texts, 1):
46
+ pages.append({"page": number, "start": cursor, "end": cursor + len(text),
47
+ "text_sha256": release.digest(text.encode()), "text_generation": str(number) * 64,
48
+ "quality": "partially-reviewed" if number == 1 else "unreviewed"})
49
+ cursor += len(text) + 2
50
+ self.export = {"version": 1, "kind": "ccrd-ocr-text-export", "complete": True, **identity,
51
+ "generation": generation, "book_id": book_id, "title": "book", "page_count": 3,
52
+ "readable_rel_path": "VoiceOfML/Test/原目录/book.txt", "accepted_pages": 1,
53
+ "quality": "partially-reviewed", "offset_unit": "unicode-codepoint", "page_separator": "\n\n",
54
+ "pages": pages, "text": text_ref,
55
+ "retained_reader_resources": [{"bucket": "vomebook/pdf-pages-v2", "path": "objects/aa/reading-manifest.json",
56
+ "sha256": "a" * 64, "bytes": 1, "role": "provenance"}]}
57
+ self.export_path = f"text/exports/{book_id}/{generation}/manifest.json"
58
+ self.export_ref = self.ref(self.export_path, release.canonical(self.export))
59
+ core = {"version": 1, "kind": "ccrd-ocr-search-release", "base_metadata_sha256": self.base_sha,
60
+ "builder_sha256": release.digest(Path(builder.__file__).read_bytes()), "tokenizer_version": release.TOKENIZER,
61
+ "exports": [self.export_ref]}
62
+ generation = release.digest(release.canonical(core))
63
+ records = [release.export_record(self.bucket, self.export_ref)[0]]
64
+ db_relative = f"indexes/ocr/{generation}/OCR.sqlite3"
65
+ directory = self.bucket / Path(db_relative).parent
66
+ directory.mkdir(parents=True)
67
+ with patch.object(builder, "BASE_DIR", self.bucket), patch.object(builder, "FULLTEXT_DIR", directory):
68
+ builder.build_source_db("OCR", records)
69
+ self.manifest = {**core, "generation": generation, "documents": 1,
70
+ "database": self.ref(db_relative, (self.bucket / db_relative).read_bytes())}
71
+ self.manifest_path = f"releases/ocr-{generation}/manifest.json"
72
+ raw = release.canonical(self.manifest)
73
+ self.write(self.manifest_path, raw)
74
+ self.manifest_sha = release.digest(raw)
75
+
76
+ def write(self, path, body):
77
+ target = self.bucket / path
78
+ target.parent.mkdir(parents=True, exist_ok=True)
79
+ target.write_bytes(body)
80
+
81
+ def ref(self, path, body):
82
+ self.write(path, body)
83
+ return {"bucket": release.BUCKET, "path": path, "sha256": release.digest(body), "bytes": len(body), "role": "runtime"}
84
+
85
+ def load(self):
86
+ return release.load_release(self.bucket, self.manifest_path, self.manifest_sha, self.base_sha)
87
+
88
+ def test_whole_release_is_verified_and_origin_hierarchy_is_preserved(self):
89
+ candidate = self.load()
90
+ self.assertEqual(candidate["records"][0]["display_rel_path"], "VoiceOfML/Test/原目录/book.txt")
91
+ self.assertEqual(candidate["text_paths"]["OCR:1"].read_text(), self.content)
92
+ self.assertEqual(candidate["tree"][0]["count"], 1)
93
+ self.assertEqual(candidate["browser"]["VoiceOfML/Test/原目录"]["files"][0]["doc_id"], "OCR:1")
94
+
95
+ def test_corrupt_text_or_database_blocks_the_release(self):
96
+ for ref in (self.export["text"], self.manifest["database"]):
97
+ target = self.bucket / ref["path"]
98
+ original = target.read_bytes()
99
+ try:
100
+ target.write_bytes(original + b"damage")
101
+ with self.assertRaisesRegex(ValueError, "checksum"):
102
+ self.load()
103
+ finally:
104
+ target.write_bytes(original)
105
+
106
+ def test_wrong_manifest_or_base_identity_cannot_activate(self):
107
+ for manifest_sha, base_sha in (("d" * 64, self.base_sha), (self.manifest_sha, "e" * 64)):
108
+ with self.subTest(manifest_sha=manifest_sha), self.assertRaises(ValueError):
109
+ release.load_release(self.bucket, self.manifest_path, manifest_sha, base_sha)
110
+
111
+ def test_export_page_offsets_and_missing_provenance_are_rejected(self):
112
+ for change in ({"pages": []}, {"retained_reader_resources": []}, {"readable_rel_path": "wrong/book.txt"},
113
+ {"accepted_pages": 3}, {"origin_repo": "../bad"}):
114
+ value = {**copy.deepcopy(self.export), **change}
115
+ ref = self.ref(self.export_path, release.canonical(value))
116
+ with self.subTest(change=list(change)), self.assertRaises(ValueError):
117
+ release.export_record(self.bucket, ref)
118
+
119
+ def test_pin_copies_verified_data_and_is_independent_of_the_mount(self):
120
+ pinned = release.pin_release(self.bucket, self.manifest_path, self.manifest_sha, self.base_sha, self.root / "cache")
121
+ self.assertNotIn(self.bucket, pinned["database"].parents)
122
+ self.assertEqual(pinned["text_paths"]["OCR:1"].read_text(), self.content)
123
+ self.write(self.export["text"]["path"], b"changed remote mount")
124
+ self.assertEqual(pinned["text_paths"]["OCR:1"].read_text(), self.content)
125
+
126
+ def test_small_index_uses_existing_in_memory_search_contract(self):
127
+ candidate = self.load()
128
+ databases = app.FulltextDatabases(self.root / "empty-db", in_memory=True, extra_paths=[candidate["database"]])
129
+ try:
130
+ self.assertEqual(databases.search_source("OCR", "手机", exact=False), {1})
131
+ self.assertEqual(databases.search_source("OCR", "手。机", exact=True), {1})
132
+ self.assertEqual(databases.search_source("OCR", "文化 ABC", exact=False), {1})
133
+ self.assertEqual(databases.search_source("OCR", "手机 ABC", exact=True), set())
134
+ self.assertEqual(databases.connections["OCR"].execute("PRAGMA database_list").fetchone()[2], "")
135
+ finally:
136
+ databases.close()
137
+
138
+ def test_startup_loads_one_pinned_release_and_serves_plain_txt(self):
139
+ tree, browser = self.root / "tree.gz", self.root / "browser.gz"
140
+ tree.write_bytes(gzip.compress(b'{"CCRD":[]}')); browser.write_bytes(gzip.compress(b'{"CCRD":{}}'))
141
+ saved = {key: getattr(app, key) for key in ("records", "record_map", "record_map_index", "sources", "source_counts",
142
+ "folder_tree_data", "folder_browser_data", "active_ocr_release")}
143
+ try:
144
+ with patch.dict("os.environ", {"CCRD_INDEX_BUCKET_DIR": str(self.bucket), "CCRD_OCR_RELEASE_PATH": self.manifest_path,
145
+ "CCRD_OCR_RELEASE_SHA256": self.manifest_sha}), \
146
+ patch.object(app, "DATA_PATH", self.base), patch.object(app, "FOLDER_TREE_PATH", tree), \
147
+ patch.object(app, "FOLDER_BROWSER_PATH", browser), patch.object(app, "BASE_DIR", self.root):
148
+ app.load_data()
149
+ self.assertEqual(app.source_counts, {"CCRD": 1, "OCR": 1})
150
+ self.assertEqual(app.record_map["CCRD:1"]["display_name"], "旧文件")
151
+ client = TestClient(app.app)
152
+ response = client.get("/txt/OCR/VoiceOfML/Test/原目录/book.txt")
153
+ self.assertEqual(response.status_code, 200)
154
+ self.assertEqual(response.text, self.content)
155
+ self.assertIn("inline", response.headers["content-disposition"])
156
+ self.assertEqual(client.get("/txt/OCR/missing.txt").status_code, 404)
157
+ self.assertEqual(client.get("/api/preview/OCR:1").json()["text"], self.content)
158
+ self.assertEqual(json.loads(app.sidebar_payload_for_path(""))["sources"][-1]["slug"], "OCR")
159
+ self.assertEqual(app.zip_entries(["OCR:1"])[0][0], "OCR成果/VoiceOfML/Test/原目录/book.txt")
160
+ self.assertEqual(client.get("/data/sidebar/global.json").json()["sources"][-1]["slug"], "OCR")
161
+ self.assertEqual(client.get("/data/initial/sources/OCR.json").json()["total"], 1)
162
+ finally:
163
+ for key, value in saved.items(): setattr(app, key, value)
164
+ app.build_indexes()
165
+
166
+
167
+ if __name__ == "__main__":
168
+ unittest.main()
tests/test_prepare_fulltext_index.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import json
3
+ import os
4
+ import sqlite3
5
+ import subprocess
6
+ import sys
7
+ import tempfile
8
+ import unittest
9
+ from pathlib import Path
10
+
11
+
12
+ ROOT = Path(__file__).resolve().parents[1]
13
+ PREPARE = ROOT / "prepare_fulltext_index.py"
14
+ TOKENIZER_VERSION = "cjk-bigram-boundary-fts5-v6-snippet-anchors"
15
+
16
+
17
+ def create_database(path: Path, doc_id: str) -> str:
18
+ with sqlite3.connect(path) as connection:
19
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
20
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (TOKENIZER_VERSION,))
21
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
22
+ connection.execute("INSERT INTO documents VALUES (?, 'summary')", (doc_id,))
23
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
24
+ connection.execute("INSERT INTO content_fts (rowid, content) VALUES (1, 'content')")
25
+ return hashlib.sha256(path.read_bytes()).hexdigest()
26
+
27
+
28
+ class PrepareFulltextIndexTests(unittest.TestCase):
29
+ def setUp(self):
30
+ self.temporary = tempfile.TemporaryDirectory()
31
+ self.root = Path(self.temporary.name)
32
+ self.bucket = self.root / "bucket"
33
+ self.generation = self.bucket / "generations/test"
34
+ self.generation.mkdir(parents=True)
35
+ self.fulltext = self.root / "fulltext"
36
+
37
+ def tearDown(self):
38
+ self.temporary.cleanup()
39
+
40
+ def write_manifest(self, hashes):
41
+ self.bucket.joinpath("current.json").write_text(json.dumps({
42
+ "format": 1,
43
+ "tokenizer_version": TOKENIZER_VERSION,
44
+ "databases": {
45
+ source: {
46
+ "path": f"generations/test/{source}.sqlite3",
47
+ "sha256": digest,
48
+ "bytes": (self.generation / f"{source}.sqlite3").stat().st_size,
49
+ "documents": 1,
50
+ "fts_rows": 1,
51
+ }
52
+ for source, digest in hashes.items()
53
+ },
54
+ }), encoding="utf-8")
55
+
56
+ def run_prepare(self):
57
+ environment = os.environ | {
58
+ "CCRD_INDEX_BUCKET_DIR": str(self.bucket),
59
+ "CCRD_FULLTEXT_DIR": str(self.fulltext),
60
+ }
61
+ return subprocess.run([sys.executable, str(PREPARE)], env=environment, capture_output=True, text=True)
62
+
63
+ def test_activates_verified_bucket_generation(self):
64
+ hashes = {
65
+ source: create_database(self.generation / f"{source}.sqlite3", f"{source}:1")
66
+ for source in ("CCRD", "CW")
67
+ }
68
+ self.write_manifest(hashes)
69
+ newer = self.bucket / "generations/ccrd-newer"
70
+ newer.mkdir(parents=True)
71
+ for source in ("CCRD", "CW"):
72
+ create_database(newer / f"{source}.sqlite3", f"{source}:newer")
73
+
74
+ result = self.run_prepare()
75
+
76
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
77
+ status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
78
+ self.assertEqual(status['state'], 'ready')
79
+ for source in hashes:
80
+ target = self.fulltext / f"{source}.sqlite3"
81
+ self.assertTrue(target.is_file())
82
+ self.assertFalse(target.is_symlink())
83
+ self.assertEqual(target.read_bytes(), (self.generation / f"{source}.sqlite3").read_bytes())
84
+
85
+ def test_rejects_invalid_hash_format(self):
86
+ hashes = {
87
+ source: create_database(self.generation / f"{source}.sqlite3", f"{source}:1")
88
+ for source in ("CCRD", "CW")
89
+ }
90
+ hashes["CW"] = "not-a-sha256"
91
+ self.write_manifest(hashes)
92
+
93
+ result = self.run_prepare()
94
+
95
+ self.assertNotEqual(result.returncode, 0)
96
+ status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
97
+ self.assertEqual(status['state'], 'unavailable')
98
+ self.assertEqual(status['error'], 'ValueError')
99
+ self.assertFalse((self.fulltext / "CCRD.sqlite3").exists())
100
+
101
+ def test_recovers_from_stale_pointer_with_latest_complete_generation(self):
102
+ latest = self.bucket / "generations/ccrd-new"
103
+ latest.mkdir(parents=True)
104
+ for source in ("CCRD", "CW"):
105
+ create_database(latest / f"{source}.sqlite3", f"{source}:latest")
106
+ self.bucket.joinpath("current.json").write_text(json.dumps({
107
+ "format": 1,
108
+ "tokenizer_version": TOKENIZER_VERSION,
109
+ "databases": {
110
+ source: {
111
+ "path": f"generations/ccrd-old/{source}.sqlite3",
112
+ "sha256": "0" * 64,
113
+ "bytes": 1,
114
+ "documents": 1,
115
+ "fts_rows": 1,
116
+ }
117
+ for source in ("CCRD", "CW")
118
+ },
119
+ }), encoding="utf-8")
120
+
121
+ result = self.run_prepare()
122
+
123
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
124
+ status = json.loads((self.fulltext / "bucket-index-status.json").read_text())
125
+ self.assertEqual(status["state"], "ready")
126
+ self.assertEqual(status["pointer_error"]["error"], "ValueError")
127
+ for source in ("CCRD", "CW"):
128
+ self.assertEqual(
129
+ (self.fulltext / f"{source}.sqlite3").read_bytes(),
130
+ (latest / f"{source}.sqlite3").read_bytes(),
131
+ )
132
+
133
+
134
+ if __name__ == "__main__":
135
+ unittest.main()
tests/test_search.py ADDED
@@ -0,0 +1,1214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import sqlite3
5
+ import tempfile
6
+ import unittest
7
+ from html.parser import HTMLParser
8
+ from pathlib import Path
9
+ from unittest.mock import MagicMock, patch
10
+
11
+ import app
12
+ import build_fulltext_db
13
+
14
+
15
+ class InteractiveIdParser(HTMLParser):
16
+ def __init__(self):
17
+ super().__init__()
18
+ self.ids = set()
19
+
20
+ def handle_starttag(self, tag, attrs):
21
+ values = dict(attrs)
22
+ if tag in {"button", "input", "select", "a"} and values.get("id"):
23
+ self.ids.add(values["id"])
24
+
25
+
26
+ def encode_doc_numbers(numbers: list[int]) -> bytes:
27
+ return build_fulltext_db.encode_doc_ids(numbers)
28
+
29
+
30
+ class TokenizationTests(unittest.TestCase):
31
+ def test_normalized_snippet_payload_matches_current_path(self):
32
+ text = build_fulltext_db.fts_content("前文\nABC 与 文化 革命,后文。")
33
+ self.assertEqual(
34
+ app.build_normalized_snippet_payload(text, "文化 革命"),
35
+ app.build_snippet_payload(text, "文化 革命"),
36
+ )
37
+
38
+ def test_normalized_exact_snippet_payload_matches_current_path(self):
39
+ text = build_fulltext_db.fts_content("前文\nABC 与 文化革命,后文。")
40
+ self.assertEqual(
41
+ app.build_normalized_exact_snippet_payload(text, "文化革命"),
42
+ app.build_snippet_payload(text, "文化革命", exact=True),
43
+ )
44
+
45
+ def test_normalizes_latin_and_indexes_chinese_unigrams_and_bigrams(self):
46
+ self.assertEqual(app.query_terms("ABC 手机"), ["abc", "手机"])
47
+ self.assertEqual(app.query_tokens("文化革命"), ["文化", "化革", "革命"])
48
+ self.assertEqual(app.index_tokens("手机"), {"手", "机", "手机"})
49
+
50
+ def test_builder_and_runtime_tokenizers_stay_in_sync(self):
51
+ text = "文化革命 ABC 手机"
52
+ self.assertEqual(app.index_tokens(text), build_fulltext_db.index_tokens(text))
53
+ expected_hashes = {
54
+ hashlib.sha256(token.encode("utf-8")).digest()[:16]
55
+ for token in build_fulltext_db.index_tokens(text) | build_fulltext_db.literal_tokens(text)
56
+ }
57
+ self.assertEqual(build_fulltext_db.token_hashes(text), expected_hashes)
58
+
59
+ def test_literal_tokens_preserve_punctuation_boundaries(self):
60
+ self.assertEqual(app.literal_query_tokens("手。机"), ["\0literal:手。机"])
61
+ self.assertEqual(
62
+ set(app.literal_query_tokens("手。机")),
63
+ build_fulltext_db.literal_tokens("手。机"),
64
+ )
65
+ self.assertEqual(app.literal_query_tokens("文化革命"), [])
66
+ self.assertEqual(app.literal_query_tokens("手。机文化"), ["\0literal:手。机"])
67
+
68
+ def test_snippet_anchors_use_normalized_character_offsets_for_all_token_types(self):
69
+ anchors = build_fulltext_db.snippet_anchors("ABC 手机 手。机")
70
+ offset_for = lambda token: anchors[hashlib.sha256(token.encode("utf-8")).digest()[:16]]
71
+ self.assertEqual(offset_for("abc"), 0)
72
+ self.assertEqual(offset_for("手机"), 4)
73
+ self.assertEqual(offset_for("手"), 4)
74
+ self.assertEqual(offset_for("\0literal:手。机"), 7)
75
+
76
+ def test_exact_wildcards_are_non_greedy_and_cross_newlines(self):
77
+ self.assertTrue(app.matches_exact_query("手\n持电话机", "手*机"))
78
+ self.assertTrue(app.matches_exact_query("手机", "手?"))
79
+ self.assertFalse(app.matches_exact_query("手持电话机", "手?机"))
80
+
81
+ def test_wildcard_required_tokens_keep_only_cjk_fixed_parts(self):
82
+ self.assertEqual(app.wildcard_required_tokens("手*机"), ["手", "机"])
83
+ self.assertEqual(app.wildcard_required_tokens("文化*资料"), ["文化", "资料"])
84
+ self.assertEqual(app.wildcard_required_tokens("ABC*文化"), ["文化"])
85
+
86
+
87
+ class MetadataSearchTests(unittest.TestCase):
88
+ def setUp(self):
89
+ self.original_records = app.records
90
+ self.original_sources = app.source_counts
91
+ app.records = [
92
+ self.record("CCRD:0", "文化革命资料", ["历史"]),
93
+ self.record("CCRD:1", "文化资料", ["革命文献"]),
94
+ self.record("CW:2", "革命资料", ["其他"]),
95
+ ]
96
+ app.source_counts = {"CCRD": 2, "CW": 1}
97
+ app.build_indexes()
98
+
99
+ def tearDown(self):
100
+ app.records = self.original_records
101
+ app.source_counts = self.original_sources
102
+ app.build_indexes()
103
+
104
+ @staticmethod
105
+ def record(doc_id: str, name: str, dirs: list[str]) -> dict:
106
+ source = doc_id.split(":", 1)[0]
107
+ path = "/".join([*dirs, name + ".txt"])
108
+ return {
109
+ "doc_id": doc_id,
110
+ "source": source,
111
+ "source_name": source,
112
+ "display_name": name,
113
+ "display_dirs": dirs,
114
+ "display_rel_path": path,
115
+ "extension": ".txt",
116
+ "size": 100,
117
+ "storage_root": source,
118
+ "storage_rel_path": path,
119
+ }
120
+
121
+ def test_multiple_terms_are_and_not_or(self):
122
+ result = app.search("文化 革命")
123
+ self.assertEqual(result["total"], 2)
124
+ self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0", "CCRD:1"})
125
+
126
+ def test_metadata_prefilter_keeps_shared_postings_and_matches_direct_intersection(self):
127
+ snapshot = {token: set(indices) for token, indices in app.word_index.items()}
128
+ for terms in (["文化", "革命"], ["革命", "文化"], ["文化革命"], ["不存在"], ["文化", "文化"], ["。"], []):
129
+ expected = None
130
+ for term in terms:
131
+ for token in app.query_tokens(term):
132
+ posting = snapshot.get(token, set())
133
+ expected = set(posting) if expected is None else expected & posting
134
+ with self.subTest(terms=terms):
135
+ self.assertEqual(app.metadata_prefilter(terms, True), expected)
136
+ self.assertEqual(app.word_index, snapshot)
137
+
138
+ def test_mixed_latin_metadata_candidates_keep_path_option_and_substrings(self):
139
+ app.records = [
140
+ self.record("CCRD:0", "文化 ABC123", []),
141
+ self.record("CCRD:1", "文化", ["ABC123"]),
142
+ self.record("CW:2", "革命 ABC123", []),
143
+ ]
144
+ app.build_indexes()
145
+ for paths, expected in ((True, {0, 1}), (False, {0})):
146
+ for terms in (["文化", "abc"], ["abc", "文化"]):
147
+ self.assertEqual(app.metadata_prefilter(terms, paths), expected)
148
+
149
+ def test_search_paths_false_excludes_path_only_match(self):
150
+ result = app.search("革命", search_paths=False)
151
+ self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0", "CW:2"})
152
+
153
+ def test_filters_sorting_and_pagination_are_stable(self):
154
+ app.records[0]["size"] = 50
155
+ app.records[1]["size"] = 200
156
+ app.build_indexes()
157
+ result = app.search("文化", sources_filter=["CCRD"], min_size=100, sort="size", page_size=1)
158
+ self.assertEqual(result["total"], 1)
159
+ self.assertEqual(result["results"][0]["doc_id"], "CCRD:1")
160
+
161
+ def test_directory_filter_preserves_boundaries_and_root_semantics(self):
162
+ app.records = [
163
+ self.record("CCRD:0", "one", []),
164
+ self.record("CCRD:1", "two", ["a"]),
165
+ self.record("CCRD:2", "three", ["a", "b"]),
166
+ self.record("CW:3", "four", ["ab"]),
167
+ self.record("CW:4", "five", ["x", "y"]),
168
+ self.record("CW:5", "six", ["", "a"]),
169
+ self.record("CW:6", "seven", ["a", "", "b"]),
170
+ ]
171
+ for folders in ([], [""], ["a"], ["a", "x"], ["", "a"], ["a/", "x/y"], ["missing", "ab"], ["a", "a"]):
172
+ for sources in (None, [], ["CCRD"], ["CCRD", "CW"]):
173
+ with self.subTest(folders=folders, sources=sources):
174
+ expected = [
175
+ idx for idx, rec in enumerate(app.records)
176
+ if (sources is None or rec["source"] in sources)
177
+ and (not folders or any(
178
+ "/".join(rec["display_dirs"]) == folder
179
+ or "/".join(rec["display_dirs"]).startswith(folder + "/")
180
+ for folder in folders
181
+ ))
182
+ ]
183
+ self.assertEqual(app.apply_filters(range(len(app.records)), sources, folders), expected)
184
+
185
+ def test_unfiltered_candidates_do_not_read_record_fields(self):
186
+ with patch.object(app, "records", [None, None, None]):
187
+ self.assertEqual(app.apply_filters(iter([2, 0, 1])), [2, 0, 1])
188
+
189
+ def test_precomputed_name_and_size_orders_match_previous_sort_keys(self):
190
+ app.records[0]["size"] = 50
191
+ app.records[1]["size"] = 200
192
+ app.records[2]["size"] = 200
193
+ app.build_indexes()
194
+ expected_name = sorted(
195
+ range(len(app.records)),
196
+ key=lambda idx: (app.records[idx]["display_rel_path"].lower(), app.records[idx]["doc_id"]),
197
+ )
198
+ expected_size = sorted(
199
+ range(len(app.records)),
200
+ key=lambda idx: (-app.records[idx]["size"], app.records[idx]["display_rel_path"].lower(), app.records[idx]["doc_id"]),
201
+ )
202
+ self.assertEqual(app.name_order, expected_name)
203
+ self.assertEqual(app.size_order, expected_size)
204
+ self.assertEqual(app.source_name_order["CCRD"], [idx for idx in expected_name if app.records[idx]["source"] == "CCRD"])
205
+ self.assertEqual(app.source_size_order["CCRD"], [idx for idx in expected_size if app.records[idx]["source"] == "CCRD"])
206
+ self.assertEqual(app.ordered_page(set(range(len(app.records))), app.size_order, 1, 2), expected_size[1:3])
207
+
208
+ def test_sparse_ordered_pages_match_global_order_with_ties(self):
209
+ app.records = [
210
+ self.record(f"CCRD:{number}", "Name" if number % 2 else "name", [str(number % 3)])
211
+ for number in range(40)
212
+ ]
213
+ app.build_indexes()
214
+ for ordering in (app.name_order, app.size_order, list(reversed(app.name_order))):
215
+ for candidates in ([39, 2, 10, 1], list(range(40)), []):
216
+ expected = [idx for idx in ordering if idx in set(candidates)]
217
+ for start, page_size in ((0, 1), (0, 10), (1, 10), (4, 10), (30, 10)):
218
+ with self.subTest(candidates=len(candidates), start=start, page_size=page_size):
219
+ self.assertEqual(
220
+ app.ordered_page(candidates, ordering, start, page_size),
221
+ expected[start:start + page_size],
222
+ )
223
+
224
+ def test_empty_global_and_single_source_queries_bypass_filtering_and_order_scan(self):
225
+ with patch.object(app, "apply_filters", side_effect=AssertionError("fast path scanned records")), \
226
+ patch.object(app, "ordered_page", side_effect=AssertionError("fast path scanned order")):
227
+ global_name = app.search("", sort="name", page=2, page_size=1)
228
+ source_size = app.search("", sources_filter=["CCRD"], sort="size", page_size=2)
229
+ self.assertEqual(global_name["results"][0]["doc_id"], app.records[app.name_order[1]]["doc_id"])
230
+ self.assertEqual(
231
+ [item["doc_id"] for item in source_size["results"]],
232
+ [app.records[idx]["doc_id"] for idx in app.source_size_order["CCRD"][:2]],
233
+ )
234
+
235
+ def test_empty_query_with_record_filters_keeps_general_path(self):
236
+ with patch.object(app, "apply_filters", wraps=app.apply_filters) as filtered:
237
+ result = app.search("", sources_filter=["CCRD"], min_size=100, sort="size")
238
+ filtered.assert_called_once()
239
+ self.assertEqual(result["total"], 2)
240
+
241
+ def test_bounded_relevance_page_matches_full_sort(self):
242
+ indices = list(range(len(app.records)))
243
+ tokens = app.query_tokens("文化 革命")
244
+ for search_paths in (False, True):
245
+ expected = sorted(
246
+ indices,
247
+ key=lambda idx: (
248
+ -app.score_record(idx, tokens, search_paths),
249
+ app.records[idx]["display_rel_path"].lower(),
250
+ app.records[idx]["doc_id"],
251
+ ),
252
+ )
253
+ for start, page_size in ((0, 1), (1, 2), (2, 2)):
254
+ self.assertEqual(
255
+ app.relevance_page(indices, tokens, search_paths, start, page_size),
256
+ expected[start:start + page_size],
257
+ )
258
+
259
+ def test_heavy_relevance_pages_preserve_full_order_and_duplicates(self):
260
+ app.records = [self.record(f"CCRD:{number:05d}", ("文化" if number % 3 else "革命"), [str(number % 7)])
261
+ for number in range(10020)]
262
+ app.build_indexes()
263
+ indices = list(range(len(app.records)))
264
+ for paths in (True, False):
265
+ tokens = app.query_tokens("文化 革命")
266
+ expected = sorted(indices, key=lambda idx: (
267
+ -app.score_record(idx, tokens, paths), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
268
+ ))
269
+ for start in (0, 100, 999, 1000, 5000, 9900, 10020):
270
+ self.assertEqual(app.relevance_page(indices, tokens, paths, start, 200), expected[start:start + 200])
271
+ duplicate_indices = indices + [0]
272
+ expected = sorted(duplicate_indices, key=lambda idx: (
273
+ -app.score_record(idx, tokens, True), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
274
+ ))
275
+ self.assertEqual(app.relevance_page(duplicate_indices, tokens, True, 1000, 200), expected[1000:1200])
276
+ self.assertEqual(app.relevance_page(duplicate_indices, tokens, True, 0, 200), expected[:200])
277
+
278
+ def test_shallow_relevance_stops_only_after_enough_maximum_score_hits(self):
279
+ app.records = [self.record(f"CCRD:{number}", "文化革命", ["same"])
280
+ for number in range(10020)]
281
+ app.build_indexes()
282
+ candidates = list(range(len(app.records)))
283
+ for paths in (True, False):
284
+ with self.subTest(paths=paths), patch.object(app, "score_record", wraps=app.score_record) as scored:
285
+ self.assertEqual(app.relevance_page(candidates, ["文化", "革命"], paths, 100, 100), app.name_order[100:200])
286
+ self.assertEqual(scored.call_count, 200)
287
+ # A file-only source-name match cannot attain the path-inclusive maximum.
288
+ with patch.object(app, "score_record", wraps=app.score_record) as scored:
289
+ self.assertEqual(app.relevance_page(candidates, ["ccrd"], True, 0, 100), app.name_order[:100])
290
+ self.assertEqual(scored.call_count, len(candidates))
291
+
292
+ def test_numeric_fulltext_mapping_preserves_source_identity_and_missing_ids(self):
293
+ app.records = [
294
+ self.record("CW:0", "first", []),
295
+ self.record("CCRD:0", "second", []),
296
+ self.record("CCRD:7", "third", []),
297
+ self.record("CCRD:007", "padded", []),
298
+ self.record("CCRD:invalid", "invalid", []),
299
+ ]
300
+ app.build_indexes()
301
+ fake = MagicMock()
302
+ fake.search_source.side_effect = lambda source, q, exact: {0, 7, 99}
303
+ with patch.object(app, "fulltext_databases", fake), \
304
+ patch.object(app, "add_summaries", side_effect=lambda items, *args, **kwargs: items):
305
+ result = app.fulltext_search("missing", sort="name")
306
+ self.assertEqual(result["total"], 3)
307
+ self.assertEqual({item["doc_id"] for item in result["results"]}, {"CW:0", "CCRD:0", "CCRD:7"})
308
+ cw = app.fulltext_search("missing", sources_filter=["CW"])
309
+ self.assertEqual([item["doc_id"] for item in cw["results"]], ["CW:0"])
310
+ self.assertEqual(app.fulltext_search("missing", sources_filter=["unknown"])["total"], 0)
311
+ app.records = [self.record("CCRD:99", "replacement", [])]
312
+ app.build_indexes()
313
+ self.assertEqual(app.source_doc_number_index, {"CCRD": {99: 0}})
314
+
315
+ def test_shallow_relevance_falls_back_when_maximum_score_hits_are_insufficient(self):
316
+ app.records = [self.record(f"CCRD:{number}", "文化革命" if number < 3 else "文化", ["same"])
317
+ for number in range(10020)]
318
+ for record in app.records:
319
+ record["source_name"] = "文化 革命"
320
+ app.build_indexes()
321
+ candidates = list(reversed(range(len(app.records))))
322
+ tokens = ["文化", "革命"]
323
+ expected = sorted(candidates, key=lambda idx: (
324
+ -app.score_record(idx, tokens, True), app.records[idx]["_rank_path_key"], app.records[idx]["doc_id"],
325
+ ))
326
+ for start in (0, 100):
327
+ self.assertEqual(app.relevance_page(candidates, tokens, True, start, 100), expected[start:start + 100])
328
+
329
+ def test_fulltext_relevance_respects_search_paths_false(self):
330
+ original_databases = app.fulltext_databases
331
+ fake_databases = MagicMock()
332
+ fake_databases.search_source.return_value = set()
333
+ fake_databases.has_content_fts = {"CCRD": False, "CW": False}
334
+ fake_databases.summaries.return_value = {}
335
+ fake_databases.matched_snippets.return_value = {}
336
+ app.fulltext_databases = fake_databases
337
+ try:
338
+ result = app.fulltext_search("文化", search_paths=False)
339
+ expected = sorted(
340
+ app.metadata_matches("文化", False, False),
341
+ key=lambda idx: (-app.score_record(idx, app.query_tokens("文化"), False), app.records[idx]["display_rel_path"].lower()),
342
+ )
343
+ self.assertEqual([item["doc_id"] for item in result["results"]], [app.records[idx]["doc_id"] for idx in expected])
344
+ finally:
345
+ app.fulltext_databases = original_databases
346
+
347
+ def test_exact_metadata_index_matches_full_scan(self):
348
+ for query in ("文化革命", "革命资料", "不存在"):
349
+ indexed = app.metadata_matches(query, True, True)
350
+ scanned = {
351
+ idx for idx, record in enumerate(app.records)
352
+ if app.matches_exact_query(record["_search_text"], query)
353
+ }
354
+ self.assertEqual(indexed, scanned)
355
+
356
+ def test_exact_search_uses_metadata_candidate_path(self):
357
+ with patch.object(app, "metadata_matches", wraps=app.metadata_matches) as metadata_matches:
358
+ result = app.search("文化革命", exact=True)
359
+ metadata_matches.assert_called_once_with("文化革命", True, True)
360
+ self.assertEqual({item["doc_id"] for item in result["results"]}, {"CCRD:0"})
361
+
362
+ def test_metadata_match_cache_is_option_scoped_bounded_and_cleared_on_rebuild(self):
363
+ app.metadata_match_cache.clear()
364
+ with patch.object(app, "_metadata_matches_uncached", wraps=app._metadata_matches_uncached) as uncached:
365
+ first = app.metadata_matches("文化革命", True, True)
366
+ second = app.metadata_matches("文化革命", True, True)
367
+ self.assertEqual(first, second)
368
+ first.clear()
369
+ self.assertEqual(app.metadata_matches("文化革命", True, True), second)
370
+ self.assertEqual(uncached.call_count, 1)
371
+ app.metadata_matches("文化革命", True, False)
372
+ app.metadata_matches("文化 革命", False, True)
373
+ self.assertEqual(uncached.call_count, 3)
374
+ with patch.object(app, "METADATA_MATCH_CACHE_MAX_ENTRIES", 2):
375
+ app.metadata_matches("甲", False, True)
376
+ app.metadata_matches("乙", False, True)
377
+ app.metadata_matches("丙", False, True)
378
+ self.assertLessEqual(len(app.metadata_match_cache), 2)
379
+ app.build_indexes()
380
+ self.assertEqual(app.metadata_match_cache, {})
381
+
382
+ def test_metadata_cache_id_budget_never_truncates_results(self):
383
+ app.clear_metadata_match_cache()
384
+ with patch.object(app, "METADATA_MATCH_CACHE_MAX_TOTAL_IDS", 3), \
385
+ patch.object(app, "_metadata_matches_uncached", return_value={0, 1}):
386
+ for query in ("甲", "乙", "丙"):
387
+ self.assertEqual(app.metadata_matches(query, False, True), {0, 1})
388
+ self.assertEqual(sum(map(len, app.metadata_match_cache.values())), 2)
389
+ app.clear_metadata_match_cache()
390
+ with patch.object(app, "_metadata_matches_uncached", return_value={0, 1, 2, 3}) as search:
391
+ for _ in range(2):
392
+ self.assertEqual(app.metadata_matches("oversized", False, True), {0, 1, 2, 3})
393
+ self.assertEqual(search.call_count, 2)
394
+ self.assertEqual(app.metadata_match_cache, {})
395
+
396
+ def test_old_metadata_computation_cannot_repopulate_after_invalidation(self):
397
+ app.clear_metadata_match_cache()
398
+ def old_result(*_args):
399
+ app.clear_metadata_match_cache()
400
+ return {99}
401
+ with patch.object(app, "_metadata_matches_uncached", side_effect=old_result):
402
+ self.assertEqual(app.metadata_matches("文化", False, True), {99})
403
+ self.assertEqual(app.metadata_match_cache, {})
404
+ with patch.object(app, "_metadata_matches_uncached", side_effect=RuntimeError("failure")):
405
+ with self.assertRaises(RuntimeError):
406
+ app.metadata_matches("文化", False, True)
407
+ self.assertEqual(app.metadata_match_cache, {})
408
+
409
+ def test_exact_metadata_wildcard_preserves_full_scan_semantics(self):
410
+ for query in ("文化*资料", "手?机", "*文化*"):
411
+ indexed = app.metadata_matches(query, True, True)
412
+ scanned = {
413
+ idx for idx, record in enumerate(app.records)
414
+ if app.matches_exact_query(record["_search_text"], query)
415
+ }
416
+ self.assertEqual(indexed, scanned)
417
+
418
+ def test_exact_and_normal_latin_metadata_substrings_are_not_lost(self):
419
+ app.records[0]["display_name"] = "smartphone"
420
+ app.records[0]["display_rel_path"] = "folder/smartphone.txt"
421
+ app.build_indexes()
422
+ self.assertIn(0, app.metadata_matches("phone", True, False))
423
+ self.assertIn(0, app.metadata_matches("phone", False, False))
424
+
425
+
426
+ class FulltextDatabaseTests(unittest.TestCase):
427
+ def test_uses_rarest_posting_first_and_intersects_all_tokens(self):
428
+ with tempfile.TemporaryDirectory() as directory:
429
+ path = Path(directory) / "CCRD.sqlite3"
430
+ with sqlite3.connect(path) as connection:
431
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
432
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
433
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
434
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
435
+ for token, numbers in (("文化", [1, 2, 3]), ("化革", [2]), ("革命", [2, 3])):
436
+ token_hash = hashlib.sha256(token.encode()).digest()[:16]
437
+ connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
438
+ databases = app.FulltextDatabases(Path(directory))
439
+ try:
440
+ self.assertEqual(databases.search_source("CCRD", "文化革命"), {2})
441
+ finally:
442
+ databases.close()
443
+
444
+ def test_exact_phrase_uses_literal_postings(self):
445
+ with tempfile.TemporaryDirectory() as directory:
446
+ path = Path(directory) / "CCRD.sqlite3"
447
+ with sqlite3.connect(path) as connection:
448
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
449
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
450
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
451
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
452
+ for token in [*app.query_tokens("手。机"), *app.literal_query_tokens("手。机")]:
453
+ token_hash = hashlib.sha256(token.encode()).digest()[:16]
454
+ connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers([7]), 1))
455
+ databases = app.FulltextDatabases(Path(directory))
456
+ try:
457
+ self.assertEqual(databases.search_source("CCRD", "手。机", exact=True), {7})
458
+ self.assertEqual(databases.search_source("CCRD", "手机", exact=True), set())
459
+ finally:
460
+ databases.close()
461
+
462
+ def test_short_exact_query_uses_postings_before_verification(self):
463
+ with tempfile.TemporaryDirectory() as directory:
464
+ path = Path(directory) / "CCRD.sqlite3"
465
+ with sqlite3.connect(path) as connection:
466
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
467
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
468
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
469
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
470
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
471
+ for token in [*app.query_tokens("手机"), *app.literal_query_tokens("手机")]:
472
+ token_hash = hashlib.sha256(token.encode()).digest()[:16]
473
+ connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers([7]), 1))
474
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", (7, "这是手机测试"))
475
+ databases = app.FulltextDatabases(Path(directory))
476
+ try:
477
+ self.assertEqual(databases.search_source("CCRD", "手机", exact=True), {7})
478
+ finally:
479
+ databases.close()
480
+
481
+ def test_fts_content_strictly_verifies_wildcard_candidates(self):
482
+ with tempfile.TemporaryDirectory() as directory:
483
+ path = Path(directory) / "CCRD.sqlite3"
484
+ with sqlite3.connect(path) as connection:
485
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
486
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
487
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
488
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
489
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
490
+ connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
491
+ (1, "手持电话机"),
492
+ (2, "手工劳动和机器"),
493
+ (3, "只有手持机"),
494
+ (4, "字面[手持机"),
495
+ ])
496
+ databases = app.FulltextDatabases(Path(directory))
497
+ try:
498
+ connection = databases.connections["CCRD"]
499
+ self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "手?机"), {3, 4})
500
+ self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "手*机"), {1, 2, 3, 4})
501
+ self.assertEqual(databases.verify_wildcard_content(connection, {1, 2, 3, 4}, "[手?机"), {4})
502
+ finally:
503
+ databases.close()
504
+
505
+ def test_large_wildcard_candidates_use_one_fts_scan_without_widening_results(self):
506
+ with tempfile.TemporaryDirectory() as directory:
507
+ path = Path(directory) / "CCRD.sqlite3"
508
+ with sqlite3.connect(path) as connection:
509
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
510
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
511
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
512
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
513
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
514
+ connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)",
515
+ [(number, "文化资料" if number % 2 else "其他") for number in range(1, 10001)])
516
+ databases = app.FulltextDatabases(Path(directory))
517
+ try:
518
+ connection = databases.connections["CCRD"]
519
+ with patch.object(databases, "verify_wildcard_content", wraps=databases.verify_wildcard_content) as verifier:
520
+ result = databases.verify_wildcard_content(connection, set(range(1, 10001)), "文化*资料", fts_filtered=True)
521
+ self.assertEqual(result, set(range(1, 10001, 2)))
522
+ verifier.assert_called_once()
523
+ finally:
524
+ databases.close()
525
+
526
+ def test_short_wildcard_uses_required_postings_before_fts_verification(self):
527
+ with tempfile.TemporaryDirectory() as directory:
528
+ path = Path(directory) / "CCRD.sqlite3"
529
+ with sqlite3.connect(path) as connection:
530
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
531
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
532
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
533
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
534
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
535
+ for token, numbers in (("手", [1, 2]), ("机", [1, 3])):
536
+ token_hash = hashlib.sha256(token.encode()).digest()[:16]
537
+ connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
538
+ connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
539
+ (1, "手持机"), (2, "手持电话"), (3, "机器"), (4, "手机"),
540
+ ])
541
+ databases = app.FulltextDatabases(Path(directory))
542
+ try:
543
+ self.assertEqual(databases.search_source("CCRD", "手*机", exact=True), {1})
544
+ finally:
545
+ databases.close()
546
+
547
+ def test_content_fts_returns_contiguous_cjk_phrase(self):
548
+ with tempfile.TemporaryDirectory() as directory:
549
+ path = Path(directory) / "CCRD.sqlite3"
550
+ with sqlite3.connect(path) as connection:
551
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
552
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
553
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
554
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
555
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
556
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", ("这是文化革命资料",))
557
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (8, ?)", ("这是文化和革命资料",))
558
+ databases = app.FulltextDatabases(Path(directory))
559
+ try:
560
+ self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), {7})
561
+ finally:
562
+ databases.close()
563
+
564
+ def test_exact_short_non_cjk_literals_verify_against_fts_content(self):
565
+ with tempfile.TemporaryDirectory() as directory:
566
+ path = Path(directory) / "CCRD.sqlite3"
567
+ with sqlite3.connect(path) as connection:
568
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
569
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
570
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
571
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
572
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
573
+ connection.executemany("INSERT INTO content_fts(rowid, content) VALUES (?, ?)", [
574
+ (1, "smartphone abc123xyz"),
575
+ (2, "手 单独"),
576
+ (3, "手。句号"),
577
+ (4, "手机 没有句号"),
578
+ ])
579
+ for token, numbers in (("手", [2, 3, 4]), ("机", [4])):
580
+ token_hash = hashlib.sha256(token.encode()).digest()[:16]
581
+ connection.execute("INSERT INTO postings VALUES (?, ?, ?)", (token_hash, encode_doc_numbers(numbers), len(numbers)))
582
+ databases = app.FulltextDatabases(Path(directory))
583
+ try:
584
+ # Latin substrings shorter than three characters must not be lost.
585
+ self.assertEqual(databases.search_source("CCRD", "ab", exact=True), {1})
586
+ self.assertEqual(databases.search_source("CCRD", "a", exact=True), {1})
587
+ self.assertEqual(databases.search_source("CCRD", "12", exact=True), {1})
588
+ # Short punctuation-separated CJK must not collapse to the bare character.
589
+ self.assertEqual(databases.search_source("CCRD", "手。", exact=True), {3})
590
+ # Pure CJK single characters still resolve through postings.
591
+ self.assertEqual(databases.search_source("CCRD", "手", exact=True), {2, 3, 4})
592
+ finally:
593
+ databases.close()
594
+
595
+ def test_builder_sanitizes_nul_before_fts_indexing(self):
596
+ self.assertEqual(build_fulltext_db.fts_content("ABC\0文化革命"), "abc 文化革命")
597
+
598
+ def test_metadata_only_result_does_not_read_original_text_for_snippet(self):
599
+ original_records = app.records
600
+ original_map = app.record_map
601
+ original_indices = app.record_map_index
602
+ original_databases = app.fulltext_databases
603
+ app.records = [{"doc_id": "CCRD:0", "storage_root": "unused", "storage_rel_path": "unused"}]
604
+ app.record_map = {"CCRD:0": app.records[0]}
605
+ app.record_map_index = {"CCRD:0": 0}
606
+ try:
607
+ item = {"doc_id": "CCRD:0"}
608
+ fake_databases = MagicMock()
609
+ fake_databases.summaries.return_value = {"CCRD:0": "摘要没有查询词"}
610
+ with patch.object(app, "get_doc_storage_path", side_effect=AssertionError("metadata hit read source text")):
611
+ app.fulltext_databases = fake_databases
612
+ self.assertEqual(app.add_summaries([item], "手机", True, set()), [item])
613
+ finally:
614
+ app.records = original_records
615
+ app.record_map = original_map
616
+ app.record_map_index = original_indices
617
+ app.fulltext_databases = original_databases
618
+
619
+ def test_fulltext_result_list_uses_stored_summaries_without_source_reads(self):
620
+ original_databases = app.fulltext_databases
621
+ fake_databases = MagicMock()
622
+ fake_databases.summaries.return_value = {"CCRD:0": "文档摘要"}
623
+ fake_databases.matched_snippets.return_value = {}
624
+ app.fulltext_databases = fake_databases
625
+ try:
626
+ item = {"doc_id": "CCRD:0"}
627
+ with patch.object(app, "get_doc_storage_path", side_effect=AssertionError("list summary read source text")):
628
+ result = app.add_summaries([item], "手机", matched_snippets=False, content_indices={0})
629
+ self.assertEqual(result[0]["snippet"], "文档摘要")
630
+ finally:
631
+ app.fulltext_databases = original_databases
632
+
633
+ def test_current_page_snippets_are_bounded_and_read_from_fts(self):
634
+ with tempfile.TemporaryDirectory() as directory:
635
+ path = Path(directory) / "CCRD.sqlite3"
636
+ with sqlite3.connect(path) as connection:
637
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
638
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
639
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
640
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
641
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
642
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", ("甲" * 1000 + "文化革命" + "乙" * 1000,))
643
+ databases = app.FulltextDatabases(Path(directory))
644
+ try:
645
+ calls = []
646
+ def counted_instr(content, needle):
647
+ calls.append(needle)
648
+ return content.find(needle) + 1
649
+ databases.connections["CCRD"].create_function("instr", 2, counted_instr)
650
+ payload = databases.matched_snippets(["CCRD:7"], "文化革命", exact=True)["CCRD:7"]
651
+ self.assertIn("文化革命", payload["snippet"])
652
+ self.assertIn("文化革命", payload["highlights"])
653
+ self.assertLessEqual(len(payload["snippet"]), 230)
654
+ self.assertEqual(calls, ["文化革命"])
655
+ finally:
656
+ databases.close()
657
+
658
+ def test_normal_snippets_seek_token_anchors_without_instr_scanning(self):
659
+ with tempfile.TemporaryDirectory() as directory:
660
+ path = Path(directory) / "CCRD.sqlite3"
661
+ content = "甲" * 1000 + "文化革命" + "乙" * 1000
662
+ with sqlite3.connect(path) as connection:
663
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
664
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
665
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
666
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
667
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
668
+ connection.execute("CREATE TABLE snippet_anchors (token_hash BLOB NOT NULL, doc_number INTEGER NOT NULL, char_offset INTEGER NOT NULL, PRIMARY KEY (token_hash, doc_number)) WITHOUT ROWID")
669
+ connection.execute("INSERT INTO content_fts(rowid, content) VALUES (7, ?)", (content,))
670
+ for token_hash, char_offset in build_fulltext_db.snippet_anchors(content).items():
671
+ connection.execute("INSERT INTO snippet_anchors VALUES (?, 7, ?)", (token_hash, char_offset))
672
+ databases = app.FulltextDatabases(Path(directory))
673
+ statements = []
674
+ databases.connections["CCRD"].set_trace_callback(statements.append)
675
+ try:
676
+ payload = databases.matched_snippets(["CCRD:7"], "文化革命")["CCRD:7"]
677
+ self.assertIn("文化", payload["highlights"])
678
+ self.assertIn("snippet_anchors", "\n".join(statements))
679
+ self.assertNotIn("instr(content", "\n".join(statements).lower())
680
+ finally:
681
+ databases.close()
682
+
683
+ def test_fulltext_results_use_synchronous_fts_snippets(self):
684
+ original_records = app.records
685
+ original_map = app.record_map
686
+ original_indices = app.record_map_index
687
+ original_counts = app.source_counts
688
+ original_databases = app.fulltext_databases
689
+ record = {
690
+ "doc_id": "CCRD:7", "source": "CCRD", "source_name": "CCRD",
691
+ "display_name": "文档", "extension": "txt", "display_dirs": [],
692
+ "display_rel_path": "文档.txt", "size": 10,
693
+ "_file_search_text": "文档 ccrd", "_path_search_text": "文档.txt",
694
+ "_search_text": "文档 ccrd 文档.txt",
695
+ }
696
+ fake_databases = MagicMock()
697
+ fake_databases.search_source.return_value = {7}
698
+ fake_databases.summaries.return_value = {"CCRD:7": "开头摘要"}
699
+ fake_databases.matched_snippets.return_value = {
700
+ "CCRD:7": {"snippet": "...命中文化革命正文...", "highlights": ["文化革命"]}
701
+ }
702
+ app.records = [record]
703
+ app.record_map = {"CCRD:7": record}
704
+ app.record_map_index = {"CCRD:7": 0}
705
+ app.source_counts = {"CCRD": 1}
706
+ app.build_indexes()
707
+ app.fulltext_databases = fake_databases
708
+ try:
709
+ result = app.fulltext_search("文化革命", exact=True)
710
+ self.assertEqual(result["results"][0]["snippet"], "...命中文化革命正文...")
711
+ self.assertEqual(result["results"][0]["snippet_highlights"], ["文化革命"])
712
+ self.assertTrue(result["results"][0]["snippet_complete"])
713
+ fake_databases.matched_snippets.assert_called_once_with(["CCRD:7"], "文化革命", exact=True, timings=None)
714
+ finally:
715
+ app.records = original_records
716
+ app.record_map = original_map
717
+ app.record_map_index = original_indices
718
+ app.source_counts = original_counts
719
+ app.build_indexes()
720
+ app.fulltext_databases = original_databases
721
+
722
+ def test_normal_snippet_uses_actual_bigram_match(self):
723
+ payload = app.build_snippet_payload("前文很多内容,文化在这里,后面另有革命。", "文化革命")
724
+ self.assertIn("文化", payload["snippet"])
725
+ self.assertIn("文化", payload["highlights"])
726
+
727
+ def test_exact_snippet_requires_contiguous_query(self):
728
+ text = "手。机只是干扰,后文才出现手机。"
729
+ payload = app.build_snippet_payload(text, "手机", exact=True)
730
+ self.assertIn("后文才出现手机", payload["snippet"])
731
+ self.assertEqual(payload["highlights"], ["手机"])
732
+
733
+ def test_snippet_returns_original_text_for_normalized_highlight(self):
734
+ payload = app.build_snippet_payload("正文使用全角ABC字符。", "ABC", exact=True)
735
+ self.assertEqual(payload["highlights"], ["ABC"])
736
+
737
+ def test_wildcard_snippet_highlights_fixed_original_parts(self):
738
+ payload = app.build_snippet_payload("这是手持电话机测试。", "手*机", exact=True)
739
+ self.assertEqual(payload["highlights"], ["手", "机"])
740
+
741
+ def test_missing_token_returns_no_matches(self):
742
+ with tempfile.TemporaryDirectory() as directory:
743
+ path = Path(directory) / "CCRD.sqlite3"
744
+ with sqlite3.connect(path) as connection:
745
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
746
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
747
+ databases = app.FulltextDatabases(Path(directory))
748
+ try:
749
+ self.assertEqual(databases.search_source("CCRD", "不存在"), set())
750
+ finally:
751
+ databases.close()
752
+
753
+ def test_missing_runtime_database_returns_no_matches_until_hot_loaded(self):
754
+ with tempfile.TemporaryDirectory() as directory:
755
+ databases = app.FulltextDatabases(Path(directory))
756
+ try:
757
+ self.assertEqual(databases.search_source("CCRD", "文化革命", exact=True), set())
758
+ finally:
759
+ databases.close()
760
+
761
+ def test_hot_loading_increments_cache_generation(self):
762
+ with tempfile.TemporaryDirectory() as directory:
763
+ root = Path(directory)
764
+ databases = app.FulltextDatabases(root)
765
+ self.assertEqual(databases.generation, 0)
766
+ path = root / "CCRD.sqlite3"
767
+ with sqlite3.connect(path) as connection:
768
+ connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
769
+ connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
770
+ connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
771
+ connection.execute("INSERT INTO metadata VALUES ('tokenizer_version', ?)", (app.TOKENIZER_VERSION,))
772
+ connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
773
+ databases.refresh()
774
+ try:
775
+ self.assertEqual(databases.generation, 1)
776
+ finally:
777
+ databases.close()
778
+
779
+
780
+ class RequestValidationTests(unittest.TestCase):
781
+ def test_search_request_limits_expensive_inputs(self):
782
+ with self.assertRaises(ValueError):
783
+ app.SearchRequest(q="x" * 257)
784
+ with self.assertRaises(ValueError):
785
+ app.SearchRequest(page=0)
786
+ with self.assertRaises(ValueError):
787
+ app.SearchRequest(page_size=201)
788
+ with self.assertRaises(ValueError):
789
+ app.SearchRequest(sort="unknown")
790
+
791
+ def test_zip_request_has_independent_default_list(self):
792
+ first = app.ZipRequest()
793
+ second = app.ZipRequest()
794
+ first.doc_ids.append("CCRD:1")
795
+ self.assertEqual(second.doc_ids, [])
796
+
797
+ def test_zip_entries_deduplicate_ids_and_preserve_safe_paths(self):
798
+ original_map = app.record_map
799
+ try:
800
+ with tempfile.TemporaryDirectory() as directory:
801
+ path = Path(directory) / "article.txt"
802
+ path.write_bytes(b"x" * 70000)
803
+ record = {
804
+ "source": "CCRD", "display_rel_path": "folder/article.txt",
805
+ "storage_root": "unused", "storage_rel_path": "unused",
806
+ }
807
+ app.record_map = {"CCRD:1": record}
808
+ with patch.object(app, "get_doc_storage_path", return_value=path):
809
+ entries = app.zip_entries(["CCRD:1", "CCRD:1"])
810
+ self.assertEqual(entries, [("CCRD/article [CCRD_1].txt", path)])
811
+ finally:
812
+ app.record_map = original_map
813
+
814
+ def test_unknown_source_returns_404_shape(self):
815
+ original_counts = app.source_counts
816
+ app.source_counts = {"CCRD": 1}
817
+ try:
818
+ response = app.api_search_source("missing", app.SearchRequest())
819
+ self.assertEqual(response.status_code, 404)
820
+ self.assertIn(b'"total":0', response.body)
821
+ finally:
822
+ app.source_counts = original_counts
823
+
824
+ def test_index_status_reports_background_build_progress(self):
825
+ original_databases = app.fulltext_databases
826
+ original_counts = app.source_counts
827
+ fake_databases = MagicMock()
828
+ fake_databases.connections = {"CCRD": MagicMock()}
829
+ fake_databases.tokenizer_versions = {"CCRD": app.TOKENIZER_VERSION}
830
+ fake_databases.has_content_fts = {"CCRD": True}
831
+ fake_databases.document_counts = {"CCRD": 1}
832
+ fake_databases.fts_counts = {"CCRD": 1}
833
+ app.fulltext_databases = fake_databases
834
+ app.source_counts = {"CCRD": 1, "CW": 1}
835
+ try:
836
+ response = app.api_index_status()
837
+ payload = __import__("json").loads(response.body)
838
+ self.assertFalse(payload["ready"])
839
+ self.assertEqual(payload["loaded_sources"], ["CCRD"])
840
+ self.assertEqual(payload["building_sources"], ["CW"])
841
+ finally:
842
+ app.fulltext_databases = original_databases
843
+ app.source_counts = original_counts
844
+
845
+ def test_index_status_rejects_wrong_version_or_incomplete_database(self):
846
+ original_databases = app.fulltext_databases
847
+ original_counts = app.source_counts
848
+ fake = MagicMock()
849
+ fake.connections = {"CCRD": MagicMock()}
850
+ fake.tokenizer_versions = {"CCRD": "wrong-version"}
851
+ fake.has_content_fts = {"CCRD": True}
852
+ fake.document_counts = {"CCRD": 1}
853
+ fake.fts_counts = {"CCRD": 1}
854
+ app.fulltext_databases = fake
855
+ app.source_counts = {"CCRD": 1}
856
+ try:
857
+ payload = __import__("json").loads(app.api_index_status().body)
858
+ self.assertFalse(payload["ready"])
859
+ self.assertEqual(payload["loaded_sources"], [])
860
+ self.assertEqual(payload["building_sources"], ["CCRD"])
861
+ finally:
862
+ app.fulltext_databases = original_databases
863
+ app.source_counts = original_counts
864
+
865
+ def test_index_status_rejects_database_without_snippet_anchors(self):
866
+ original_databases = app.fulltext_databases
867
+ original_counts = app.source_counts
868
+ fake = MagicMock()
869
+ fake.connections = {"CCRD": MagicMock()}
870
+ fake.tokenizer_versions = {"CCRD": app.TOKENIZER_VERSION}
871
+ fake.has_content_fts = {"CCRD": True}
872
+ fake.has_snippet_anchors = {"CCRD": False}
873
+ fake.document_counts = {"CCRD": 1}
874
+ fake.fts_counts = {"CCRD": 1}
875
+ app.fulltext_databases = fake
876
+ app.source_counts = {"CCRD": 1}
877
+ try:
878
+ payload = __import__("json").loads(app.api_index_status().body)
879
+ self.assertFalse(payload["ready"])
880
+ self.assertEqual(payload["building_sources"], ["CCRD"])
881
+ finally:
882
+ app.fulltext_databases = original_databases
883
+ app.source_counts = original_counts
884
+
885
+ def test_identical_search_requests_use_bounded_response_cache(self):
886
+ original_records = app.records
887
+ original_sources = app.source_counts
888
+ original_cache = app.search_response_cache
889
+ app.records = []
890
+ app.source_counts = {}
891
+ app.search_response_cache = {}
892
+ try:
893
+ with patch.object(app, "search", wraps=app.search) as search_call:
894
+ first = app.run_search(app.SearchRequest(q="手机"))
895
+ second = app.run_search(app.SearchRequest(q="手机"))
896
+ self.assertEqual(first, second)
897
+ self.assertEqual(search_call.call_count, 1)
898
+ self.assertTrue(any(key[0] == "search" for key in app.search_response_cache))
899
+ finally:
900
+ app.search_response_cache = original_cache
901
+ app.records = original_records
902
+ app.source_counts = original_sources
903
+
904
+ def test_search_cache_key_isolates_every_result_option(self):
905
+ original_cache = app.search_response_cache
906
+ app.search_response_cache = {}
907
+ try:
908
+ base = app.SearchRequest(q="手机")
909
+ variants = [
910
+ app.SearchRequest(q="手机", page=2),
911
+ app.SearchRequest(q="手机", page_size=1),
912
+ app.SearchRequest(q="手机", exact=True),
913
+ app.SearchRequest(q="手机", fulltext=True),
914
+ app.SearchRequest(q="手机", search_paths=False),
915
+ app.SearchRequest(q="手机", sort="name"),
916
+ app.SearchRequest(q="手机", sources=["CCRD"]),
917
+ app.SearchRequest(q="手机", folders=["历史"]),
918
+ app.SearchRequest(q="手机", min_size=1),
919
+ app.SearchRequest(q="手机", max_size=100),
920
+ ]
921
+ app.run_search(base)
922
+ for variant in variants:
923
+ app.run_search(variant)
924
+ keys = [key for key in app.search_response_cache if key[0] == "search"]
925
+ self.assertEqual(len(keys), 1 + len(variants))
926
+ finally:
927
+ app.search_response_cache = original_cache
928
+
929
+ def test_api_cache_expires_and_evicts_oldest_entry(self):
930
+ original_cache = app.api_response_cache
931
+ app.api_response_cache = {}
932
+ try:
933
+ with patch.object(app.time, "monotonic", side_effect=[0, 0, 0, app.API_CACHE_TTL_SECONDS + 1, app.API_CACHE_TTL_SECONDS + 1]):
934
+ first = app.cached_payload(("sources",), lambda: object())
935
+ cached = app.cached_payload(("sources",), lambda: object())
936
+ expired = app.cached_payload(("sources",), lambda: object())
937
+ self.assertIs(first, cached)
938
+ self.assertIsNot(first, expired)
939
+ app.api_response_cache = {
940
+ ("entry", index): (float(index), index) for index in range(501)
941
+ }
942
+ with patch.object(app.time, "monotonic", return_value=1000):
943
+ app.cached_payload(("new",), lambda: "value")
944
+ self.assertLessEqual(len(app.api_response_cache), app.API_CACHE_MAX_ENTRIES)
945
+ self.assertNotIn(("entry", 0), app.api_response_cache)
946
+ finally:
947
+ app.api_response_cache = original_cache
948
+
949
+ def test_search_and_api_caches_evict_independently(self):
950
+ original_api_cache = app.api_response_cache
951
+ original_search_cache = app.search_response_cache
952
+ app.api_response_cache = {("sources",): (1.0, "sources")}
953
+ app.search_response_cache = {("search", 1): (1.0, "results")}
954
+ try:
955
+ with patch.object(app, "API_CACHE_MAX_ENTRIES", 1):
956
+ app.cached_payload(("folders", "CCRD"), lambda: [])
957
+ self.assertEqual(app.search_response_cache[("search", 1)][1], "results")
958
+ with patch.object(app, "SEARCH_CACHE_MAX_ENTRIES", 1):
959
+ app.cached_search_payload(("search", 2), lambda: {"results": []})
960
+ self.assertIn(("folders", "CCRD"), app.api_response_cache)
961
+ finally:
962
+ app.api_response_cache = original_api_cache
963
+ app.search_response_cache = original_search_cache
964
+
965
+
966
+ class FrontendContractTests(unittest.TestCase):
967
+ def test_every_static_control_exists_is_cached_and_has_behavior(self):
968
+ html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
969
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
970
+ parser = InteractiveIdParser()
971
+ parser.feed(html)
972
+ expected = {
973
+ "hamburger-btn", "header-logo", "search-input", "settings-btn", "mobile-toggle-btn",
974
+ "theme-btn", "sidebar-expand-btn", "multi-select-toggle", "clear-filters-btn", "sort-select",
975
+ "multi-select-all", "multi-deselect", "multi-batch-download", "multi-zip-download",
976
+ "empty-random-btn", "close-filters-btn", "fulltext-toggle", "search-paths-toggle",
977
+ "history-toggle", "exact-search-toggle", "folder-select-all", "folder-deselect-all",
978
+ "source-filter-cancel", "folder-filter-cancel",
979
+ "filter-min-size", "filter-min-unit", "filter-max-size", "filter-max-unit", "random-book-btn",
980
+ }
981
+ self.assertEqual(parser.ids, expected)
982
+ selector_exceptions = {"header-logo", "search-input"}
983
+ for control_id in expected - selector_exceptions:
984
+ self.assertIn(f'$("#{control_id}")', source, control_id)
985
+ self.assertIn('DOM.headerLogo = $("#header-logo")', source)
986
+ self.assertIn('DOM.searchInput = $("#search-input")', source)
987
+ event_objects = {
988
+ "hamburger-btn": "DOM.hamburgerBtn", "header-logo": "DOM.headerLogo",
989
+ "search-input": "DOM.searchInput", "settings-btn": "DOM.settingsBtn",
990
+ "mobile-toggle-btn": "DOM.mobileToggleBtn", "theme-btn": "DOM.themeBtn",
991
+ "sidebar-expand-btn": "DOM.sidebarExpandBtn", "multi-select-toggle": "DOM.multiSelectToggle",
992
+ "clear-filters-btn": "DOM.clearFiltersBtn", "sort-select": "DOM.sortSelect",
993
+ "multi-select-all": "DOM.multiSelectAll", "multi-deselect": "DOM.multiDeselect",
994
+ "multi-batch-download": "DOM.multiBatchDownload", "multi-zip-download": "DOM.multiZipDownload",
995
+ "empty-random-btn": "DOM.emptyRandomBtn", "close-filters-btn": "DOM.closeFiltersBtn",
996
+ "fulltext-toggle": "DOM.fulltextToggle", "search-paths-toggle": "DOM.searchPathsToggle",
997
+ "history-toggle": "DOM.historyToggle", "exact-search-toggle": "DOM.exactToggle",
998
+ "folder-select-all": "DOM.folderSelectAll", "folder-deselect-all": "DOM.folderDeselectAll",
999
+ "source-filter-cancel": "DOM.sourceFilterCancel", "folder-filter-cancel": "DOM.folderFilterCancel",
1000
+ "filter-min-size": "DOM.filterMinSize", "filter-min-unit": "DOM.filterMinUnit",
1001
+ "filter-max-size": "DOM.filterMaxSize", "filter-max-unit": "DOM.filterMaxUnit",
1002
+ "random-book-btn": "DOM.randomBookBtn",
1003
+ }
1004
+ for control_id, object_name in event_objects.items():
1005
+ self.assertIn(f'{object_name}.addEventListener(', source, control_id)
1006
+
1007
+ def test_section_filter_cancel_buttons_clear_only_their_selected_state(self):
1008
+ html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
1009
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1010
+ css = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
1011
+ for control_id in ("source-filter-cancel", "folder-filter-cancel"):
1012
+ self.assertRegex(html, rf'id="{control_id}"[^>]*hidden>取消<')
1013
+ self.assertIn("DOM.sourceFilterCancel.hidden = STATE.selectedSources.length === 0", source)
1014
+ self.assertIn("DOM.folderFilterCancel.hidden = STATE.folderSelections.length === 0", source)
1015
+ self.assertIn('DOM.folderFilterCancel.addEventListener("click", () => applyFolderSelections([]))', source)
1016
+ self.assertIn(".filter-title-leading", css)
1017
+
1018
+ def test_dynamic_controls_have_delegated_event_roots(self):
1019
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1020
+ self.assertIn('DOM.resultsList.addEventListener("click"', source)
1021
+ self.assertIn('DOM.sidebarContent.addEventListener("click"', source)
1022
+ self.assertIn('DOM.previewPanel.addEventListener("click"', source)
1023
+ self.assertIn('DOM.searchHistoryDropdown.addEventListener("click"', source)
1024
+ for selector in (
1025
+ ".result-checkbox", "[data-action='preview']", ".path-folder[data-folder]",
1026
+ "[data-source]", "#back-home", "[data-folder-open]", "[data-folder-nav]",
1027
+ "[data-doc-id]", "[data-action='close-preview']", "[data-action='toggle-preview-theme']", "[data-query]",
1028
+ ):
1029
+ self.assertIn(selector, source)
1030
+
1031
+ def test_mobile_result_paths_keep_separators_with_their_folder(self):
1032
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1033
+ css = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
1034
+ self.assertIn("${highlightText(part, STATE.query)}${separator}</span>", source)
1035
+ self.assertNotIn('<span class="path-sep">/</span><span class="path-folder"', source)
1036
+ self.assertIn("body.mobile .result-path", css)
1037
+ self.assertIn("word-break: break-all", css)
1038
+ self.assertIn("overflow-wrap: normal", css)
1039
+
1040
+ def test_highlighting_is_bounded_for_common_single_character_queries(self):
1041
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1042
+ self.assertIn("const MAX_HIGHLIGHTS_PER_TEXT = 16", source)
1043
+ self.assertIn("if (!remaining) return match", source)
1044
+
1045
+ def test_search_debounce_allows_ime_input_to_settle(self):
1046
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1047
+ self.assertIn("const SEARCH_DEBOUNCE_MS = 120", source)
1048
+ self.assertIn("}, SEARCH_DEBOUNCE_MS);", source)
1049
+
1050
+ def test_visible_page_keeps_the_search_connection_warm(self):
1051
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1052
+ html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
1053
+ self.assertIn("const KEEPALIVE_INTERVAL_MS = 45 * 1000", source)
1054
+ self.assertIn('fetch("/api/ping"', source)
1055
+ self.assertIn('fetch("/api/ping", { cache: "no-store" }).catch(() => {})', html)
1056
+ self.assertLess(html.index('fetch("/api/ping"'), html.index('<link rel="stylesheet"'))
1057
+ self.assertIn("if (document.hidden || !navigator.onLine || keepalivePending)", source)
1058
+ self.assertIn('window.addEventListener("online", () => warmConnection(true))', source)
1059
+ self.assertIn('document.addEventListener("visibilitychange"', source)
1060
+ self.assertIn("window.setInterval(() => warmConnection(), KEEPALIVE_INTERVAL_MS)", source)
1061
+
1062
+ def test_visible_results_lazily_load_exact_snippets(self):
1063
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1064
+ self.assertIn("const SNIPPET_CONCURRENCY = 4", source)
1065
+ self.assertIn('priority: "low"', source)
1066
+ self.assertIn("function pauseSnippetLoadingForPreview()", source)
1067
+ self.assertIn("pauseSnippetLoadingForPreview();", source)
1068
+ self.assertIn('priority: "high"', source)
1069
+ append_results = source[source.index("function appendResults(items)") : source.index("function renderResults({ animate = false } = {})")]
1070
+ self.assertNotIn("animateResultRows", append_results)
1071
+ self.assertNotIn("preview-loading", source)
1072
+ self.assertIn("new IntersectionObserver", source)
1073
+ self.assertIn("API.snippet(task.docId, task.query, task.exact, controller.signal)", source)
1074
+ self.assertIn("for (const controller of snippetControllers) controller.abort()", source)
1075
+ self.assertIn("task.sequence !== STATE.searchSequence", source)
1076
+ self.assertIn("data.highlights || []", source)
1077
+ self.assertIn('if (node.dataset.snippetPending !== "1") return', source)
1078
+ self.assertIn('node.querySelector(".snippet-loading")', source)
1079
+ self.assertIn("正在提取摘要...", source)
1080
+ self.assertIn("function renderResults({ animate = false } = {}) {\n resetSnippetObserver();", source)
1081
+
1082
+ def test_zip_download_uses_native_form_instead_of_buffering_blob(self):
1083
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1084
+ self.assertIn('fetch("/api/zip-prepare"', source)
1085
+ self.assertIn("window.location.assign(data.download_url)", source)
1086
+ self.assertNotIn('const blob = await resp.blob()', source)
1087
+ self.assertIn("合并下载最多支持 500 个文件", source)
1088
+ self.assertNotIn("zip-download-frame", source)
1089
+
1090
+ def test_results_reserve_snippet_height_and_clear_stale_selection(self):
1091
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1092
+ style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
1093
+ html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
1094
+ self.assertIn("if (STATE.page === 1 && STATE.selectedIds.size)", source)
1095
+ self.assertIn("min-height: 1.5em", style)
1096
+ self.assertIn("-webkit-line-clamp: 3", style)
1097
+ self.assertIn("content-visibility: auto", style)
1098
+ self.assertIn("contain-intrinsic-size: auto 112px", style)
1099
+ self.assertIn('id="multi-select-all" class="multi-action-btn">全选</button>', html)
1100
+ self.assertNotIn("全选已加载", html)
1101
+
1102
+ def test_frontend_motion_preserves_loading_and_accessibility_contracts(self):
1103
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1104
+ style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
1105
+ html = (Path(app.BASE_DIR) / "static" / "index.html").read_text(encoding="utf-8")
1106
+ self.assertIn('matchMedia("(prefers-reduced-motion: reduce)")', source)
1107
+ self.assertIn('setSearchMotion(true, STATE.page === 1 && STATE.results.length > 0)', source)
1108
+ self.assertIn('DOM.resultsContainer.setAttribute("aria-busy", loading ? "true" : "false")', source)
1109
+ self.assertIn("function renderResults({ animate = false } = {})", source)
1110
+ self.assertIn("if (animate) animateResultRows", source)
1111
+ self.assertIn("renderResults({ animate: true })", source)
1112
+ self.assertIn("Math.round(24 * Math.sqrt(index))", source)
1113
+ self.assertIn('classList.add("preview-enter")', source)
1114
+ self.assertIn(".result-item.result-enter", style)
1115
+ self.assertIn("animation: result-enter 320ms cubic-bezier(0.2, 0, 0, 1) both", style)
1116
+ self.assertIn(".results-container.is-refreshing .results-list", style)
1117
+ self.assertIn("--motion-emphasized-easing: cubic-bezier(0.05, 0.7, 0.1, 1)", style)
1118
+ self.assertIn("--motion-spring-easing: cubic-bezier(0.18, 1.32, 0.32, 1)", style)
1119
+ self.assertIn(".preview-panel.preview-enter .preview-body", style)
1120
+ self.assertIn("transition: width 150ms ease, opacity 150ms ease", style)
1121
+ self.assertIn('document.querySelectorAll(".theme-icon-svg")', source)
1122
+ self.assertIn("icon.getAnimations().forEach", source)
1123
+ self.assertIn('DOM.resultsList.classList.add("theme-static")', source)
1124
+ self.assertIn(".results-list.theme-static *", style)
1125
+ self.assertNotIn("document.body.offsetWidth", source)
1126
+ self.assertNotIn("theme-transitioning", source + style)
1127
+ self.assertNotIn("sidebar-opening", source + style)
1128
+ self.assertIn("@media (prefers-reduced-motion: reduce)", style)
1129
+ self.assertIn('id="results-container" class="results-container" aria-busy="false"', html)
1130
+ self.assertIn('id="toast" class="toast" role="status" aria-live="polite"', html)
1131
+
1132
+ def test_preview_open_limits_text_selection_to_the_preview_panel(self):
1133
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1134
+ style = (Path(app.BASE_DIR) / "static" / "style.css").read_text(encoding="utf-8")
1135
+ do_search = source[source.index("async function doSearch()") : source.index("function resultPathHtml")]
1136
+ self.assertNotIn('DOM.previewPanel.style.display = "none"', do_search)
1137
+ self.assertNotIn('document.body.classList.remove("preview-open")', do_search)
1138
+ self.assertIn('document.body.classList.add("preview-open")', source)
1139
+ self.assertIn('document.body.classList.remove("preview-open")', source)
1140
+ self.assertIn("body.preview-open {\n user-select: none;", style)
1141
+ self.assertIn("body.preview-open #preview-panel {\n user-select: text;", style)
1142
+ self.assertIn("body.preview-open #preview-panel .preview-header {\n user-select: none;", style)
1143
+
1144
+ def test_frontend_guards_initial_search_races_and_popup_blocking(self):
1145
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1146
+ self.assertIn("async function applyInitialSearch(sequence)", source)
1147
+ self.assertIn("sequence !== STATE.searchSequence", source)
1148
+ self.assertIn("if (await applyInitialSearch(sequence)) return", source)
1149
+ self.assertIn('frame.src = `/api/download/${encodeURIComponent(id)}`', source)
1150
+ self.assertNotIn("window.open(`/api/download/", source)
1151
+ self.assertIn('if (event.key === "Escape")', source)
1152
+ self.assertIn('showToast("随机文章加载失败")', source)
1153
+ self.assertIn('DOM.searchInput.placeholder = STATE.fulltext ?', source)
1154
+ self.assertIn('if (!resp.ok) throw new Error(`HTTP ${resp.status}`)', source)
1155
+ self.assertIn('if (value && value.indexing) return', source)
1156
+ self.assertIn('if (STATE.page > 1) STATE.page -= 1', source)
1157
+ self.assertIn('showToast(timedOut ? "搜索超时,请重试" : "搜索失败")', source)
1158
+ self.assertIn("text = highlightText(rawText, keyword)", source)
1159
+
1160
+ def test_sidebar_uses_inline_payload_before_network(self):
1161
+ source = (Path(app.BASE_DIR) / "static" / "app.js").read_text(encoding="utf-8")
1162
+ self.assertIn('readInlineJson("initial-sidebar-data")', source)
1163
+ self.assertIn('readInlineJson("initial-sidebar-sources")', source)
1164
+
1165
+ def test_server_embeds_route_sidebar_payload(self):
1166
+ html = app.render_index_html("CCRD")
1167
+ self.assertIn('id="initial-sidebar-data"', html)
1168
+ self.assertIn('id="initial-sidebar-sources"', html)
1169
+ self.assertIn('"source":"CCRD"', html)
1170
+
1171
+ def test_docker_build_generates_indexes_in_image(self):
1172
+ dockerfile = (Path(app.BASE_DIR) / "Dockerfile").read_text(encoding="utf-8")
1173
+ self.assertIn("FROM node:26.11.1-bookworm-slim AS static-assets", dockerfile)
1174
+ self.assertIn("ARG ESBUILD_VERSION=0.28.2", dockerfile)
1175
+ self.assertIn("esbuild static/app.js --minify --target=es2019 --charset=utf8 --legal-comments=none", dockerfile)
1176
+ self.assertIn("esbuild static/style.css --minify", dockerfile)
1177
+ self.assertIn("esbuild static/sw.js --minify --target=es2019 --charset=utf8 --legal-comments=none", dockerfile)
1178
+ self.assertIn("COPY --from=static-assets /out/ static/", dockerfile)
1179
+ self.assertEqual(dockerfile.count("FROM node:"), 1)
1180
+ self.assertEqual(dockerfile.count("FROM python:"), 1)
1181
+ self.assertIn("COPY CCRD/ CCRD/", dockerfile)
1182
+ self.assertIn("COPY CW/ CW/", dockerfile)
1183
+ self.assertIn("COPY app.py build_fulltext_db.py prepare_fulltext_index.py start.sh ./", dockerfile)
1184
+ self.assertIn('CMD ["sh", "/app/start.sh"]', dockerfile)
1185
+ self.assertNotIn("RUN python build_fulltext_db.py", dockerfile)
1186
+ start_script = (Path(app.BASE_DIR) / "start.sh").read_text(encoding="utf-8")
1187
+ self.assertIn("python /app/prepare_fulltext_index.py", start_script)
1188
+ self.assertIn("python /app/build_fulltext_db.py", start_script)
1189
+ self.assertIn(") &", start_script)
1190
+ self.assertIn("exec uvicorn", start_script)
1191
+
1192
+ def test_generated_indexes_are_excluded_from_build_context(self):
1193
+ dockerignore = (Path(app.BASE_DIR) / ".dockerignore").read_text(encoding="utf-8")
1194
+ gitattributes = (Path(app.BASE_DIR) / ".gitattributes").read_text(encoding="utf-8")
1195
+ self.assertIn("data/fulltext/", dockerignore)
1196
+ self.assertNotIn("data/fulltext/CCRD.sqlite3", gitattributes)
1197
+ self.assertNotIn("data/fulltext/CW.sqlite3", gitattributes)
1198
+
1199
+ def test_service_worker_only_caches_explicit_static_assets(self):
1200
+ source = (Path(app.BASE_DIR) / "static" / "sw.js").read_text(encoding="utf-8")
1201
+ precache = source.split("];", 1)[0]
1202
+ self.assertNotIn('"/",', precache)
1203
+ self.assertNotIn("/data/initial/manifest.json", precache)
1204
+ self.assertNotIn("/data/sidebar/manifest.json", precache)
1205
+ self.assertNotIn("cacheManifestUrls", source)
1206
+ self.assertIn('event.request.method !== "GET"', source)
1207
+ self.assertIn('url.pathname.startsWith("/static/")', source)
1208
+ self.assertIn('url.pathname.startsWith("/data/initial/")', source)
1209
+ self.assertIn('url.pathname.startsWith("/data/sidebar/")', source)
1210
+ self.assertIn("if (!cacheable)", source)
1211
+
1212
+
1213
+ if __name__ == "__main__":
1214
+ unittest.main()
tests/test_search_oracle.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import gzip
4
+ import json
5
+ import os
6
+ import re
7
+ import time
8
+ import unicodedata
9
+ import unittest
10
+ from concurrent.futures import ThreadPoolExecutor
11
+ from pathlib import Path
12
+ from urllib.error import HTTPError, URLError
13
+ from urllib.request import Request, urlopen
14
+
15
+
16
+ BASE_DIR = Path(__file__).resolve().parents[1]
17
+ BASE_URL = os.environ.get("CCRD_ORACLE_BASE_URL", "").rstrip("/")
18
+ TOKEN_RE = re.compile(r"[a-z0-9]+|[\u4e00-\u9fff\u3400-\u4dbf]+")
19
+
20
+
21
+ def normalize(value: str) -> str:
22
+ return unicodedata.normalize("NFKC", value or "").casefold()
23
+
24
+
25
+ def unique(values):
26
+ return list(dict.fromkeys(values))
27
+
28
+
29
+ def parts(query: str) -> list[str]:
30
+ return unique(TOKEN_RE.findall(normalize(query)))
31
+
32
+
33
+ def query_tokens(query: str) -> list[str]:
34
+ result = []
35
+ for part in parts(query):
36
+ if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", part) and len(part) > 1:
37
+ result.extend(part[index:index + 2] for index in range(len(part) - 1))
38
+ else:
39
+ result.append(part)
40
+ return unique(result)
41
+
42
+
43
+ def body_tokens(text: str) -> set[str]:
44
+ result = set()
45
+ for part in TOKEN_RE.findall(text):
46
+ if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", part):
47
+ result.update(part)
48
+ result.update(part[index:index + 2] for index in range(len(part) - 1))
49
+ else:
50
+ result.add(part)
51
+ return result
52
+
53
+
54
+ def wildcard_regex(query: str):
55
+ pattern = []
56
+ for character in normalize(query):
57
+ pattern.append(".*?" if character == "*" else "." if character == "?" else re.escape(character))
58
+ return re.compile("".join(pattern), re.DOTALL)
59
+
60
+
61
+ def metadata_text(record: dict, search_paths: bool) -> str:
62
+ fields = [record["display_name"], record["source_name"]]
63
+ if search_paths:
64
+ fields.append(record["display_rel_path"])
65
+ return normalize(" ".join(fields))
66
+
67
+
68
+ def metadata_match(record: dict, query: str, exact: bool, search_paths: bool) -> bool:
69
+ text = metadata_text(record, search_paths)
70
+ if exact:
71
+ return bool(wildcard_regex(query).search(text)) if "*" in query or "?" in query else normalize(query) in text
72
+ terms = parts(query)
73
+ return bool(terms) and all(term in text for term in terms)
74
+
75
+
76
+ @unittest.skipUnless(BASE_URL, "set CCRD_ORACLE_BASE_URL to run brute-force corpus oracle")
77
+ class ProductionSearchOracleTests(unittest.TestCase):
78
+ @classmethod
79
+ def setUpClass(cls):
80
+ payload = json.loads(gzip.decompress((BASE_DIR / "data/search_data.json.gz").read_bytes()))
81
+ cls.records = payload["records"]
82
+ cls.by_id = {record["doc_id"]: record for record in cls.records}
83
+ status = cls.request_json("/api/index-status")
84
+ if not status.get("ready"):
85
+ raise unittest.SkipTest(f'fulltext indexes are not ready: {status.get("building_sources", [])}')
86
+ cls.matrix = [
87
+ ("手机", True), ("手。机", True), ("文化革命", True),
88
+ ("手*机", True), ("手?机", True), ("文化 革命", False),
89
+ ("ABC", True), ("ABC", True), ("hair", True), ("hair", False),
90
+ ]
91
+ cls.body_matches = {(query, exact): set() for query, exact in cls.matrix}
92
+ started = time.monotonic()
93
+ for record in cls.records:
94
+ text = normalize((BASE_DIR / record["storage_root"] / record["storage_rel_path"]).read_text(encoding="utf-8", errors="ignore"))
95
+ tokens = None
96
+ for query, exact in cls.matrix:
97
+ if exact:
98
+ matched = bool(wildcard_regex(query).search(text)) if "*" in query or "?" in query else normalize(query) in text
99
+ else:
100
+ if tokens is None:
101
+ tokens = body_tokens(text)
102
+ matched = all(token in tokens for token in query_tokens(query))
103
+ if matched:
104
+ cls.body_matches[(query, exact)].add(record["doc_id"])
105
+ print(f"oracle_scanned={len(cls.records)} elapsed_seconds={time.monotonic() - started:.2f}")
106
+
107
+ @classmethod
108
+ def request_json(cls, path: str, payload: dict | None = None):
109
+ data = json.dumps(payload).encode() if payload is not None else None
110
+ for attempt in range(3):
111
+ try:
112
+ with urlopen(Request(BASE_URL + path, data=data, headers={"Content-Type": "application/json"} if data else {}), timeout=120) as response:
113
+ return json.load(response)
114
+ except (HTTPError, URLError, TimeoutError):
115
+ if attempt == 2:
116
+ raise
117
+ time.sleep(attempt + 1)
118
+
119
+ @classmethod
120
+ def api_ids(cls, body: dict, path: str = "/api/search") -> tuple[set[str], int]:
121
+ first = cls.request_json(path, {**body, "page": 1, "page_size": 200})
122
+ total = first["total"]
123
+ page_count = (total + 199) // 200
124
+ ids_by_page = {1: [item["doc_id"] for item in first["results"]]}
125
+
126
+ def fetch(page: int):
127
+ payload = cls.request_json(path, {**body, "page": page, "page_size": 200})
128
+ if payload["total"] != total:
129
+ raise AssertionError(f"total changed during pagination for {body}: {payload['total']} != {total}")
130
+ return page, [item["doc_id"] for item in payload["results"]]
131
+
132
+ with ThreadPoolExecutor(max_workers=6) as executor:
133
+ for page, page_ids in executor.map(fetch, range(2, page_count + 1)):
134
+ ids_by_page[page] = page_ids
135
+ ids = [doc_id for page in range(1, page_count + 1) for doc_id in ids_by_page[page]]
136
+ if len(ids) != len(set(ids)):
137
+ raise AssertionError(f"duplicate IDs for {body}")
138
+ if len(ids) != total:
139
+ raise AssertionError(f"pagination did not cover total for {body}: {len(ids)} != {total}")
140
+ return set(ids), total
141
+
142
+ def expected_ids(self, query: str, exact: bool, fulltext: bool = True, search_paths: bool = True, sources=None, folders=None, min_size=None, max_size=None):
143
+ body = self.body_matches[(query, exact)] if fulltext else set()
144
+ result = set()
145
+ for record in self.records:
146
+ if record["doc_id"] not in body and not metadata_match(record, query, exact, search_paths):
147
+ continue
148
+ if sources and record["source"] not in sources:
149
+ continue
150
+ rel_dir = "/".join(record["display_dirs"])
151
+ if folders and not any(rel_dir == folder or rel_dir.startswith(folder + "/") for folder in folders):
152
+ continue
153
+ if min_size is not None and record["size"] < min_size:
154
+ continue
155
+ if max_size is not None and record["size"] > max_size:
156
+ continue
157
+ result.add(record["doc_id"])
158
+ return result
159
+
160
+ def test_full_result_sets_and_totals_match_independent_oracle(self):
161
+ for query, exact in unique(self.matrix):
162
+ with self.subTest(query=query, exact=exact):
163
+ body = {"q": query, "exact": exact, "fulltext": True, "search_paths": True, "sort": "name"}
164
+ actual, total = self.api_ids(body)
165
+ expected = self.expected_ids(query, exact)
166
+ self.assertEqual(total, len(expected))
167
+ self.assertEqual(actual, expected)
168
+
169
+ def test_filters_modes_and_normalization_match_oracle(self):
170
+ query, exact = "文化 革命", False
171
+ folder = self.records[0]["display_dirs"][0]
172
+ variants = [
173
+ {"fulltext": False, "search_paths": True},
174
+ {"fulltext": False, "search_paths": False},
175
+ {"fulltext": True, "search_paths": False, "sources": ["CCRD"]},
176
+ {"fulltext": True, "search_paths": True, "sources": ["CW"]},
177
+ {"fulltext": True, "search_paths": True, "sources": ["CCRD"], "folders": [folder]},
178
+ {"fulltext": True, "search_paths": True, "min_size": 1000, "max_size": 10000},
179
+ ]
180
+ for options in variants:
181
+ with self.subTest(options=options):
182
+ body = {"q": query, "exact": exact, "sort": "name", **options}
183
+ actual, total = self.api_ids(body)
184
+ expected = self.expected_ids(query, exact, **options)
185
+ self.assertEqual(total, len(expected))
186
+ self.assertEqual(actual, expected)
187
+ self.assertEqual(self.body_matches[("ABC", True)], self.body_matches[("ABC", True)])
188
+
189
+ def test_sorting_page_sizes_source_route_and_exact_snippets(self):
190
+ base = {"q": "手机", "exact": True, "fulltext": True, "search_paths": True, "sources": ["CW"]}
191
+ expected = self.expected_ids("手机", True, sources=["CW"])
192
+ reconstructed = []
193
+ for page_size in (17, 100, 200):
194
+ page = 1
195
+ ids = []
196
+ while len(ids) < len(expected):
197
+ payload = self.request_json("/api/search", {**base, "sort": "size", "page": page, "page_size": page_size})
198
+ batch = payload["results"]
199
+ if not batch:
200
+ break
201
+ ids.extend(item["doc_id"] for item in batch)
202
+ sizes = [item["Size"] for item in batch]
203
+ self.assertEqual(sizes, sorted(sizes, reverse=True))
204
+ page += 1
205
+ self.assertEqual(set(ids), expected)
206
+ self.assertEqual(len(ids), len(expected))
207
+ reconstructed.append(set(ids))
208
+ self.assertTrue(all(result == reconstructed[0] for result in reconstructed))
209
+
210
+ route_ids, route_total = self.api_ids({**base, "sort": "name", "sources": ["CW"]}, "/api/search/CCRD")
211
+ expected_ccrd = self.expected_ids("手机", True, sources=["CCRD"])
212
+ self.assertEqual(route_total, len(expected_ccrd))
213
+ self.assertEqual(route_ids, expected_ccrd)
214
+
215
+ for query in ("手机", "文化革命", "手*机", "手?机"):
216
+ payload = self.request_json("/api/search", {
217
+ "q": query, "exact": True, "fulltext": True, "search_paths": True,
218
+ "page": 1, "page_size": 20,
219
+ })
220
+ pattern = wildcard_regex(query) if "*" in query or "?" in query else None
221
+ for item in payload["results"]:
222
+ record = self.by_id[item["doc_id"]]
223
+ content = normalize((BASE_DIR / record["storage_root"] / record["storage_rel_path"]).read_text(encoding="utf-8", errors="ignore"))
224
+ metadata = metadata_text(record, True)
225
+ self.assertTrue(bool(pattern.search(content) or pattern.search(metadata)) if pattern else normalize(query) in content or normalize(query) in metadata)
226
+
227
+
228
+ if __name__ == "__main__":
229
+ unittest.main()