Spaces:
Running
Running
Add parse crop controls and motion
Browse files- AGENTS.md +456 -0
- static/app.js +285 -12
- static/style.css +104 -0
- tests/test_api.py +1062 -0
- tests/test_browser.py +627 -0
- tests/test_frontend.py +242 -0
AGENTS.md
ADDED
|
@@ -0,0 +1,456 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BHA Search Agent Guide
|
| 2 |
+
|
| 3 |
+
Read this file before changing `bha-search-lite`. This project is a Chinese-focused, accuracy-first search service for the BHA article corpus. It is deployed as a Hugging Face Space and combines Elasticsearch with SQLite-derived metadata, summaries, facets, and warmup data.
|
| 4 |
+
|
| 5 |
+
## Deployment
|
| 6 |
+
|
| 7 |
+
- Production: `https://vomebook-bha-search.hf.space/`
|
| 8 |
+
- Runtime: FastAPI, Uvicorn, and Elasticsearch on Hugging Face Spaces
|
| 9 |
+
- Search engine: Elasticsearch
|
| 10 |
+
- Derived local data: SQLite stores for documents, summaries, facets, and warmup paths
|
| 11 |
+
- Checkout index-version contract: `INDEX_VERSION = "1.0.0"`; confirm the deployed version through production readiness metadata after deployment.
|
| 12 |
+
- Parsed repositories default to `https://github.com/anftm`; only `BHA_PARSED_REPO_PREFIX` may override this. The legacy generic `REPO_PREFIX` is intentionally ignored so a stale Space variable cannot silently switch production back to the upstream archives.
|
| 13 |
+
- This index version is not a cache-busting version. Do not bump it merely because files were pushed; Hugging Face rebuilds the Space after deployment.
|
| 14 |
+
|
| 15 |
+
## Important Files
|
| 16 |
+
|
| 17 |
+
- `app/main.py`: FastAPI composition root, middleware/lifecycle wiring, router registration, static files, and SPA fallback.
|
| 18 |
+
- `app/search_logic.py`: `SearchRequest`, normal/exact query DSL, literal `*`/`?` handling, date filters, sorting, local highlighting, and snippets.
|
| 19 |
+
- `app/proofread_core.py`: pure PatchV2 validation, UTF-16 delta replay, change previews, and auto-merge cost policy evaluation.
|
| 20 |
+
- `app/parse_core.py`: parse request models, queue DTOs, page-range coverage, date/OCR validation, and generated article-ID collision checks.
|
| 21 |
+
- `app/pipeline_dispatch.py`: repository-dispatch transport, Git blob chunking, and payload integrity metadata.
|
| 22 |
+
- `app/document_identity.py`: stable content identity shared by preview, parse, and proofreading submissions.
|
| 23 |
+
- `app/source_files.py`: source mirror state, SSRF-safe upstream access, source previews, and TXT/source/ZIP download routes.
|
| 24 |
+
- `app/runtime.py`: Elasticsearch client, readiness/status, bounded response caches, facet/source routes, activity tracking, and auxiliary warmup.
|
| 25 |
+
- `app/bootstrap_api.py`: generation-scoped default search and initial facet payload for first-screen hydration.
|
| 26 |
+
- `app/search_api.py`: search HTTP route, Elasticsearch execution, literal candidate verification, and SQLite result assembly.
|
| 27 |
+
- `app/preview_api.py`: article/random preview routes, ETag handling, variants, and source-preview metadata assembly.
|
| 28 |
+
- `app/parse_api.py`: parse queue/OCR/submission routes and parse pipeline dispatch policy.
|
| 29 |
+
- `app/proofread_api.py`: proofreading validation policy, rate limiting, idempotency, submission route, and pipeline dispatch.
|
| 30 |
+
- `app/reindex_api.py`: authenticated reindex jobs, deferred bucket snapshot publication, and generation recovery coordination.
|
| 31 |
+
- `app/indexer.py`: Elasticsearch mappings, document ingestion, bulk indexing, readiness metadata, and final refresh.
|
| 32 |
+
- `app/data_loader.py`: source loading, normalization, metadata token fields, and record preparation.
|
| 33 |
+
- `app/doc_store.py`: SQLite document access and warmup reads.
|
| 34 |
+
- `app/facet_store.py`: SQLite facet storage and paginated facet reads.
|
| 35 |
+
- `scripts/start.sh`: Elasticsearch/FastAPI startup configuration.
|
| 36 |
+
- `Dockerfile`: multi-stage Elasticsearch extraction, esbuild static minification, and the final Python runtime image.
|
| 37 |
+
- `scripts/install-elasticsearch.sh`: build-stage Elasticsearch download and SHA-512 verification.
|
| 38 |
+
- `requirements.txt`: runtime dependencies, including the pinned jieba version.
|
| 39 |
+
- `tests/test_search.py`: query DSL, tokenization, mapping, filtering, highlighting, snippets, and indexing contracts.
|
| 40 |
+
- `tests/test_api.py`: FastAPI behavior, mocked Elasticsearch, error paths, and warmup contracts.
|
| 41 |
+
- `tests/test_data_loader.py`: parsed-corpus normalization and generated document contracts.
|
| 42 |
+
- `tests/test_indexer.py`: mapping metadata, activation/rollback, facet, and build contracts.
|
| 43 |
+
- `tests/test_stores.py`: real temporary SQLite document/facet round trips, exact counts, pagination, variants, and years.
|
| 44 |
+
- `tests/test_storage_lock.py`: serving/build generation lock contracts.
|
| 45 |
+
- `tests/test_downloads.py`: TXT/source/ZIP download contracts, archive integrity, limits, cleanup, and SSRF defenses.
|
| 46 |
+
- `tests/test_frontend.py`: static frontend control, escaping, cache, timeout, race, IME, preview, and filter contracts.
|
| 47 |
+
- `tests/test_browser.py`: optional Playwright desktop/mobile interaction and browser download E2E with mocked API responses.
|
| 48 |
+
- `tests/test_live_smoke.py`: read-only production smoke tests.
|
| 49 |
+
- `TESTING.md`: local, live, and manual testing instructions.
|
| 50 |
+
|
| 51 |
+
## Accuracy Priority
|
| 52 |
+
|
| 53 |
+
Search accuracy is more important than isolated indexing or query benchmarks. Do not make a performance change if it changes result semantics, exact totals, field coverage, or user-visible correctness without explicit approval.
|
| 54 |
+
|
| 55 |
+
This project does not use semantic or vector search. Do not add it as a “quality” improvement.
|
| 56 |
+
|
| 57 |
+
## Search Semantics
|
| 58 |
+
|
| 59 |
+
### Normal Search
|
| 60 |
+
|
| 61 |
+
- Multiple query terms must use AND semantics.
|
| 62 |
+
- Do not restore single-character OR behavior.
|
| 63 |
+
- Short metadata fields use normalized jieba-derived `*_words` fields.
|
| 64 |
+
- Full article content uses the Elasticsearch CJK analyzer.
|
| 65 |
+
- Normal query routes use boundary-aware CJK matching.
|
| 66 |
+
- Phrase matches may receive a relevance boost, but phrase boosts must not widen the result set.
|
| 67 |
+
- `手机` must not be treated as equivalent to `手。机`.
|
| 68 |
+
- Searching `文化 革命` must require both terms according to the existing route semantics.
|
| 69 |
+
- Do not reduce searched fields to improve latency.
|
| 70 |
+
- Do not use truncated candidates, approximate totals, or sampled results.
|
| 71 |
+
|
| 72 |
+
### Exact Search
|
| 73 |
+
|
| 74 |
+
- Exact title and publication/source matching uses literal fields. User-supplied `*` and `?`
|
| 75 |
+
are escaped before querying Elasticsearch wildcard fields.
|
| 76 |
+
- Exact content uses phrase matching through the single CJK-analyzed `content` field.
|
| 77 |
+
- Exact queries containing non-whitespace punctuation use Elasticsearch as a complete candidate/ranking oracle, then verify every candidate literally against SQLite metadata and content before recomputing total and pagination. This preserves literal `*`, `?`, and punctuation without a duplicate content index.
|
| 78 |
+
- `authors.literal` is reserved for exact-search author substring matching; author filters continue to use `authors.keyword`, and normal search continues to use `authors_words`.
|
| 79 |
+
- `*` and `?` are literal characters in both metadata and content searches; they are not
|
| 80 |
+
wildcard operators. Searches such as `手*机` and `手?机` must match those characters
|
| 81 |
+
literally in search results, snippets, and previews.
|
| 82 |
+
- Do not reintroduce `query_string`, unrestricted Lucene syntax, or arbitrary Elasticsearch query input.
|
| 83 |
+
- Do not describe the current implementation as intervals-based or as supporting content
|
| 84 |
+
wildcards. Those are removed behaviors.
|
| 85 |
+
|
| 86 |
+
### Dates, Filters, and Facets
|
| 87 |
+
|
| 88 |
+
- Date filters must preserve exact year/month/day shape semantics.
|
| 89 |
+
- Do not add broader date fallbacks to an exact date filter.
|
| 90 |
+
- Archive, author, source, tag, date, and other filters must remain composable with search mode.
|
| 91 |
+
- Facet pagination must return exact SQLite-backed counts according to the existing data model.
|
| 92 |
+
- Do not replace exact facet counts with estimates.
|
| 93 |
+
- Invalid dates and unknown facets must retain their tested behavior.
|
| 94 |
+
|
| 95 |
+
## Index and Data Rules
|
| 96 |
+
|
| 97 |
+
- Elasticsearch index data is derived and rebuildable.
|
| 98 |
+
- Index metadata `archive_revisions` contains parsed Git tree SHAs, not commit SHAs. Commit SHAs remain transport identities used to fetch exact refs; identical trees must not trigger indexing.
|
| 99 |
+
- Local parsed data, SQLite sidecars, serving locks, and Elasticsearch indexes are generated snapshots and may be stale. Never report local document or facet counts as production counts.
|
| 100 |
+
- Free Hugging Face Spaces do not provide dependable persistent volumes; do not assume Elasticsearch data survives a rebuild.
|
| 101 |
+
- A pushed file change rebuilds the Space, but a valid bucket snapshot is restored instead of regenerating Elasticsearch and SQLite. A stale snapshot remains available while a revision-scoped convergence plan runs in the background. If remote tree revisions cannot be verified, startup keeps the restored snapshot and waits for the pipeline's explicit revision update rather than cloning every archive.
|
| 102 |
+
- Incremental failure must preserve the serving generation and fail the update. A complete rebuild is allowed only when explicitly requested or when no valid serving generation exists; never turn an incremental safety failure into an implicit full rebuild.
|
| 103 |
+
- Keep `INDEX_VERSION = "1.0.0"` unless the user explicitly requests a logical index-version change.
|
| 104 |
+
- A file upload alone does not require an `INDEX_VERSION` bump.
|
| 105 |
+
- `build_fingerprint()` intentionally hashes the mapping plus explicit document/index/SQLite schema versions, not whole Python modules. Runtime-only edits must not force a rebuild. Bump the relevant schema constant when changing indexed document generation, facet generation, or SQLite schema semantics.
|
| 106 |
+
- Both mappings and query builders must be changed together when a field changes.
|
| 107 |
+
- If a mapping or analyzer changes, rebuild and run the complete index/query tests before deployment.
|
| 108 |
+
- Do not run jieba over the complete article body. The current design intentionally limits Python tokenization to short metadata.
|
| 109 |
+
- Do not add `content_words` or duplicate full-content CJK fields merely to chase a benchmark.
|
| 110 |
+
- Do not restore removed full-content wildcard or literal fields without measuring index size, build time, memory, and exactness.
|
| 111 |
+
- Preserve the current single primary CJK content field unless a deliberate mapping migration is approved.
|
| 112 |
+
- The approved `content.index_phrases` accelerator preserves the single content field and exact phrase semantics; assess its rebuild time, index size, and production phrase timings after each mapping generation.
|
| 113 |
+
- Elasticsearch indexes `content` for CJK search but excludes `content` from `_source`; the
|
| 114 |
+
field is not separately stored and has no term vectors. Elasticsearch therefore returns
|
| 115 |
+
matching IDs but is not the production source of result-list content highlights.
|
| 116 |
+
- Bulk ingestion limits are intentionally bounded:
|
| 117 |
+
|
| 118 |
+
```text
|
| 119 |
+
BULK_CHUNK_SIZE = 100
|
| 120 |
+
BULK_MAX_BYTES = 16 MiB
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
Do not increase them blindly for remote Elasticsearch. Verify memory and failure behavior first.
|
| 124 |
+
|
| 125 |
+
## Snapshot And Production Boundary
|
| 126 |
+
|
| 127 |
+
Generated files, parsed corpora, local SQLite sidecars, local Elasticsearch indexes, counts, timings, and discrepancies describe only the checked-out or locally rebuilt snapshot and may be stale. Do not report them as production facts. Use `https://vomebook-bha-search.hf.space/` as the post-deployment acceptance layer and require a ready index before accepting document counts or analyzer behavior.
|
| 128 |
+
|
| 129 |
+
## Index Build Settings
|
| 130 |
+
|
| 131 |
+
The current build configuration intentionally favors safe, observable rebuilds:
|
| 132 |
+
|
| 133 |
+
- `refresh_interval` remains `30s` during indexing.
|
| 134 |
+
- Elasticsearch translog uses the default synchronous behavior.
|
| 135 |
+
- Elasticsearch index buffer uses the default setting.
|
| 136 |
+
- The final build performs an explicit refresh before marking the index ready.
|
| 137 |
+
- Every completed build performs one `max_num_segments=1` force merge before marking ready; it preserves query semantics. Detailed phase and segment timing logs were removed after production acceptance.
|
| 138 |
+
- Progressive builds reset live sidecars and activate the empty build index under the exclusive serving lock. Any progressive failure removes the partial physical index, alias, and sidecars instead of serving incomplete results.
|
| 139 |
+
- Complete-generation activation checkpoints both incoming and currently served SQLite files, removes WAL/SHM companions, and preserves switch state when the Elasticsearch alias commit result is uncertain.
|
| 140 |
+
- Documents should become progressively visible during a rebuild.
|
| 141 |
+
|
| 142 |
+
Do not reintroduce these previously tested-and-rejected settings without new measurements and explicit approval:
|
| 143 |
+
|
| 144 |
+
```text
|
| 145 |
+
refresh_interval = -1
|
| 146 |
+
translog.durability = async
|
| 147 |
+
translog.sync_interval = 30s
|
| 148 |
+
indices.memory.index_buffer_size = 20%
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Do not add a final flush or dynamic `put_settings` cleanup solely to claim a faster build.
|
| 152 |
+
|
| 153 |
+
## Readiness Initialization And Connection Keepalive
|
| 154 |
+
|
| 155 |
+
- The server does not run fixed-query or periodic Elasticsearch search warmup. A concrete
|
| 156 |
+
query warms query-specific Lucene data and is not a general fix for browser connection
|
| 157 |
+
setup latency. Do not reintroduce representative search terms without new measurements.
|
| 158 |
+
- Jieba is initialized during FastAPI startup without issuing a search. This moves its
|
| 159 |
+
one-time dictionary load out of the first ordinary user query; it is tokenizer
|
| 160 |
+
initialization, not fixed-query Elasticsearch warmup.
|
| 161 |
+
- `initialize_when_ready()` polls index readiness, then runs one best-effort initialization
|
| 162 |
+
of SQLite document access, sources, years, facets, and static files.
|
| 163 |
+
- The auxiliary warmup continues every 5-9 minutes while no user request was active in the
|
| 164 |
+
prior 90 seconds. It warms SQLite/facets/static files and performs one result-free
|
| 165 |
+
Elasticsearch `match_all` query (`size: 0`, no total tracking, no `_source`) to reduce
|
| 166 |
+
cold working-set recovery. It must not issue a query containing user or fixed search terms.
|
| 167 |
+
- `warmup_db()` is intentional preview-path warmup, not search-term warmup. It reads document
|
| 168 |
+
content and performs a title lookup to cover the SQLite access patterns used by `get_doc()`
|
| 169 |
+
and `variants_for_title()` when opening a preview. Keep it in both ready-state and periodic
|
| 170 |
+
auxiliary warmup unless current preview measurements show it is unnecessary.
|
| 171 |
+
- Ready-state initialization must clear stale status, source, facet, and search response
|
| 172 |
+
caches before rebuilding auxiliary cached data. It must not issue Elasticsearch searches.
|
| 173 |
+
- Ready-state initialization must remain best-effort and must not make the API unavailable
|
| 174 |
+
when one auxiliary component fails.
|
| 175 |
+
- While the page is visible, the frontend sends `GET /api/ping` approximately every 45
|
| 176 |
+
seconds. It also pings after the page becomes visible, the browser comes online, or the
|
| 177 |
+
search input receives focus after the minimum gap.
|
| 178 |
+
- `/api/ping` returns an empty `204` with `Cache-Control: no-store`. It must not inspect
|
| 179 |
+
Elasticsearch, SQLite, readiness, facets, or source data.
|
| 180 |
+
- Browser connection keepalive does not prevent Space suspension and cannot preserve a
|
| 181 |
+
connection after the page closes, the browser freezes it, the network changes, or Hugging
|
| 182 |
+
Face replaces the route.
|
| 183 |
+
|
| 184 |
+
## Snippets and Highlighting
|
| 185 |
+
|
| 186 |
+
- Result-list metadata and complete article content are read from SQLite after Elasticsearch
|
| 187 |
+
returns the current page of document IDs.
|
| 188 |
+
- The search-specific SQLite projection excludes preview/download-only columns, including
|
| 189 |
+
source-file URLs and internal IDs. Do not replace it with `SELECT *` on the search path.
|
| 190 |
+
- Search result payloads include only fields used by the shipped frontend. They deliberately
|
| 191 |
+
omit `article_id`, `publication_id`, `archive_id`, `path`, and Elasticsearch `score`.
|
| 192 |
+
- `title_highlight` is included only when the title contains an actual `<mark>` match. The
|
| 193 |
+
frontend escapes and displays `title` when that optional field is absent.
|
| 194 |
+
- Production search requests deliberately do not request Elasticsearch highlights. The
|
| 195 |
+
`highlight` consumption and maximum-fragment guard in `trim_hit()` are compatibility
|
| 196 |
+
defenses for supplied hit payloads, not evidence that production Elasticsearch provides
|
| 197 |
+
snippets.
|
| 198 |
+
- Python builds bounded result snippets and `<mark>` highlights from the SQLite content.
|
| 199 |
+
`*` and `?` remain literal characters in this local path.
|
| 200 |
+
- Result-list SQLite reads use `app/search_store.py` to return only the first 220 characters or a bounded window around the earliest exact/normal highlight term. This is a query-only optimization and is intentionally excluded from the index build fingerprint; preview and download paths still read complete content.
|
| 201 |
+
- Do not restore stored Elasticsearch content, term vectors, a duplicate content field, or
|
| 202 |
+
Elasticsearch highlighting without measuring index size, build time, search memory, and
|
| 203 |
+
result semantics and completing a coherent index rebuild.
|
| 204 |
+
- Do not read entire large article bodies repeatedly for every page without first measuring
|
| 205 |
+
the SQLite read and local snippet phases. Any optimization must preserve useful snippets
|
| 206 |
+
for matches located beyond a short content prefix.
|
| 207 |
+
- Preserve HTML escaping in title, content, and snippet highlighting.
|
| 208 |
+
- Highlighting must never create an XSS path.
|
| 209 |
+
|
| 210 |
+
## Frontend Motion
|
| 211 |
+
|
| 212 |
+
- Successful search and parse-queue responses animate only the first 15 result rows. Non-request rerenders, including closing a preview or returning from the parse editor, must remain static.
|
| 213 |
+
- Preview entrance motion runs only when the user actively opens an article. URL restoration, theme/highlight changes, compare/proofread rerenders, and history restoration must not replay it.
|
| 214 |
+
- Preserve `prefers-reduced-motion` behavior for all result and preview entrance motion.
|
| 215 |
+
|
| 216 |
+
## Date Controls
|
| 217 |
+
|
| 218 |
+
- The custom year picker must include every year returned by the facets API. Keep the list scrollable rather than truncating available years.
|
| 219 |
+
- Default year-picker options include only four-digit numeric years. Do not guess, rewrite, or expose malformed year values as normal options.
|
| 220 |
+
- Before year facets are available, the picker fallback range is 1843 through 2025 inclusive.
|
| 221 |
+
|
| 222 |
+
## Caching and Readiness
|
| 223 |
+
|
| 224 |
+
- Preserve readiness response semantics and stable health responses.
|
| 225 |
+
- Search/status/source/facet caches must have bounded size and explicit TTL behavior.
|
| 226 |
+
- Responses of at least 500 bytes use application-level Brotli quality 5 when accepted and
|
| 227 |
+
fall back to GZip for older clients. Download routes are excluded. Hugging Face did not
|
| 228 |
+
compress measured production search responses on behalf of the app, so preserve this
|
| 229 |
+
middleware unless current end-to-end measurements establish an equivalent replacement.
|
| 230 |
+
- Source and facet response caches use a 600-second TTL. Search responses remain at 30
|
| 231 |
+
seconds and health/status responses remain at 5 seconds.
|
| 232 |
+
- `/api/bootstrap` combines the default empty-query page with archives/types/years and the
|
| 233 |
+
first source/author/tag facet pages. It keeps one server-side payload without a time TTL,
|
| 234 |
+
keyed by the current SQLite serving-generation token; a generation change replaces it.
|
| 235 |
+
Its HTTP response remains `no-store`, so browsers and intermediaries cannot serve an old
|
| 236 |
+
generation. The default frontend must hydrate from this one request and fall back to the
|
| 237 |
+
ordinary search/facet APIs when bootstrap is unavailable or URL state is non-default.
|
| 238 |
+
- The source-mirror manifest cache (BHA source-file proxy) is refreshed in the background every
|
| 239 |
+
`BHA_SOURCE_MIRROR_TTL_SECONDS` (default 600, configurable; e.g. 86400 once the mirror fill is
|
| 240 |
+
complete) and never stalls a request that finds a fresh cache. A failed refresh keeps the last
|
| 241 |
+
good manifest and backs off for the same TTL; requests fall back to the upstream URL either way.
|
| 242 |
+
- Mirrored source files are fetched directly from the mirror base URL with `follow_redirects=True`,
|
| 243 |
+
skipping the per-hop `open_source_response` IP validation. This is safe because the mirror URL is
|
| 244 |
+
constructed from the cached manifest (`mirrored_source_url` pins the host to
|
| 245 |
+
`SOURCE_MIRROR_BASE_URL` and restricts paths to `archives*` without `..`). Non-mirror URLs still go
|
| 246 |
+
through the full SSRF validation in `open_source_response`.
|
| 247 |
+
- Cache keys must include all query and filter options that affect results.
|
| 248 |
+
- Do not cache partial-build search responses as if they were final data.
|
| 249 |
+
- Do not allow stale pre-ready facet or source responses to survive into the ready state.
|
| 250 |
+
- Do not use cache invalidation as a substitute for correct index readiness.
|
| 251 |
+
- Temporary request timing headers and Elasticsearch profile routes were removed after production acceptance. Do not restore them as permanent public diagnostics.
|
| 252 |
+
|
| 253 |
+
## API Safety
|
| 254 |
+
|
| 255 |
+
Current safety constraints are intentional:
|
| 256 |
+
|
| 257 |
+
```text
|
| 258 |
+
query length <= 256 characters
|
| 259 |
+
page >= 1
|
| 260 |
+
page_size bounded by the existing API model
|
| 261 |
+
sort values restricted to supported modes
|
| 262 |
+
date and size values validated
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
- Keep Elasticsearch query construction server-side.
|
| 266 |
+
- Do not expose arbitrary DSL, query string, script, or field injection through request parameters.
|
| 267 |
+
- Unknown facet and invalid route behavior must remain stable.
|
| 268 |
+
- Preserve CORS and compression behavior unless explicitly requested.
|
| 269 |
+
- `POST /api/reindex` is an administrative index-update route. It performs an incremental update by default; `POST /api/reindex?full=true` explicitly requests the complete rebuild fallback. Both require the `BHA_REINDEX_TOKEN` Space Secret as a Bearer token, use a ten-minute accidental-repeat cooldown, and must never expose the token in URLs, frontend code, responses, or logs.
|
| 270 |
+
- Automated reindex callers send complete `archive_commits` and `archive_revisions` maps. The service fetches exact commits, verifies their tree SHAs, and skips network fetches for already matching local archives before indexing.
|
| 271 |
+
- An API-triggered index becomes `ready` after atomic index/sidecar activation. Bucket snapshot publication then runs in a background thread and reports its own status; snapshot latency or failure must not hold online revision acceptance open.
|
| 272 |
+
- `POST /api/proofread` accepts proofreading changes only for a `doc_id` present in the served SQLite generation. It derives archive/article/publication routing server-side, rate-limits submissions, and dispatches the pipeline `submit-proofread` workflow. Configure the Space Secret `BHA_PROOFREAD_GITHUB_TOKEN`; `BHA_PROOFREAD_PIPELINE_REPOSITORY` defaults to `anftm/pipeline`. Never expose the GitHub token in frontend code, responses, or logs.
|
| 273 |
+
- The initial parsing workbench is limited to archive 25 documents whose titles begin with `【文章待拆分】`. `GET /api/parse/queue` reads that queue from the served SQLite generation, and per-page OCR is fetched through the same-origin parse endpoint. The original-image pane provides a normalized draggable OCR crop for an article and optional current-page overrides; crop changes filter the OCR/body preview and are submitted as `content_thresholds`. `POST /api/parse` accepts only structured article metadata, page ranges, edited content, OCR options, and the bounded OCR baseline size; it requires complete page coverage and rejects duplicate generated article IDs. The separate pipeline `submit-parse` workflow creates one config PR and one batch `ocr_patch` PR so edited bodies persist and later proofreading can continue from them. It must never accept complete TypeScript config content from the browser, and both parsing PRs always require manual review.
|
| 274 |
+
- Proofreading body changes use the upstream PatchV2 structure and target `ocr_patch`. Supported metadata changes target `config`: single-value title/source name/source author/source type and multi-value authors/dates/tags/source files. Keep these targets separate.
|
| 275 |
+
- Proofreading text deltas are generated with the SRI-pinned `diff_match_patch` 20121119 browser build and may contain any number of ordered `=`, `-`, and `+` operations. The frontend preview, API change preview, and pipeline submission code must all replay the same multi-operation UTF-16 delta semantics. Inserted text remains Unicode and escapes only delta delimiters; do not restore whole-middle replacement or URI-encode every Chinese character.
|
| 276 |
+
- Applying the proofreading full-text editor back to structured parts must sequence-align unchanged lines before creating patches. Inserting, splitting, joining, or deleting a paragraph must preserve the original DOM identity and PatchV2 index of later unchanged parts; never map full-text lines to existing parts by array position alone.
|
| 277 |
+
- Proofreading is limited to 30 accepted submissions per client IP per hour. Only body/comment corrections are eligible for pipeline auto-merge: parts may be unlimited in number, total diff cost (including the complete text of inserted paragraphs) must stay within 500 UTF-16 code units, and the net paragraph delta (deletes minus inserts) must stay within ±3. Semantically empty operations such as identity diffs, unchanged types, empty insert lists, and `delete: false` are removed during normalization; a request with no effective change is rejected. Every metadata change, including tags, requires manual review. Branch protection or merge conflicts may still leave an otherwise eligible PR for manual review.
|
| 278 |
+
- Proofreading payload limits use serialized Unicode character counts, not UTF-8 byte counts. Direct `repository_dispatch` payloads stay below 60,000 characters and at most 10 top-level properties; larger accepted submissions are split into ordered Git blobs and dispatched by SHA, then reassembled and SHA-256 verified by the pipeline workflow. Keep the service and pipeline transport contracts in sync.
|
| 279 |
+
- Proofreading tracker issues and auto-merge log comments retain complete original/edited full text. When the combined body exceeds GitHub's limit, the pipeline moves each full-text section into idempotent, size-bounded follow-up comments and splits oversized sections without silently dropping text.
|
| 280 |
+
|
| 281 |
+
## Testing
|
| 282 |
+
|
| 283 |
+
### Fast Default
|
| 284 |
+
|
| 285 |
+
Install runtime dependencies and run deterministic local tests. Elasticsearch calls are mocked; SQLite tests use real temporary databases:
|
| 286 |
+
|
| 287 |
+
```bash
|
| 288 |
+
python3 -m pip install -r requirements.txt
|
| 289 |
+
python3 -m unittest \
|
| 290 |
+
tests.test_search tests.test_api tests.test_data_loader tests.test_indexer \
|
| 291 |
+
tests.test_stores tests.test_storage_lock tests.test_downloads tests.test_frontend \
|
| 292 |
+
tests.test_bucket_probe tests.test_bucket_snapshot tests.test_search_store -v
|
| 293 |
+
```
|
| 294 |
+
|
| 295 |
+
Run syntax checks:
|
| 296 |
+
|
| 297 |
+
```bash
|
| 298 |
+
python3 -m py_compile app/main.py app/indexer.py app/data_loader.py app/doc_store.py app/facet_store.py
|
| 299 |
+
bash -n scripts/start.sh
|
| 300 |
+
```
|
| 301 |
+
|
| 302 |
+
### Browser Integration
|
| 303 |
+
|
| 304 |
+
Run optional browser E2E tests explicitly:
|
| 305 |
+
|
| 306 |
+
```bash
|
| 307 |
+
python3 -m pip install -r requirements-test.txt
|
| 308 |
+
playwright install chromium
|
| 309 |
+
APP_ROOT="$PWD" python3 -m unittest tests.test_browser -v
|
| 310 |
+
```
|
| 311 |
+
|
| 312 |
+
### Rebuild And Real Elasticsearch
|
| 313 |
+
|
| 314 |
+
A local real-Elasticsearch index build is explicit heavy acceptance, not part of the default suite. Require it for mapping, analyzer, ingestion, alias/activation, or index-generation changes when a provisioned Elasticsearch and representative parsed corpus are available. Full-corpus rebuilds remain release work.
|
| 315 |
+
|
| 316 |
+
### Production Acceptance
|
| 317 |
+
|
| 318 |
+
Run read-only production smoke tests:
|
| 319 |
+
|
| 320 |
+
```bash
|
| 321 |
+
BHA_LIVE_BASE_URL="https://vomebook-bha-search.hf.space" python3 -m unittest tests.test_live_smoke -v
|
| 322 |
+
```
|
| 323 |
+
|
| 324 |
+
The local suite deliberately mocks Elasticsearch. Local SQLite and corpus artifacts may be stale and do not prove deployed index state. The live smoke suite is the acceptance
|
| 325 |
+
layer for the real CJK analyzer, phrase and literal-character behavior, exact totals, stable pagination,
|
| 326 |
+
facet composition, previews, and TXT downloads. It must remain read-only and must never
|
| 327 |
+
call `/api/reindex` or remote source-file proxy downloads.
|
| 328 |
+
|
| 329 |
+
Minimum manual query matrix:
|
| 330 |
+
|
| 331 |
+
```text
|
| 332 |
+
手机
|
| 333 |
+
手。机
|
| 334 |
+
文化 革命
|
| 335 |
+
文化革命
|
| 336 |
+
手*机
|
| 337 |
+
手?机
|
| 338 |
+
normal mode
|
| 339 |
+
exact mode
|
| 340 |
+
fulltext on/off
|
| 341 |
+
metadata-only search
|
| 342 |
+
archive/source/author/tag filters
|
| 343 |
+
date filters
|
| 344 |
+
relevance/date sorting
|
| 345 |
+
pagination
|
| 346 |
+
facet pagination
|
| 347 |
+
preview and snippets
|
| 348 |
+
Chinese IME input
|
| 349 |
+
mobile layout
|
| 350 |
+
```
|
| 351 |
+
|
| 352 |
+
For live exact-content tests, verify the original article or preview text when necessary. Do not decide correctness from a list snippet alone.
|
| 353 |
+
|
| 354 |
+
## Deployment Upload Matrix
|
| 355 |
+
|
| 356 |
+
Query/API/runtime-only change:
|
| 357 |
+
|
| 358 |
+
```text
|
| 359 |
+
app/main.py
|
| 360 |
+
app/runtime.py, for readiness, caches, facets, sources, or warmup
|
| 361 |
+
app/bootstrap_api.py, for default first-screen hydration
|
| 362 |
+
app/search_api.py, for the search route
|
| 363 |
+
app/preview_api.py, for article previews
|
| 364 |
+
app/source_files.py, for source previews or downloads
|
| 365 |
+
app/reindex_api.py, for reindex or snapshot coordination
|
| 366 |
+
```
|
| 367 |
+
|
| 368 |
+
Search request/query/result semantics change:
|
| 369 |
+
|
| 370 |
+
```text
|
| 371 |
+
app/main.py
|
| 372 |
+
app/search_api.py
|
| 373 |
+
app/search_logic.py
|
| 374 |
+
```
|
| 375 |
+
|
| 376 |
+
Proofreading patch semantics change:
|
| 377 |
+
|
| 378 |
+
```text
|
| 379 |
+
app/main.py
|
| 380 |
+
app/proofread_api.py
|
| 381 |
+
app/proofread_core.py
|
| 382 |
+
app/pipeline_dispatch.py, when dispatch transport changes
|
| 383 |
+
app/document_identity.py, when content identity changes
|
| 384 |
+
```
|
| 385 |
+
|
| 386 |
+
Parse workbench semantics change:
|
| 387 |
+
|
| 388 |
+
```text
|
| 389 |
+
app/main.py
|
| 390 |
+
app/parse_api.py
|
| 391 |
+
app/parse_core.py
|
| 392 |
+
app/pipeline_dispatch.py, when dispatch transport changes
|
| 393 |
+
app/document_identity.py, when content identity changes
|
| 394 |
+
```
|
| 395 |
+
|
| 396 |
+
Mapping/indexer change:
|
| 397 |
+
|
| 398 |
+
```text
|
| 399 |
+
app/main.py
|
| 400 |
+
app/indexer.py
|
| 401 |
+
app/data_loader.py, if data fields change
|
| 402 |
+
scripts/start.sh, if startup settings change
|
| 403 |
+
```
|
| 404 |
+
|
| 405 |
+
Dependency change:
|
| 406 |
+
|
| 407 |
+
```text
|
| 408 |
+
requirements.txt
|
| 409 |
+
```
|
| 410 |
+
|
| 411 |
+
Container build change:
|
| 412 |
+
|
| 413 |
+
```text
|
| 414 |
+
Dockerfile
|
| 415 |
+
.dockerignore, when build-context rules change
|
| 416 |
+
scripts/install-elasticsearch.sh, when the Elasticsearch artifact changes
|
| 417 |
+
requirements.txt, when Python runtime dependencies change
|
| 418 |
+
```
|
| 419 |
+
|
| 420 |
+
Keep `ES_VERSION` and `ES_SHA512` in the Dockerfile synchronized. The final image intentionally excludes build-only `tar`, `curl`, Node.js, esbuild, and the Elasticsearch installer; runtime readiness uses Python's standard library. Frontend source remains readable under `static/`; Docker serves only the minified builder output with the same filenames.
|
| 421 |
+
|
| 422 |
+
Tests/documentation:
|
| 423 |
+
|
| 424 |
+
```text
|
| 425 |
+
tests/
|
| 426 |
+
TESTING.md
|
| 427 |
+
AGENTS.md
|
| 428 |
+
```
|
| 429 |
+
|
| 430 |
+
Under the current deployment workflow, HF is expected to rebuild the Space after deployment. Verify `/api/health`, document count, index readiness, deployed index metadata, and the live smoke suite rather than assuming the rebuild completed correctly.
|
| 431 |
+
|
| 432 |
+
## Explicit Prohibitions
|
| 433 |
+
|
| 434 |
+
- Do not add semantic/vector search.
|
| 435 |
+
- Do not weaken exactness for speed.
|
| 436 |
+
- Do not change AND to OR.
|
| 437 |
+
- Do not make `手机` equal `手。机`.
|
| 438 |
+
- Do not truncate candidates, totals, facets, or fields.
|
| 439 |
+
- Do not use approximate counts.
|
| 440 |
+
- Do not reduce fields or filters to make queries faster.
|
| 441 |
+
- Do not bump `INDEX_VERSION = "1.0.0"` just because files were uploaded or the Space rebuilt.
|
| 442 |
+
- Do not assume a free HF Space has persistent storage.
|
| 443 |
+
- Do not use Elasticsearch async translog settings or a 20% index buffer without explicit new evidence and approval.
|
| 444 |
+
- Do not restore `refresh_interval=-1` without a measured benefit and a rebuild-visibility review.
|
| 445 |
+
- Do not run jieba over full article content.
|
| 446 |
+
- Do not add arbitrary `query_string` or raw Elasticsearch DSL input.
|
| 447 |
+
- Do not disable bounded local snippets or HTML escaping.
|
| 448 |
+
- Do not add fixed-query or periodic search warmup without current measurements.
|
| 449 |
+
- Do not introduce unbounded caches.
|
| 450 |
+
- Do not commit credentials, tokens, secrets, or private data.
|
| 451 |
+
- Do not run destructive Git commands or revert user changes.
|
| 452 |
+
- Do not modify CCRD, GitHub Search, Hugging Face Search, or unrelated projects while working here.
|
| 453 |
+
|
| 454 |
+
## Engineering Priority
|
| 455 |
+
|
| 456 |
+
Prefer the smallest correct change. Preserve Chinese search semantics, exact totals, stable filters, progressive index visibility, and deployability. If a performance optimization conflicts with accuracy, stop and ask rather than guessing.
|
static/app.js
CHANGED
|
@@ -121,6 +121,7 @@ let previewCloseUrl = "/";
|
|
| 121 |
let previewResultsScrollTop = 0;
|
| 122 |
let animateNextPreview = false;
|
| 123 |
let previewAnimationCleanup = null;
|
|
|
|
| 124 |
let activeYearInput = null;
|
| 125 |
let yearPickerPointer = null;
|
| 126 |
let searchComposing = false;
|
|
@@ -1517,7 +1518,7 @@ function parseQueueHTML(item) {
|
|
| 1517 |
return `<div class="result-item parse-queue-item"><div class="result-file-icon"><span class="ui-icon ui-icon-file" aria-hidden="true"></span></div><div class="result-main"><div class="result-line"><div class="result-info"><div class="result-title">${escapeHTML(item.publication_name || item.title)}</div><div class="result-path"><span>Archive ${escapeHTML(item.archive_id)}</span><span class="path-sep">/</span><span>${start === end ? `第 ${start} 页` : `第 ${start}-${end} 页`}</span>${dates ? `<span class="path-sep">/</span><span>${escapeHTML(dates)}</span>` : ""}</div></div><div class="result-actions"><button class="result-action-btn primary" data-parse-open="${escapeHTML(item.doc_id)}">开始解析</button></div></div></div></div>`;
|
| 1518 |
}
|
| 1519 |
|
| 1520 |
-
function renderParseQueue() {
|
| 1521 |
const pageSize = STATE.parsePageSize;
|
| 1522 |
const totalPages = Math.max(1, Math.ceil(STATE.parseTotal / pageSize));
|
| 1523 |
DOM.resultCount.textContent = `待解析 ${STATE.parseTotal.toLocaleString()} 份`;
|
|
@@ -1530,6 +1531,7 @@ function renderParseQueue() {
|
|
| 1530 |
DOM.results.hidden = !STATE.parseItems.length;
|
| 1531 |
DOM.emptyState.style.display = STATE.parseItems.length ? "none" : "flex";
|
| 1532 |
DOM.results.innerHTML = STATE.parseItems.map(parseQueueHTML).join("");
|
|
|
|
| 1533 |
}
|
| 1534 |
|
| 1535 |
async function loadParseQueue() {
|
|
@@ -1542,7 +1544,7 @@ async function loadParseQueue() {
|
|
| 1542 |
if (!STATE.parseMode) return;
|
| 1543 |
STATE.parseTotal = Number(data.total) || 0;
|
| 1544 |
STATE.parseItems = Array.isArray(data.items) ? data.items : [];
|
| 1545 |
-
renderParseQueue();
|
| 1546 |
} catch (error) {
|
| 1547 |
DOM.statusText.textContent = "待解析队列加载失败";
|
| 1548 |
showToast(`待解析队列加载失败:${error.message || "未知错误"}`);
|
|
@@ -1556,14 +1558,16 @@ function parseDateParts(article) {
|
|
| 1556 |
|
| 1557 |
function parseArticleRowHTML(article, index, maxPage) {
|
| 1558 |
const date = parseDateParts(article);
|
| 1559 |
-
|
|
|
|
|
|
|
| 1560 |
}
|
| 1561 |
|
| 1562 |
function parseEditorHTML(item) {
|
| 1563 |
const article = item.article || {};
|
| 1564 |
const maxPage = Number(article.page_end) || (item.source_files || []).length || 1;
|
| 1565 |
const initial = { title: "", authors: [], dates: article.dates || [], page_start: 1, page_end: maxPage };
|
| 1566 |
-
return `<div class="preview-header"><div class="preview-heading"><div class="preview-title">${escapeHTML(item.publication_name || item.title)}</div><div class="preview-compare-note">待解析 / ${maxPage} 页</div></div><div class="preview-actions"><button type="button" class="text-btn-sm" data-parse-back>返回队列</button><button type="button" class="icon-btn-sm" data-parse-close aria-label="关闭">×</button></div></div><div class="parse-workbench"><section class="parse-source-pane"><div class="proofread-pane-header"><span>原图与 OCR</span><label class="proofread-page-control">页码 <input data-parse-page type="number" min="1" max="${maxPage}" value="1"></label></div><div class="parse-page-tools"><button type="button" class="text-btn-sm" data-parse-page-prev disabled>上一页</button><button type="button" class="text-btn-sm" data-parse-page-next ${maxPage === 1 ? "disabled" : ""}>下一页</button><span data-parse-page-status>1 / ${maxPage}</span></div><div class="parse-page-content"><div class="parse-image-stage"><img data-parse-image alt="原文件第 1 页"></div><pre class="parse-ocr-text" data-parse-ocr>正在加载 OCR...</pre></div></section><form class="proofread-editor parse-editor" data-parse-form data-max-page="${maxPage}"><div class="proofread-pane-header"><span>文章列表</span><button type="submit" class="result-action-btn primary">提交解析</button></div><div data-parse-articles>${parseArticleRowHTML(initial, 0, maxPage)}</div><div class="proofread-form-actions"><button type="button" class="text-btn-sm" data-parse-add>添加文章</button></div><label class="proofread-field"><span>解析备注</span><textarea data-parse-description></textarea></label><div class="date-error" data-parse-error hidden></div></form></div>`;
|
| 1567 |
}
|
| 1568 |
|
| 1569 |
async function loadParsePage(page) {
|
|
@@ -1572,6 +1576,17 @@ async function loadParsePage(page) {
|
|
| 1572 |
const maxPage = Number((item.article || {}).page_end) || (item.source_files || []).length || 1;
|
| 1573 |
page = Math.min(maxPage, Math.max(1, Number(page) || 1));
|
| 1574 |
STATE.parseCurrentPage = page;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1575 |
const image = DOM.previewPanel.querySelector("[data-parse-image]");
|
| 1576 |
const text = DOM.previewPanel.querySelector("[data-parse-ocr]");
|
| 1577 |
const input = DOM.previewPanel.querySelector("[data-parse-page]");
|
|
@@ -1587,18 +1602,229 @@ async function loadParsePage(page) {
|
|
| 1587 |
try {
|
| 1588 |
const data = await parseOCRPage(item, page);
|
| 1589 |
if (!STATE.parseItem || STATE.parseItem.doc_id !== item.doc_id || STATE.parseCurrentPage !== page) return;
|
| 1590 |
-
if (text) text.textContent = parseOCRText(data);
|
|
|
|
| 1591 |
} catch (error) {
|
| 1592 |
if (text) text.textContent = `OCR 加载失败:${error.message || "未知错误"}`;
|
| 1593 |
}
|
| 1594 |
}
|
| 1595 |
|
| 1596 |
-
function
|
| 1597 |
-
|
| 1598 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1599 |
.map((row) => row.text || "").filter(Boolean).join("\n");
|
| 1600 |
}
|
| 1601 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1602 |
async function parseOCRPage(item, page) {
|
| 1603 |
const key = `${item.doc_id}:${page}`;
|
| 1604 |
if (parseOCRCache.has(key)) {
|
|
@@ -1629,8 +1855,9 @@ async function refreshParseArticleBody(row, force = false) {
|
|
| 1629 |
if (status) status.textContent = "正在读取...";
|
| 1630 |
try {
|
| 1631 |
const pages = await Promise.all(Array.from({ length: end - start + 1 }, (_value, index) => parseOCRPage(STATE.parseItem, start + index)));
|
| 1632 |
-
const
|
| 1633 |
-
const
|
|
|
|
| 1634 |
textarea.value = body;
|
| 1635 |
textarea.dataset.edited = "0";
|
| 1636 |
row.dataset.basePartCount = String(boxCount);
|
|
@@ -1655,6 +1882,7 @@ async function openParseEditor(docId) {
|
|
| 1655 |
DOM.previewPanel.innerHTML = parseEditorHTML(item);
|
| 1656 |
DOM.results.hidden = true;
|
| 1657 |
DOM.emptyState.style.display = "none";
|
|
|
|
| 1658 |
loadParsePage(1);
|
| 1659 |
DOM.previewPanel.querySelectorAll("[data-parse-article]").forEach((row) => refreshParseArticleBody(row));
|
| 1660 |
} catch (error) {
|
|
@@ -1671,7 +1899,11 @@ function collectParseArticles(form) {
|
|
| 1671 |
if (year) date.year = year;
|
| 1672 |
if (month) date.month = month;
|
| 1673 |
if (day) date.day = day;
|
| 1674 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1675 |
title: row.querySelector("[data-parse-title]").value.trim(),
|
| 1676 |
authors: row.querySelector("[data-parse-authors]").value.split(/[、,,/]/).map((value) => value.trim()).filter(Boolean),
|
| 1677 |
dates: Object.keys(date).length ? [date] : [],
|
|
@@ -1681,6 +1913,9 @@ function collectParseArticles(form) {
|
|
| 1681 |
content: row.querySelector("[data-parse-content]").value,
|
| 1682 |
base_part_count: Number(row.dataset.basePartCount) || 0,
|
| 1683 |
};
|
|
|
|
|
|
|
|
|
|
| 1684 |
});
|
| 1685 |
}
|
| 1686 |
|
|
@@ -3445,6 +3680,34 @@ function attachEvents() {
|
|
| 3445 |
renderParseQueue();
|
| 3446 |
return;
|
| 3447 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3448 |
if (event.target.closest("[data-parse-page-prev]")) {
|
| 3449 |
loadParsePage(STATE.parseCurrentPage - 1);
|
| 3450 |
return;
|
|
@@ -3474,8 +3737,10 @@ function attachEvents() {
|
|
| 3474 |
const removeArticle = event.target.closest("[data-parse-remove]");
|
| 3475 |
if (removeArticle && !removeArticle.disabled) {
|
| 3476 |
const row = removeArticle.closest("[data-parse-article]");
|
|
|
|
| 3477 |
if (row) row.remove();
|
| 3478 |
DOM.previewPanel.querySelectorAll("[data-parse-article] legend").forEach((legend, index) => { legend.textContent = `文章 ${index + 1}`; });
|
|
|
|
| 3479 |
return;
|
| 3480 |
}
|
| 3481 |
const partInsert = event.target.closest("[data-add-part-before], [data-add-part-after]");
|
|
@@ -3600,6 +3865,12 @@ function attachEvents() {
|
|
| 3600 |
if (event.target.closest("[data-preview-theme]")) toggleTheme();
|
| 3601 |
if (event.target.closest("[data-preview-mobile]")) toggleMobileMode();
|
| 3602 |
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3603 |
DOM.previewPanel.addEventListener("change", (event) => {
|
| 3604 |
const parsePage = event.target.closest("[data-parse-page]");
|
| 3605 |
if (parsePage) {
|
|
@@ -3608,7 +3879,9 @@ function attachEvents() {
|
|
| 3608 |
}
|
| 3609 |
const parseRange = event.target.closest("[data-parse-start], [data-parse-end]");
|
| 3610 |
if (parseRange) {
|
| 3611 |
-
|
|
|
|
|
|
|
| 3612 |
return;
|
| 3613 |
}
|
| 3614 |
const select = event.target.closest("[data-compare-select]");
|
|
|
|
| 121 |
let previewResultsScrollTop = 0;
|
| 122 |
let animateNextPreview = false;
|
| 123 |
let previewAnimationCleanup = null;
|
| 124 |
+
let parseCropDrag = null;
|
| 125 |
let activeYearInput = null;
|
| 126 |
let yearPickerPointer = null;
|
| 127 |
let searchComposing = false;
|
|
|
|
| 1518 |
return `<div class="result-item parse-queue-item"><div class="result-file-icon"><span class="ui-icon ui-icon-file" aria-hidden="true"></span></div><div class="result-main"><div class="result-line"><div class="result-info"><div class="result-title">${escapeHTML(item.publication_name || item.title)}</div><div class="result-path"><span>Archive ${escapeHTML(item.archive_id)}</span><span class="path-sep">/</span><span>${start === end ? `第 ${start} 页` : `第 ${start}-${end} 页`}</span>${dates ? `<span class="path-sep">/</span><span>${escapeHTML(dates)}</span>` : ""}</div></div><div class="result-actions"><button class="result-action-btn primary" data-parse-open="${escapeHTML(item.doc_id)}">开始解析</button></div></div></div></div>`;
|
| 1519 |
}
|
| 1520 |
|
| 1521 |
+
function renderParseQueue({ animate = false } = {}) {
|
| 1522 |
const pageSize = STATE.parsePageSize;
|
| 1523 |
const totalPages = Math.max(1, Math.ceil(STATE.parseTotal / pageSize));
|
| 1524 |
DOM.resultCount.textContent = `待解析 ${STATE.parseTotal.toLocaleString()} 份`;
|
|
|
|
| 1531 |
DOM.results.hidden = !STATE.parseItems.length;
|
| 1532 |
DOM.emptyState.style.display = STATE.parseItems.length ? "none" : "flex";
|
| 1533 |
DOM.results.innerHTML = STATE.parseItems.map(parseQueueHTML).join("");
|
| 1534 |
+
if (animate) animateResultRows(Array.from(DOM.results.children).slice(0, 15));
|
| 1535 |
}
|
| 1536 |
|
| 1537 |
async function loadParseQueue() {
|
|
|
|
| 1544 |
if (!STATE.parseMode) return;
|
| 1545 |
STATE.parseTotal = Number(data.total) || 0;
|
| 1546 |
STATE.parseItems = Array.isArray(data.items) ? data.items : [];
|
| 1547 |
+
renderParseQueue({ animate: true });
|
| 1548 |
} catch (error) {
|
| 1549 |
DOM.statusText.textContent = "待解析队列加载失败";
|
| 1550 |
showToast(`待解析队列加载失败:${error.message || "未知错误"}`);
|
|
|
|
| 1558 |
|
| 1559 |
function parseArticleRowHTML(article, index, maxPage) {
|
| 1560 |
const date = parseDateParts(article);
|
| 1561 |
+
const ocr = escapeHTML(JSON.stringify(article.ocr || {}));
|
| 1562 |
+
const exceptions = escapeHTML(JSON.stringify(article.ocr_exceptions || {}));
|
| 1563 |
+
return `<fieldset class="parse-article" data-parse-article data-parse-ocr="${ocr}" data-parse-ocr-exceptions="${exceptions}"><legend>文章 ${index + 1}</legend><div class="parse-article-actions"><button type="button" class="text-btn-sm" data-parse-crop-open>裁剪</button><button type="button" class="text-btn-sm" data-parse-remove ${index === 0 ? "disabled" : ""}>删除</button></div><label class="proofread-field"><span>标题</span><input data-parse-title value="${escapeHTML(article.title || "")}" required></label><label class="proofread-field"><span>作者</span><input data-parse-authors value="${escapeHTML((article.authors || []).join("、"))}"></label><div class="parse-date-row"><label class="proofread-field"><span>年</span><input data-parse-year type="number" min="1" max="9999" value="${escapeHTML(date.year)}"></label><label class="proofread-field"><span>月</span><input data-parse-month type="number" min="1" max="12" value="${escapeHTML(date.month)}"></label><label class="proofread-field"><span>日</span><input data-parse-day type="number" min="1" max="31" value="${escapeHTML(date.day)}"></label></div><div class="parse-range-row"><label class="proofread-field"><span>开始页</span><input data-parse-start type="number" min="1" max="${maxPage}" value="${article.page_start || 1}" required></label><label class="proofread-field"><span>结束页</span><input data-parse-end type="number" min="1" max="${maxPage}" value="${article.page_end || maxPage}" required></label></div><label class="proofread-field parse-body-field"><span class="parse-body-heading"><span>正文</span><span><span class="parse-body-status" data-parse-body-status></span><button type="button" class="text-btn-sm" data-parse-refresh-body>从 OCR 刷新</button></span></span><textarea data-parse-content required>${escapeHTML(article.content || "")}</textarea></label></fieldset>`;
|
| 1564 |
}
|
| 1565 |
|
| 1566 |
function parseEditorHTML(item) {
|
| 1567 |
const article = item.article || {};
|
| 1568 |
const maxPage = Number(article.page_end) || (item.source_files || []).length || 1;
|
| 1569 |
const initial = { title: "", authors: [], dates: article.dates || [], page_start: 1, page_end: maxPage };
|
| 1570 |
+
return `<div class="preview-header"><div class="preview-heading"><div class="preview-title">${escapeHTML(item.publication_name || item.title)}</div><div class="preview-compare-note">待解析 / ${maxPage} 页</div></div><div class="preview-actions"><button type="button" class="text-btn-sm" data-parse-back>返回队列</button><button type="button" class="icon-btn-sm" data-parse-close aria-label="关闭">×</button></div></div><div class="parse-workbench"><section class="parse-source-pane"><div class="proofread-pane-header"><span>原图与 OCR</span><label class="proofread-page-control">页码 <input data-parse-page type="number" min="1" max="${maxPage}" value="1"></label></div><div class="parse-page-tools"><button type="button" class="text-btn-sm" data-parse-page-prev disabled>上一页</button><button type="button" class="text-btn-sm" data-parse-page-next ${maxPage === 1 ? "disabled" : ""}>下一页</button><span data-parse-page-status>1 / ${maxPage}</span><button type="button" class="text-btn-sm parse-crop-toggle" data-parse-crop-toggle>裁剪 OCR</button></div><div class="parse-crop-toolbar" data-parse-crop-toolbar hidden><span data-parse-crop-target>文章 1</span><div class="parse-crop-scope" role="group" aria-label="裁剪范围"><button type="button" class="text-btn-sm primary" data-parse-crop-scope="article">整篇</button><button type="button" class="text-btn-sm" data-parse-crop-scope="page">仅本页</button></div><span data-parse-crop-values>上 0% 下 0% 左 0% 右 0%</span><button type="button" class="text-btn-sm" data-parse-crop-reset>重置</button><button type="button" class="text-btn-sm" data-parse-crop-close>完成</button></div><div class="parse-page-content"><div class="parse-image-stage"><div class="parse-image-wrap"><img data-parse-image alt="原文件第 1 页"><div class="parse-crop-overlay" data-parse-crop-overlay hidden><div class="parse-crop-box" data-parse-crop-box><span class="parse-crop-handle top" data-crop-handle="top"></span><span class="parse-crop-handle right" data-crop-handle="right"></span><span class="parse-crop-handle bottom" data-crop-handle="bottom"></span><span class="parse-crop-handle left" data-crop-handle="left"></span><span class="parse-crop-handle top-left" data-crop-handle="top-left"></span><span class="parse-crop-handle top-right" data-crop-handle="top-right"></span><span class="parse-crop-handle bottom-right" data-crop-handle="bottom-right"></span><span class="parse-crop-handle bottom-left" data-crop-handle="bottom-left"></span></div></div></div></div><pre class="parse-ocr-text" data-parse-ocr>正在加载 OCR...</pre></div></section><form class="proofread-editor parse-editor" data-parse-form data-max-page="${maxPage}"><div class="proofread-pane-header"><span>文章列表</span><button type="submit" class="result-action-btn primary">提交解析</button></div><div data-parse-articles>${parseArticleRowHTML(initial, 0, maxPage)}</div><div class="proofread-form-actions"><button type="button" class="text-btn-sm" data-parse-add>添加文章</button></div><label class="proofread-field"><span>解析备注</span><textarea data-parse-description></textarea></label><div class="date-error" data-parse-error hidden></div></form></div>`;
|
| 1571 |
}
|
| 1572 |
|
| 1573 |
async function loadParsePage(page) {
|
|
|
|
| 1576 |
const maxPage = Number((item.article || {}).page_end) || (item.source_files || []).length || 1;
|
| 1577 |
page = Math.min(maxPage, Math.max(1, Number(page) || 1));
|
| 1578 |
STATE.parseCurrentPage = page;
|
| 1579 |
+
const cropToolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1580 |
+
if (cropToolbar && !cropToolbar.hidden) {
|
| 1581 |
+
const selected = parseCropTargetRow();
|
| 1582 |
+
const coversPage = selected
|
| 1583 |
+
&& Number(selected.querySelector("[data-parse-start]").value) <= page
|
| 1584 |
+
&& Number(selected.querySelector("[data-parse-end]").value) >= page;
|
| 1585 |
+
if (!coversPage) {
|
| 1586 |
+
if (selected) selected.classList.remove("is-crop-target");
|
| 1587 |
+
setParseCropTarget(parseCropTargetRow());
|
| 1588 |
+
}
|
| 1589 |
+
}
|
| 1590 |
const image = DOM.previewPanel.querySelector("[data-parse-image]");
|
| 1591 |
const text = DOM.previewPanel.querySelector("[data-parse-ocr]");
|
| 1592 |
const input = DOM.previewPanel.querySelector("[data-parse-page]");
|
|
|
|
| 1602 |
try {
|
| 1603 |
const data = await parseOCRPage(item, page);
|
| 1604 |
if (!STATE.parseItem || STATE.parseItem.doc_id !== item.doc_id || STATE.parseCurrentPage !== page) return;
|
| 1605 |
+
if (text) text.textContent = parseOCRText(data, parseOCRThresholds(parseCropTargetRow(), page));
|
| 1606 |
+
updateParseCropUI();
|
| 1607 |
} catch (error) {
|
| 1608 |
if (text) text.textContent = `OCR 加载失败:${error.message || "未知错误"}`;
|
| 1609 |
}
|
| 1610 |
}
|
| 1611 |
|
| 1612 |
+
function parseOCRThresholds(row, page) {
|
| 1613 |
+
if (!row) return [0, 0, 0, 0];
|
| 1614 |
+
let ocr = {};
|
| 1615 |
+
let exceptions = {};
|
| 1616 |
+
try { ocr = JSON.parse(row.dataset.parseOcr || "{}"); } catch (_error) { /* invalid local state falls back to no crop */ }
|
| 1617 |
+
try { exceptions = JSON.parse(row.dataset.parseOcrExceptions || "{}"); } catch (_error) { /* invalid local state falls back to no crop */ }
|
| 1618 |
+
const options = { ...ocr, ...(exceptions[String(page)] || {}) };
|
| 1619 |
+
const values = Array.isArray(options.content_thresholds) ? options.content_thresholds : [0, 0, 0, 0];
|
| 1620 |
+
return values.map((value) => Math.min(0.98, Math.max(0, Number(value) || 0)));
|
| 1621 |
+
}
|
| 1622 |
+
|
| 1623 |
+
function parseOCRRows(data, thresholds = [0, 0, 0, 0]) {
|
| 1624 |
+
const dimensions = data.dimensions || {};
|
| 1625 |
+
const width = Number(dimensions.width) || 0;
|
| 1626 |
+
const height = Number(dimensions.height) || 0;
|
| 1627 |
+
const [top, bottom, left, right] = thresholds;
|
| 1628 |
+
return (data.ocr_results || []).filter((row) => {
|
| 1629 |
+
if (!width || !height || !Array.isArray(row.box) || !row.box.length) return true;
|
| 1630 |
+
const xs = row.box.map((point) => Number(point[0]) || 0);
|
| 1631 |
+
const ys = row.box.map((point) => Number(point[1]) || 0);
|
| 1632 |
+
const centerX = (Math.min(...xs) + Math.max(...xs)) / 2;
|
| 1633 |
+
const centerY = (Math.min(...ys) + Math.max(...ys)) / 2;
|
| 1634 |
+
return centerX >= left * width && centerX <= (1 - right) * width
|
| 1635 |
+
&& centerY >= top * height && centerY <= (1 - bottom) * height;
|
| 1636 |
+
}).slice().sort((a, b) => ((a.box || [[0, 0]])[0][1] - (b.box || [[0, 0]])[0][1]) || ((a.box || [[0, 0]])[0][0] - (b.box || [[0, 0]])[0][0]));
|
| 1637 |
+
}
|
| 1638 |
+
|
| 1639 |
+
function parseOCRText(data, thresholds = [0, 0, 0, 0]) {
|
| 1640 |
+
return parseOCRRows(data, thresholds)
|
| 1641 |
.map((row) => row.text || "").filter(Boolean).join("\n");
|
| 1642 |
}
|
| 1643 |
|
| 1644 |
+
function parseCropTargetRow() {
|
| 1645 |
+
const rows = Array.from(DOM.previewPanel.querySelectorAll("[data-parse-article]"));
|
| 1646 |
+
const selected = rows.find((row) => row.classList.contains("is-crop-target"));
|
| 1647 |
+
if (selected) return selected;
|
| 1648 |
+
return rows.find((row) => {
|
| 1649 |
+
const start = Number(row.querySelector("[data-parse-start]").value);
|
| 1650 |
+
const end = Number(row.querySelector("[data-parse-end]").value);
|
| 1651 |
+
return start <= STATE.parseCurrentPage && end >= STATE.parseCurrentPage;
|
| 1652 |
+
}) || rows[0] || null;
|
| 1653 |
+
}
|
| 1654 |
+
|
| 1655 |
+
function setParseCropTarget(row) {
|
| 1656 |
+
DOM.previewPanel.querySelectorAll("[data-parse-article]").forEach((candidate) => candidate.classList.toggle("is-crop-target", candidate === row));
|
| 1657 |
+
updateParseCropUI();
|
| 1658 |
+
}
|
| 1659 |
+
|
| 1660 |
+
function parseCropScope() {
|
| 1661 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1662 |
+
return toolbar && toolbar.dataset.scope === "page" ? "page" : "article";
|
| 1663 |
+
}
|
| 1664 |
+
|
| 1665 |
+
function setParseCropScope(scope) {
|
| 1666 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1667 |
+
if (!toolbar) return;
|
| 1668 |
+
toolbar.dataset.scope = scope === "page" ? "page" : "article";
|
| 1669 |
+
toolbar.querySelectorAll("[data-parse-crop-scope]").forEach((button) => button.classList.toggle("primary", button.dataset.parseCropScope === toolbar.dataset.scope));
|
| 1670 |
+
updateParseCropUI();
|
| 1671 |
+
}
|
| 1672 |
+
|
| 1673 |
+
function cropOptions(row, scope, page) {
|
| 1674 |
+
let options = {};
|
| 1675 |
+
try {
|
| 1676 |
+
if (scope === "page") {
|
| 1677 |
+
const exceptions = JSON.parse(row.dataset.parseOcrExceptions || "{}");
|
| 1678 |
+
options = exceptions[String(page)] || {};
|
| 1679 |
+
} else {
|
| 1680 |
+
options = JSON.parse(row.dataset.parseOcr || "{}");
|
| 1681 |
+
}
|
| 1682 |
+
} catch (_error) { /* invalid local state falls back to defaults */ }
|
| 1683 |
+
return options;
|
| 1684 |
+
}
|
| 1685 |
+
|
| 1686 |
+
function setParseCropThresholds(row, scope, page, thresholds, reset = false) {
|
| 1687 |
+
if (!row) return;
|
| 1688 |
+
const key = scope === "page" ? "parseOcrExceptions" : "parseOcr";
|
| 1689 |
+
let value = {};
|
| 1690 |
+
try { value = JSON.parse(row.dataset[key] || "{}"); } catch (_error) { value = {}; }
|
| 1691 |
+
if (scope === "page") {
|
| 1692 |
+
if (reset) delete value[String(page)];
|
| 1693 |
+
else value[String(page)] = { ...(value[String(page)] || {}), content_thresholds: thresholds };
|
| 1694 |
+
} else if (reset) {
|
| 1695 |
+
delete value.content_thresholds;
|
| 1696 |
+
} else {
|
| 1697 |
+
value.content_thresholds = thresholds;
|
| 1698 |
+
}
|
| 1699 |
+
row.dataset[key] = JSON.stringify(value);
|
| 1700 |
+
}
|
| 1701 |
+
|
| 1702 |
+
function updateParseCropUI() {
|
| 1703 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1704 |
+
const box = DOM.previewPanel.querySelector("[data-parse-crop-box]");
|
| 1705 |
+
if (!toolbar || !box || toolbar.hidden) return;
|
| 1706 |
+
const row = parseCropTargetRow();
|
| 1707 |
+
if (!row) return;
|
| 1708 |
+
const rows = Array.from(DOM.previewPanel.querySelectorAll("[data-parse-article]"));
|
| 1709 |
+
const label = toolbar.querySelector("[data-parse-crop-target]");
|
| 1710 |
+
const scope = parseCropScope();
|
| 1711 |
+
const page = STATE.parseCurrentPage;
|
| 1712 |
+
const options = cropOptions(row, scope, page);
|
| 1713 |
+
const thresholds = Array.isArray(options.content_thresholds)
|
| 1714 |
+
? options.content_thresholds.map((value) => Number(value) || 0)
|
| 1715 |
+
: parseOCRThresholds(row, page);
|
| 1716 |
+
if (label) label.textContent = `文章 ${rows.indexOf(row) + 1}${scope === "page" ? ` / 第 ${page} 页` : ""}`;
|
| 1717 |
+
box.style.top = `${thresholds[0] * 100}%`;
|
| 1718 |
+
box.style.bottom = `${thresholds[1] * 100}%`;
|
| 1719 |
+
box.style.left = `${thresholds[2] * 100}%`;
|
| 1720 |
+
box.style.right = `${thresholds[3] * 100}%`;
|
| 1721 |
+
const values = toolbar.querySelector("[data-parse-crop-values]");
|
| 1722 |
+
if (values) values.textContent = `上 ${Math.round(thresholds[0] * 100)}% 下 ${Math.round(thresholds[1] * 100)}% 左 ${Math.round(thresholds[2] * 100)}% 右 ${Math.round(thresholds[3] * 100)}%`;
|
| 1723 |
+
}
|
| 1724 |
+
|
| 1725 |
+
function openParseCrop(row = null) {
|
| 1726 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1727 |
+
const overlay = DOM.previewPanel.querySelector("[data-parse-crop-overlay]");
|
| 1728 |
+
if (!toolbar || !overlay) return;
|
| 1729 |
+
row = row || parseCropTargetRow();
|
| 1730 |
+
if (!row) return;
|
| 1731 |
+
const start = Number(row.querySelector("[data-parse-start]").value) || 1;
|
| 1732 |
+
const end = Number(row.querySelector("[data-parse-end]").value) || start;
|
| 1733 |
+
if (STATE.parseCurrentPage < start || STATE.parseCurrentPage > end) loadParsePage(start);
|
| 1734 |
+
toolbar.hidden = false;
|
| 1735 |
+
overlay.hidden = false;
|
| 1736 |
+
setParseCropTarget(row);
|
| 1737 |
+
setParseCropScope("article");
|
| 1738 |
+
}
|
| 1739 |
+
|
| 1740 |
+
function closeParseCrop() {
|
| 1741 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 1742 |
+
const overlay = DOM.previewPanel.querySelector("[data-parse-crop-overlay]");
|
| 1743 |
+
if (toolbar) toolbar.hidden = true;
|
| 1744 |
+
if (overlay) overlay.hidden = true;
|
| 1745 |
+
parseCropDrag = null;
|
| 1746 |
+
}
|
| 1747 |
+
|
| 1748 |
+
async function updateParsePageOCR() {
|
| 1749 |
+
const item = STATE.parseItem;
|
| 1750 |
+
const text = DOM.previewPanel.querySelector("[data-parse-ocr]");
|
| 1751 |
+
if (!item || !text) return;
|
| 1752 |
+
try {
|
| 1753 |
+
const page = STATE.parseCurrentPage;
|
| 1754 |
+
const data = await parseOCRPage(item, page);
|
| 1755 |
+
if (!STATE.parseItem || STATE.parseItem.doc_id !== item.doc_id || STATE.parseCurrentPage !== page) return;
|
| 1756 |
+
text.textContent = parseOCRText(data, parseOCRThresholds(parseCropTargetRow(), page));
|
| 1757 |
+
} catch (error) {
|
| 1758 |
+
text.textContent = `OCR 加载失败:${error.message || "未知错误"}`;
|
| 1759 |
+
}
|
| 1760 |
+
}
|
| 1761 |
+
|
| 1762 |
+
function finishParseCropChange(row) {
|
| 1763 |
+
if (!row) return;
|
| 1764 |
+
const textarea = row.querySelector("[data-parse-content]");
|
| 1765 |
+
const status = row.querySelector("[data-parse-body-status]");
|
| 1766 |
+
if (textarea && textarea.dataset.edited === "1") {
|
| 1767 |
+
if (status) status.textContent = "裁剪已更改,需刷新正文";
|
| 1768 |
+
} else {
|
| 1769 |
+
refreshParseArticleBody(row, true);
|
| 1770 |
+
}
|
| 1771 |
+
}
|
| 1772 |
+
|
| 1773 |
+
function beginParseCropDrag(event) {
|
| 1774 |
+
const box = event.target.closest("[data-parse-crop-box]");
|
| 1775 |
+
const overlay = event.target.closest("[data-parse-crop-overlay]");
|
| 1776 |
+
const row = parseCropTargetRow();
|
| 1777 |
+
if (!box || !overlay || !row || event.button !== 0) return;
|
| 1778 |
+
event.preventDefault();
|
| 1779 |
+
const handle = event.target.closest("[data-crop-handle]");
|
| 1780 |
+
const page = STATE.parseCurrentPage;
|
| 1781 |
+
const scope = parseCropScope();
|
| 1782 |
+
const options = cropOptions(row, scope, page);
|
| 1783 |
+
const thresholds = Array.isArray(options.content_thresholds)
|
| 1784 |
+
? options.content_thresholds.map((value) => Number(value) || 0)
|
| 1785 |
+
: parseOCRThresholds(row, page);
|
| 1786 |
+
parseCropDrag = {
|
| 1787 |
+
row, scope, page, thresholds, handle: handle ? handle.dataset.cropHandle : "move",
|
| 1788 |
+
startX: event.clientX, startY: event.clientY, rect: overlay.getBoundingClientRect(),
|
| 1789 |
+
};
|
| 1790 |
+
if (box.setPointerCapture) box.setPointerCapture(event.pointerId);
|
| 1791 |
+
}
|
| 1792 |
+
|
| 1793 |
+
function moveParseCrop(event) {
|
| 1794 |
+
const drag = parseCropDrag;
|
| 1795 |
+
if (!drag || !drag.rect.width || !drag.rect.height) return;
|
| 1796 |
+
event.preventDefault();
|
| 1797 |
+
const dx = (event.clientX - drag.startX) / drag.rect.width;
|
| 1798 |
+
const dy = (event.clientY - drag.startY) / drag.rect.height;
|
| 1799 |
+
const values = drag.thresholds.slice();
|
| 1800 |
+
const minimum = 0.02;
|
| 1801 |
+
const clamp = (value, opposite) => Math.min(1 - opposite - minimum, Math.max(0, value));
|
| 1802 |
+
if (drag.handle === "move") {
|
| 1803 |
+
const moveX = Math.min(values[3], Math.max(-values[2], dx));
|
| 1804 |
+
const moveY = Math.min(values[1], Math.max(-values[0], dy));
|
| 1805 |
+
values[0] += moveY;
|
| 1806 |
+
values[1] -= moveY;
|
| 1807 |
+
values[2] += moveX;
|
| 1808 |
+
values[3] -= moveX;
|
| 1809 |
+
} else {
|
| 1810 |
+
if (drag.handle.includes("top")) values[0] = clamp(values[0] + dy, values[1]);
|
| 1811 |
+
if (drag.handle.includes("bottom")) values[1] = clamp(values[1] - dy, values[0]);
|
| 1812 |
+
if (drag.handle.includes("left")) values[2] = clamp(values[2] + dx, values[3]);
|
| 1813 |
+
if (drag.handle.includes("right")) values[3] = clamp(values[3] - dx, values[2]);
|
| 1814 |
+
}
|
| 1815 |
+
const rounded = values.map((value) => Math.round(value * 10000) / 10000);
|
| 1816 |
+
setParseCropThresholds(drag.row, drag.scope, drag.page, rounded);
|
| 1817 |
+
updateParseCropUI();
|
| 1818 |
+
}
|
| 1819 |
+
|
| 1820 |
+
function endParseCrop() {
|
| 1821 |
+
if (!parseCropDrag) return;
|
| 1822 |
+
const row = parseCropDrag.row;
|
| 1823 |
+
parseCropDrag = null;
|
| 1824 |
+
updateParsePageOCR();
|
| 1825 |
+
finishParseCropChange(row);
|
| 1826 |
+
}
|
| 1827 |
+
|
| 1828 |
async function parseOCRPage(item, page) {
|
| 1829 |
const key = `${item.doc_id}:${page}`;
|
| 1830 |
if (parseOCRCache.has(key)) {
|
|
|
|
| 1855 |
if (status) status.textContent = "正在读取...";
|
| 1856 |
try {
|
| 1857 |
const pages = await Promise.all(Array.from({ length: end - start + 1 }, (_value, index) => parseOCRPage(STATE.parseItem, start + index)));
|
| 1858 |
+
const pageTexts = pages.map((data, index) => parseOCRText(data, parseOCRThresholds(row, start + index)));
|
| 1859 |
+
const body = pageTexts.filter(Boolean).join("\n");
|
| 1860 |
+
const boxCount = pages.reduce((total, data, index) => total + parseOCRRows(data, parseOCRThresholds(row, start + index)).length, 0);
|
| 1861 |
textarea.value = body;
|
| 1862 |
textarea.dataset.edited = "0";
|
| 1863 |
row.dataset.basePartCount = String(boxCount);
|
|
|
|
| 1882 |
DOM.previewPanel.innerHTML = parseEditorHTML(item);
|
| 1883 |
DOM.results.hidden = true;
|
| 1884 |
DOM.emptyState.style.display = "none";
|
| 1885 |
+
animatePreviewPanel();
|
| 1886 |
loadParsePage(1);
|
| 1887 |
DOM.previewPanel.querySelectorAll("[data-parse-article]").forEach((row) => refreshParseArticleBody(row));
|
| 1888 |
} catch (error) {
|
|
|
|
| 1899 |
if (year) date.year = year;
|
| 1900 |
if (month) date.month = month;
|
| 1901 |
if (day) date.day = day;
|
| 1902 |
+
let ocr = {};
|
| 1903 |
+
let ocrExceptions = {};
|
| 1904 |
+
try { ocr = JSON.parse(row.dataset.parseOcr || "{}"); } catch (_error) { /* validation reports malformed state */ }
|
| 1905 |
+
try { ocrExceptions = JSON.parse(row.dataset.parseOcrExceptions || "{}"); } catch (_error) { /* validation reports malformed state */ }
|
| 1906 |
+
const article = {
|
| 1907 |
title: row.querySelector("[data-parse-title]").value.trim(),
|
| 1908 |
authors: row.querySelector("[data-parse-authors]").value.split(/[、,,/]/).map((value) => value.trim()).filter(Boolean),
|
| 1909 |
dates: Object.keys(date).length ? [date] : [],
|
|
|
|
| 1913 |
content: row.querySelector("[data-parse-content]").value,
|
| 1914 |
base_part_count: Number(row.dataset.basePartCount) || 0,
|
| 1915 |
};
|
| 1916 |
+
if (Object.keys(ocr).length) article.ocr = ocr;
|
| 1917 |
+
if (Object.keys(ocrExceptions).length) article.ocr_exceptions = ocrExceptions;
|
| 1918 |
+
return article;
|
| 1919 |
});
|
| 1920 |
}
|
| 1921 |
|
|
|
|
| 3680 |
renderParseQueue();
|
| 3681 |
return;
|
| 3682 |
}
|
| 3683 |
+
const cropOpen = event.target.closest("[data-parse-crop-open]");
|
| 3684 |
+
if (cropOpen) {
|
| 3685 |
+
openParseCrop(cropOpen.closest("[data-parse-article]"));
|
| 3686 |
+
return;
|
| 3687 |
+
}
|
| 3688 |
+
if (event.target.closest("[data-parse-crop-toggle]")) {
|
| 3689 |
+
const toolbar = DOM.previewPanel.querySelector("[data-parse-crop-toolbar]");
|
| 3690 |
+
if (toolbar && toolbar.hidden) openParseCrop();
|
| 3691 |
+
else closeParseCrop();
|
| 3692 |
+
return;
|
| 3693 |
+
}
|
| 3694 |
+
const cropScope = event.target.closest("[data-parse-crop-scope]");
|
| 3695 |
+
if (cropScope) {
|
| 3696 |
+
setParseCropScope(cropScope.dataset.parseCropScope);
|
| 3697 |
+
return;
|
| 3698 |
+
}
|
| 3699 |
+
if (event.target.closest("[data-parse-crop-reset]")) {
|
| 3700 |
+
const row = parseCropTargetRow();
|
| 3701 |
+
setParseCropThresholds(row, parseCropScope(), STATE.parseCurrentPage, [0, 0, 0, 0], true);
|
| 3702 |
+
updateParseCropUI();
|
| 3703 |
+
updateParsePageOCR();
|
| 3704 |
+
finishParseCropChange(row);
|
| 3705 |
+
return;
|
| 3706 |
+
}
|
| 3707 |
+
if (event.target.closest("[data-parse-crop-close]")) {
|
| 3708 |
+
closeParseCrop();
|
| 3709 |
+
return;
|
| 3710 |
+
}
|
| 3711 |
if (event.target.closest("[data-parse-page-prev]")) {
|
| 3712 |
loadParsePage(STATE.parseCurrentPage - 1);
|
| 3713 |
return;
|
|
|
|
| 3737 |
const removeArticle = event.target.closest("[data-parse-remove]");
|
| 3738 |
if (removeArticle && !removeArticle.disabled) {
|
| 3739 |
const row = removeArticle.closest("[data-parse-article]");
|
| 3740 |
+
const wasCropTarget = row && row.classList.contains("is-crop-target");
|
| 3741 |
if (row) row.remove();
|
| 3742 |
DOM.previewPanel.querySelectorAll("[data-parse-article] legend").forEach((legend, index) => { legend.textContent = `文章 ${index + 1}`; });
|
| 3743 |
+
if (wasCropTarget) setParseCropTarget(parseCropTargetRow());
|
| 3744 |
return;
|
| 3745 |
}
|
| 3746 |
const partInsert = event.target.closest("[data-add-part-before], [data-add-part-after]");
|
|
|
|
| 3865 |
if (event.target.closest("[data-preview-theme]")) toggleTheme();
|
| 3866 |
if (event.target.closest("[data-preview-mobile]")) toggleMobileMode();
|
| 3867 |
});
|
| 3868 |
+
DOM.previewPanel.addEventListener("pointerdown", (event) => {
|
| 3869 |
+
if (event.target.closest("[data-parse-crop-box]")) beginParseCropDrag(event);
|
| 3870 |
+
});
|
| 3871 |
+
window.addEventListener("pointermove", moveParseCrop, { passive: false });
|
| 3872 |
+
window.addEventListener("pointerup", endParseCrop);
|
| 3873 |
+
window.addEventListener("pointercancel", endParseCrop);
|
| 3874 |
DOM.previewPanel.addEventListener("change", (event) => {
|
| 3875 |
const parsePage = event.target.closest("[data-parse-page]");
|
| 3876 |
if (parsePage) {
|
|
|
|
| 3879 |
}
|
| 3880 |
const parseRange = event.target.closest("[data-parse-start], [data-parse-end]");
|
| 3881 |
if (parseRange) {
|
| 3882 |
+
const row = parseRange.closest("[data-parse-article]");
|
| 3883 |
+
refreshParseArticleBody(row);
|
| 3884 |
+
if (row && row.classList.contains("is-crop-target")) updateParseCropUI();
|
| 3885 |
return;
|
| 3886 |
}
|
| 3887 |
const select = event.target.closest("[data-compare-select]");
|
static/style.css
CHANGED
|
@@ -713,6 +713,48 @@ body.parse-mode .sidebar-content > #parse-page-size-section {
|
|
| 713 |
font-size: 12px;
|
| 714 |
}
|
| 715 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 716 |
.parse-page-content {
|
| 717 |
display: grid;
|
| 718 |
grid-template-columns: minmax(0, 1fr) minmax(220px, 34%);
|
|
@@ -727,12 +769,60 @@ body.parse-mode .sidebar-content > #parse-page-size-section {
|
|
| 727 |
background: #fff;
|
| 728 |
}
|
| 729 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 730 |
.parse-image-stage img {
|
| 731 |
display: block;
|
| 732 |
width: 100%;
|
| 733 |
height: auto;
|
| 734 |
}
|
| 735 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
.parse-ocr-text {
|
| 737 |
min-width: 0;
|
| 738 |
overflow: auto;
|
|
@@ -762,6 +852,11 @@ body.parse-mode .sidebar-content > #parse-page-size-section {
|
|
| 762 |
position: relative;
|
| 763 |
}
|
| 764 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 765 |
.parse-article-actions {
|
| 766 |
position: absolute;
|
| 767 |
top: 5px;
|
|
@@ -824,6 +919,15 @@ body.mobile .parse-page-content {
|
|
| 824 |
grid-template-columns: 1fr;
|
| 825 |
}
|
| 826 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
body.mobile .parse-ocr-text {
|
| 828 |
max-height: 280px;
|
| 829 |
border-top: 1px solid var(--outline-variant);
|
|
|
|
| 713 |
font-size: 12px;
|
| 714 |
}
|
| 715 |
|
| 716 |
+
.parse-crop-toggle {
|
| 717 |
+
margin-left: auto;
|
| 718 |
+
}
|
| 719 |
+
|
| 720 |
+
.parse-crop-toolbar {
|
| 721 |
+
display: flex;
|
| 722 |
+
min-height: 42px;
|
| 723 |
+
align-items: center;
|
| 724 |
+
gap: 8px;
|
| 725 |
+
padding: 6px 10px;
|
| 726 |
+
border-bottom: 1px solid var(--outline-variant);
|
| 727 |
+
background: var(--surface-container);
|
| 728 |
+
color: var(--on-surface-variant);
|
| 729 |
+
font-size: 12px;
|
| 730 |
+
}
|
| 731 |
+
|
| 732 |
+
.parse-crop-toolbar[hidden] {
|
| 733 |
+
display: none;
|
| 734 |
+
}
|
| 735 |
+
|
| 736 |
+
.parse-crop-scope {
|
| 737 |
+
display: inline-flex;
|
| 738 |
+
border: 1px solid var(--outline-variant);
|
| 739 |
+
border-radius: 5px;
|
| 740 |
+
overflow: hidden;
|
| 741 |
+
}
|
| 742 |
+
|
| 743 |
+
.parse-crop-scope .text-btn-sm {
|
| 744 |
+
border-radius: 0;
|
| 745 |
+
}
|
| 746 |
+
|
| 747 |
+
.parse-crop-scope .text-btn-sm.primary {
|
| 748 |
+
background: var(--primary-container);
|
| 749 |
+
color: var(--on-primary-container);
|
| 750 |
+
}
|
| 751 |
+
|
| 752 |
+
[data-parse-crop-values] {
|
| 753 |
+
flex: 1;
|
| 754 |
+
min-width: 180px;
|
| 755 |
+
text-align: center;
|
| 756 |
+
}
|
| 757 |
+
|
| 758 |
.parse-page-content {
|
| 759 |
display: grid;
|
| 760 |
grid-template-columns: minmax(0, 1fr) minmax(220px, 34%);
|
|
|
|
| 769 |
background: #fff;
|
| 770 |
}
|
| 771 |
|
| 772 |
+
.parse-image-wrap {
|
| 773 |
+
position: relative;
|
| 774 |
+
width: 100%;
|
| 775 |
+
overflow: hidden;
|
| 776 |
+
}
|
| 777 |
+
|
| 778 |
.parse-image-stage img {
|
| 779 |
display: block;
|
| 780 |
width: 100%;
|
| 781 |
height: auto;
|
| 782 |
}
|
| 783 |
|
| 784 |
+
.parse-crop-overlay {
|
| 785 |
+
position: absolute;
|
| 786 |
+
inset: 0;
|
| 787 |
+
touch-action: none;
|
| 788 |
+
}
|
| 789 |
+
|
| 790 |
+
.parse-crop-overlay[hidden] {
|
| 791 |
+
display: none;
|
| 792 |
+
}
|
| 793 |
+
|
| 794 |
+
.parse-crop-box {
|
| 795 |
+
position: absolute;
|
| 796 |
+
top: 0;
|
| 797 |
+
right: 0;
|
| 798 |
+
bottom: 0;
|
| 799 |
+
left: 0;
|
| 800 |
+
min-width: 20px;
|
| 801 |
+
min-height: 20px;
|
| 802 |
+
border: 2px solid #00a878;
|
| 803 |
+
box-shadow: 0 0 0 9999px rgb(0 0 0 / 48%);
|
| 804 |
+
cursor: move;
|
| 805 |
+
}
|
| 806 |
+
|
| 807 |
+
.parse-crop-handle {
|
| 808 |
+
position: absolute;
|
| 809 |
+
width: 14px;
|
| 810 |
+
height: 14px;
|
| 811 |
+
border: 2px solid #fff;
|
| 812 |
+
border-radius: 50%;
|
| 813 |
+
background: #00a878;
|
| 814 |
+
box-shadow: 0 1px 4px rgb(0 0 0 / 40%);
|
| 815 |
+
}
|
| 816 |
+
|
| 817 |
+
.parse-crop-handle.top { top: -8px; left: 50%; cursor: ns-resize; transform: translateX(-50%); }
|
| 818 |
+
.parse-crop-handle.right { top: 50%; right: -8px; cursor: ew-resize; transform: translateY(-50%); }
|
| 819 |
+
.parse-crop-handle.bottom { bottom: -8px; left: 50%; cursor: ns-resize; transform: translateX(-50%); }
|
| 820 |
+
.parse-crop-handle.left { top: 50%; left: -8px; cursor: ew-resize; transform: translateY(-50%); }
|
| 821 |
+
.parse-crop-handle.top-left { top: -8px; left: -8px; cursor: nwse-resize; }
|
| 822 |
+
.parse-crop-handle.top-right { top: -8px; right: -8px; cursor: nesw-resize; }
|
| 823 |
+
.parse-crop-handle.bottom-right { right: -8px; bottom: -8px; cursor: nwse-resize; }
|
| 824 |
+
.parse-crop-handle.bottom-left { bottom: -8px; left: -8px; cursor: nesw-resize; }
|
| 825 |
+
|
| 826 |
.parse-ocr-text {
|
| 827 |
min-width: 0;
|
| 828 |
overflow: auto;
|
|
|
|
| 852 |
position: relative;
|
| 853 |
}
|
| 854 |
|
| 855 |
+
.parse-article.is-crop-target {
|
| 856 |
+
border-color: #00a878;
|
| 857 |
+
box-shadow: inset 3px 0 #00a878;
|
| 858 |
+
}
|
| 859 |
+
|
| 860 |
.parse-article-actions {
|
| 861 |
position: absolute;
|
| 862 |
top: 5px;
|
|
|
|
| 919 |
grid-template-columns: 1fr;
|
| 920 |
}
|
| 921 |
|
| 922 |
+
body.mobile .parse-crop-toolbar {
|
| 923 |
+
flex-wrap: wrap;
|
| 924 |
+
}
|
| 925 |
+
|
| 926 |
+
body.mobile [data-parse-crop-values] {
|
| 927 |
+
order: 4;
|
| 928 |
+
width: 100%;
|
| 929 |
+
}
|
| 930 |
+
|
| 931 |
body.mobile .parse-ocr-text {
|
| 932 |
max-height: 280px;
|
| 933 |
border-top: 1px solid var(--outline-variant);
|
tests/test_api.py
ADDED
|
@@ -0,0 +1,1062 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import json
|
| 3 |
+
import unittest
|
| 4 |
+
import tempfile
|
| 5 |
+
from contextlib import nullcontext
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from unittest.mock import AsyncMock, MagicMock, patch
|
| 8 |
+
|
| 9 |
+
from fastapi.testclient import TestClient
|
| 10 |
+
from elasticsearch.exceptions import BadRequestError
|
| 11 |
+
|
| 12 |
+
from app import bootstrap_api, document_identity, main, parse_api, pipeline_dispatch, preview_api, proofread_api, proofread_core, reindex_api, runtime, search_api, search_logic, source_files
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class SearchApiTests(unittest.TestCase):
|
| 16 |
+
@classmethod
|
| 17 |
+
def setUpClass(cls):
|
| 18 |
+
cls.client = TestClient(main.app)
|
| 19 |
+
|
| 20 |
+
def setUp(self):
|
| 21 |
+
runtime.search_response_cache.clear()
|
| 22 |
+
bootstrap_api.bootstrap_cache = None
|
| 23 |
+
proofread_api.proofread_requests.clear()
|
| 24 |
+
proofread_api.proofread_dispatches.clear()
|
| 25 |
+
|
| 26 |
+
def test_preview_etag_and_proofread_content_hash(self):
|
| 27 |
+
stored = {"doc_id": "doc-1", "title": "标题", "article": {"parts": [{"text": "正文"}]}}
|
| 28 |
+
content_hash = document_identity.content_hash(stored)
|
| 29 |
+
payload = {"content_hash": content_hash, "source_files": []}
|
| 30 |
+
with patch.object(runtime, "index_status", return_value="ready"), \
|
| 31 |
+
patch.object(preview_api, "serving_lock", return_value=nullcontext()), \
|
| 32 |
+
patch.object(preview_api, "get_doc", return_value=stored), \
|
| 33 |
+
patch.object(preview_api, "preview_payload", AsyncMock(return_value=payload)):
|
| 34 |
+
response = self.client.get("/api/preview/doc-1")
|
| 35 |
+
cached = self.client.get("/api/preview/doc-1", headers={"If-None-Match": response.headers["etag"]})
|
| 36 |
+
self.assertEqual(response.status_code, 200)
|
| 37 |
+
self.assertEqual(response.json()["content_hash"], content_hash)
|
| 38 |
+
self.assertEqual(cached.status_code, 304)
|
| 39 |
+
|
| 40 |
+
def test_proofread_rejects_stale_hash_and_deduplicates_dispatch(self):
|
| 41 |
+
stored = {
|
| 42 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 43 |
+
"publication_id": "publication-1", "title": "标题",
|
| 44 |
+
"article": {"parts": [{"text": "旧", "type": "paragraph"}], "comments": []},
|
| 45 |
+
}
|
| 46 |
+
request = {
|
| 47 |
+
"doc_id": "doc-1",
|
| 48 |
+
"patch": {"version": 2, "parts": {"0": {"diff": "-1\t+新"}}, "comments": {}, "description": ""},
|
| 49 |
+
}
|
| 50 |
+
dispatch = AsyncMock()
|
| 51 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 52 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 53 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 54 |
+
stale = self.client.post("/api/proofread", json={**request, "base_content_hash": "0" * 64})
|
| 55 |
+
current_hash = document_identity.content_hash(stored)
|
| 56 |
+
first = self.client.post("/api/proofread", json={**request, "base_content_hash": current_hash})
|
| 57 |
+
duplicate = self.client.post("/api/proofread", json={**request, "base_content_hash": current_hash})
|
| 58 |
+
self.assertEqual(stale.status_code, 409)
|
| 59 |
+
self.assertEqual(first.status_code, 202)
|
| 60 |
+
self.assertEqual(duplicate.status_code, 200)
|
| 61 |
+
self.assertEqual(duplicate.json()["status"], "duplicate")
|
| 62 |
+
self.assertEqual(first.json()["request_id"], duplicate.json()["request_id"])
|
| 63 |
+
dispatch.assert_awaited_once()
|
| 64 |
+
|
| 65 |
+
def test_proofread_requires_server_github_token(self):
|
| 66 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", ""):
|
| 67 |
+
response = self.client.post("/api/proofread", json={"doc_id": "doc-1", "patch": {"version": 2}})
|
| 68 |
+
self.assertEqual(response.status_code, 503)
|
| 69 |
+
|
| 70 |
+
def test_parse_queue_uses_placeholder_store(self):
|
| 71 |
+
stored = {
|
| 72 |
+
"doc_id": "doc-1", "archive_id": 25, "publication_id": "a" * 36,
|
| 73 |
+
"publication_name": "小报", "title": "【文章待拆分】小报",
|
| 74 |
+
"article": {"page_start": 1, "page_end": 2}, "source_files": ["one", "two"],
|
| 75 |
+
}
|
| 76 |
+
with patch.object(parse_api, "serving_lock", return_value=nullcontext()), \
|
| 77 |
+
patch.object(parse_api, "parse_queue_docs", return_value=(1, [stored])):
|
| 78 |
+
response = self.client.get("/api/parse/queue?page=1&page_size=20")
|
| 79 |
+
self.assertEqual(response.status_code, 200)
|
| 80 |
+
self.assertEqual(response.json()["total"], 1)
|
| 81 |
+
self.assertEqual(response.json()["items"][0]["page_end"], 2)
|
| 82 |
+
|
| 83 |
+
def test_parse_submission_dispatches_structured_articles(self):
|
| 84 |
+
stored = {
|
| 85 |
+
"doc_id": "doc-1", "archive_id": 25, "article_id": "placeholder",
|
| 86 |
+
"publication_id": "052417de-42fb-4781-9885-af4fb006e9b6",
|
| 87 |
+
"publication_name": "革命楼通讯", "title": "【文章待拆分】革命楼通讯",
|
| 88 |
+
"source_files": ["one", "two"],
|
| 89 |
+
"article": {
|
| 90 |
+
"title": "【文章待拆分】革命楼通讯", "page_start": 1, "page_end": 2,
|
| 91 |
+
"dates": [{"year": 1967, "month": 8, "day": 29}],
|
| 92 |
+
},
|
| 93 |
+
}
|
| 94 |
+
dispatch = AsyncMock()
|
| 95 |
+
articles = [
|
| 96 |
+
{"title": "第一篇", "authors": ["作者甲"], "dates": [{"year": 1967, "month": 8, "day": 29}], "page_start": 1, "page_end": 1, "content": "第一篇正文", "base_part_count": 12, "ocr": {"content_thresholds": [0.1, 0.05, 0.08, 0.07]}, "ocr_exceptions": {"1": {"content_thresholds": [0.12, 0.05, 0.08, 0.07]}}},
|
| 97 |
+
{"title": "第二篇", "authors": [], "dates": [{"year": 1967, "month": 8, "day": 29}], "page_start": 2, "page_end": 2, "content": "第二篇正文", "base_part_count": 15},
|
| 98 |
+
]
|
| 99 |
+
with patch.object(parse_api, "GITHUB_TOKEN", "secret"), \
|
| 100 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 101 |
+
patch.object(parse_api, "dispatch_parse", dispatch):
|
| 102 |
+
response = self.client.post("/api/parse", json={
|
| 103 |
+
"doc_id": "doc-1", "base_content_hash": document_identity.content_hash(stored),
|
| 104 |
+
"articles": articles, "description": "拆分",
|
| 105 |
+
})
|
| 106 |
+
self.assertEqual(response.status_code, 202)
|
| 107 |
+
dispatch.assert_awaited_once()
|
| 108 |
+
payload = dispatch.await_args.args[0]
|
| 109 |
+
self.assertEqual(payload["kind"], "parse")
|
| 110 |
+
self.assertEqual(payload["archive_id"], 25)
|
| 111 |
+
self.assertEqual(len(payload["body"]["articles"]), 2)
|
| 112 |
+
self.assertEqual(payload["body"]["locator"]["page_end"], 2)
|
| 113 |
+
self.assertEqual(payload["body"]["articles"][0]["ocr"]["content_thresholds"], [0.1, 0.05, 0.08, 0.07])
|
| 114 |
+
self.assertIn("1", payload["body"]["articles"][0]["ocr_exceptions"])
|
| 115 |
+
|
| 116 |
+
def test_parse_submission_rejects_missing_pages_and_duplicate_ids(self):
|
| 117 |
+
stored = {
|
| 118 |
+
"doc_id": "doc-1", "archive_id": 25,
|
| 119 |
+
"publication_id": "052417de-42fb-4781-9885-af4fb006e9b6",
|
| 120 |
+
"title": "【文章待拆分】革命楼通讯",
|
| 121 |
+
"article": {"title": "【文章待拆分】革命楼通讯", "page_start": 1, "page_end": 2},
|
| 122 |
+
}
|
| 123 |
+
base = {"doc_id": "doc-1", "articles": [{"title": "同题", "page_start": 1, "page_end": 1, "content": "正文", "base_part_count": 1}]}
|
| 124 |
+
duplicate = {"doc_id": "doc-1", "articles": [
|
| 125 |
+
{"title": "同题", "page_start": 1, "page_end": 1, "content": "正文一", "base_part_count": 1},
|
| 126 |
+
{"title": "同题", "page_start": 2, "page_end": 2, "content": "正文二", "base_part_count": 1},
|
| 127 |
+
]}
|
| 128 |
+
with patch.object(parse_api, "GITHUB_TOKEN", "secret"), \
|
| 129 |
+
patch.object(source_files, "get_source_doc", return_value=stored):
|
| 130 |
+
missing = self.client.post("/api/parse", json=base)
|
| 131 |
+
collision = self.client.post("/api/parse", json=duplicate)
|
| 132 |
+
self.assertEqual(missing.status_code, 400)
|
| 133 |
+
self.assertIn("cover", missing.json()["error"])
|
| 134 |
+
self.assertEqual(collision.status_code, 400)
|
| 135 |
+
self.assertIn("duplicate", collision.json()["error"])
|
| 136 |
+
|
| 137 |
+
def test_proofread_rate_limit_uses_forwarded_client_on_space(self):
|
| 138 |
+
request = MagicMock()
|
| 139 |
+
request.headers = {"x-forwarded-for": "198.51.100.8, 203.0.113.7"}
|
| 140 |
+
request.client.host = "10.0.0.2"
|
| 141 |
+
with patch.object(proofread_api, "TRUST_PROXY", True), patch.object(proofread_api, "PROXY_NETWORKS", (proofread_api.ipaddress.ip_network("10.0.0.0/8"),)):
|
| 142 |
+
self.assertIsNone(proofread_api.proofread_rate_error(request))
|
| 143 |
+
self.assertIn("198.51.100.8", proofread_api.proofread_requests)
|
| 144 |
+
|
| 145 |
+
def test_proofread_dispatches_body_and_metadata_tasks(self):
|
| 146 |
+
stored = {
|
| 147 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 148 |
+
"publication_id": "publication-1", "title": "旧标题",
|
| 149 |
+
"article": {"title": "旧标题", "page_start": 1, "page_end": 2, "parts": [{"text": "旧", "type": "paragraph"}], "comments": []},
|
| 150 |
+
}
|
| 151 |
+
dispatch = AsyncMock()
|
| 152 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 153 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 154 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 155 |
+
response = self.client.post("/api/proofread", json={
|
| 156 |
+
"doc_id": "doc-1",
|
| 157 |
+
"patch": {"version": 2, "parts": {"0": {"diff": "-1\t+新"}}, "comments": {}, "description": ""},
|
| 158 |
+
"metadata": {"article": {"title": "新标题", "authors": ["作者"], "dates": [{"year": 1967}], "tags": [{"name": "标签", "type": "主题/事件"}]}},
|
| 159 |
+
})
|
| 160 |
+
self.assertEqual(response.status_code, 202)
|
| 161 |
+
self.assertEqual(response.json()["tasks"], 1)
|
| 162 |
+
dispatch.assert_awaited_once()
|
| 163 |
+
payload = dispatch.await_args.args[0]
|
| 164 |
+
self.assertEqual(payload["kind"], "proofread")
|
| 165 |
+
self.assertEqual(payload["archive_id"], 3)
|
| 166 |
+
self.assertLessEqual(len(payload), 10)
|
| 167 |
+
self.assertNotIn("patch", payload)
|
| 168 |
+
self.assertNotIn("metadata", payload)
|
| 169 |
+
self.assertNotIn("locator", payload)
|
| 170 |
+
self.assertIn("patch", payload["body"])
|
| 171 |
+
self.assertIn("metadata", payload["body"])
|
| 172 |
+
self.assertIn("locator", payload["body"])
|
| 173 |
+
self.assertFalse(payload.get("auto_merge"))
|
| 174 |
+
changed = payload["changed"]
|
| 175 |
+
self.assertEqual(changed[0], {"kind": "part", "index": 1, "original": "旧", "edited": "新"})
|
| 176 |
+
self.assertEqual(changed[1], {"kind": "metadata", "field": "title", "old": "旧标题", "new": "新标题"})
|
| 177 |
+
self.assertEqual(changed[2]["field"], "authors")
|
| 178 |
+
self.assertEqual(changed[3]["field"], "dates")
|
| 179 |
+
self.assertEqual(changed[4]["field"], "tags")
|
| 180 |
+
|
| 181 |
+
def test_proofread_change_preview_covers_comments_description_and_inserts(self):
|
| 182 |
+
patch = {
|
| 183 |
+
"version": 2,
|
| 184 |
+
"parts": {"0": {"diff": "=1\t-1\t+新", "insertAfter": [{"text": "插入段", "type": "paragraph"}]}, "2": {"delete": True}},
|
| 185 |
+
"comments": {"1": {"diff": "=1\t+注\t=1"}},
|
| 186 |
+
"description": "+描述",
|
| 187 |
+
}
|
| 188 |
+
article = {
|
| 189 |
+
"parts": [{"text": "旧文", "type": "paragraph"}, {"text": "二", "type": "paragraph"}, {"text": "三", "type": "paragraph"}],
|
| 190 |
+
"comments": ["注释", "已删注释"],
|
| 191 |
+
"description": "",
|
| 192 |
+
}
|
| 193 |
+
source = {"publication_name": "来源甲", "publication_author": "著者", "publication_type": "书", "source_files": [{"url": "https://x/a"}]}
|
| 194 |
+
changes = proofread_core.proofread_change_preview(patch, article, {"source": {"name": "来源乙"}}, source)
|
| 195 |
+
self.assertEqual(changes[0]["kind"], "part")
|
| 196 |
+
self.assertEqual(changes[0]["index"], 1)
|
| 197 |
+
self.assertEqual(changes[0]["original"], "旧文")
|
| 198 |
+
self.assertEqual(changes[0]["edited"], "旧新")
|
| 199 |
+
self.assertEqual(changes[1], {"kind": "part", "index": 1, "insert": True, "text": "插入段", "part_type": "paragraph"})
|
| 200 |
+
self.assertEqual(changes[2], {"kind": "part", "index": 3, "delete": True, "original": "三"})
|
| 201 |
+
self.assertEqual(changes[3], {"kind": "comment", "index": 1, "original": "注释", "edited": "注注释"})
|
| 202 |
+
self.assertEqual(changes[4], {"kind": "description", "original": "", "edited": "描述"})
|
| 203 |
+
self.assertEqual(changes[5], {"kind": "metadata", "field": "name", "old": "来源甲", "new": "来源乙"})
|
| 204 |
+
|
| 205 |
+
def test_multi_operation_delta_replays_moved_text_and_utf16(self):
|
| 206 |
+
self.assertEqual(proofread_core.apply_text_delta("甲乙丙丁", "-2\t=2\t+甲乙"), "丙丁甲乙")
|
| 207 |
+
self.assertEqual(proofread_core.apply_text_delta("甲😀乙", "=1\t-2\t+校\t=1"), "甲校乙")
|
| 208 |
+
|
| 209 |
+
def test_change_preview_includes_type_and_new_comment_but_skips_unchanged_metadata(self):
|
| 210 |
+
article = {
|
| 211 |
+
"title": "标题",
|
| 212 |
+
"parts": [{"text": "正文", "type": "paragraph"}],
|
| 213 |
+
"comments": [],
|
| 214 |
+
}
|
| 215 |
+
changes = proofread_core.proofread_change_preview(
|
| 216 |
+
{"version": 2, "parts": {"0": {"type": "quotation"}}, "comments": {}, "description": "", "newComments": ["新增注释"]},
|
| 217 |
+
article,
|
| 218 |
+
{"article": {"title": "标题"}},
|
| 219 |
+
{},
|
| 220 |
+
)
|
| 221 |
+
self.assertEqual(changes, [
|
| 222 |
+
{"kind": "part_type", "index": 1, "old": "paragraph", "new": "quotation"},
|
| 223 |
+
{"kind": "new_comment", "index": 1, "text": "新增注释"},
|
| 224 |
+
])
|
| 225 |
+
|
| 226 |
+
def test_change_preview_keeps_full_text_within_budget(self):
|
| 227 |
+
long_text = "长文" * 1000
|
| 228 |
+
stored = {
|
| 229 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 230 |
+
"publication_id": "publication-1", "title": "标题",
|
| 231 |
+
"article": {"title": "标题", "page_start": 1, "page_end": 1, "parts": [{"text": long_text, "type": "paragraph"}], "comments": []},
|
| 232 |
+
}
|
| 233 |
+
dispatch = AsyncMock()
|
| 234 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 235 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 236 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 237 |
+
response = self.client.post("/api/proofread", json={
|
| 238 |
+
"doc_id": "doc-1",
|
| 239 |
+
"patch": {"version": 2, "parts": {"0": {"diff": "=1999\t-1\t+新"}}, "comments": {}, "description": ""},
|
| 240 |
+
})
|
| 241 |
+
self.assertEqual(response.status_code, 202)
|
| 242 |
+
changed = dispatch.await_args.args[0]["changed"]
|
| 243 |
+
self.assertEqual(len(changed[0]["original"]), 2000)
|
| 244 |
+
self.assertNotIn("…", changed[0]["original"])
|
| 245 |
+
self.assertEqual(changed[0]["edited"], long_text[:1999] + "新")
|
| 246 |
+
|
| 247 |
+
def test_oversized_change_preview_is_clipped_before_dispatch(self):
|
| 248 |
+
stored = {
|
| 249 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 250 |
+
"publication_id": "publication-1", "title": "标题",
|
| 251 |
+
"article": {"title": "标题", "page_start": 1, "page_end": 1, "parts": [{"text": "长文" * 15000, "type": "paragraph"}], "comments": []},
|
| 252 |
+
}
|
| 253 |
+
dispatch = AsyncMock()
|
| 254 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 255 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 256 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 257 |
+
response = self.client.post("/api/proofread", json={
|
| 258 |
+
"doc_id": "doc-1",
|
| 259 |
+
"patch": {"version": 2, "parts": {"0": {"diff": "=29999\t-1\t+新"}}, "comments": {}, "description": ""},
|
| 260 |
+
})
|
| 261 |
+
self.assertEqual(response.status_code, 202)
|
| 262 |
+
payload = dispatch.await_args.args[0]
|
| 263 |
+
self.assertLessEqual(len(json.dumps(payload, ensure_ascii=False)), proofread_api.CHANGE_PREVIEW_BUDGET_CHARACTERS)
|
| 264 |
+
original = payload["changed"][0]["original"]
|
| 265 |
+
self.assertEqual(len(original), 2001)
|
| 266 |
+
self.assertTrue(original.endswith("…"))
|
| 267 |
+
|
| 268 |
+
def test_unicode_payload_over_old_byte_limit_is_accepted(self):
|
| 269 |
+
inserted = "校" * 18000
|
| 270 |
+
stored = {
|
| 271 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 272 |
+
"publication_id": "publication-1", "title": "标题",
|
| 273 |
+
"article": {"title": "标题", "parts": [{"text": "", "type": "paragraph"}], "comments": []},
|
| 274 |
+
}
|
| 275 |
+
dispatch = AsyncMock()
|
| 276 |
+
request = {
|
| 277 |
+
"doc_id": "doc-1",
|
| 278 |
+
"patch": {"version": 2, "parts": {"0": {"diff": f"+{inserted}"}}, "comments": {}, "description": ""},
|
| 279 |
+
}
|
| 280 |
+
self.assertGreater(len(json.dumps(request, ensure_ascii=False).encode("utf-8")), 50 * 1024)
|
| 281 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 282 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 283 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 284 |
+
response = self.client.post("/api/proofread", json=request)
|
| 285 |
+
self.assertEqual(response.status_code, 202)
|
| 286 |
+
dispatch.assert_awaited_once()
|
| 287 |
+
|
| 288 |
+
def test_dispatch_segments_oversized_payload_into_git_blobs(self):
|
| 289 |
+
calls = []
|
| 290 |
+
|
| 291 |
+
class Response:
|
| 292 |
+
def __init__(self, status_code, payload=None):
|
| 293 |
+
self.status_code = status_code
|
| 294 |
+
self._payload = payload or {}
|
| 295 |
+
|
| 296 |
+
def json(self):
|
| 297 |
+
return self._payload
|
| 298 |
+
|
| 299 |
+
class Client:
|
| 300 |
+
async def __aenter__(self):
|
| 301 |
+
return self
|
| 302 |
+
|
| 303 |
+
async def __aexit__(self, *_args):
|
| 304 |
+
return None
|
| 305 |
+
|
| 306 |
+
async def post(self, url, **kwargs):
|
| 307 |
+
calls.append((url, kwargs["json"]))
|
| 308 |
+
if url.endswith("/git/blobs"):
|
| 309 |
+
return Response(201, {"sha": f"{len(calls):040x}"})
|
| 310 |
+
return Response(204)
|
| 311 |
+
|
| 312 |
+
payload = {"request_id": "request-1", "body": {"patch": "中文" * 100}}
|
| 313 |
+
with patch.object(proofread_api, "DISPATCH_MAX_CHARACTERS", 80), \
|
| 314 |
+
patch.object(proofread_api, "BLOB_CHUNK_CHARACTERS", 40), \
|
| 315 |
+
patch.object(pipeline_dispatch.httpx, "AsyncClient", return_value=Client()):
|
| 316 |
+
asyncio.run(proofread_api.dispatch_proofread(payload))
|
| 317 |
+
blob_calls = [item for item in calls if item[0].endswith("/git/blobs")]
|
| 318 |
+
self.assertGreater(len(blob_calls), 1)
|
| 319 |
+
serialized = "".join(item[1]["content"] for item in blob_calls)
|
| 320 |
+
self.assertEqual(json.loads(serialized), payload)
|
| 321 |
+
dispatch = calls[-1][1]["client_payload"]
|
| 322 |
+
self.assertEqual(dispatch["payload_characters"], len(serialized))
|
| 323 |
+
self.assertEqual(dispatch["payload_sha256"], proofread_api.hashlib.sha256(serialized.encode()).hexdigest())
|
| 324 |
+
self.assertEqual(dispatch["request_id"], "request-1")
|
| 325 |
+
|
| 326 |
+
calls.clear()
|
| 327 |
+
property_payload = {f"field_{index}": index for index in range(11)}
|
| 328 |
+
property_payload["request_id"] = "request-2"
|
| 329 |
+
with patch.object(proofread_api, "DISPATCH_MAX_CHARACTERS", 1_000_000), \
|
| 330 |
+
patch.object(pipeline_dispatch.httpx, "AsyncClient", return_value=Client()):
|
| 331 |
+
asyncio.run(proofread_api.dispatch_proofread(property_payload))
|
| 332 |
+
self.assertTrue(any(item[0].endswith("/git/blobs") for item in calls))
|
| 333 |
+
self.assertEqual(calls[-1][1]["client_payload"]["request_id"], "request-2")
|
| 334 |
+
|
| 335 |
+
def test_small_body_correction_is_marked_for_auto_merge(self):
|
| 336 |
+
stored = {
|
| 337 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 338 |
+
"publication_id": "publication-1", "title": "标题",
|
| 339 |
+
"article": {
|
| 340 |
+
"title": "标题", "page_start": 1, "page_end": 1,
|
| 341 |
+
"parts": [{"text": "旧文", "type": "paragraph"}],
|
| 342 |
+
"comments": [], "comment_pivots": [], "description": "",
|
| 343 |
+
},
|
| 344 |
+
}
|
| 345 |
+
dispatch = AsyncMock()
|
| 346 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 347 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 348 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 349 |
+
response = self.client.post("/api/proofread", json={
|
| 350 |
+
"doc_id": "doc-1",
|
| 351 |
+
"patch": {"version": 2, "parts": {"0": {"diff": "=1\t-1\t+新"}}, "comments": {}, "description": ""},
|
| 352 |
+
})
|
| 353 |
+
self.assertEqual(response.status_code, 202)
|
| 354 |
+
self.assertTrue(response.json()["auto_merge"])
|
| 355 |
+
self.assertTrue(dispatch.await_args.args[0]["auto_merge"])
|
| 356 |
+
|
| 357 |
+
def test_multi_part_correction_with_type_and_insert_is_marked_for_auto_merge(self):
|
| 358 |
+
stored = {
|
| 359 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 360 |
+
"publication_id": "publication-1", "title": "标题",
|
| 361 |
+
"article": {
|
| 362 |
+
"title": "标题", "page_start": 1, "page_end": 1,
|
| 363 |
+
"parts": [{"text": "第一段", "type": "paragraph"}, {"text": "第二段", "type": "paragraph"}],
|
| 364 |
+
"comments": [], "comment_pivots": [], "description": "",
|
| 365 |
+
},
|
| 366 |
+
}
|
| 367 |
+
dispatch = AsyncMock()
|
| 368 |
+
patch_doc = {
|
| 369 |
+
"version": 2,
|
| 370 |
+
"parts": {
|
| 371 |
+
"0": {"diff": "=2\t-1\t+改", "insertAfter": [{"text": "插入段", "type": "paragraph"}]},
|
| 372 |
+
"1": {"type": "quotation"},
|
| 373 |
+
},
|
| 374 |
+
"comments": {}, "description": "",
|
| 375 |
+
}
|
| 376 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 377 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 378 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 379 |
+
response = self.client.post("/api/proofread", json={"doc_id": "doc-1", "patch": patch_doc})
|
| 380 |
+
self.assertEqual(response.status_code, 202)
|
| 381 |
+
self.assertTrue(response.json()["auto_merge"])
|
| 382 |
+
self.assertTrue(dispatch.await_args.args[0]["auto_merge"])
|
| 383 |
+
|
| 384 |
+
def test_auto_merge_allows_500_cost_but_rejects_501_or_large_delta(self):
|
| 385 |
+
stored = {
|
| 386 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 387 |
+
"publication_id": "publication-1", "title": "标题",
|
| 388 |
+
"article": {
|
| 389 |
+
"title": "标题", "page_start": 1, "page_end": 1,
|
| 390 |
+
"parts": [{"text": "第一段", "type": "paragraph"}],
|
| 391 |
+
"comments": [], "comment_pivots": [], "description": "",
|
| 392 |
+
},
|
| 393 |
+
}
|
| 394 |
+
dispatch = AsyncMock()
|
| 395 |
+
cases = [
|
| 396 |
+
({"0": {"diff": "=3\t+" + "改" * 500}}, True),
|
| 397 |
+
({"0": {"diff": "=3\t+" + "改" * 501}}, False),
|
| 398 |
+
({"0": {"insertAfter": [{"text": "改" * 500, "type": "paragraph"}]}}, True),
|
| 399 |
+
({"0": {"insertAfter": [{"text": "改" * 501, "type": "paragraph"}]}}, False),
|
| 400 |
+
({"0": {"diff": "=3\t+改", "insertAfter": [{"text": "a", "type": "paragraph"} for _ in range(4)]}}, False),
|
| 401 |
+
]
|
| 402 |
+
for patch_doc, expected_auto_merge in cases:
|
| 403 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 404 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 405 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 406 |
+
response = self.client.post("/api/proofread", json={"doc_id": "doc-1", "patch": {"version": 2, "parts": patch_doc, "comments": {}, "description": ""}})
|
| 407 |
+
self.assertEqual(response.status_code, 202)
|
| 408 |
+
self.assertEqual(response.json()["auto_merge"], expected_auto_merge)
|
| 409 |
+
self.assertEqual(dispatch.await_args.args[0].get("auto_merge") is True, expected_auto_merge)
|
| 410 |
+
|
| 411 |
+
def test_malformed_patch_is_rejected_before_dispatch(self):
|
| 412 |
+
stored = {"doc_id": "doc-1", "archive_id": 3, "article_id": "a", "publication_id": "p", "article": {"parts": [], "comments": []}}
|
| 413 |
+
dispatch = AsyncMock()
|
| 414 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), patch.object(source_files, "get_source_doc", return_value=stored), patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 415 |
+
response = self.client.post("/api/proofread", json={"doc_id": "doc-1", "patch": {"version": 2}})
|
| 416 |
+
self.assertEqual(response.status_code, 400)
|
| 417 |
+
dispatch.assert_not_awaited()
|
| 418 |
+
|
| 419 |
+
def test_semantically_empty_patch_is_rejected_before_dispatch(self):
|
| 420 |
+
stored = {
|
| 421 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "a", "publication_id": "p",
|
| 422 |
+
"article": {"parts": [{"text": "正文", "type": "paragraph"}], "comments": ["注释"]},
|
| 423 |
+
}
|
| 424 |
+
patches = [
|
| 425 |
+
{"version": 2, "parts": {"0": {"delete": False}}, "comments": {}, "description": ""},
|
| 426 |
+
{"version": 2, "parts": {"0": {"diff": "=2"}}, "comments": {}, "description": ""},
|
| 427 |
+
{"version": 2, "parts": {"0": {"type": "paragraph"}}, "comments": {}, "description": ""},
|
| 428 |
+
{"version": 2, "parts": {"0": {"insertAfter": []}}, "comments": {}, "description": ""},
|
| 429 |
+
{"version": 2, "parts": {}, "comments": {"1": {"diff": "=2"}}, "description": ""},
|
| 430 |
+
{"version": 2, "parts": {}, "comments": {}, "description": "=0"},
|
| 431 |
+
]
|
| 432 |
+
dispatch = AsyncMock()
|
| 433 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 434 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 435 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 436 |
+
responses = [self.client.post("/api/proofread", json={"doc_id": "doc-1", "patch": value}) for value in patches]
|
| 437 |
+
self.assertTrue(all(response.status_code == 400 for response in responses))
|
| 438 |
+
dispatch.assert_not_awaited()
|
| 439 |
+
|
| 440 |
+
def test_unchanged_metadata_is_rejected_before_dispatch(self):
|
| 441 |
+
stored = {
|
| 442 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "a", "publication_id": "p",
|
| 443 |
+
"title": "标题", "article": {"title": "标题", "parts": [], "comments": []},
|
| 444 |
+
}
|
| 445 |
+
dispatch = AsyncMock()
|
| 446 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 447 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 448 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 449 |
+
response = self.client.post("/api/proofread", json={
|
| 450 |
+
"doc_id": "doc-1", "metadata": {"article": {"title": "标题"}},
|
| 451 |
+
})
|
| 452 |
+
self.assertEqual(response.status_code, 400)
|
| 453 |
+
dispatch.assert_not_awaited()
|
| 454 |
+
|
| 455 |
+
def test_metadata_requires_exact_string_types(self):
|
| 456 |
+
stored = {"doc_id": "doc-1", "archive_id": 3, "article_id": "a", "publication_id": "p", "article": {}}
|
| 457 |
+
dispatch = AsyncMock()
|
| 458 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), patch.object(source_files, "get_source_doc", return_value=stored), patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 459 |
+
response = self.client.post("/api/proofread", json={"doc_id": "doc-1", "metadata": {"article": {"title": []}}})
|
| 460 |
+
self.assertEqual(response.status_code, 400)
|
| 461 |
+
dispatch.assert_not_awaited()
|
| 462 |
+
|
| 463 |
+
def test_tag_correction_requires_manual_review(self):
|
| 464 |
+
stored = {
|
| 465 |
+
"doc_id": "doc-1", "archive_id": 3, "article_id": "article-1",
|
| 466 |
+
"publication_id": "publication-1", "title": "标题",
|
| 467 |
+
"article": {"title": "标题", "page_start": 1, "page_end": 1, "tags": []},
|
| 468 |
+
}
|
| 469 |
+
dispatch = AsyncMock()
|
| 470 |
+
with patch.object(proofread_api, "GITHUB_TOKEN", "secret"), \
|
| 471 |
+
patch.object(source_files, "get_source_doc", return_value=stored), \
|
| 472 |
+
patch.object(proofread_api, "dispatch_proofread", dispatch):
|
| 473 |
+
response = self.client.post("/api/proofread", json={
|
| 474 |
+
"doc_id": "doc-1",
|
| 475 |
+
"metadata": {"article": {"tags": [{"name": "标签", "type": "主题/事件"}]}},
|
| 476 |
+
})
|
| 477 |
+
self.assertEqual(response.status_code, 202)
|
| 478 |
+
self.assertFalse(response.json()["auto_merge"])
|
| 479 |
+
self.assertNotIn("auto_merge", dispatch.await_args.args[0])
|
| 480 |
+
|
| 481 |
+
def test_query_length_is_limited(self):
|
| 482 |
+
response = self.client.post("/api/search", json={"q": "甲" * 257})
|
| 483 |
+
self.assertEqual(response.status_code, 422)
|
| 484 |
+
self.assertNotIn("server-timing", response.headers)
|
| 485 |
+
|
| 486 |
+
def test_search_rejects_invalid_sort_and_filter_lists_over_500(self):
|
| 487 |
+
invalid_sort = self.client.post("/api/search", json={"q": "手机", "sort": "dat_desc"})
|
| 488 |
+
too_many_filters = self.client.post("/api/search", json={"q": "手机", "sources": [str(index) for index in range(501)]})
|
| 489 |
+
self.assertEqual(invalid_sort.status_code, 422)
|
| 490 |
+
self.assertEqual(too_many_filters.status_code, 422)
|
| 491 |
+
|
| 492 |
+
def test_search_request_mutable_defaults_are_isolated(self):
|
| 493 |
+
first = search_logic.SearchRequest()
|
| 494 |
+
second = search_logic.SearchRequest()
|
| 495 |
+
first.sources.append("刊物")
|
| 496 |
+
first.date_terms.append({"year": 1966})
|
| 497 |
+
self.assertEqual(second.sources, [])
|
| 498 |
+
self.assertEqual(second.date_terms, [])
|
| 499 |
+
|
| 500 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 501 |
+
@patch.object(runtime, "has_index_data", return_value=False)
|
| 502 |
+
def test_search_without_index_returns_stable_shape(self, _has_index, _status):
|
| 503 |
+
response = self.client.post("/api/search", json={"q": "手机", "page": -2, "page_size": 500})
|
| 504 |
+
self.assertEqual(response.status_code, 200)
|
| 505 |
+
self.assertEqual(response.json()["results"], [])
|
| 506 |
+
self.assertEqual(response.json()["page"], 1)
|
| 507 |
+
|
| 508 |
+
@patch.object(search_api, "get_search_docs")
|
| 509 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 510 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 511 |
+
def test_search_calls_es_and_trims_results(self, _status, _has_index, get_search_docs):
|
| 512 |
+
stored = {
|
| 513 |
+
"doc-1": {
|
| 514 |
+
"doc_id": "doc-1",
|
| 515 |
+
"title": "手机资料",
|
| 516 |
+
"publication_name": "测试刊物",
|
| 517 |
+
"content": "这是手机资料",
|
| 518 |
+
}
|
| 519 |
+
}
|
| 520 |
+
get_search_docs.return_value = stored
|
| 521 |
+
fake_es = MagicMock()
|
| 522 |
+
fake_es.search.return_value = {
|
| 523 |
+
"hits": {
|
| 524 |
+
"total": {"value": 1},
|
| 525 |
+
"hits": [{"_id": "doc-1", "_score": 2.0, "highlight": {"content": ["这是<mark>手机</mark>资料"]}}],
|
| 526 |
+
}
|
| 527 |
+
}
|
| 528 |
+
with patch.object(runtime, "es", fake_es):
|
| 529 |
+
response = self.client.post("/api/search", json={"q": "手机", "exact": False, "page_size": 500})
|
| 530 |
+
self.assertEqual(response.status_code, 200)
|
| 531 |
+
payload = response.json()
|
| 532 |
+
self.assertEqual(payload["total"], 1)
|
| 533 |
+
self.assertEqual(payload["page_size"], 100)
|
| 534 |
+
self.assertEqual(payload["results"][0]["doc_id"], "doc-1")
|
| 535 |
+
self.assertEqual(payload["results"][0]["title_highlight"], "<mark>手机</mark>资料")
|
| 536 |
+
for field in ("article_id", "publication_id", "archive_id", "path", "score"):
|
| 537 |
+
self.assertNotIn(field, payload["results"][0])
|
| 538 |
+
kwargs = fake_es.search.call_args.kwargs
|
| 539 |
+
self.assertEqual(kwargs["size"], 100)
|
| 540 |
+
self.assertNotIn("highlight", kwargs)
|
| 541 |
+
|
| 542 |
+
@patch.object(search_api, "get_search_docs", return_value={})
|
| 543 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 544 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 545 |
+
def test_empty_relevance_search_uses_stable_doc_id_sort(self, _status, _has_index, _get_docs):
|
| 546 |
+
fake_es = MagicMock()
|
| 547 |
+
fake_es.search.return_value = {"hits": {"total": {"value": 0}, "hits": []}}
|
| 548 |
+
with patch.object(runtime, "es", fake_es):
|
| 549 |
+
response = self.client.post("/api/search", json={"q": ""})
|
| 550 |
+
self.assertEqual(response.status_code, 200)
|
| 551 |
+
self.assertEqual(
|
| 552 |
+
fake_es.search.call_args.kwargs["sort"],
|
| 553 |
+
[{"doc_id": {"order": "asc"}}],
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
@patch.object(search_api, "get_search_docs", return_value={
|
| 557 |
+
"doc-1": {"doc_id": "doc-1", "title": "无关标题", "content": "正文手机内容"},
|
| 558 |
+
})
|
| 559 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 560 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 561 |
+
def test_search_omits_redundant_title_highlight(self, _status, _has_index, _get_docs):
|
| 562 |
+
fake_es = MagicMock()
|
| 563 |
+
fake_es.search.return_value = {
|
| 564 |
+
"hits": {"total": {"value": 1}, "hits": [{"_id": "doc-1", "_score": 1.0}]},
|
| 565 |
+
}
|
| 566 |
+
with patch.object(runtime, "es", fake_es):
|
| 567 |
+
response = self.client.post("/api/search", json={"q": "手机"})
|
| 568 |
+
item = response.json()["results"][0]
|
| 569 |
+
self.assertEqual(item["title"], "无关标题")
|
| 570 |
+
self.assertNotIn("title_highlight", item)
|
| 571 |
+
|
| 572 |
+
@patch.object(search_api, "get_search_docs", return_value={})
|
| 573 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 574 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 575 |
+
def test_identical_search_requests_use_bounded_response_cache(self, _status, _has_index, _get_docs):
|
| 576 |
+
fake_es = MagicMock()
|
| 577 |
+
fake_es.search.return_value = {"hits": {"total": {"value": 0}, "hits": []}}
|
| 578 |
+
with patch.object(runtime, "es", fake_es):
|
| 579 |
+
first = self.client.post("/api/search", json={"q": "手机", "page": 1, "page_size": 20})
|
| 580 |
+
second = self.client.post("/api/search", json={"q": "手机", "page": 1, "page_size": 20})
|
| 581 |
+
self.assertEqual(first.status_code, 200)
|
| 582 |
+
self.assertEqual(second.status_code, 200)
|
| 583 |
+
self.assertEqual(fake_es.search.call_count, 1)
|
| 584 |
+
|
| 585 |
+
@patch.object(search_api, "get_search_docs", return_value={})
|
| 586 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 587 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 588 |
+
def test_search_cache_key_separates_page_and_filter_options(self, _status, _has_index, _get_docs):
|
| 589 |
+
fake_es = MagicMock()
|
| 590 |
+
fake_es.search.return_value = {"hits": {"total": {"value": 0}, "hits": []}}
|
| 591 |
+
with patch.object(runtime, "es", fake_es):
|
| 592 |
+
self.client.post("/api/search", json={"q": "手机", "page": 1, "page_size": 20})
|
| 593 |
+
self.client.post("/api/search", json={"q": "手机", "page": 2, "page_size": 20})
|
| 594 |
+
self.client.post("/api/search", json={"q": "手机", "page": 1, "page_size": 20, "archive_id": 1})
|
| 595 |
+
self.assertEqual(fake_es.search.call_count, 3)
|
| 596 |
+
|
| 597 |
+
def test_search_cache_key_covers_every_result_option(self):
|
| 598 |
+
base = search_logic.SearchRequest(q="手机")
|
| 599 |
+
variants = [
|
| 600 |
+
search_logic.SearchRequest(q="手机", page=2), search_logic.SearchRequest(q="手机", page_size=50),
|
| 601 |
+
search_logic.SearchRequest(q="手机", exact=False), search_logic.SearchRequest(q="手机", fulltext=False),
|
| 602 |
+
search_logic.SearchRequest(q="手机", sort="title"), search_logic.SearchRequest(q="手机", source="刊物甲"),
|
| 603 |
+
search_logic.SearchRequest(q="手机", sources=["刊物甲"]), search_logic.SearchRequest(q="手机", exclude_sources=["刊物乙"]),
|
| 604 |
+
search_logic.SearchRequest(q="手机", author="作者甲"), search_logic.SearchRequest(q="手机", authors=["作者甲"]),
|
| 605 |
+
search_logic.SearchRequest(q="手机", exclude_authors=["作者乙"]), search_logic.SearchRequest(q="手机", tag="标签甲"),
|
| 606 |
+
search_logic.SearchRequest(q="手机", tags=["标签甲"]), search_logic.SearchRequest(q="手机", exclude_tags=["标签乙"]),
|
| 607 |
+
search_logic.SearchRequest(q="手机", archive_id=3), search_logic.SearchRequest(q="手机", publication_type="期刊"),
|
| 608 |
+
search_logic.SearchRequest(q="手机", date_from=19660101), search_logic.SearchRequest(q="手机", date_to=19661231),
|
| 609 |
+
search_logic.SearchRequest(q="手机", date_terms=[{"year": 1966, "month": 5}]),
|
| 610 |
+
]
|
| 611 |
+
keys = {runtime.search_cache_key(item, max(1, item.page), min(max(1, item.page_size), 100)) for item in [base, *variants]}
|
| 612 |
+
self.assertEqual(len(keys), 1 + len(variants))
|
| 613 |
+
|
| 614 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 615 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 616 |
+
def test_invalid_es_query_returns_400(self, _status, _has_index):
|
| 617 |
+
fake_es = MagicMock()
|
| 618 |
+
fake_es.search.side_effect = BadRequestError("bad query", meta=MagicMock(status=400), body={"error": "bad query"})
|
| 619 |
+
with patch.object(runtime, "es", fake_es):
|
| 620 |
+
response = self.client.post("/api/search", json={"q": "手机"})
|
| 621 |
+
self.assertEqual(response.status_code, 400)
|
| 622 |
+
self.assertEqual(response.json()["error"], "invalid search query")
|
| 623 |
+
|
| 624 |
+
@patch.object(search_api, "get_search_docs", return_value={})
|
| 625 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 626 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 627 |
+
def test_fulltext_off_does_not_request_highlights(self, _status, _has_index, _get_docs):
|
| 628 |
+
fake_es = MagicMock()
|
| 629 |
+
fake_es.search.return_value = {"hits": {"total": {"value": 0}, "hits": []}}
|
| 630 |
+
with patch.object(runtime, "es", fake_es):
|
| 631 |
+
response = self.client.post("/api/search", json={"q": "手机", "fulltext": False})
|
| 632 |
+
self.assertEqual(response.status_code, 200)
|
| 633 |
+
self.assertNotIn("highlight", fake_es.search.call_args.kwargs)
|
| 634 |
+
|
| 635 |
+
@patch.object(search_api, "get_search_docs", side_effect=search_api.SearchStoreError("sidecar failed"))
|
| 636 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 637 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 638 |
+
def test_search_storage_failure_returns_503_and_is_not_cached(self, _status, _has_index, _get_docs):
|
| 639 |
+
fake_es = MagicMock()
|
| 640 |
+
fake_es.search.return_value = {"hits": {"total": {"value": 1}, "hits": [{"_id": "doc-1"}]}}
|
| 641 |
+
with patch.object(runtime, "es", fake_es):
|
| 642 |
+
response = self.client.post("/api/search", json={"q": "手机"})
|
| 643 |
+
self.assertEqual(response.status_code, 503)
|
| 644 |
+
self.assertEqual(response.json()["error"], "search storage unavailable")
|
| 645 |
+
self.assertEqual(runtime.search_response_cache, {})
|
| 646 |
+
|
| 647 |
+
@patch.object(search_api, "get_search_docs", return_value={
|
| 648 |
+
"doc-3": {"doc_id": "doc-3", "title": "手。机资料", "content": "正文手。机内容"},
|
| 649 |
+
})
|
| 650 |
+
@patch.object(search_api, "literal_match_ids", return_value={"doc-3"})
|
| 651 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 652 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 653 |
+
def test_literal_exact_verification_recomputes_total_across_all_candidates(
|
| 654 |
+
self, _status, _has_index, _literal_ids, _get_docs,
|
| 655 |
+
):
|
| 656 |
+
fake_es = MagicMock()
|
| 657 |
+
fake_es.search.side_effect = [
|
| 658 |
+
{"hits": {"total": {"value": 3}, "hits": [{"_id": "doc-1", "_score": 3.0}]}},
|
| 659 |
+
{"hits": {"total": {"value": 3}, "hits": [
|
| 660 |
+
{"_id": "doc-1", "_score": 3.0},
|
| 661 |
+
{"_id": "doc-2", "_score": 2.0},
|
| 662 |
+
{"_id": "doc-3", "_score": 1.0},
|
| 663 |
+
]}},
|
| 664 |
+
]
|
| 665 |
+
with patch.object(runtime, "es", fake_es):
|
| 666 |
+
response = self.client.post(
|
| 667 |
+
"/api/search", json={"q": "手。机", "exact": True, "fulltext": True, "page_size": 1},
|
| 668 |
+
)
|
| 669 |
+
self.assertEqual(response.status_code, 200)
|
| 670 |
+
self.assertEqual(response.json()["total"], 1)
|
| 671 |
+
self.assertEqual([item["doc_id"] for item in response.json()["results"]], ["doc-3"])
|
| 672 |
+
self.assertEqual(fake_es.search.call_count, 2)
|
| 673 |
+
|
| 674 |
+
@patch.object(search_api, "get_search_docs")
|
| 675 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 676 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 677 |
+
def test_missing_or_oversized_highlight_fetches_full_fallback_content(self, _status, _has_index, get_search_docs):
|
| 678 |
+
stored = {"doc-1": {"doc_id": "doc-1", "title": "标题", "content": "后文手机正文"}}
|
| 679 |
+
get_search_docs.return_value = stored
|
| 680 |
+
fake_es = MagicMock()
|
| 681 |
+
fake_es.search.return_value = {
|
| 682 |
+
"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [
|
| 683 |
+
{"_id": "doc-1", "highlight": {"content": ["甲" * (search_logic.MAX_HIGHLIGHT_FRAGMENT_CHARS + 1)]}}
|
| 684 |
+
]}
|
| 685 |
+
}
|
| 686 |
+
with patch.object(runtime, "es", fake_es):
|
| 687 |
+
response = self.client.post("/api/search", json={"q": "手机", "fulltext": True})
|
| 688 |
+
self.assertEqual(response.status_code, 200)
|
| 689 |
+
self.assertIn("<mark>手机</mark>", response.json()["results"][0]["snippet"][0])
|
| 690 |
+
self.assertEqual(get_search_docs.call_count, 1)
|
| 691 |
+
|
| 692 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 693 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": True, "index_status": "ready"})
|
| 694 |
+
def test_result_window_limit(self, _status, _has_index):
|
| 695 |
+
response = self.client.post("/api/search", json={"q": "手机", "page": 5001, "page_size": 100})
|
| 696 |
+
self.assertEqual(response.status_code, 400)
|
| 697 |
+
self.assertEqual(response.json()["error"], "result window exceeded")
|
| 698 |
+
|
| 699 |
+
def test_unknown_facet_returns_404(self):
|
| 700 |
+
response = self.client.get("/api/facet/not-a-facet")
|
| 701 |
+
self.assertEqual(response.status_code, 404)
|
| 702 |
+
|
| 703 |
+
@patch.object(runtime, "FACET_DB")
|
| 704 |
+
@patch.object(runtime, "cached_status_payload", return_value={"index_ready": False, "index_status": "indexing"})
|
| 705 |
+
def test_sources_and_facets_have_stable_building_shapes_without_sidecar(self, _status, facet_db):
|
| 706 |
+
facet_db.exists.return_value = False
|
| 707 |
+
sources = self.client.get("/api/sources").json()
|
| 708 |
+
facet = self.client.get("/api/facet/source?page=-2&page_size=999").json()
|
| 709 |
+
self.assertEqual(sources["sources"], [])
|
| 710 |
+
self.assertTrue(sources["indexing"])
|
| 711 |
+
self.assertEqual(facet["items"], [])
|
| 712 |
+
self.assertEqual(facet["page"], 1)
|
| 713 |
+
|
| 714 |
+
@patch.object(preview_api, "variants_for_title", return_value=[{"doc_id": "doc-2", "publication_name": "另一刊物"}])
|
| 715 |
+
@patch.object(preview_api, "get_doc")
|
| 716 |
+
def test_preview_random_and_spa_routes(self, get_doc, _variants):
|
| 717 |
+
get_doc.return_value = {
|
| 718 |
+
"doc_id": "doc-1", "title": "标题", "publication_name": "刊物", "publication_type": "期刊",
|
| 719 |
+
"authors": ["作者"], "tag_names": ["标签"], "date_display": ["1966/5/16"],
|
| 720 |
+
"source_files": [], "source_file_count": 0, "content": "正文",
|
| 721 |
+
}
|
| 722 |
+
preview = self.client.get("/api/preview/doc-1")
|
| 723 |
+
self.assertEqual(preview.status_code, 200)
|
| 724 |
+
self.assertEqual(preview.json()["variants"][0]["doc_id"], "doc-2")
|
| 725 |
+
with patch.object(preview_api, "random_doc", return_value=get_doc.return_value):
|
| 726 |
+
self.assertEqual(self.client.get("/api/random").json()["doc_id"], "doc-1")
|
| 727 |
+
with patch.object(preview_api, "get_doc", return_value=None), patch.object(preview_api, "random_doc", return_value=None):
|
| 728 |
+
self.assertEqual(self.client.get("/api/preview/missing").status_code, 404)
|
| 729 |
+
self.assertEqual(self.client.get("/api/random").status_code, 404)
|
| 730 |
+
self.assertEqual(self.client.get("/").status_code, 200)
|
| 731 |
+
self.assertEqual(self.client.get("/route/that/does/not/exist").status_code, 200)
|
| 732 |
+
self.assertIn("BHA Search", self.client.get("/route/that/does/not/exist").text)
|
| 733 |
+
|
| 734 |
+
def test_cors_contract(self):
|
| 735 |
+
response = self.client.options(
|
| 736 |
+
"/api/search",
|
| 737 |
+
headers={"Origin": "https://example.com", "Access-Control-Request-Method": "POST"},
|
| 738 |
+
)
|
| 739 |
+
self.assertEqual(response.status_code, 200)
|
| 740 |
+
self.assertEqual(response.headers["access-control-allow-origin"], "*")
|
| 741 |
+
self.assertNotIn("server-timing", response.headers)
|
| 742 |
+
|
| 743 |
+
def test_large_responses_prefer_brotli_and_fall_back_to_gzip(self):
|
| 744 |
+
brotli = self.client.get("/", headers={"Accept-Encoding": "br, gzip"})
|
| 745 |
+
gzip = self.client.get("/", headers={"Accept-Encoding": "gzip"})
|
| 746 |
+
identity = self.client.get("/", headers={"Accept-Encoding": "identity"})
|
| 747 |
+
self.assertEqual(brotli.status_code, 200)
|
| 748 |
+
self.assertEqual(brotli.headers["content-encoding"], "br")
|
| 749 |
+
self.assertEqual(gzip.headers["content-encoding"], "gzip")
|
| 750 |
+
self.assertNotIn("content-encoding", identity.headers)
|
| 751 |
+
self.assertEqual(brotli.content, identity.content)
|
| 752 |
+
self.assertEqual(gzip.content, identity.content)
|
| 753 |
+
|
| 754 |
+
@patch.object(runtime, "index_status", return_value="failed")
|
| 755 |
+
@patch.object(runtime, "index_ready", return_value=False)
|
| 756 |
+
@patch.object(runtime, "document_count", return_value=0)
|
| 757 |
+
def test_health_ok_reflects_readiness(self, _count, _ready, _status):
|
| 758 |
+
runtime.status_response_cache = None
|
| 759 |
+
with patch.object(runtime, "bucket_snapshot_status", return_value={"state": "unavailable", "stage": "validate_sidecars"}):
|
| 760 |
+
payload = runtime.cached_status_payload()
|
| 761 |
+
self.assertFalse(payload["ok"])
|
| 762 |
+
self.assertFalse(payload["index_ready"])
|
| 763 |
+
self.assertEqual(payload["bucket_snapshot"]["stage"], "validate_sidecars")
|
| 764 |
+
|
| 765 |
+
def test_external_generation_switch_changes_response_cache_keys(self):
|
| 766 |
+
old_generation = ((1, 2, 3, 4), (5, 6, 7, 8))
|
| 767 |
+
new_generation = ((1, 9, 3, 10), (5, 11, 7, 12))
|
| 768 |
+
request = search_logic.SearchRequest(q="test")
|
| 769 |
+
with patch.object(runtime, "serving_generation_token", return_value=old_generation):
|
| 770 |
+
old_key = runtime.search_cache_key(request, 1, 20)
|
| 771 |
+
with patch.object(runtime, "serving_generation_token", return_value=new_generation):
|
| 772 |
+
new_key = runtime.search_cache_key(request, 1, 20)
|
| 773 |
+
self.assertNotEqual(old_key, new_key)
|
| 774 |
+
|
| 775 |
+
def test_serving_generation_is_not_ready_until_shell_status_is_ready(self):
|
| 776 |
+
with patch.object(runtime, "index_ready", return_value=True), patch.object(runtime, "index_status", return_value="restoring"):
|
| 777 |
+
self.assertFalse(runtime.serving_generation_ready())
|
| 778 |
+
with patch.object(runtime, "index_ready", return_value=True), patch.object(runtime, "index_status", return_value="ready"):
|
| 779 |
+
self.assertTrue(runtime.serving_generation_ready())
|
| 780 |
+
|
| 781 |
+
@patch.object(runtime, "cached_status_payload", return_value={
|
| 782 |
+
"ok": False,
|
| 783 |
+
"index_ready": False,
|
| 784 |
+
"document_count": 0,
|
| 785 |
+
"index_status": "indexing",
|
| 786 |
+
"progress": {"status": "indexing", "indexed": 42},
|
| 787 |
+
"es": True,
|
| 788 |
+
})
|
| 789 |
+
def test_health_returns_progress_json_with_not_ready_status(self, _status):
|
| 790 |
+
response = self.client.get("/api/health")
|
| 791 |
+
self.assertEqual(response.status_code, 503)
|
| 792 |
+
self.assertEqual(response.json()["progress"]["indexed"], 42)
|
| 793 |
+
self.assertNotIn("server-timing", response.headers)
|
| 794 |
+
|
| 795 |
+
def test_ping_is_empty_and_not_cached(self):
|
| 796 |
+
response = self.client.get("/api/ping")
|
| 797 |
+
self.assertEqual(response.status_code, 204)
|
| 798 |
+
self.assertEqual(response.content, b"")
|
| 799 |
+
self.assertEqual(response.headers["cache-control"], "no-store")
|
| 800 |
+
|
| 801 |
+
def test_bootstrap_is_cached_until_serving_generation_changes(self):
|
| 802 |
+
generation = ((1, 2, 3, 4), (5, 6, 7, 8))
|
| 803 |
+
search = bootstrap_api.JSONResponse({
|
| 804 |
+
"results": [{"doc_id": "doc-1"}], "total": 1, "page": 1, "page_size": 20,
|
| 805 |
+
"indexing": False, "index_status": "ready",
|
| 806 |
+
})
|
| 807 |
+
facet_payloads = {
|
| 808 |
+
kind: {"items": [{"name": kind}], "total": 1, "page": 1, "page_size": 200, "has_more": False}
|
| 809 |
+
for kind in ("source", "author", "tag")
|
| 810 |
+
}
|
| 811 |
+
with patch.object(runtime, "index_status", return_value="ready"), \
|
| 812 |
+
patch.object(runtime, "serving_generation_ready", return_value=True), \
|
| 813 |
+
patch.object(runtime, "serving_generation_token", return_value=generation) as token, \
|
| 814 |
+
patch.object(runtime, "cached_sources_payload", return_value={"archives": [], "types": [], "years": []}), \
|
| 815 |
+
patch.object(runtime, "cached_facet_payload", side_effect=lambda kind, *_args: facet_payloads[kind]), \
|
| 816 |
+
patch.object(bootstrap_api.search_api, "search", return_value=search) as default_search, \
|
| 817 |
+
patch.object(bootstrap_api, "serving_lock", return_value=nullcontext()):
|
| 818 |
+
first = bootstrap_api.bootstrap()
|
| 819 |
+
second = bootstrap_api.bootstrap()
|
| 820 |
+
token.return_value = ((9, 10, 11, 12), generation[1])
|
| 821 |
+
third = bootstrap_api.bootstrap()
|
| 822 |
+
self.assertEqual(first.status_code, 200)
|
| 823 |
+
self.assertEqual(second.body, first.body)
|
| 824 |
+
self.assertNotEqual(json.loads(third.body)["generation"], json.loads(first.body)["generation"])
|
| 825 |
+
self.assertEqual(default_search.call_count, 2)
|
| 826 |
+
self.assertEqual(first.headers["cache-control"], "no-store")
|
| 827 |
+
|
| 828 |
+
def test_reindex_requires_configured_bearer_token(self):
|
| 829 |
+
with patch.object(reindex_api, "REINDEX_TOKEN", ""), patch.object(reindex_api, "acquire_index_lock") as acquire_lock:
|
| 830 |
+
response = self.client.post("/api/reindex")
|
| 831 |
+
self.assertEqual(response.status_code, 503)
|
| 832 |
+
self.assertEqual(response.headers["cache-control"], "no-store")
|
| 833 |
+
acquire_lock.assert_not_called()
|
| 834 |
+
|
| 835 |
+
with patch.object(reindex_api, "REINDEX_TOKEN", "secret"), patch.object(reindex_api, "acquire_index_lock") as acquire_lock:
|
| 836 |
+
response = self.client.post("/api/reindex", headers={"Authorization": "Bearer wrong"})
|
| 837 |
+
self.assertEqual(response.status_code, 401)
|
| 838 |
+
self.assertEqual(response.headers["www-authenticate"], "Bearer")
|
| 839 |
+
acquire_lock.assert_not_called()
|
| 840 |
+
|
| 841 |
+
def test_authenticated_reindex_starts_once_with_short_cooldown(self):
|
| 842 |
+
lock_file = MagicMock()
|
| 843 |
+
with tempfile.TemporaryDirectory() as directory, \
|
| 844 |
+
patch.object(reindex_api, "REINDEX_TOKEN", "secret"), \
|
| 845 |
+
patch.object(reindex_api, "REINDEX_LAST_START", Path(directory) / "last-start"), \
|
| 846 |
+
patch.object(reindex_api, "acquire_index_lock", return_value=lock_file), \
|
| 847 |
+
patch.object(reindex_api, "start_reindex_worker") as start_worker:
|
| 848 |
+
response = self.client.post("/api/reindex", headers={"Authorization": "Bearer secret"})
|
| 849 |
+
self.assertEqual(response.status_code, 202)
|
| 850 |
+
self.assertEqual(response.json()["cooldown_seconds"], 600)
|
| 851 |
+
start_worker.assert_called_once_with(lock_file, reset=False)
|
| 852 |
+
self.assertEqual(response.json()["mode"], "incremental")
|
| 853 |
+
lock_file.close.assert_not_called()
|
| 854 |
+
|
| 855 |
+
def test_authenticated_reindex_can_request_explicit_full_rebuild(self):
|
| 856 |
+
lock_file = MagicMock()
|
| 857 |
+
with tempfile.TemporaryDirectory() as directory, \
|
| 858 |
+
patch.object(reindex_api, "REINDEX_TOKEN", "secret"), \
|
| 859 |
+
patch.object(reindex_api, "REINDEX_LAST_START", Path(directory) / "last-start"), \
|
| 860 |
+
patch.object(reindex_api, "acquire_index_lock", return_value=lock_file), \
|
| 861 |
+
patch.object(reindex_api, "start_reindex_worker") as start_worker:
|
| 862 |
+
response = self.client.post(
|
| 863 |
+
"/api/reindex?full=true", headers={"Authorization": "Bearer secret"}
|
| 864 |
+
)
|
| 865 |
+
self.assertEqual(response.status_code, 202)
|
| 866 |
+
self.assertEqual(response.json()["mode"], "full")
|
| 867 |
+
start_worker.assert_called_once_with(lock_file, reset=True)
|
| 868 |
+
|
| 869 |
+
def test_reindex_passes_exact_archive_identities_to_worker(self):
|
| 870 |
+
lock_file = MagicMock()
|
| 871 |
+
commits = {str(value): "a" * 40 for value in range(32)}
|
| 872 |
+
revisions = {str(value): "b" * 40 for value in range(32)}
|
| 873 |
+
with tempfile.TemporaryDirectory() as directory, \
|
| 874 |
+
patch.object(reindex_api, "REINDEX_TOKEN", "secret"), \
|
| 875 |
+
patch.object(reindex_api, "REINDEX_LAST_START", Path(directory) / "last-start"), \
|
| 876 |
+
patch.object(reindex_api, "acquire_index_lock", return_value=lock_file), \
|
| 877 |
+
patch.object(reindex_api, "start_reindex_worker") as start_worker:
|
| 878 |
+
response = self.client.post("/api/reindex", headers={"Authorization": "Bearer secret"}, json={
|
| 879 |
+
"archive_commits": commits,
|
| 880 |
+
"archive_revisions": revisions,
|
| 881 |
+
"changed_archives": [0, 4],
|
| 882 |
+
})
|
| 883 |
+
self.assertEqual(response.status_code, 202)
|
| 884 |
+
start_worker.assert_called_once_with(
|
| 885 |
+
lock_file, reset=False,
|
| 886 |
+
expected_commits={value: "a" * 40 for value in range(32)},
|
| 887 |
+
expected_revisions={value: "b" * 40 for value in range(32)},
|
| 888 |
+
changed_archives=[0, 4],
|
| 889 |
+
)
|
| 890 |
+
|
| 891 |
+
def test_reindex_marks_ready_before_starting_snapshot_publish(self):
|
| 892 |
+
lock_file = MagicMock()
|
| 893 |
+
thread = MagicMock()
|
| 894 |
+
with tempfile.TemporaryDirectory() as directory, \
|
| 895 |
+
patch.object(runtime, "INDEX_STATUS_PATH", Path(directory) / "status"), \
|
| 896 |
+
patch.object(reindex_api, "BUCKET_DIR", Path(directory)), \
|
| 897 |
+
patch.object(reindex_api, "ensure_index", return_value=True), \
|
| 898 |
+
patch.object(reindex_api.threading, "Thread", return_value=thread):
|
| 899 |
+
reindex_api.run_reindex(lock_file)
|
| 900 |
+
self.assertEqual(runtime.INDEX_STATUS_PATH.read_text(encoding="utf-8"), "ready")
|
| 901 |
+
thread.start.assert_called_once_with()
|
| 902 |
+
lock_file.close.assert_called_once_with()
|
| 903 |
+
|
| 904 |
+
def test_reindex_noop_does_not_publish_snapshot(self):
|
| 905 |
+
lock_file = MagicMock()
|
| 906 |
+
thread = MagicMock()
|
| 907 |
+
with tempfile.TemporaryDirectory() as directory, \
|
| 908 |
+
patch.object(runtime, "INDEX_STATUS_PATH", Path(directory) / "status"), \
|
| 909 |
+
patch.object(reindex_api, "BUCKET_DIR", Path(directory)), \
|
| 910 |
+
patch.object(reindex_api, "ensure_index", return_value=False), \
|
| 911 |
+
patch.object(reindex_api.threading, "Thread", return_value=thread):
|
| 912 |
+
reindex_api.run_reindex(lock_file)
|
| 913 |
+
thread.assert_not_called()
|
| 914 |
+
lock_file.close.assert_called_once_with()
|
| 915 |
+
|
| 916 |
+
def test_reindex_failure_with_pending_switch_is_not_marked_ready(self):
|
| 917 |
+
lock_file = MagicMock()
|
| 918 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 919 |
+
root = Path(directory)
|
| 920 |
+
switch_state = root / "index-switch.json"
|
| 921 |
+
switch_state.write_text("{}", encoding="utf-8")
|
| 922 |
+
status = root / "index-status"
|
| 923 |
+
with patch.object(runtime, "INDEX_SWITCH_STATE_PATH", switch_state), \
|
| 924 |
+
patch.object(runtime, "INDEX_STATUS_PATH", status), \
|
| 925 |
+
patch.object(reindex_api, "BUCKET_DIR", None), \
|
| 926 |
+
patch.object(reindex_api, "ensure_index", side_effect=RuntimeError("uncertain switch")), \
|
| 927 |
+
patch.object(runtime, "index_ready", return_value=True):
|
| 928 |
+
reindex_api.run_reindex(lock_file)
|
| 929 |
+
self.assertEqual(status.read_text(encoding="utf-8"), "failed")
|
| 930 |
+
lock_file.close.assert_called_once_with()
|
| 931 |
+
|
| 932 |
+
def test_reindex_failure_keeps_valid_serving_generation_ready(self):
|
| 933 |
+
lock_file = MagicMock()
|
| 934 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 935 |
+
status = Path(directory) / "index-status"
|
| 936 |
+
with patch.object(runtime, "INDEX_SWITCH_STATE_PATH", Path(directory) / "missing-switch.json"), \
|
| 937 |
+
patch.object(runtime, "INDEX_STATUS_PATH", status), \
|
| 938 |
+
patch.object(reindex_api, "BUCKET_DIR", None), \
|
| 939 |
+
patch.object(reindex_api, "ensure_index", side_effect=RuntimeError("database is locked")), \
|
| 940 |
+
patch.object(runtime, "serving_generation_valid", return_value=True):
|
| 941 |
+
reindex_api.run_reindex(lock_file)
|
| 942 |
+
self.assertEqual(status.read_text(encoding="utf-8"), "ready")
|
| 943 |
+
lock_file.close.assert_called_once_with()
|
| 944 |
+
|
| 945 |
+
def test_serving_generation_validation_retries_transient_sidecar_lock(self):
|
| 946 |
+
fake_es = MagicMock()
|
| 947 |
+
fake_es.get.return_value = {"_source": {"version": "1"}}
|
| 948 |
+
with patch.object(runtime, "es", fake_es), \
|
| 949 |
+
patch.object(runtime, "sidecars_ready", side_effect=[False, False, True]) as sidecars_ready, \
|
| 950 |
+
patch.object(runtime.time, "sleep") as sleep:
|
| 951 |
+
self.assertTrue(runtime.serving_generation_valid())
|
| 952 |
+
self.assertEqual(sidecars_ready.call_count, 3)
|
| 953 |
+
self.assertEqual(sleep.call_count, 2)
|
| 954 |
+
|
| 955 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 956 |
+
@patch.object(runtime, "index_status", return_value="restoring")
|
| 957 |
+
def test_search_does_not_read_restoring_generation(self, _status, _has_data):
|
| 958 |
+
fake_es = MagicMock()
|
| 959 |
+
with patch.object(runtime, "es", fake_es):
|
| 960 |
+
response = self.client.post("/api/search", json={"q": "文化革命"})
|
| 961 |
+
self.assertEqual(response.status_code, 200)
|
| 962 |
+
self.assertEqual(response.json()["results"], [])
|
| 963 |
+
self.assertTrue(response.json()["indexing"])
|
| 964 |
+
fake_es.search.assert_not_called()
|
| 965 |
+
|
| 966 |
+
@patch.object(runtime, "has_index_data", return_value=True)
|
| 967 |
+
@patch.object(runtime, "index_status", return_value="failed")
|
| 968 |
+
def test_search_does_not_serve_failed_generation(self, _status, _has_data):
|
| 969 |
+
fake_es = MagicMock()
|
| 970 |
+
with patch.object(runtime, "es", fake_es):
|
| 971 |
+
response = self.client.post("/api/search", json={"q": "文化革命"})
|
| 972 |
+
self.assertEqual(response.status_code, 200)
|
| 973 |
+
self.assertEqual(response.json()["results"], [])
|
| 974 |
+
self.assertEqual(response.json()["index_status"], "failed")
|
| 975 |
+
fake_es.search.assert_not_called()
|
| 976 |
+
|
| 977 |
+
@patch.object(runtime, "index_status", return_value="restoring")
|
| 978 |
+
@patch.object(preview_api, "get_doc")
|
| 979 |
+
def test_preview_is_blocked_during_restore(self, get_doc, _status):
|
| 980 |
+
response = self.client.get("/api/preview/doc-1")
|
| 981 |
+
self.assertEqual(response.status_code, 503)
|
| 982 |
+
get_doc.assert_not_called()
|
| 983 |
+
|
| 984 |
+
def test_preview_payload_exposes_mirrored_pdf_page_previews(self):
|
| 985 |
+
stored = {
|
| 986 |
+
"doc_id": "doc-1", "title": "资料",
|
| 987 |
+
"source_files": ["https://raw.githubusercontent.com/o/r/main/a.pdf"],
|
| 988 |
+
}
|
| 989 |
+
entry = {"page_previews": {"paths": ["archives0/aa/one.webp", "archives0/bb/two.webp"]}}
|
| 990 |
+
with patch.object(source_files, "source_mirror_entry", AsyncMock(return_value=entry)), \
|
| 991 |
+
patch.object(preview_api, "serving_lock", return_value=nullcontext()):
|
| 992 |
+
payload = asyncio.run(preview_api.preview_payload(stored))
|
| 993 |
+
self.assertEqual(
|
| 994 |
+
payload["source_files"][0]["pages"],
|
| 995 |
+
[
|
| 996 |
+
"https://huggingface.co/datasets/vomebook/BHA-Source-Files/resolve/main/archives0/aa/one.webp",
|
| 997 |
+
"https://huggingface.co/datasets/vomebook/BHA-Source-Files/resolve/main/archives0/bb/two.webp",
|
| 998 |
+
],
|
| 999 |
+
)
|
| 1000 |
+
|
| 1001 |
+
def test_source_and_facet_cache_ttl_is_ten_minutes(self):
|
| 1002 |
+
self.assertEqual(runtime.FACET_CACHE_TTL_SECONDS, 600)
|
| 1003 |
+
|
| 1004 |
+
@patch.object(runtime, "warm_static_files", return_value=True)
|
| 1005 |
+
@patch.object(runtime, "clear_response_caches")
|
| 1006 |
+
@patch.object(runtime, "cached_facet_payload", return_value={"items": []})
|
| 1007 |
+
@patch.object(runtime, "available_years", return_value=[1966])
|
| 1008 |
+
@patch.object(runtime, "cached_sources_payload", return_value={})
|
| 1009 |
+
@patch.object(runtime, "warmup_db", return_value=True)
|
| 1010 |
+
@patch.object(runtime, "warmup_elasticsearch", return_value=True)
|
| 1011 |
+
def test_full_auxiliary_warmup_clears_caches_and_loads_sidecars(
|
| 1012 |
+
self,
|
| 1013 |
+
_elasticsearch,
|
| 1014 |
+
_warmup_db,
|
| 1015 |
+
_sources,
|
| 1016 |
+
_years,
|
| 1017 |
+
facet,
|
| 1018 |
+
clear_caches,
|
| 1019 |
+
_static,
|
| 1020 |
+
):
|
| 1021 |
+
result = runtime.run_auxiliary_warmup(full=True)
|
| 1022 |
+
self.assertTrue(result["ok"])
|
| 1023 |
+
self.assertTrue(result["elasticsearch_warmed"])
|
| 1024 |
+
self.assertEqual(facet.call_count, 3)
|
| 1025 |
+
clear_caches.assert_called_once()
|
| 1026 |
+
|
| 1027 |
+
@patch.object(runtime, "warm_static_files", return_value=True)
|
| 1028 |
+
@patch.object(runtime, "clear_response_caches")
|
| 1029 |
+
@patch.object(runtime, "cached_facet_payload", return_value={"items": []})
|
| 1030 |
+
@patch.object(runtime, "available_years", return_value=[])
|
| 1031 |
+
@patch.object(runtime, "cached_sources_payload", return_value={})
|
| 1032 |
+
@patch.object(runtime, "warmup_db", return_value=True)
|
| 1033 |
+
@patch.object(runtime, "warmup_elasticsearch", return_value=True)
|
| 1034 |
+
def test_periodic_auxiliary_warmup_preserves_response_caches(
|
| 1035 |
+
self,
|
| 1036 |
+
_elasticsearch,
|
| 1037 |
+
_warmup_db,
|
| 1038 |
+
_sources,
|
| 1039 |
+
_years,
|
| 1040 |
+
facet,
|
| 1041 |
+
clear_caches,
|
| 1042 |
+
_static,
|
| 1043 |
+
):
|
| 1044 |
+
result = runtime.run_auxiliary_warmup(full=False)
|
| 1045 |
+
self.assertTrue(result["ok"])
|
| 1046 |
+
self.assertEqual(facet.call_count, 3)
|
| 1047 |
+
clear_caches.assert_not_called()
|
| 1048 |
+
|
| 1049 |
+
@patch.object(runtime, "es")
|
| 1050 |
+
def test_elasticsearch_warmup_is_a_result_free_match_all_query(self, fake_es):
|
| 1051 |
+
self.assertTrue(runtime.warmup_elasticsearch())
|
| 1052 |
+
fake_es.search.assert_called_once_with(
|
| 1053 |
+
index=runtime.INDEX_NAME,
|
| 1054 |
+
size=0,
|
| 1055 |
+
track_total_hits=False,
|
| 1056 |
+
query={"match_all": {}},
|
| 1057 |
+
_source=False,
|
| 1058 |
+
)
|
| 1059 |
+
|
| 1060 |
+
|
| 1061 |
+
if __name__ == "__main__":
|
| 1062 |
+
unittest.main()
|
tests/test_browser.py
ADDED
|
@@ -0,0 +1,627 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import base64
|
| 5 |
+
import socket
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
import time
|
| 9 |
+
import unittest
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from urllib.parse import parse_qs, urlparse
|
| 12 |
+
from urllib.request import urlopen
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
from playwright.sync_api import sync_playwright
|
| 16 |
+
except ImportError:
|
| 17 |
+
sync_playwright = None
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def free_port():
|
| 24 |
+
with socket.socket() as sock:
|
| 25 |
+
sock.bind(("127.0.0.1", 0))
|
| 26 |
+
return sock.getsockname()[1]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def minimal_pdf():
|
| 30 |
+
stream = b"BT /F1 18 Tf 20 100 Td (BHA PDF) Tj ET"
|
| 31 |
+
objects = [
|
| 32 |
+
b"<< /Type /Catalog /Pages 2 0 R >>",
|
| 33 |
+
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
| 34 |
+
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
|
| 35 |
+
b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream",
|
| 36 |
+
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
| 37 |
+
]
|
| 38 |
+
payload = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
|
| 39 |
+
offsets = [0]
|
| 40 |
+
for number, body in enumerate(objects, 1):
|
| 41 |
+
offsets.append(len(payload))
|
| 42 |
+
payload.extend(f"{number} 0 obj\n".encode() + body + b"\nendobj\n")
|
| 43 |
+
xref = len(payload)
|
| 44 |
+
payload.extend(f"xref\n0 {len(objects) + 1}\n".encode())
|
| 45 |
+
payload.extend(b"0000000000 65535 f \n")
|
| 46 |
+
for offset in offsets[1:]:
|
| 47 |
+
payload.extend(f"{offset:010d} 00000 n \n".encode())
|
| 48 |
+
payload.extend(f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode())
|
| 49 |
+
return bytes(payload)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@unittest.skipUnless(sync_playwright, "install requirements-test.txt and run: playwright install chromium")
|
| 53 |
+
class BrowserInteractionTests(unittest.TestCase):
|
| 54 |
+
@classmethod
|
| 55 |
+
def setUpClass(cls):
|
| 56 |
+
cls.port = free_port()
|
| 57 |
+
cls.base_url = f"http://127.0.0.1:{cls.port}"
|
| 58 |
+
env = {**os.environ, "APP_ROOT": str(BASE_DIR), "DATA_ROOT": str(BASE_DIR / "data")}
|
| 59 |
+
cls.server = subprocess.Popen(
|
| 60 |
+
[sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(cls.port), "--log-level", "warning"],
|
| 61 |
+
cwd=BASE_DIR, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
| 62 |
+
)
|
| 63 |
+
for _ in range(80):
|
| 64 |
+
try:
|
| 65 |
+
with urlopen(cls.base_url + "/", timeout=1) as response:
|
| 66 |
+
if response.status == 200:
|
| 67 |
+
break
|
| 68 |
+
except Exception:
|
| 69 |
+
time.sleep(0.25)
|
| 70 |
+
else:
|
| 71 |
+
cls.server.terminate()
|
| 72 |
+
raise RuntimeError("BHA browser test server did not start")
|
| 73 |
+
cls.playwright = sync_playwright().start()
|
| 74 |
+
cls.browser = cls.playwright.chromium.launch(headless=True, args=["--no-sandbox"])
|
| 75 |
+
|
| 76 |
+
@classmethod
|
| 77 |
+
def tearDownClass(cls):
|
| 78 |
+
cls.browser.close()
|
| 79 |
+
cls.playwright.stop()
|
| 80 |
+
cls.server.terminate()
|
| 81 |
+
try:
|
| 82 |
+
cls.server.wait(timeout=10)
|
| 83 |
+
except subprocess.TimeoutExpired:
|
| 84 |
+
cls.server.kill()
|
| 85 |
+
|
| 86 |
+
def open_page(self, viewport=None, preview_parts=None, track_empty=False, bootstrap_total=1):
|
| 87 |
+
context = self.browser.new_context(viewport=viewport or {"width": 1440, "height": 900}, accept_downloads=True)
|
| 88 |
+
page = context.new_page()
|
| 89 |
+
errors = []
|
| 90 |
+
if track_empty:
|
| 91 |
+
page.add_init_script("""
|
| 92 |
+
window.__emptyStateWasVisible = false;
|
| 93 |
+
document.addEventListener("DOMContentLoaded", () => {
|
| 94 |
+
const empty = document.querySelector("#empty-state");
|
| 95 |
+
new MutationObserver(() => {
|
| 96 |
+
if (getComputedStyle(empty).display !== "none") window.__emptyStateWasVisible = true;
|
| 97 |
+
}).observe(empty, { attributes: true, attributeFilter: ["style", "hidden"] });
|
| 98 |
+
});
|
| 99 |
+
""")
|
| 100 |
+
page.on("pageerror", lambda error: errors.append(str(error)))
|
| 101 |
+
results = [{
|
| 102 |
+
"doc_id": "doc-1", "title": "手机资料", "title_highlight": "<mark>手机</mark>资料",
|
| 103 |
+
"publication_name": "测试刊物", "publication_type": "期刊", "authors": ["作者甲"],
|
| 104 |
+
"tag_names": ["标签甲"], "date_display": ["1966/5/16"], "source_file_count": 0,
|
| 105 |
+
"snippet": ["这是<mark>手机</mark>正文"], "score": 1,
|
| 106 |
+
}]
|
| 107 |
+
preview = {
|
| 108 |
+
"doc_id": "doc-1", "title": "手机资料", "publication_name": "测试刊物",
|
| 109 |
+
"publication_author": "出版者", "publication_type": "期刊", "authors": ["作者甲"], "tag_names": ["标签甲"],
|
| 110 |
+
"date_display": ["1966/5/16"], "source_file_count": 1, "content": "这是手机正文", "variants": [],
|
| 111 |
+
"article_id": "article-1", "publication_id": "publication-1", "archive_id": 1,
|
| 112 |
+
"source_files": [{"index": 0, "name": "原稿.pdf", "url": "https://example.com/source.pdf"}],
|
| 113 |
+
"article": {"title": "手机资料", "authors": ["作者甲"], "dates": [{"year": 1966, "month": 5, "day": 16}], "tags": [{"type": "主题/事件", "name": "标签甲"}, {"name": "标签乙"}], "page_start": 2, "page_end": 2, "description": "", "parts": preview_parts or [{"type": "paragraph", "text": "这是手机正文"}], "comments": [], "comment_pivots": []},
|
| 114 |
+
}
|
| 115 |
+
pdf = minimal_pdf()
|
| 116 |
+
|
| 117 |
+
def route_api(route, request):
|
| 118 |
+
path = request.url.split(self.base_url, 1)[-1]
|
| 119 |
+
if path.startswith("/api/bootstrap"):
|
| 120 |
+
facet = lambda name: {"items": [{"name": name, "value": name, "count": 1}], "total": 1, "page": 1, "page_size": 200, "has_more": False}
|
| 121 |
+
payload = {
|
| 122 |
+
"generation": "test-generation",
|
| 123 |
+
"search": {"results": results, "total": bootstrap_total, "page": 1, "page_size": 20, "indexing": False, "index_status": "ready"},
|
| 124 |
+
"sources": {"archives": [{"name": "3", "value": 3, "count": 1}], "types": [{"name": "期刊", "value": "期刊", "count": 1}], "years": [{"name": "1966", "value": 1966, "count": 0}]},
|
| 125 |
+
"facets": {"source": facet("测试刊物"), "author": facet("作者甲"), "tag": facet("标签甲")},
|
| 126 |
+
}
|
| 127 |
+
elif path.startswith("/api/search"):
|
| 128 |
+
body = request.post_data_json or {}
|
| 129 |
+
payload = {"results": results, "total": bootstrap_total, "page": body.get("page", 1), "page_size": body.get("page_size", 20), "indexing": False, "index_status": "ready"}
|
| 130 |
+
elif path.startswith("/api/sources"):
|
| 131 |
+
payload = {"sources": [], "authors": [], "tags": [], "archives": [{"name": "3", "value": 3, "count": 1}], "types": [{"name": "期刊", "value": "期刊", "count": 1}], "years": [{"name": "1966", "value": 1966, "count": 0}], "indexing": False, "index_status": "ready"}
|
| 132 |
+
elif path.startswith("/api/facet/"):
|
| 133 |
+
kind = path.split("/api/facet/", 1)[1].split("?", 1)[0]
|
| 134 |
+
name = {"source": "测试刊物", "author": "作者甲", "tag": "标签甲"}[kind]
|
| 135 |
+
has_query = bool(parse_qs(urlparse(path).query).get("q"))
|
| 136 |
+
show_item = has_query or kind != "tag"
|
| 137 |
+
payload = {"items": [{"name": name, "value": name, "count": 1}] if show_item else [], "total": 1 if show_item else 0, "page": 1, "page_size": 500 if has_query else 200, "has_more": False, "indexing": False, "index_status": "ready"}
|
| 138 |
+
elif path.startswith("/api/preview/source/"):
|
| 139 |
+
match = re.fullmatch(r"bytes=(\d+)-(\d+)", request.headers.get("range", ""))
|
| 140 |
+
if not match:
|
| 141 |
+
route.fulfill(status=416, body=b"")
|
| 142 |
+
return
|
| 143 |
+
begin = int(match.group(1))
|
| 144 |
+
end = min(int(match.group(2)), len(pdf) - 1)
|
| 145 |
+
chunk = pdf[begin:end + 1]
|
| 146 |
+
route.fulfill(status=206, body=chunk, headers={
|
| 147 |
+
"Content-Type": "application/pdf",
|
| 148 |
+
"Content-Range": f"bytes {begin}-{end}/{len(pdf)}",
|
| 149 |
+
"Content-Length": str(len(chunk)),
|
| 150 |
+
"Accept-Ranges": "bytes",
|
| 151 |
+
})
|
| 152 |
+
return
|
| 153 |
+
elif path.startswith("/api/preview/") or path.startswith("/api/random"):
|
| 154 |
+
payload = preview
|
| 155 |
+
elif path.startswith("/api/health"):
|
| 156 |
+
payload = {"ok": True, "es": True, "index_ready": True, "index_status": "ready", "document_count": 1, "progress": None}
|
| 157 |
+
elif path.startswith("/api/download/txt/"):
|
| 158 |
+
route.fulfill(status=200, body="标题:手机资料\n\n这是手机正文", headers={"Content-Type": "text/plain; charset=utf-8", "Content-Disposition": "attachment; filename*=UTF-8''test.txt"})
|
| 159 |
+
return
|
| 160 |
+
else:
|
| 161 |
+
payload = {}
|
| 162 |
+
route.fulfill(status=200, content_type="application/json", body=json.dumps(payload, ensure_ascii=False))
|
| 163 |
+
|
| 164 |
+
page.route("**/api/**", route_api)
|
| 165 |
+
page.goto(self.base_url, wait_until="domcontentloaded")
|
| 166 |
+
page.locator(".result-item").wait_for(timeout=15000)
|
| 167 |
+
return context, page, errors
|
| 168 |
+
|
| 169 |
+
def test_default_screen_hydrates_then_prefetches_the_next_page(self):
|
| 170 |
+
context, page, errors = self.open_page(track_empty=True, bootstrap_total=21)
|
| 171 |
+
try:
|
| 172 |
+
page.wait_for_function("""performance.getEntriesByType('resource')
|
| 173 |
+
.some((entry) => new URL(entry.name).pathname === '/api/search')""")
|
| 174 |
+
api_paths = page.evaluate("""performance.getEntriesByType('resource')
|
| 175 |
+
.map((entry) => new URL(entry.name).pathname).filter((path) => path.startsWith('/api/'))""")
|
| 176 |
+
self.assertEqual(api_paths, ["/api/bootstrap", "/api/search"])
|
| 177 |
+
self.assertEqual(page.locator("#source-filter [data-source]").count(), 1)
|
| 178 |
+
self.assertFalse(page.evaluate("window.__emptyStateWasVisible"))
|
| 179 |
+
self.assertEqual(errors, [])
|
| 180 |
+
finally:
|
| 181 |
+
context.close()
|
| 182 |
+
|
| 183 |
+
def test_parse_crop_filters_ocr_and_submits_page_thresholds(self):
|
| 184 |
+
context = self.browser.new_context(viewport={"width": 1440, "height": 900})
|
| 185 |
+
page = context.new_page()
|
| 186 |
+
errors = []
|
| 187 |
+
submitted = []
|
| 188 |
+
page.on("pageerror", lambda error: errors.append(str(error)))
|
| 189 |
+
page.on("dialog", lambda dialog: dialog.accept())
|
| 190 |
+
png = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
|
| 191 |
+
queue_item = {
|
| 192 |
+
"doc_id": "parse-doc", "archive_id": 25, "publication_id": "052417de-42fb-4781-9885-af4fb006e9b6",
|
| 193 |
+
"publication_name": "待拆小报", "title": "【文章待拆分】待拆小报", "publication_type": "img",
|
| 194 |
+
"authors": [], "date_display": ["1967/5/12"], "page_start": 1, "page_end": 1, "source_files": ["one"],
|
| 195 |
+
}
|
| 196 |
+
preview = {
|
| 197 |
+
**queue_item, "content_hash": "a" * 64, "content": "页眉\n正文", "variants": [],
|
| 198 |
+
"article": {"title": queue_item["title"], "authors": [], "dates": [{"year": 1967, "month": 5, "day": 12}], "page_start": 1, "page_end": 1},
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
def route_api(route, request):
|
| 202 |
+
path = request.url.split(self.base_url, 1)[-1]
|
| 203 |
+
if path.startswith("/api/parse/queue"):
|
| 204 |
+
payload = {"total": 1, "page": 1, "page_size": 100, "items": [queue_item]}
|
| 205 |
+
elif path.startswith("/api/parse/parse-doc/ocr"):
|
| 206 |
+
payload = {"page": 1, "dimensions": {"width": 1000, "height": 1000}, "ocr_results": [
|
| 207 |
+
{"text": "页眉", "box": [[400, 20], [600, 20], [600, 80], [400, 80]]},
|
| 208 |
+
{"text": "正文", "box": [[400, 450], [600, 450], [600, 550], [400, 550]]},
|
| 209 |
+
]}
|
| 210 |
+
elif path.startswith("/api/preview/source/parse-doc"):
|
| 211 |
+
route.fulfill(status=200, body=png, headers={"Content-Type": "image/png"})
|
| 212 |
+
return
|
| 213 |
+
elif path.startswith("/api/preview/parse-doc"):
|
| 214 |
+
payload = preview
|
| 215 |
+
elif path == "/api/parse" and request.method == "POST":
|
| 216 |
+
submitted.append(request.post_data_json)
|
| 217 |
+
route.fulfill(status=202, content_type="application/json", body='{"ok":true}')
|
| 218 |
+
return
|
| 219 |
+
elif path.startswith("/api/health"):
|
| 220 |
+
payload = {"ok": True, "es": True, "index_status": "ready", "progress": None}
|
| 221 |
+
elif path.startswith("/api/sources"):
|
| 222 |
+
payload = {"sources": [], "authors": [], "tags": [], "archives": [], "types": [], "years": [], "indexing": False, "index_status": "ready"}
|
| 223 |
+
elif path.startswith("/api/facet/"):
|
| 224 |
+
payload = {"items": [], "total": 0, "page": 1, "page_size": 200, "has_more": False}
|
| 225 |
+
else:
|
| 226 |
+
payload = {}
|
| 227 |
+
route.fulfill(status=200, content_type="application/json", body=json.dumps(payload, ensure_ascii=False))
|
| 228 |
+
|
| 229 |
+
page.route("**/api/**", route_api)
|
| 230 |
+
try:
|
| 231 |
+
page.goto(self.base_url + "/?parse=1", wait_until="domcontentloaded")
|
| 232 |
+
page.locator("[data-parse-open]").click()
|
| 233 |
+
page.locator("[data-parse-content]").wait_for()
|
| 234 |
+
page.wait_for_function("document.querySelector('[data-parse-content]').value.includes('页眉')")
|
| 235 |
+
page.locator("[data-parse-crop-toggle]").click()
|
| 236 |
+
page.locator("[data-parse-crop-scope='page']").click()
|
| 237 |
+
overlay = page.locator("[data-parse-crop-overlay]").bounding_box()
|
| 238 |
+
handle = page.locator("[data-crop-handle='top']").bounding_box()
|
| 239 |
+
page.mouse.move(handle["x"] + handle["width"] / 2, handle["y"] + handle["height"] / 2)
|
| 240 |
+
page.mouse.down()
|
| 241 |
+
page.mouse.move(handle["x"] + handle["width"] / 2, handle["y"] + overlay["height"] * 0.2)
|
| 242 |
+
page.mouse.up()
|
| 243 |
+
page.wait_for_function("document.querySelector('[data-parse-content]').value === '正文'")
|
| 244 |
+
page.set_viewport_size({"width": 390, "height": 844})
|
| 245 |
+
page.locator("#mobile-toggle-btn").click()
|
| 246 |
+
self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 247 |
+
self.assertLessEqual(page.evaluate("document.documentElement.scrollWidth"), 390)
|
| 248 |
+
page.locator("[data-parse-title]").fill("第一篇")
|
| 249 |
+
page.locator("[data-parse-form] button[type='submit']").click()
|
| 250 |
+
page.wait_for_function("document.querySelector('#toast').textContent.includes('解析已提交')")
|
| 251 |
+
thresholds = submitted[0]["articles"][0]["ocr_exceptions"]["1"]["content_thresholds"]
|
| 252 |
+
self.assertGreater(thresholds[0], 0.15)
|
| 253 |
+
self.assertEqual(submitted[0]["articles"][0]["content"], "正文")
|
| 254 |
+
self.assertEqual(errors, [])
|
| 255 |
+
finally:
|
| 256 |
+
context.close()
|
| 257 |
+
|
| 258 |
+
def test_desktop_search_filters_preview_theme_and_download(self):
|
| 259 |
+
context, page, errors = self.open_page()
|
| 260 |
+
try:
|
| 261 |
+
page.locator("#search-input").fill("文化革命")
|
| 262 |
+
page.wait_for_timeout(250)
|
| 263 |
+
self.assertIn("q=", page.url)
|
| 264 |
+
page.locator("label:has(#exact-toggle)").click()
|
| 265 |
+
page.locator("label:has(#fulltext-toggle)").click()
|
| 266 |
+
page.locator("#sort-filter").select_option("title")
|
| 267 |
+
page.locator("#source-filter [data-source]").click()
|
| 268 |
+
self.assertIn("source=", page.url)
|
| 269 |
+
original = page.locator("body").get_attribute("class")
|
| 270 |
+
page.locator("#theme-btn").click()
|
| 271 |
+
self.assertNotEqual(original, page.locator("body").get_attribute("class"))
|
| 272 |
+
page.locator("[data-preview]").click()
|
| 273 |
+
page.locator("#preview-panel").wait_for(state="visible")
|
| 274 |
+
self.assertIn("这是", page.locator(".preview-body").text_content())
|
| 275 |
+
page.locator("[data-close-preview]").click()
|
| 276 |
+
with page.expect_download(timeout=10000) as info:
|
| 277 |
+
page.locator('a[href^="/api/download/txt/"]').click()
|
| 278 |
+
self.assertTrue(info.value.suggested_filename.endswith(".txt"))
|
| 279 |
+
page.locator("#random-book-btn").click()
|
| 280 |
+
page.locator("#preview-panel").wait_for(state="visible")
|
| 281 |
+
self.assertEqual(errors, [])
|
| 282 |
+
finally:
|
| 283 |
+
context.close()
|
| 284 |
+
|
| 285 |
+
def test_result_date_click_syncs_date_terms_ui(self):
|
| 286 |
+
context, page, errors = self.open_page()
|
| 287 |
+
try:
|
| 288 |
+
page.locator("#left-toggle").click()
|
| 289 |
+
page.locator("[data-date-term-label='1966/5/16']").click()
|
| 290 |
+
term = page.locator("[data-date-term-index='0']")
|
| 291 |
+
self.assertEqual(term.locator("[data-date-term-field='year']").input_value(), "1966")
|
| 292 |
+
self.assertEqual(term.locator("[data-date-term-field='month']").input_value(), "5")
|
| 293 |
+
self.assertEqual(term.locator("[data-date-term-field='day']").input_value(), "16")
|
| 294 |
+
self.assertIn("date_term=year%3A1966-month%3A5-day%3A16", page.url)
|
| 295 |
+
self.assertEqual(errors, [])
|
| 296 |
+
finally:
|
| 297 |
+
context.close()
|
| 298 |
+
|
| 299 |
+
def test_proofread_compare_mode_is_usable_on_desktop_and_mobile(self):
|
| 300 |
+
for viewport in ({"width": 1440, "height": 900}, {"width": 390, "height": 844}):
|
| 301 |
+
context, page, errors = self.open_page(viewport)
|
| 302 |
+
try:
|
| 303 |
+
if page.locator("body").evaluate("node => node.classList.contains('mobile')") and page.locator("#left-sidebar").evaluate("node => node.classList.contains('open')"):
|
| 304 |
+
page.locator("#left-toggle").click()
|
| 305 |
+
page.locator("[data-preview]").click()
|
| 306 |
+
if viewport["width"] > 600:
|
| 307 |
+
page.locator("[data-proofread-select]").evaluate("node => { const range = document.createRange(); range.selectNodeContents(node); const selection = window.getSelection(); selection.removeAllRanges(); selection.addRange(range); node.dispatchEvent(new MouseEvent('mouseup', { bubbles: true })); }")
|
| 308 |
+
else:
|
| 309 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 310 |
+
page.locator("[data-proofread-form]").wait_for(state="visible")
|
| 311 |
+
page.wait_for_function("typeof diff_match_patch !== 'undefined'")
|
| 312 |
+
self.assertTrue(page.locator("[data-proofread-back]").is_visible())
|
| 313 |
+
self.assertTrue(page.locator("[data-preview-mobile]").is_visible() if viewport["width"] <= 600 else True)
|
| 314 |
+
self.assertEqual(page.locator("[data-meta-title]").input_value(), "手机资料")
|
| 315 |
+
self.assertEqual(page.locator("[data-proofread-part-input='0']").input_value(), "这是手机正文")
|
| 316 |
+
delta = page.evaluate("textDelta('旧〔1〕文', '新〔1〕文')")
|
| 317 |
+
self.assertIn("新", delta)
|
| 318 |
+
self.assertNotIn("%E6%96%B0", delta)
|
| 319 |
+
applied = page.evaluate("([original, delta]) => applyTextDelta(original, delta)", ["旧〔1〕文", delta])
|
| 320 |
+
self.assertEqual(applied, "新〔1〕文")
|
| 321 |
+
moved = page.evaluate("() => { const original = '甲乙共同段丙丁'; const edited = '丙丁甲乙共同段'; const delta = textDelta(original, edited); return { delta, applied: applyTextDelta(original, delta) }; }")
|
| 322 |
+
self.assertEqual(moved["applied"], "丙丁甲乙共同段")
|
| 323 |
+
self.assertIn("=", moved["delta"], (viewport, moved))
|
| 324 |
+
self.assertGreaterEqual(len(moved["delta"].split("\t")), 3)
|
| 325 |
+
escaped = page.evaluate("() => { const original = '甲乙'; const edited = '甲%\t校乙'; const delta = textDelta(original, edited); return { delta, applied: applyTextDelta(original, delta) }; }")
|
| 326 |
+
self.assertEqual(escaped["applied"], "甲%\t校乙")
|
| 327 |
+
self.assertIn("%25", escaped["delta"])
|
| 328 |
+
self.assertIn("%09", escaped["delta"])
|
| 329 |
+
source_box = page.locator(".proofread-source").bounding_box()
|
| 330 |
+
editor_box = page.locator(".proofread-editor").bounding_box()
|
| 331 |
+
self.assertIsNotNone(source_box)
|
| 332 |
+
self.assertIsNotNone(editor_box)
|
| 333 |
+
if viewport["width"] > 600:
|
| 334 |
+
self.assertLessEqual(source_box["x"] + source_box["width"], editor_box["x"] + 1)
|
| 335 |
+
else:
|
| 336 |
+
self.assertLessEqual(source_box["y"] + source_box["height"], editor_box["y"] + 1)
|
| 337 |
+
self.assertEqual(errors, [])
|
| 338 |
+
finally:
|
| 339 |
+
context.close()
|
| 340 |
+
|
| 341 |
+
def test_proofread_page_images_refit_and_recover_after_error(self):
|
| 342 |
+
context, page, errors = self.open_page()
|
| 343 |
+
first = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='80'%3E%3Crect width='120' height='80' fill='white'/%3E%3C/svg%3E"
|
| 344 |
+
broken = "data:image/webp;base64,AAAA"
|
| 345 |
+
third = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='800' height='100'%3E%3Crect width='800' height='100' fill='white'/%3E%3C/svg%3E"
|
| 346 |
+
try:
|
| 347 |
+
page.locator("[data-preview]").click()
|
| 348 |
+
page.locator("#preview-panel").wait_for(state="visible")
|
| 349 |
+
page.evaluate("""([first, broken, third]) => {
|
| 350 |
+
const item = getCachedPreview(STATE.preview);
|
| 351 |
+
item.source_files = [{ index: 0, name: "原稿.pdf", url: "https://example.com/source.pdf", pages: [first, broken, third] }];
|
| 352 |
+
STATE.proofreadFile = "0";
|
| 353 |
+
STATE.proofreadPage = "1";
|
| 354 |
+
openProofread(item, null, true);
|
| 355 |
+
}""", [first, broken, third])
|
| 356 |
+
image = page.locator("[data-proofread-source-image]")
|
| 357 |
+
page.wait_for_function("document.querySelector('[data-proofread-source-image]').naturalWidth === 120")
|
| 358 |
+
first_width = image.get_attribute("data-image-base-width")
|
| 359 |
+
page.locator("[data-page-image-next]").click()
|
| 360 |
+
page.locator("[data-image-error]").wait_for(state="visible")
|
| 361 |
+
self.assertTrue(image.is_hidden())
|
| 362 |
+
page.locator("[data-page-image-next]").click()
|
| 363 |
+
page.wait_for_function("document.querySelector('[data-proofread-source-image]').naturalWidth === 800 && !document.querySelector('[data-proofread-source-image]').hidden")
|
| 364 |
+
self.assertNotEqual(first_width, image.get_attribute("data-image-base-width"))
|
| 365 |
+
self.assertEqual(page.locator("[data-page-image-input]").input_value(), "3")
|
| 366 |
+
self.assertIn("proofread_page=3", page.url)
|
| 367 |
+
self.assertEqual(errors, [])
|
| 368 |
+
finally:
|
| 369 |
+
context.close()
|
| 370 |
+
|
| 371 |
+
def test_pdf_range_transport_rejects_invalid_followup_ranges(self):
|
| 372 |
+
context, page, errors = self.open_page()
|
| 373 |
+
try:
|
| 374 |
+
result = page.evaluate("""async () => {
|
| 375 |
+
class Transport {
|
| 376 |
+
constructor(length, initialData) { this.length = length; this.initialData = initialData; }
|
| 377 |
+
onDataRange() {}
|
| 378 |
+
}
|
| 379 |
+
const originalFetch = window.fetch;
|
| 380 |
+
let request = 0;
|
| 381 |
+
window.fetch = async () => {
|
| 382 |
+
request += 1;
|
| 383 |
+
if (request === 1) {
|
| 384 |
+
return new Response(new Uint8Array(8), { status: 206, headers: { "Content-Range": "bytes 0-7/8" } });
|
| 385 |
+
}
|
| 386 |
+
return new Response(new Uint8Array(4), { status: 206, headers: { "Content-Range": "bytes 0-3/8" } });
|
| 387 |
+
};
|
| 388 |
+
try {
|
| 389 |
+
const range = await createPDFRangeTransport({ PDFDataRangeTransport: Transport }, "https://example.test/file.pdf");
|
| 390 |
+
range.requestDataRange(4, 8);
|
| 391 |
+
try { await range.failurePromise; return false; } catch (_error) { return true; }
|
| 392 |
+
} finally {
|
| 393 |
+
window.fetch = originalFetch;
|
| 394 |
+
}
|
| 395 |
+
}""")
|
| 396 |
+
self.assertTrue(result)
|
| 397 |
+
self.assertEqual(errors, [])
|
| 398 |
+
finally:
|
| 399 |
+
context.close()
|
| 400 |
+
|
| 401 |
+
def test_real_pdf_worker_renders_and_download_stays_in_toolbar(self):
|
| 402 |
+
context, page, errors = self.open_page()
|
| 403 |
+
try:
|
| 404 |
+
page.locator("[data-preview]").click()
|
| 405 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 406 |
+
page.wait_for_function("document.querySelector('[data-pdf-status]')?.textContent === '1 / 1'", timeout=20000)
|
| 407 |
+
self.assertGreater(page.locator("[data-pdf-canvas]").evaluate("canvas => canvas.width"), 0)
|
| 408 |
+
download = page.locator(".proofread-pdf-tools [data-pdf-download-link]")
|
| 409 |
+
self.assertTrue(download.is_visible())
|
| 410 |
+
page.locator('[data-pdf-action="zoom-in"]').evaluate("button => { button.click(); button.click(); button.click(); }")
|
| 411 |
+
page.wait_for_timeout(300)
|
| 412 |
+
self.assertEqual(page.locator("[data-pdf-status]").text_content(), "1 / 1")
|
| 413 |
+
self.assertEqual(page.evaluate("pdfDocumentCache.size"), 1)
|
| 414 |
+
page.evaluate("window.dispatchEvent(new PageTransitionEvent('pagehide'))")
|
| 415 |
+
page.wait_for_function("pdfDocumentCache.size === 0")
|
| 416 |
+
self.assertEqual(errors, [])
|
| 417 |
+
finally:
|
| 418 |
+
context.close()
|
| 419 |
+
|
| 420 |
+
def test_proofread_draft_writes_are_debounced(self):
|
| 421 |
+
context, page, errors = self.open_page()
|
| 422 |
+
try:
|
| 423 |
+
page.locator("[data-preview]").click()
|
| 424 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 425 |
+
page.evaluate("sessionStorage.removeItem(`bha-proofread-draft:${STATE.preview}`)")
|
| 426 |
+
page.locator('[data-proofread-part-input="0"]').fill("防抖后的正文")
|
| 427 |
+
self.assertIsNone(page.evaluate("sessionStorage.getItem(`bha-proofread-draft:${STATE.preview}`)"))
|
| 428 |
+
page.wait_for_timeout(500)
|
| 429 |
+
draft = json.loads(page.evaluate("sessionStorage.getItem(`bha-proofread-draft:${STATE.preview}`)"))
|
| 430 |
+
self.assertEqual(draft["parts"]["0"], "防抖后的正文")
|
| 431 |
+
self.assertEqual(errors, [])
|
| 432 |
+
finally:
|
| 433 |
+
context.close()
|
| 434 |
+
|
| 435 |
+
def test_fulltext_apply_is_idempotent_and_syncs_with_parts(self):
|
| 436 |
+
context, page, errors = self.open_page()
|
| 437 |
+
try:
|
| 438 |
+
page.locator("[data-preview]").click()
|
| 439 |
+
page.locator("#preview-panel").wait_for(state="visible")
|
| 440 |
+
page.locator("[data-proofread-select]").evaluate("node => { const range = document.createRange(); range.selectNodeContents(node); const selection = window.getSelection(); selection.removeAllRanges(); selection.addRange(range); node.dispatchEvent(new MouseEvent('mouseup', { bubbles: true })); }")
|
| 441 |
+
page.locator("[data-proofread-form]").wait_for(state="visible")
|
| 442 |
+
page.locator("[data-fulltext-input]").fill("第一段\n第二段\n第三段")
|
| 443 |
+
page.locator("[data-fulltext-apply]").click()
|
| 444 |
+
page.wait_for_timeout(150)
|
| 445 |
+
self.assertEqual(page.locator("[data-proofread-new-part]").count(), 2)
|
| 446 |
+
self.assertEqual(page.locator("[data-proofread-new-part]").last.locator("textarea").input_value(), "第三段")
|
| 447 |
+
page.locator("[data-fulltext-apply]").click()
|
| 448 |
+
page.wait_for_timeout(150)
|
| 449 |
+
self.assertEqual(page.locator("[data-proofread-new-part]").count(), 2)
|
| 450 |
+
page.locator("[data-fulltext-toggle]").click()
|
| 451 |
+
page.locator("[data-part-editor-toggle]").click()
|
| 452 |
+
page.locator("[data-add-part-after='0']").click()
|
| 453 |
+
page.locator("[data-proofread-part-group='0'] [data-part-insert-after] [data-proofread-new-part]").last.locator("textarea").fill("中间段")
|
| 454 |
+
page.locator("[data-fulltext-toggle]").click()
|
| 455 |
+
self.assertIn("中间段", page.locator("[data-fulltext-input]").input_value())
|
| 456 |
+
page.locator("[data-fulltext-input]").fill("临时修改未应用")
|
| 457 |
+
page.locator("[data-fulltext-toggle]").click()
|
| 458 |
+
page.locator("[data-fulltext-toggle]").click()
|
| 459 |
+
self.assertEqual(page.locator("[data-fulltext-input]").input_value(), "临时修改未应用")
|
| 460 |
+
self.assertEqual(errors, [])
|
| 461 |
+
finally:
|
| 462 |
+
context.close()
|
| 463 |
+
|
| 464 |
+
def test_result_path_clicks_sync_filters_and_url(self):
|
| 465 |
+
context, page, errors = self.open_page()
|
| 466 |
+
try:
|
| 467 |
+
page.locator(".result-item [data-source]").click()
|
| 468 |
+
self.assertIn("source=%E6%B5%8B%E8%AF%95%E5%88%8A%E7%89%A9", page.url)
|
| 469 |
+
page.locator(".result-item [data-author]").click()
|
| 470 |
+
self.assertIn("author=%E4%BD%9C%E8%80%85%E7%94%B2", page.url)
|
| 471 |
+
page.locator(".result-item [data-tag]").click()
|
| 472 |
+
self.assertIn("tag=%E6%A0%87%E7%AD%BE%E7%94%B2", page.url)
|
| 473 |
+
page.locator("#tag-filter [data-tag='标签甲'] .checkbox-count").wait_for(state="visible")
|
| 474 |
+
self.assertEqual(page.locator("#tag-filter [data-tag='标签甲'] .checkbox-count").text_content(), "1")
|
| 475 |
+
self.assertEqual(errors, [])
|
| 476 |
+
finally:
|
| 477 |
+
context.close()
|
| 478 |
+
|
| 479 |
+
def test_mobile_sidebar_date_and_pagination_controls(self):
|
| 480 |
+
context, page, errors = self.open_page({"width": 390, "height": 844})
|
| 481 |
+
try:
|
| 482 |
+
self.assertTrue(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 483 |
+
if page.locator("#left-sidebar").evaluate("node => node.classList.contains('open')"):
|
| 484 |
+
page.locator("#left-toggle").click()
|
| 485 |
+
page.locator("#left-toggle").click()
|
| 486 |
+
self.assertTrue(page.locator("#left-sidebar").evaluate("node => node.classList.contains('open')"))
|
| 487 |
+
page.locator("#date-from-year").fill("1966")
|
| 488 |
+
page.locator("#date-from-year").press("Enter")
|
| 489 |
+
page.locator("#date-from-month").select_option("5")
|
| 490 |
+
page.locator("#date-from-day").select_option("16")
|
| 491 |
+
self.assertIn("date_from=19660516", page.url)
|
| 492 |
+
page.locator("#date-clear").click()
|
| 493 |
+
self.assertNotIn("date_from=", page.url)
|
| 494 |
+
if page.locator("#left-sidebar").evaluate("node => node.classList.contains('open')"):
|
| 495 |
+
page.locator("#left-toggle").click()
|
| 496 |
+
page.locator("[data-date-term-label='1966/5/16']").click()
|
| 497 |
+
term = page.locator("[data-date-term-index='0']")
|
| 498 |
+
self.assertEqual(term.locator("[data-date-term-field='year']").input_value(), "1966")
|
| 499 |
+
self.assertEqual(term.locator("[data-date-term-field='month']").input_value(), "5")
|
| 500 |
+
self.assertEqual(term.locator("[data-date-term-field='day']").input_value(), "16")
|
| 501 |
+
self.assertIn("date_term=year%3A1966-month%3A5-day%3A16", page.url)
|
| 502 |
+
page.locator("#mobile-toggle-btn").click()
|
| 503 |
+
self.assertFalse(page.locator("body").evaluate("node => node.classList.contains('mobile')"))
|
| 504 |
+
page.keyboard.press("/")
|
| 505 |
+
self.assertTrue(page.locator("#search-input").evaluate("node => document.activeElement === node"))
|
| 506 |
+
self.assertEqual(errors, [])
|
| 507 |
+
finally:
|
| 508 |
+
context.close()
|
| 509 |
+
|
| 510 |
+
def test_year_picker_lists_and_selects_every_available_year(self):
|
| 511 |
+
context, page, errors = self.open_page()
|
| 512 |
+
try:
|
| 513 |
+
page.evaluate("STATE.facets.years = [...Array.from({ length: 140 }, (_, index) => ({ value: 1880 + index })), { value: '66' }, { value: '未知' }, { value: '12345' }]")
|
| 514 |
+
page.locator("#date-from-year").click()
|
| 515 |
+
options = page.locator("#year-picker .year-option")
|
| 516 |
+
self.assertEqual(options.count(), 140)
|
| 517 |
+
self.assertEqual(page.locator("#year-picker [data-year-value='66']").count(), 0)
|
| 518 |
+
self.assertEqual(page.locator("#year-picker [data-year-value='12345']").count(), 0)
|
| 519 |
+
page.locator("#year-picker").evaluate("node => { node.scrollTop = node.scrollHeight; }")
|
| 520 |
+
options.last.click()
|
| 521 |
+
self.assertEqual(page.locator("#date-from-year").input_value(), "2019")
|
| 522 |
+
page.evaluate("STATE.facets.years = []")
|
| 523 |
+
page.locator("#date-to-year").click()
|
| 524 |
+
fallback_options = page.locator("#year-picker .year-option")
|
| 525 |
+
self.assertEqual(fallback_options.count(), 183)
|
| 526 |
+
self.assertEqual(fallback_options.first.get_attribute("data-year-value"), "1843")
|
| 527 |
+
self.assertEqual(fallback_options.last.get_attribute("data-year-value"), "2025")
|
| 528 |
+
self.assertEqual(errors, [])
|
| 529 |
+
finally:
|
| 530 |
+
context.close()
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def test_proofread_submit_requires_confirm_and_posts_payload(self):
|
| 534 |
+
context, page, errors = self.open_page()
|
| 535 |
+
try:
|
| 536 |
+
submitted = []
|
| 537 |
+
|
| 538 |
+
def route_proofread(route, request):
|
| 539 |
+
if request.method == "POST":
|
| 540 |
+
submitted.append(request.post_data_json)
|
| 541 |
+
route.fulfill(status=202, content_type="application/json", body=json.dumps({"ok": True, "status": "accepted", "tasks": 1, "auto_merge": True}))
|
| 542 |
+
|
| 543 |
+
page.route("**/api/proofread", route_proofread)
|
| 544 |
+
page.locator("[data-preview]").click()
|
| 545 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 546 |
+
page.locator("[data-proofread-form]").wait_for(state="visible")
|
| 547 |
+
page.locator("[data-proofread-part-input]").fill("这是手机正文(已校订)")
|
| 548 |
+
page.locator('button[type="submit"]').click()
|
| 549 |
+
confirm = page.locator("[data-proofread-confirm]")
|
| 550 |
+
confirm.wait_for(state="visible")
|
| 551 |
+
self.assertIn("段落 1", confirm.text_content())
|
| 552 |
+
self.assertIn("diff-del", confirm.inner_html())
|
| 553 |
+
page.locator("[data-cancel-proofread]").first.click()
|
| 554 |
+
confirm.wait_for(state="hidden")
|
| 555 |
+
self.assertEqual(submitted, [])
|
| 556 |
+
page.locator('button[type="submit"]').click()
|
| 557 |
+
page.locator("[data-confirm-proofread]").click()
|
| 558 |
+
page.wait_for_function("document.querySelector('[data-proofread-confirm]') === null")
|
| 559 |
+
self.assertEqual(len(submitted), 1)
|
| 560 |
+
body = submitted[0]
|
| 561 |
+
self.assertEqual(body["doc_id"], "doc-1")
|
| 562 |
+
self.assertIn("0", body["patch"]["parts"])
|
| 563 |
+
self.assertIn("diff", body["patch"]["parts"]["0"])
|
| 564 |
+
self.assertEqual(errors, [])
|
| 565 |
+
finally:
|
| 566 |
+
context.close()
|
| 567 |
+
|
| 568 |
+
def test_fulltext_split_preserves_following_part_indices(self):
|
| 569 |
+
parts = [
|
| 570 |
+
{"type": "paragraph", "text": "第一段完整内容"},
|
| 571 |
+
{"type": "paragraph", "text": "第二段保持不变"},
|
| 572 |
+
{"type": "paragraph", "text": "第三段保持不变"},
|
| 573 |
+
]
|
| 574 |
+
context, page, errors = self.open_page(preview_parts=parts)
|
| 575 |
+
try:
|
| 576 |
+
page.locator("[data-preview]").click()
|
| 577 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 578 |
+
form = page.locator("[data-proofread-form]")
|
| 579 |
+
form.wait_for(state="visible")
|
| 580 |
+
form.locator("[data-fulltext-input]").fill("第一段\n完整内容\n第二段保持不变\n第三段保持不变")
|
| 581 |
+
form.locator("[data-fulltext-apply]").click()
|
| 582 |
+
payload = page.evaluate("() => proofreadPayload(STATE.proofreadItem, document.querySelector('[data-proofread-form]'))")
|
| 583 |
+
self.assertEqual(set(payload["patch"]["parts"]), {"0"})
|
| 584 |
+
change = payload["patch"]["parts"]["0"]
|
| 585 |
+
self.assertIn("diff", change)
|
| 586 |
+
self.assertEqual(change["insertAfter"], [{"text": "��整内容", "type": "paragraph"}])
|
| 587 |
+
self.assertEqual(form.locator("[data-proofread-part-input='1']").input_value(), "第二段保持不变")
|
| 588 |
+
self.assertEqual(form.locator("[data-proofread-part-input='2']").input_value(), "第三段保持不变")
|
| 589 |
+
self.assertEqual(errors, [])
|
| 590 |
+
finally:
|
| 591 |
+
context.close()
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def test_proofread_review_ignores_untouched_tags_with_varying_key_order(self):
|
| 595 |
+
context, page, errors = self.open_page()
|
| 596 |
+
try:
|
| 597 |
+
submitted = []
|
| 598 |
+
|
| 599 |
+
def route_proofread(route, request):
|
| 600 |
+
if request.method == "POST":
|
| 601 |
+
submitted.append(request.post_data_json)
|
| 602 |
+
route.fulfill(status=202, content_type="application/json", body=json.dumps({"ok": True, "status": "accepted", "tasks": 1, "auto_merge": True}))
|
| 603 |
+
|
| 604 |
+
page.route("**/api/proofread", route_proofread)
|
| 605 |
+
page.locator("[data-preview]").click()
|
| 606 |
+
page.locator("[data-compare-select]").select_option("__proofread__")
|
| 607 |
+
page.locator("[data-proofread-form]").wait_for(state="visible")
|
| 608 |
+
page.locator('button[type="submit"]').click()
|
| 609 |
+
self.assertEqual(page.locator("[data-proofread-confirm]").count(), 0)
|
| 610 |
+
self.assertEqual(page.evaluate("document.querySelector('#toast').textContent"), "没有需要提交的修改")
|
| 611 |
+
page.locator("[data-proofread-part-input]").fill("这是手机正文(已校订)")
|
| 612 |
+
page.locator('button[type="submit"]').click()
|
| 613 |
+
confirm = page.locator("[data-proofread-confirm]")
|
| 614 |
+
confirm.wait_for(state="visible")
|
| 615 |
+
self.assertNotIn("标签", confirm.text_content())
|
| 616 |
+
page.locator("[data-confirm-proofread]").click()
|
| 617 |
+
page.wait_for_function("document.querySelector('[data-proofread-confirm]') === null")
|
| 618 |
+
self.assertEqual(len(submitted), 1)
|
| 619 |
+
self.assertIsNone(submitted[0].get("metadata"))
|
| 620 |
+
self.assertIn("0", submitted[0]["patch"]["parts"])
|
| 621 |
+
self.assertEqual(errors, [])
|
| 622 |
+
finally:
|
| 623 |
+
context.close()
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
if __name__ == "__main__":
|
| 627 |
+
unittest.main()
|
tests/test_frontend.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import unittest
|
| 2 |
+
from html.parser import HTMLParser
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from app.config import APP_ROOT
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class InteractiveIdParser(HTMLParser):
|
| 9 |
+
def __init__(self):
|
| 10 |
+
super().__init__()
|
| 11 |
+
self.ids = set()
|
| 12 |
+
|
| 13 |
+
def handle_starttag(self, tag, attrs):
|
| 14 |
+
values = dict(attrs)
|
| 15 |
+
if tag in {"button", "input", "select", "form"} and values.get("id"):
|
| 16 |
+
self.ids.add(values["id"])
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class FrontendContractTests(unittest.TestCase):
|
| 20 |
+
@classmethod
|
| 21 |
+
def setUpClass(cls):
|
| 22 |
+
cls.static = Path(APP_ROOT) / "static"
|
| 23 |
+
cls.html = (cls.static / "index.html").read_text(encoding="utf-8")
|
| 24 |
+
cls.source = (cls.static / "app.js").read_text(encoding="utf-8")
|
| 25 |
+
cls.css = (cls.static / "style.css").read_text(encoding="utf-8")
|
| 26 |
+
|
| 27 |
+
def test_every_static_control_is_cached_and_bound(self):
|
| 28 |
+
parser = InteractiveIdParser()
|
| 29 |
+
parser.feed(self.html)
|
| 30 |
+
expected = {
|
| 31 |
+
"search-form", "search-input", "left-toggle", "mobile-toggle-btn", "theme-btn",
|
| 32 |
+
"parse-btn",
|
| 33 |
+
"sidebar-expand", "exact-toggle", "preview-highlight-toggle", "fulltext-toggle",
|
| 34 |
+
"page-size", "sort-filter", "source-search", "author-search", "tag-search",
|
| 35 |
+
"archive-filter", "type-filter", "date-from-year", "date-from-month",
|
| 36 |
+
"date-from-day", "date-to-year", "date-to-month", "date-to-day", "date-clear",
|
| 37 |
+
"date-term-add", "clear-filters", "empty-random-btn", "random-book-btn",
|
| 38 |
+
"prev-page", "next-page", "page-input",
|
| 39 |
+
}
|
| 40 |
+
self.assertTrue(expected.issubset(parser.ids), expected - parser.ids)
|
| 41 |
+
for control_id in expected - {"search-form"}:
|
| 42 |
+
self.assertIn(f'$("#{control_id}")', self.source, control_id)
|
| 43 |
+
self.assertIn("DOM.searchForm.addEventListener", self.source)
|
| 44 |
+
|
| 45 |
+
def test_parse_queue_editor_uses_structured_same_origin_contracts(self):
|
| 46 |
+
for contract in (
|
| 47 |
+
'apiWithTimeout(`/api/parse/queue?page=${STATE.parsePage}&page_size=${STATE.parsePageSize}`)',
|
| 48 |
+
'apiWithTimeout(`/api/parse/${encodeURIComponent(item.doc_id)}/ocr?page=${page}`',
|
| 49 |
+
'apiWithTimeout("/api/parse", { method: "POST"',
|
| 50 |
+
"function collectParseArticles(form)",
|
| 51 |
+
"function parseCoverageError(articles, maxPage)",
|
| 52 |
+
"function refreshParseArticleBody(row, force = false)",
|
| 53 |
+
"base_part_count: Number(row.dataset.basePartCount) || 0",
|
| 54 |
+
"data-parse-content",
|
| 55 |
+
'data-parse-page-prev',
|
| 56 |
+
'data-parse-add',
|
| 57 |
+
'data-parse-crop-toggle',
|
| 58 |
+
'data-parse-crop-scope="page"',
|
| 59 |
+
"function beginParseCropDrag(event)",
|
| 60 |
+
"content_thresholds: thresholds",
|
| 61 |
+
"article.ocr_exceptions = ocrExceptions",
|
| 62 |
+
"parseOCRText(data, parseOCRThresholds(row, start + index))",
|
| 63 |
+
"renderParseQueue({ animate: true })",
|
| 64 |
+
):
|
| 65 |
+
self.assertIn(contract, self.source)
|
| 66 |
+
self.assertNotIn("replace_articles", self.source)
|
| 67 |
+
|
| 68 |
+
def test_search_race_timeout_cache_and_ime_guards_are_present(self):
|
| 69 |
+
for contract in (
|
| 70 |
+
"const REQUEST_TIMEOUT_MS = 12000", "const SEARCH_CACHE_LIMIT = 60",
|
| 71 |
+
"const SEARCH_CACHE_TTL = 8 * 60 * 1000", "searchRequestSeq",
|
| 72 |
+
"searchAbortController.abort()", 'addEventListener("compositionstart"',
|
| 73 |
+
'addEventListener("compositionend"', "requestSeq !== searchRequestSeq",
|
| 74 |
+
'data.index_status === "ready" && !data.indexing',
|
| 75 |
+
):
|
| 76 |
+
self.assertIn(contract, self.source)
|
| 77 |
+
|
| 78 |
+
def test_default_screen_uses_one_generation_scoped_bootstrap(self):
|
| 79 |
+
for contract in (
|
| 80 |
+
'window.__bhaBootstrapPromise = fetch("/api/bootstrap"',
|
| 81 |
+
"function canUseInitialBootstrap()",
|
| 82 |
+
"async function loadInitialBootstrap()",
|
| 83 |
+
"earlyBootstrap ? await earlyBootstrap : await apiWithTimeout(\"/api/bootstrap\", {}, 25000)",
|
| 84 |
+
"setCachedSearch(stableStringify(searchPayload()), search)",
|
| 85 |
+
"scheduleNextSearchPagePrefetch(searchPayload())",
|
| 86 |
+
"if (!searchAbortController && !bootstrapPending) renderResults()",
|
| 87 |
+
"canUseInitialBootstrap() ? loadInitialBootstrap() : Promise.all([doSearch(), loadFacets()])",
|
| 88 |
+
):
|
| 89 |
+
self.assertIn(contract, self.html + self.source)
|
| 90 |
+
|
| 91 |
+
def test_preview_cache_has_ttl_and_bounded_size(self):
|
| 92 |
+
for contract in (
|
| 93 |
+
"const PREVIEW_CACHE_TTL = 10 * 60 * 1000",
|
| 94 |
+
"Date.now() - entry.time > PREVIEW_CACHE_TTL",
|
| 95 |
+
"previewCache.delete(docId)",
|
| 96 |
+
"PREVIEW_CACHE_MAX_CHARS",
|
| 97 |
+
):
|
| 98 |
+
self.assertIn(contract, self.source)
|
| 99 |
+
self.assertIn("function getCachedPreview(docId)", self.source)
|
| 100 |
+
self.assertIn("function setCachedPreview(docId, value)", self.source)
|
| 101 |
+
|
| 102 |
+
def test_visible_page_keeps_the_search_connection_warm(self):
|
| 103 |
+
for contract in (
|
| 104 |
+
"const KEEPALIVE_INTERVAL_MS = 45 * 1000",
|
| 105 |
+
'fetch("/api/ping"',
|
| 106 |
+
"if (document.hidden || !navigator.onLine || keepalivePending)",
|
| 107 |
+
'window.addEventListener("online", () => warmConnection(true))',
|
| 108 |
+
'document.addEventListener("visibilitychange"',
|
| 109 |
+
"window.setInterval(() => warmConnection(), KEEPALIVE_INTERVAL_MS)",
|
| 110 |
+
):
|
| 111 |
+
self.assertIn(contract, self.source)
|
| 112 |
+
|
| 113 |
+
def test_dynamic_output_is_escaped_and_download_routes_are_explicit(self):
|
| 114 |
+
self.assertIn("function escapeHTML(value)", self.source)
|
| 115 |
+
self.assertIn("highlightPreviewText(item.content || \"\")", self.source)
|
| 116 |
+
self.assertIn("/api/download/txt/", self.source)
|
| 117 |
+
self.assertIn("/api/download/source-zip/", self.source)
|
| 118 |
+
self.assertIn("/api/download/source/", self.source)
|
| 119 |
+
self.assertIn("/api/preview/source/", self.source)
|
| 120 |
+
self.assertIn("encodeURIComponent(item.doc_id)", self.source)
|
| 121 |
+
|
| 122 |
+
def test_result_tags_keep_compact_metadata_text(self):
|
| 123 |
+
self.assertIn('class="result-size"', self.source)
|
| 124 |
+
self.assertRegex(self.css, r"\.result-size\s*\{[^}]*font-size:\s*11px;")
|
| 125 |
+
|
| 126 |
+
def test_proofread_delta_keeps_unicode_and_escapes_only_delimiters(self):
|
| 127 |
+
self.assertNotIn("diff_match_patch/20121119/diff_match_patch.js", self.html)
|
| 128 |
+
self.assertIn("diff_match_patch/20121119/diff_match_patch.js", self.source)
|
| 129 |
+
self.assertIn("sha512-5YZxUisfaneUbwv58nPp10qwt6DefHuJ", self.source)
|
| 130 |
+
self.assertIn("function loadDiffMatchPatch()", self.source)
|
| 131 |
+
self.assertIn("loadDiffMatchPatch();", self.source)
|
| 132 |
+
self.assertIn("differ.diff_main(original, edited, true)", self.source)
|
| 133 |
+
self.assertIn('inserted.replace(/%/g, "%25").replace(/\\t/g, "%09")', self.source)
|
| 134 |
+
self.assertNotIn('encodeURI(inserted)', self.source)
|
| 135 |
+
self.assertIn('if (cursor !== source.length) throw new Error("delta length mismatch")', self.source)
|
| 136 |
+
|
| 137 |
+
def test_preview_filters_dates_pagination_and_mobile_have_handlers(self):
|
| 138 |
+
for contract in (
|
| 139 |
+
'DOM.results.addEventListener("click"', 'DOM.previewPanel.addEventListener("click"',
|
| 140 |
+
'window.addEventListener("popstate"', "toggleResultFilter", "toggleResultDateTerm",
|
| 141 |
+
"refreshSelectedFacetItem", "commitResultPage", "commitFilterPage", "validateDateRange", "localStorage.setItem(\"mobileMode\"",
|
| 142 |
+
'showToast("随机文章加载失败")',
|
| 143 |
+
):
|
| 144 |
+
self.assertIn(contract, self.source)
|
| 145 |
+
|
| 146 |
+
def test_result_and_preview_motion_is_bounded_and_reduced_motion_safe(self):
|
| 147 |
+
for contract in (
|
| 148 |
+
'window.matchMedia("(prefers-reduced-motion: reduce)")',
|
| 149 |
+
"renderResults({ animate: true })",
|
| 150 |
+
"children).slice(0, 15)",
|
| 151 |
+
"let animateNextPreview = false",
|
| 152 |
+
"if (animateNextPreview) animatePreviewPanel()",
|
| 153 |
+
"function stopPreviewAnimation()",
|
| 154 |
+
"function openProofread(item, focusPart = null, restoring = false) {\n if (!item) return;\n stopPreviewAnimation();",
|
| 155 |
+
):
|
| 156 |
+
self.assertIn(contract, self.source)
|
| 157 |
+
for contract in (
|
| 158 |
+
".result-item.result-enter",
|
| 159 |
+
"animation: result-enter 320ms",
|
| 160 |
+
".preview-panel.preview-enter",
|
| 161 |
+
"animation: preview-enter 360ms",
|
| 162 |
+
):
|
| 163 |
+
self.assertIn(contract, self.css)
|
| 164 |
+
|
| 165 |
+
def test_year_picker_does_not_truncate_available_years(self):
|
| 166 |
+
self.assertIn('const isFourDigitYear = (value) => /^\\d{4}$/.test(String(value))', self.source)
|
| 167 |
+
self.assertIn("Array.from({ length: 183 }, (_, index) => String(1843 + index))", self.source)
|
| 168 |
+
self.assertIn("const years = yearItems().filter", self.source)
|
| 169 |
+
self.assertNotIn(".filter((year) => !keyword || String(year).includes(keyword)).slice", self.source)
|
| 170 |
+
self.assertIn("const openAbove = spaceBelow", self.source)
|
| 171 |
+
self.assertIn('DOM.yearPicker.style.bottom = openAbove ?', self.source)
|
| 172 |
+
|
| 173 |
+
def test_proofreading_uses_compare_menu_selection_and_source_proxy(self):
|
| 174 |
+
for contract in (
|
| 175 |
+
"对比原文件并校对", 'value="__proofread__"', "data-proofread-select",
|
| 176 |
+
'addEventListener("mouseup"', "/api/download/source/", "/api/preview/source/", "inline=true",
|
| 177 |
+
'apiWithTimeout("/api/proofread"', "data-meta-title", "data-list=",
|
| 178 |
+
):
|
| 179 |
+
self.assertIn(contract, self.source)
|
| 180 |
+
|
| 181 |
+
def test_proofreading_persists_state_and_supports_structural_editing(self):
|
| 182 |
+
for contract in ("sessionStorage", "STATE.compare", "STATE.proofread", "STATE.proofreadItem", "STATE.proofreadPage", "data-add-part-before", "data-add-part-after", "data-delete-part", "data-fulltext-input", "flushProofreadDraft", "setTimeout(flushProofreadDraft, 400)", 'addEventListener("pagehide"', "校订内容已过期,请重新打开校对页"):
|
| 183 |
+
self.assertIn(contract, self.source)
|
| 184 |
+
self.assertIn('data-fulltext-toggle>收起全文编辑</button>', self.source)
|
| 185 |
+
self.assertIn('<div data-fulltext-box><textarea data-fulltext-input>', self.source)
|
| 186 |
+
self.assertNotIn('<div data-fulltext-box hidden>', self.source)
|
| 187 |
+
|
| 188 |
+
def test_proofreading_shows_change_preview_before_submitting(self):
|
| 189 |
+
for contract in (
|
| 190 |
+
"校订提交预览", "data-confirm-proofread", "data-cancel-proofread",
|
| 191 |
+
"确认提交", "返回继续编辑", "function proofreadReviewHTML",
|
| 192 |
+
"function applyTextDelta", "function postProofread",
|
| 193 |
+
"元数据对照", "proofread-metadata-table", "原元数据", "修改后元数据",
|
| 194 |
+
"function metadataItemText", "function metadataCellHTML",
|
| 195 |
+
):
|
| 196 |
+
self.assertIn(contract, self.source)
|
| 197 |
+
self.assertIn("function matchingLineAnchors(current, desired)", self.source)
|
| 198 |
+
self.assertIn("const anchors = [...matchingLineAnchors(current, lines)", self.source)
|
| 199 |
+
self.assertIn('apiWithTimeout("/api/proofread"', self.source)
|
| 200 |
+
self.assertNotIn("JSON.stringify(oldValues[key])", self.source)
|
| 201 |
+
|
| 202 |
+
def test_proofreading_mobile_places_fulltext_editor_at_top(self):
|
| 203 |
+
for contract in (
|
| 204 |
+
"data-fulltext-section", "function relocateFulltextEditor",
|
| 205 |
+
"STATE.isMobile ? fulltextSection :", "STATE.isMobile ? partToggle :",
|
| 206 |
+
"relocateFulltextEditor();",
|
| 207 |
+
):
|
| 208 |
+
self.assertIn(contract, self.source)
|
| 209 |
+
|
| 210 |
+
def test_proofreading_image_fallback_and_compact_body_editor(self):
|
| 211 |
+
for contract in (
|
| 212 |
+
"data-proofread-source-image", "data-image-zoom", "loading=\"eager\"", "fetchpriority=\"high\"", "data-proofread-back", "data-preview-mobile", "图片无法在线显示",
|
| 213 |
+
"没有原文件", "仍可编辑正文并提交校对", "data-part-editor-toggle", "收起段落编辑",
|
| 214 |
+
"在上新建段落", "在下新建段落", "删除当前段落", "proofread-structure-closed",
|
| 215 |
+
"proofread-description-heading", "proofread-grid-no-source", '"校对"',
|
| 216 |
+
"isPDFSource", "pageImageViewerHTML", "initial_page", "initial_page_number", "page_numbers", "data-page-image-viewer", "data-page-count", "source-page", "data-page-image-prev", "data-page-image-next", "此原文件暂不支持在线预览", "proofread-document-viewer",
|
| 217 |
+
"loadPDFJS", "data-pdf-canvas", "proofread-pdf-stage is-loading", "is-loading", "is-error", "pdf.worker.min.mjs", "getCachedPDFDocument", "PDF_DOCUMENT_CACHE_LIMIT = 8", "createPDFRangeTransport", "readPDFRange", "failurePromise", "destroyPDFCacheEntry", "PDFDataRangeTransport", "Range: \"bytes=0-524287\"", "requestDataRange(begin, end)", "disableAutoFetch", "disableStream: true", "disableRange: false", "rangeChunkSize: 512 * 1024", "RenderingCancelledException", "ResizeObserver", "data-image-error", "data-pdf-download-link", "decoding=\"async\"", "PDF 无法在线预览",
|
| 218 |
+
):
|
| 219 |
+
self.assertIn(contract, self.source)
|
| 220 |
+
|
| 221 |
+
def test_preview_open_limits_text_selection_to_the_preview_panel(self):
|
| 222 |
+
self.assertIn("body.preview-open {\n user-select: none;", self.css)
|
| 223 |
+
self.assertIn("-webkit-user-select: none;", self.css)
|
| 224 |
+
self.assertIn("body.preview-open #preview-panel {\n user-select: text;", self.css)
|
| 225 |
+
self.assertIn("body.preview-open #preview-panel .preview-header {\n user-select: none;", self.css)
|
| 226 |
+
|
| 227 |
+
def test_pdfjs_is_pinned_and_preconnected(self):
|
| 228 |
+
self.assertIn('cdnjs.cloudflare.com/ajax/libs/pdf.js/4.10.38/pdf.min.mjs', self.source)
|
| 229 |
+
self.assertIn('cdnjs.cloudflare.com/ajax/libs/pdf.js/4.10.38/pdf.worker.min.mjs', self.source)
|
| 230 |
+
self.assertIn('rel="preconnect" href="https://cdnjs.cloudflare.com"', self.html)
|
| 231 |
+
self.assertIn('rel="preconnect" href="https://huggingface.co"', self.html)
|
| 232 |
+
self.assertIn("fetchInitialPDFRange", self.source)
|
| 233 |
+
self.assertIn("Promise.all([pdfjsLoad, initialLoad])", self.source)
|
| 234 |
+
self.assertIn('navigator.serviceWorker.register("/static/sw.js"', self.source)
|
| 235 |
+
service_worker = (self.static / "sw.js").read_text(encoding="utf-8")
|
| 236 |
+
self.assertIn('bha-page-webp-v1.0.1', service_worker)
|
| 237 |
+
self.assertIn('"/api/preview/source-page/"', service_worker)
|
| 238 |
+
self.assertIn("MAX_ENTRIES = 50", service_worker)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
if __name__ == "__main__":
|
| 242 |
+
unittest.main()
|