Spaces:
Running
Running
Add generation-scoped first-screen bootstrap
Browse files- app/bootstrap_api.py +56 -0
- app/main.py +2 -1
- static/app.js +59 -4
app/bootstrap_api.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import json
|
| 3 |
+
import threading
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
from fastapi import APIRouter
|
| 7 |
+
from fastapi.responses import JSONResponse
|
| 8 |
+
|
| 9 |
+
from . import runtime, search_api
|
| 10 |
+
from .search_logic import SearchRequest
|
| 11 |
+
from .storage_lock import serving_lock
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
router = APIRouter()
|
| 15 |
+
bootstrap_cache: tuple[tuple[Any, ...], dict[str, Any]] | None = None
|
| 16 |
+
bootstrap_lock = threading.Lock()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def generation_id(generation: tuple[Any, ...]) -> str:
|
| 20 |
+
return hashlib.sha256(repr(generation).encode()).hexdigest()[:16]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@router.get("/api/bootstrap")
|
| 24 |
+
def bootstrap():
|
| 25 |
+
global bootstrap_cache
|
| 26 |
+
with bootstrap_lock:
|
| 27 |
+
generation = runtime.serving_generation_token()
|
| 28 |
+
if runtime.index_status() == "ready" and bootstrap_cache and bootstrap_cache[0] == generation:
|
| 29 |
+
return JSONResponse(bootstrap_cache[1], headers={"Cache-Control": "no-store"})
|
| 30 |
+
if not runtime.serving_generation_ready():
|
| 31 |
+
return JSONResponse({"error": "serving generation is not ready"}, status_code=503)
|
| 32 |
+
|
| 33 |
+
for _attempt in range(2):
|
| 34 |
+
generation = runtime.serving_generation_token()
|
| 35 |
+
search_response = search_api.search(SearchRequest())
|
| 36 |
+
if not isinstance(search_response, JSONResponse) or search_response.status_code != 200:
|
| 37 |
+
return JSONResponse({"error": "default search is unavailable"}, status_code=503)
|
| 38 |
+
search_payload = json.loads(search_response.body)
|
| 39 |
+
with serving_lock():
|
| 40 |
+
sources = runtime.cached_sources_payload()
|
| 41 |
+
facets = {
|
| 42 |
+
kind: runtime.cached_facet_payload(kind, 1, 200, "")
|
| 43 |
+
for kind in ("source", "author", "tag")
|
| 44 |
+
}
|
| 45 |
+
if runtime.index_status() != "ready" or runtime.serving_generation_token() != generation:
|
| 46 |
+
continue
|
| 47 |
+
payload = {
|
| 48 |
+
"generation": generation_id(generation),
|
| 49 |
+
"search": search_payload,
|
| 50 |
+
"sources": sources,
|
| 51 |
+
"facets": facets,
|
| 52 |
+
}
|
| 53 |
+
bootstrap_cache = (generation, payload)
|
| 54 |
+
return JSONResponse(payload, headers={"Cache-Control": "no-store"})
|
| 55 |
+
|
| 56 |
+
return JSONResponse({"error": "serving generation changed during bootstrap"}, status_code=503)
|
app/main.py
CHANGED
|
@@ -6,7 +6,7 @@ from brotli_asgi import BrotliMiddleware
|
|
| 6 |
from fastapi.staticfiles import StaticFiles
|
| 7 |
from .config import APP_ROOT
|
| 8 |
from .data_loader import initialize_search_tokenizer
|
| 9 |
-
from . import parse_api, preview_api, proofread_api, reindex_api, runtime, search_api, source_files
|
| 10 |
app = FastAPI(title="BHA Search Lite")
|
| 11 |
app.add_middleware(
|
| 12 |
CORSMiddleware,
|
|
@@ -42,6 +42,7 @@ async def close_source_client():
|
|
| 42 |
|
| 43 |
app.include_router(runtime.router)
|
| 44 |
app.include_router(search_api.router)
|
|
|
|
| 45 |
app.include_router(preview_api.router)
|
| 46 |
app.include_router(parse_api.router)
|
| 47 |
app.include_router(proofread_api.router)
|
|
|
|
| 6 |
from fastapi.staticfiles import StaticFiles
|
| 7 |
from .config import APP_ROOT
|
| 8 |
from .data_loader import initialize_search_tokenizer
|
| 9 |
+
from . import bootstrap_api, parse_api, preview_api, proofread_api, reindex_api, runtime, search_api, source_files
|
| 10 |
app = FastAPI(title="BHA Search Lite")
|
| 11 |
app.add_middleware(
|
| 12 |
CORSMiddleware,
|
|
|
|
| 42 |
|
| 43 |
app.include_router(runtime.router)
|
| 44 |
app.include_router(search_api.router)
|
| 45 |
+
app.include_router(bootstrap_api.router)
|
| 46 |
app.include_router(preview_api.router)
|
| 47 |
app.include_router(parse_api.router)
|
| 48 |
app.include_router(proofread_api.router)
|
static/app.js
CHANGED
|
@@ -73,6 +73,7 @@ let progressTimer = null;
|
|
| 73 |
let searchRequestSeq = 0;
|
| 74 |
let searchAbortController = null;
|
| 75 |
let searchPrefetchAbortController = null;
|
|
|
|
| 76 |
let pdfjsPromise = null;
|
| 77 |
const pdfDocumentCache = new Map();
|
| 78 |
const PDF_DOCUMENT_CACHE_LIMIT = 8;
|
|
@@ -923,6 +924,54 @@ function prefetchNextSearchPage(basePayload) {
|
|
| 923 |
});
|
| 924 |
}
|
| 925 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 926 |
async function loadFacets() {
|
| 927 |
const pendingKey = "sources";
|
| 928 |
if (facetPending.has(pendingKey)) return facetPending.get(pendingKey);
|
|
@@ -1164,6 +1213,7 @@ function scheduleFacetSearch(kind) {
|
|
| 1164 |
}
|
| 1165 |
|
| 1166 |
function recoverPageState() {
|
|
|
|
| 1167 |
if (DOM.statusText.textContent === "搜索中..." && !searchAbortController) DOM.statusText.textContent = "";
|
| 1168 |
let shouldLoadSources = false;
|
| 1169 |
for (const kind of ["source", "author", "tag"]) {
|
|
@@ -1330,7 +1380,8 @@ async function doSearch() {
|
|
| 1330 |
DOM.results.scrollTop = 0;
|
| 1331 |
if (STATE.preview) previewResultsScrollTop = 0;
|
| 1332 |
DOM.statusText.textContent = "";
|
| 1333 |
-
|
|
|
|
| 1334 |
prefetchNextSearchPage(payload);
|
| 1335 |
} catch (error) {
|
| 1336 |
if (error.name === "AbortError" || requestSeq !== searchRequestSeq) return;
|
|
@@ -3719,13 +3770,17 @@ function init() {
|
|
| 3719 |
}
|
| 3720 |
applyStateToDom();
|
| 3721 |
attachEvents();
|
| 3722 |
-
const initialSearch = STATE.parseMode
|
|
|
|
|
|
|
| 3723 |
if (!STATE.parseMode) restorePreview(false);
|
| 3724 |
initialSearch.finally(() => {
|
| 3725 |
lastKeepaliveAt = Date.now();
|
| 3726 |
window.setInterval(() => warmConnection(), KEEPALIVE_INTERVAL_MS);
|
| 3727 |
-
|
| 3728 |
-
|
|
|
|
|
|
|
| 3729 |
});
|
| 3730 |
}
|
| 3731 |
document.addEventListener("DOMContentLoaded", init);
|
|
|
|
| 73 |
let searchRequestSeq = 0;
|
| 74 |
let searchAbortController = null;
|
| 75 |
let searchPrefetchAbortController = null;
|
| 76 |
+
let bootstrapPending = false;
|
| 77 |
let pdfjsPromise = null;
|
| 78 |
const pdfDocumentCache = new Map();
|
| 79 |
const PDF_DOCUMENT_CACHE_LIMIT = 8;
|
|
|
|
| 924 |
});
|
| 925 |
}
|
| 926 |
|
| 927 |
+
function canUseInitialBootstrap() {
|
| 928 |
+
const payload = searchPayload();
|
| 929 |
+
return payload.q === "" && payload.page === 1 && payload.page_size === 20
|
| 930 |
+
&& payload.exact === true && payload.fulltext === true && payload.sort === "relevance"
|
| 931 |
+
&& !payload.source && !payload.author && !payload.tag && !payload.archive_id
|
| 932 |
+
&& !payload.publication_type && !payload.date_from && !payload.date_to
|
| 933 |
+
&& !payload.sources.length && !payload.authors.length && !payload.tags.length
|
| 934 |
+
&& !payload.exclude_sources.length && !payload.exclude_authors.length && !payload.exclude_tags.length
|
| 935 |
+
&& !payload.date_terms.length;
|
| 936 |
+
}
|
| 937 |
+
|
| 938 |
+
async function loadInitialBootstrap() {
|
| 939 |
+
bootstrapPending = true;
|
| 940 |
+
DOM.statusText.textContent = "搜索中...";
|
| 941 |
+
try {
|
| 942 |
+
const data = await apiWithTimeout("/api/bootstrap", {}, 25000);
|
| 943 |
+
const search = data.search || {};
|
| 944 |
+
if (search.index_status !== "ready" || search.indexing) throw new Error("bootstrap is not ready");
|
| 945 |
+
STATE.results = search.results || [];
|
| 946 |
+
STATE.total = Number(search.total || 0);
|
| 947 |
+
STATE.pageSize = Number(search.page_size || 20);
|
| 948 |
+
const sources = data.sources || {};
|
| 949 |
+
const facets = data.facets || {};
|
| 950 |
+
STATE.facets = {
|
| 951 |
+
sources: sortFacetItems((facets.source || {}).items || []),
|
| 952 |
+
authors: sortFacetItems((facets.author || {}).items || []),
|
| 953 |
+
tags: sortFacetItems((facets.tag || {}).items || []),
|
| 954 |
+
archives: sortFacetItems(sources.archives || []),
|
| 955 |
+
types: sortFacetItems(sources.types || []),
|
| 956 |
+
years: sortFacetItems(sources.years || []),
|
| 957 |
+
};
|
| 958 |
+
for (const kind of ["source", "author", "tag"]) {
|
| 959 |
+
STATE.facetTotals[kind] = Number((facets[kind] || {}).total || 0);
|
| 960 |
+
STATE.facetHasMore[kind] = Boolean((facets[kind] || {}).has_more);
|
| 961 |
+
}
|
| 962 |
+
setCachedSearch(stableStringify(searchPayload()), search);
|
| 963 |
+
renderFilterOptions({ preserveScroll: false });
|
| 964 |
+
renderResults({ animate: true });
|
| 965 |
+
DOM.results.scrollTop = 0;
|
| 966 |
+
DOM.statusText.textContent = "";
|
| 967 |
+
DOM.indexProgress.textContent = "";
|
| 968 |
+
} catch (_error) {
|
| 969 |
+
await Promise.all([doSearch(), loadFacets()]);
|
| 970 |
+
} finally {
|
| 971 |
+
bootstrapPending = false;
|
| 972 |
+
}
|
| 973 |
+
}
|
| 974 |
+
|
| 975 |
async function loadFacets() {
|
| 976 |
const pendingKey = "sources";
|
| 977 |
if (facetPending.has(pendingKey)) return facetPending.get(pendingKey);
|
|
|
|
| 1213 |
}
|
| 1214 |
|
| 1215 |
function recoverPageState() {
|
| 1216 |
+
if (bootstrapPending) return;
|
| 1217 |
if (DOM.statusText.textContent === "搜索中..." && !searchAbortController) DOM.statusText.textContent = "";
|
| 1218 |
let shouldLoadSources = false;
|
| 1219 |
for (const kind of ["source", "author", "tag"]) {
|
|
|
|
| 1380 |
DOM.results.scrollTop = 0;
|
| 1381 |
if (STATE.preview) previewResultsScrollTop = 0;
|
| 1382 |
DOM.statusText.textContent = "";
|
| 1383 |
+
if (data.index_status === "ready" && !data.indexing) DOM.indexProgress.textContent = "";
|
| 1384 |
+
else updateProgress();
|
| 1385 |
prefetchNextSearchPage(payload);
|
| 1386 |
} catch (error) {
|
| 1387 |
if (error.name === "AbortError" || requestSeq !== searchRequestSeq) return;
|
|
|
|
| 3770 |
}
|
| 3771 |
applyStateToDom();
|
| 3772 |
attachEvents();
|
| 3773 |
+
const initialSearch = STATE.parseMode
|
| 3774 |
+
? loadParseQueue()
|
| 3775 |
+
: canUseInitialBootstrap() ? loadInitialBootstrap() : Promise.all([doSearch(), loadFacets()]);
|
| 3776 |
if (!STATE.parseMode) restorePreview(false);
|
| 3777 |
initialSearch.finally(() => {
|
| 3778 |
lastKeepaliveAt = Date.now();
|
| 3779 |
window.setInterval(() => warmConnection(), KEEPALIVE_INTERVAL_MS);
|
| 3780 |
+
if (STATE.parseMode) {
|
| 3781 |
+
updateProgress();
|
| 3782 |
+
loadFacets();
|
| 3783 |
+
}
|
| 3784 |
});
|
| 3785 |
}
|
| 3786 |
document.addEventListener("DOMContentLoaded", init);
|