perf: reduce shallow search work and reuse exact counts across pages

app.py

@@ -38,6 +38,7 @@ source_name_order: dict[str, list[int]] = {}
 source_size_order: dict[str, list[int]] = {}
 record_map: dict[str, dict] = {}
 record_map_index: dict[str, int] = {}
+source_doc_number_index: dict[str, dict[int, int]] = {}
 sources: list[dict] = []
 source_counts: dict[str, int] = {}
 folder_tree_data: dict[str, list[dict]] = {}
@@ -790,17 +791,26 @@ def load_json_gz(path: Path):
     return json.loads(gzip.decompress(path.read_bytes()).decode("utf-8"))
 
 def build_indexes() -> None:
-    global word_index, latin_all_index, latin_file_index, latin_vocabulary_all, source_records_map, extension_counts, source_extension_counts, name_order, size_order, source_name_order, source_size_order
+    global word_index, latin_all_index, latin_file_index, latin_vocabulary_all, source_records_map, source_doc_number_index, extension_counts, source_extension_counts, name_order, size_order, source_name_order, source_size_order
     clear_metadata_match_cache()
     word_index = {}
     latin_all_index = {}
     latin_file_index = {}
     latin_vocabulary_all = set()
     source_records_map = {}
+    source_doc_number_index = {}
     extension_counts = {}
     source_extension_counts = {}
     for idx, rec in enumerate(records):
         source = rec["source"]
+        id_source, separator, number_text = rec["doc_id"].partition(":")
+        try:
+            number = int(number_text)
+        except ValueError:
+            pass
+        else:
+            if separator and str(number) == number_text:
+                source_doc_number_index.setdefault(id_source, {})[number] = idx
         extension = rec["extension"]
         source_records_map.setdefault(source, []).append(idx)
         extension_counts[extension] = extension_counts.get(extension, 0) + 1
@@ -870,10 +880,26 @@ def relevance_page(indices, query_tokens: list[str], search_paths: bool, start:
     limit = start + page_size
     if limit <= 0:
         return []
+    indices = list(indices)
+    if start >= len(indices):
+        return []
+    if start < 1000 and len(indices) >= 10000 and query_tokens and all(
+        re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]{1,2}", token) for token in query_tokens
+    ):
+        potential = metadata_prefilter(query_tokens, search_paths)
+        if potential is not None and len(potential) >= limit:
+            candidates = set(indices)
+            potential.intersection_update(candidates)
+            if len(candidates) == len(indices) and len(potential) >= limit:
+                # Only maximum-score hits can make unseen candidates irrelevant.
+                maximum = (5 if search_paths else 3) * len(query_tokens)
+                highest = []
+                for idx in name_order:
+                    if idx in potential and score_record(idx, query_tokens, search_paths) == maximum:
+                        highest.append(idx)
+                        if len(highest) == limit:
+                            return highest[start:limit]
     if start >= 1000:
-        indices = list(indices)
-        if start >= len(indices):
-            return []
         if len(indices) >= 10000:
             scores = {idx: score_record(idx, query_tokens, search_paths) for idx in indices}
             if len(scores) == len(indices):
@@ -1177,13 +1203,16 @@ def fulltext_search(q="", sources_filter=None, folders=None, min_size=None, max_
     source_slugs = sources_filter if sources_filter is not None else list(source_counts)
     if not source_slugs:
         return {"results": [], "total": 0, "page": max(1, page), "page_size": page_size}
-    matched_doc_ids = set()
+    content_indices = set()
     sqlite_started = time.perf_counter()
     for source_slug in source_slugs:
-        matched_doc_ids.update(f"{source_slug}:{doc_number}" for doc_number in fulltext_databases.search_source(source_slug, q, exact))
+        number_index = source_doc_number_index.get(source_slug, {})
+        content_indices.update(
+            idx for doc_number in fulltext_databases.search_source(source_slug, q, exact)
+            if (idx := number_index.get(doc_number)) is not None
+        )
     if timings is not None:
         timings["sqlite"] = (time.perf_counter() - sqlite_started) * 1000
-    content_indices = {idx for doc_id in matched_doc_ids if (idx := record_map_index.get(doc_id)) is not None}
     indices = set(content_indices)
     metadata_started = time.perf_counter()
     indices.update(metadata_matches(q, exact, search_paths))