Optimize CJK wildcard search candidates

Reuse normalized metadata text and prefilter wildcard candidates through existing CJK postings without changing tokenizer or index format.

app.py

@@ -98,6 +98,14 @@ def literal_query_tokens(text: str) -> list[str]:
         if not TOKEN_RE.fullmatch(normalized[index + 1])
     ))
 
+def wildcard_required_tokens(text: str) -> list[str]:
+    tokens = []
+    for fixed_part in re.split(r"[*?]+", normalize_text(text)):
+        for term in TOKEN_RE.findall(fixed_part):
+            if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", term):
+                tokens.extend(query_tokens(term))
+    return list(dict.fromkeys(tokens))
+
 def decode_doc_numbers(payload: bytes, candidates: set[int] | None = None) -> set[int]:
     results = set()
     number = shift = previous = 0
@@ -286,15 +294,18 @@ class FulltextDatabases:
             normalized_query = normalize_text(query)
             try:
                 if has_wildcard_query(normalized_query):
+                    required_tokens = wildcard_required_tokens(normalized_query) if version == TOKENIZER_VERSION else []
+                    candidates = self._search_postings(connection, has_doc_counts, required_tokens) if required_tokens else None
                     fixed_parts = [part for part in re.split(r"[*?]+", normalized_query) if len(part) >= 3]
                     if fixed_parts:
                         match_query = " AND ".join(f'"{part.replace(chr(34), chr(34) * 2)}"' for part in fixed_parts)
-                        candidates = {
+                        fts_candidates = {
                             int(row[0]) for row in connection.execute(
                                 "SELECT rowid FROM content_fts WHERE content_fts MATCH ?", (match_query,)
                             )
                         }
-                    else:
+                        candidates = fts_candidates if candidates is None else candidates & fts_candidates
+                    elif candidates is None:
                         candidates = {
                             int(row[0]) for row in connection.execute("SELECT rowid FROM content_fts")
                         }
@@ -581,13 +592,19 @@ fulltext_databases: Optional[FulltextDatabases] = None
 def tokenize(text: str) -> list[str]:
     return query_terms(text)
 
+def compile_exact_query(query: str) -> tuple[str, re.Pattern | None]:
+    normalized_query = normalize_text(query)
+    if has_wildcard_query(normalized_query):
+        expression = re.escape(normalized_query).replace(r"\*", ".*?").replace(r"\?", ".")
+        return normalized_query, re.compile(expression, re.DOTALL)
+    return normalized_query, None
+
+def matches_normalized_exact_query(text: str, normalized_query: str, pattern: re.Pattern | None = None) -> bool:
+    return pattern.search(text) is not None if pattern is not None else normalized_query in text
+
 def matches_exact_query(text: str, query: str) -> bool:
-    text = normalize_text(text)
-    query = normalize_text(query)
-    if "*" in query or "?" in query:
-        pattern = re.escape(query).replace(r"\*", ".*?").replace(r"\?", ".")
-        return re.search(pattern, text, re.DOTALL) is not None
-    return query in text
+    normalized_query, pattern = compile_exact_query(query)
+    return matches_normalized_exact_query(normalize_text(text), normalized_query, pattern)
 
 def has_wildcard_query(query: str) -> bool:
     return "*" in query or "?" in query
@@ -887,11 +904,20 @@ def metadata_matches(q: str, exact: bool, search_paths: bool) -> set[int]:
     field = "_search_text" if search_paths else "_file_search_text"
     if exact:
         terms = query_terms(q)
+        normalized_query, pattern = compile_exact_query(q)
         # Whole-token postings are not a lossless prefilter for Latin
         # substrings or punctuation-only queries. CJK literals necessarily
         # contain all of their indexed unigram/bigram tokens, so retain the
         # fast candidate intersection for that common path.
-        if has_wildcard_query(q) or not terms or any(re.fullmatch(r"[a-z0-9]+", term) for term in terms):
+        if has_wildcard_query(q):
+            candidate_set = None
+            for token in wildcard_required_tokens(normalized_query):
+                token_indices = word_index.get(token, set())
+                candidate_set = set(token_indices) if candidate_set is None else candidate_set & token_indices
+                if not candidate_set:
+                    return set()
+            candidates = candidate_set if candidate_set is not None else range(len(records))
+        elif not terms or any(re.fullmatch(r"[a-z0-9]+", term) for term in terms):
             candidates = range(len(records))
         else:
             candidate_set = None
@@ -901,7 +927,10 @@ def metadata_matches(q: str, exact: bool, search_paths: bool) -> set[int]:
                 if not candidate_set:
                     return set()
             candidates = candidate_set or set()
-        return {idx for idx in candidates if matches_exact_query(records[idx][field], q)}
+        return {
+            idx for idx in candidates
+            if matches_normalized_exact_query(records[idx][field], normalized_query, pattern)
+        }
     terms = query_terms(q)
     if not terms:
         return set()