Upload app.py

app.py

@@ -65,6 +65,8 @@ search_response_cache: dict[tuple, tuple[float, object]] = {}
 search_response_cache_lock = threading.Lock()
 preview_text_cache: dict[str, str] = {}
 preview_text_cache_lock = threading.Lock()
+cache_key_locks: dict[tuple, threading.Lock] = {}
+cache_key_locks_guard = threading.Lock()
 ZIP_TOKEN_TTL_SECONDS = 600
 ZIP_TOKEN_MAX_ENTRIES = 32
 zip_download_tokens: dict[str, tuple[float, list[str]]] = {}
@@ -330,7 +332,10 @@ class FulltextDatabases:
                 tokens = query_tokens(query) if version == TOKENIZER_VERSION else query_terms(query)
                 if version == TOKENIZER_VERSION:
                     tokens = [*tokens, *literal_query_tokens(query)]
-                if not tokens:
+                # Short non-CJK literals (Latin substrings or punctuation-separated
+                # characters) are not provable from whole-token/unigram postings, so
+                # verify them against the normalized FTS content directly.
+                if not tokens or not re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", normalized_query):
                     rows = connection.execute(
                         "SELECT rowid FROM content_fts WHERE instr(content, ?) > 0", (normalized_query,)
                     )
@@ -403,21 +408,6 @@ class FulltextDatabases:
                 return set()
         return matched or set()
 
-    def verify_exact_content(self, connection: sqlite3.Connection, candidates: set[int], query: str) -> set[int]:
-        if not candidates:
-            return set()
-        matched: set[int] = set()
-        ordered = sorted(candidates)
-        for offset in range(0, len(ordered), 500):
-            batch = ordered[offset:offset + 500]
-            placeholders = ",".join("?" for _ in batch)
-            rows = connection.execute(
-                f"SELECT rowid FROM content_fts WHERE rowid IN ({placeholders}) AND instr(content, ?) > 0",
-                [*batch, query],
-            )
-            matched.update(int(row[0]) for row in rows)
-        return matched
-
     def verify_wildcard_content(
         self,
         connection: sqlite3.Connection,
@@ -455,15 +445,6 @@ class FulltextDatabases:
             matched.update(int(row[0]) for row in rows)
         return matched
 
-    def can_trust_exact_content(self, source: str, query: str) -> bool:
-        with self.lock:
-            return (
-                self.tokenizer_versions.get(source) == TOKENIZER_VERSION
-                and self.has_content_fts.get(source, False)
-                and "*" not in query
-                and "?" not in query
-            )
-
     def summaries(self, doc_ids: list[str]) -> dict[str, str]:
         grouped = {}
         for doc_id in doc_ids:
@@ -986,25 +967,6 @@ def metadata_matches(q: str, exact: bool, search_paths: bool) -> set[int]:
         if all(term in records[idx][field] for term in terms)
     }
 
-def file_matches_exact_query(idx: int, query: str) -> bool:
-    file_path = get_doc_storage_path(records[idx])
-    if not file_path.exists():
-        return False
-    payload = file_path.read_bytes()
-    if "*" not in query and "?" not in query and query.encode("utf-8") in payload:
-        return True
-    return matches_exact_query(payload.decode("utf-8", errors="ignore"), query)
-
-def verify_fulltext_matches(indices: set[int], query: str) -> set[int]:
-    return set(cached_verified_fulltext_matches(tuple(sorted(indices)), query))
-
-@lru_cache(maxsize=32)
-def cached_verified_fulltext_matches(indices: tuple[int, ...], query: str) -> frozenset[int]:
-    return frozenset(
-        idx for idx in indices
-        if file_matches_exact_query(idx, query)
-    )
-
 def search(q="", sources_filter=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, sort="relevance", exact=False, search_paths=True, timings=None):
     started = time.perf_counter()
     q = q.strip()
@@ -1161,14 +1123,29 @@ def _cached_payload(cache, lock, key: tuple, builder, ttl: int, max_entries: int
     if timings is not None:
         timings["cache"] = "miss"
         timings["cache_lookup"] = (time.perf_counter() - cache_started) * 1000
-    value = builder()
-    inserted_at = time.monotonic()
-    with lock:
-        cache[key] = (inserted_at, value)
-        while len(cache) > max_entries:
-            oldest_key = min(cache, key=lambda item: cache[item][0])
-            cache.pop(oldest_key, None)
-    return value
+    # Serialize identical cold keys so concurrent requests compute once instead
+    # of stampeding the builder. The per-key lock is removed once it is no longer
+    # contended so the map stays bounded to the number of in-flight keys.
+    with cache_key_locks_guard:
+        key_lock = cache_key_locks.setdefault(key, threading.Lock())
+    try:
+        with key_lock:
+            with lock:
+                cached = cache.get(key)
+                if cached and now - cached[0] < ttl:
+                    return cached[1]
+            value = builder()
+            inserted_at = time.monotonic()
+            with lock:
+                cache[key] = (inserted_at, value)
+                while len(cache) > max_entries:
+                    oldest_key = min(cache, key=lambda item: cache[item][0])
+                    cache.pop(oldest_key, None)
+            return value
+    finally:
+        with cache_key_locks_guard:
+            if cache_key_locks.get(key) is key_lock:
+                cache_key_locks.pop(key, None)
 
 def cached_payload(key: tuple, builder, timings=None):
     return _cached_payload(