Upload 3 files

Dockerfile

@@ -3,14 +3,23 @@ FROM python:3.11-slim
 WORKDIR /app
 
 COPY requirements.txt .
+
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY app.py .
+COPY app.py build_fulltext_db.py ./
+
 COPY static/ static/
+
 COPY data/ data/
+
 COPY CCRD/ CCRD/
+
 COPY CW/ CW/
 
+# Full-text indexes are derived artifacts. Build them inside the Space image so
+# the repository never needs to store SQLite files larger than the LFS limit.
+RUN python build_fulltext_db.py
+
 EXPOSE 7860
 
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--log-level", "info"]

app.py

@@ -35,7 +35,7 @@ extension_counts: dict[str, int] = {}
 source_extension_counts: dict[str, dict[str, int]] = {}
 word_index: dict[str, set[int]] = {}
 TOKEN_RE = re.compile(r"[a-z0-9]+|[\u4e00-\u9fff\u3400-\u4dbf]+")
-TOKENIZER_VERSION = "cjk-bigram-boundary-v4"
+TOKENIZER_VERSION = "cjk-bigram-boundary-fts5-v5"
 LITERAL_PREFIX = "\0literal:"
 API_CACHE_TTL_SECONDS = 120
 SEARCH_CACHE_TTL_SECONDS = 30
@@ -94,6 +94,7 @@ class FulltextDatabases:
         self.connections = {}
         self.has_doc_counts = {}
         self.tokenizer_versions = {}
+        self.has_content_fts = {}
         for path in directory.glob("*.sqlite3"):
             uri = f"file:{path.resolve()}?mode=ro&immutable=1"
             connection = sqlite3.connect(uri, uri=True, check_same_thread=False)
@@ -107,6 +108,9 @@ class FulltextDatabases:
                 "SELECT value FROM metadata WHERE key = 'tokenizer_version'"
             ).fetchone() if has_metadata else None
             self.tokenizer_versions[path.stem] = version_row[0] if version_row else "legacy"
+            self.has_content_fts[path.stem] = bool(connection.execute(
+                "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'content_fts'"
+            ).fetchone())
 
     def close(self) -> None:
         for connection in self.connections.values():
@@ -115,6 +119,18 @@ class FulltextDatabases:
     def search_source(self, source: str, query: str, exact: bool = False) -> set[int]:
         connection = self.connections.get(source)
         version = self.tokenizer_versions.get(source)
+        if exact and self.has_content_fts.get(source) and "*" not in query and "?" not in query and len(normalize_text(query)) >= 3:
+            phrase = normalize_text(query)
+            grams = [phrase[index:index + 3] for index in range(len(phrase) - 2)]
+            try:
+                match_query = " AND ".join('"' + gram.replace('"', '""') + '"' for gram in grams)
+                rows = connection.execute(
+                    "SELECT rowid FROM content_fts WHERE content_fts MATCH ?",
+                    (f'"{phrase}"',),
+                ).fetchall()
+                return {int(row[0]) for row in rows}
+            except sqlite3.OperationalError:
+                pass
         if exact and version == TOKENIZER_VERSION and "*" not in query and "?" not in query:
             tokens = [*query_tokens(query), *literal_query_tokens(query)]
         else:
@@ -147,6 +163,15 @@ class FulltextDatabases:
                 return set()
         return matched or set()
 
+    def can_trust_exact_content(self, source: str, query: str) -> bool:
+        return (
+            self.tokenizer_versions.get(source) == TOKENIZER_VERSION
+            and self.has_content_fts.get(source, False)
+            and "*" not in query
+            and "?" not in query
+            and len(normalize_text(query)) >= 3
+        )
+
     def summaries(self, doc_ids: list[str]) -> dict[str, str]:
         grouped = {}
         for doc_id in doc_ids:
@@ -392,7 +417,8 @@ def fulltext_search(q="", sources_filter=None, folders=None, min_size=None, max_
         matched_doc_ids.update(f"{source_slug}:{doc_number}" for doc_number in fulltext_databases.search_source(source_slug, q, exact))
     content_indices = {idx for doc_id in matched_doc_ids if (idx := record_map_index.get(doc_id)) is not None}
     indices = set(content_indices)
-    if exact and ("*" in q or "?" in q or len(normalize_text(q)) > 3):
+    trust_fts = exact and all(fulltext_databases.can_trust_exact_content(source_slug, q) for source_slug in source_slugs)
+    if exact and not trust_fts and ("*" in q or "?" in q or len(normalize_text(q)) > 3):
         content_indices = verify_fulltext_matches(content_indices, q)
         indices = set(content_indices)
     indices.update(metadata_matches(q, exact, search_paths))
@@ -409,7 +435,9 @@ def fulltext_search(q="", sources_filter=None, folders=None, min_size=None, max_
     result_items = add_summaries(
         [trim_record(records[idx]) for idx in filtered[start:start + page_size]],
         q,
-        matched_snippets=True,
+        # Keep the first page fast. Exact position snippets are available from preview;
+        # do not open dozens of large source files just to decorate list results.
+        matched_snippets=False,
         content_indices=content_indices,
     )
     return {"results": result_items, "total": total, "page": page, "page_size": page_size}

build_fulltext_db.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+import gzip
+import hashlib
+import json
+import re
+import sqlite3
+import unicodedata
+import unicodedata
+from collections import defaultdict
+from pathlib import Path
+BASE_DIR = Path(__file__).resolve().parent
+DATA_PATH = BASE_DIR / "data/search_data.json.gz"
+FULLTEXT_DIR = BASE_DIR / "data/fulltext"
+TOKEN_RE = re.compile(r"[a-z0-9]+|[\u4e00-\u9fff\u3400-\u4dbf]+")
+SUMMARY_CHARS = 700
+TOKENIZER_VERSION = "cjk-bigram-boundary-fts5-v5"
+LITERAL_PREFIX = "\0literal:"
+
+def load_json_gz(path: Path):
+    return json.loads(gzip.decompress(path.read_bytes()).decode("utf-8"))
+
+def index_tokens(text: str) -> set[str]:
+    tokens = set()
+    normalized = unicodedata.normalize("NFKC", text or "").casefold()
+    for part in TOKEN_RE.findall(normalized):
+        if re.fullmatch(r"[\u4e00-\u9fff\u3400-\u4dbf]+", part):
+            tokens.update(part)
+            tokens.update(part[index:index + 2] for index in range(len(part) - 1))
+        else:
+            tokens.add(part)
+    return tokens
+
+def literal_tokens(text: str) -> set[str]:
+    normalized = normalize_literal(text)
+    return {
+        LITERAL_PREFIX + normalized[index:index + 3]
+        for index in range(len(normalized) - 2)
+        if not TOKEN_RE.fullmatch(normalized[index + 1])
+    }
+
+def normalize_literal(text: str) -> str:
+    return unicodedata.normalize("NFKC", text or "").casefold()
+
+def encode_doc_ids(doc_numbers: list[int]) -> bytes:
+    previous = 0
+    output = bytearray()
+    for doc_number in sorted(doc_numbers):
+        delta = doc_number - previous
+        previous = doc_number
+        while delta >= 0x80:
+            output.append((delta & 0x7F) | 0x80)
+            delta >>= 7
+        output.append(delta)
+    return bytes(output)
+
+def doc_number(doc_id: str) -> int:
+    try:
+        return int(doc_id.rsplit(":", 1)[1])
+    except (IndexError, ValueError) as exc:
+        raise ValueError(f"doc_id must end with ':<number>': {doc_id}") from exc
+
+def get_doc_storage_path(record: dict) -> Path:
+    return BASE_DIR / record["storage_root"] / record["storage_rel_path"]
+
+def build_summary(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip()[:SUMMARY_CHARS]
+
+def fts_content(text: str) -> str:
+    return unicodedata.normalize("NFKC", text or "").casefold().replace("\0", " ")
+
+def build_source_db(source: str, source_records: list[dict]) -> None:
+    db_path = FULLTEXT_DIR / f"{source}.sqlite3"
+    tmp_path = db_path.with_suffix(".sqlite3.tmp")
+    postings: dict[bytes, set[int]] = defaultdict(set)
+    documents = []
+    content_rows = []
+    missing = 0
+    if tmp_path.exists():
+        tmp_path.unlink()
+    for index, record in enumerate(source_records, 1):
+        doc_id = record["doc_id"]
+        file_path = get_doc_storage_path(record)
+        if not file_path.exists():
+            missing += 1
+            continue
+        text = file_path.read_text(encoding="utf-8", errors="ignore")
+        documents.append((doc_id, build_summary(text)))
+        number = doc_number(doc_id)
+        # FTS5 tokenization stops at embedded NUL bytes even though selecting the
+        # virtual column returns the remainder. Replace them only in derived index data.
+        content_rows.append((number, fts_content(text)))
+        for token in index_tokens(text) | literal_tokens(text):
+            token_hash = hashlib.sha256(token.encode("utf-8")).digest()[:16]
+            postings[token_hash].add(number)
+        if index % 1000 == 0:
+            print(f"{source}: indexed {index}/{len(source_records)} records")
+    with sqlite3.connect(tmp_path) as connection:
+        connection.execute("PRAGMA journal_mode = OFF")
+        connection.execute("PRAGMA synchronous = OFF")
+        connection.execute("PRAGMA temp_store = MEMORY")
+        connection.execute("CREATE TABLE documents (doc_id TEXT PRIMARY KEY, summary TEXT NOT NULL)")
+        connection.execute("CREATE TABLE postings (token_hash BLOB PRIMARY KEY, docs BLOB NOT NULL, doc_count INTEGER NOT NULL)")
+        connection.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
+        connection.execute("INSERT INTO metadata (key, value) VALUES ('tokenizer_version', ?)", (TOKENIZER_VERSION,))
+        connection.execute("CREATE VIRTUAL TABLE content_fts USING fts5(content, tokenize='trigram', detail='full')")
+        connection.executemany("INSERT INTO documents (doc_id, summary) VALUES (?, ?)", documents)
+        connection.executemany("INSERT INTO content_fts (rowid, content) VALUES (?, ?)", content_rows)
+        connection.executemany(
+            "INSERT INTO postings (token_hash, docs, doc_count) VALUES (?, ?, ?)",
+            (
+                (token_hash, encode_doc_ids(list(doc_numbers)), len(doc_numbers))
+                for token_hash, doc_numbers in postings.items()
+            ),
+        )
+        connection.execute("CREATE INDEX idx_documents_doc_id ON documents(doc_id)")
+    tmp_path.replace(db_path)
+    print(
+        f"{source}: wrote {db_path.relative_to(BASE_DIR)} "
+        f"with {len(documents)} documents, {len(postings)} tokens, {missing} missing files"
+    )
+
+def main() -> None:
+    payload = load_json_gz(DATA_PATH)
+    records = payload.get("records", [])
+    FULLTEXT_DIR.mkdir(parents=True, exist_ok=True)
+    records_by_source: dict[str, list[dict]] = defaultdict(list)
+    for record in records:
+        records_by_source[record["source"]].append(record)
+    for source in sorted(records_by_source):
+        build_source_db(source, records_by_source[source])
+
+if __name__ == "__main__":
+    main()