Upload prepare_fulltext_index.py

prepare_fulltext_index.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Activate a verified CCRD index generation from a mounted HF Storage Bucket."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sqlite3
+from pathlib import Path
+
+
+BASE_DIR = Path(__file__).resolve().parent
+FULLTEXT_DIR = Path(os.environ.get("CCRD_FULLTEXT_DIR", str(BASE_DIR / "data/fulltext")))
+BUCKET_DIR = os.environ.get("CCRD_INDEX_BUCKET_DIR", "").strip()
+TOKENIZER_VERSION = "cjk-bigram-boundary-fts5-v5"
+SOURCES = ("CCRD", "CW")
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def validated_database(bucket: Path, source: str, details: object) -> Path:
+    if not isinstance(details, dict):
+        raise ValueError(f"{source}: missing manifest details")
+    relative_path = details.get("path")
+    expected_hash = details.get("sha256")
+    expected_documents = details.get("documents")
+    expected_fts_rows = details.get("fts_rows")
+    if not isinstance(relative_path, str) or not relative_path.startswith("generations/"):
+        raise ValueError(f"{source}: invalid database path")
+    if not isinstance(expected_hash, str) or len(expected_hash) != 64:
+        raise ValueError(f"{source}: invalid database hash")
+    if not isinstance(expected_documents, int) or not isinstance(expected_fts_rows, int):
+        raise ValueError(f"{source}: invalid document counts")
+
+    path = (bucket / relative_path).resolve()
+    if bucket not in path.parents or not path.is_file():
+        raise ValueError(f"{source}: database is outside the bucket or missing")
+    if sha256(path) != expected_hash:
+        raise ValueError(f"{source}: database hash mismatch")
+    with sqlite3.connect(f"file:{path}?mode=ro&immutable=1", uri=True) as connection:
+        tokenizer = connection.execute("SELECT value FROM metadata WHERE key = 'tokenizer_version'").fetchone()
+        documents = int(connection.execute("SELECT COUNT(*) FROM documents").fetchone()[0])
+        fts_rows = int(connection.execute("SELECT COUNT(*) FROM content_fts").fetchone()[0])
+        integrity = connection.execute("PRAGMA integrity_check").fetchone()[0]
+    if not tokenizer or tokenizer[0] != TOKENIZER_VERSION:
+        raise ValueError(f"{source}: tokenizer version mismatch")
+    if documents != expected_documents or fts_rows != expected_fts_rows or integrity != "ok":
+        raise ValueError(f"{source}: database structure mismatch")
+    return path
+
+
+def main() -> int:
+    if not BUCKET_DIR:
+        print("ccrd_bucket_index=disabled")
+        return 1
+    bucket = Path(BUCKET_DIR).resolve()
+    manifest_path = bucket / "current.json"
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        if manifest.get("format") != 1 or manifest.get("tokenizer_version") != TOKENIZER_VERSION:
+            raise ValueError("invalid manifest version")
+        databases = manifest.get("databases")
+        if not isinstance(databases, dict):
+            raise ValueError("missing databases manifest")
+        paths = {source: validated_database(bucket, source, databases.get(source)) for source in SOURCES}
+    except Exception as exc:
+        print(f"ccrd_bucket_index=unavailable:{type(exc).__name__}:{exc}")
+        return 1
+
+    FULLTEXT_DIR.mkdir(parents=True, exist_ok=True)
+    for source, path in paths.items():
+        target = FULLTEXT_DIR / f"{source}.sqlite3"
+        temporary = target.with_suffix(".sqlite3.bucket")
+        temporary.unlink(missing_ok=True)
+        temporary.symlink_to(path)
+        temporary.replace(target)
+    (FULLTEXT_DIR / "build-status.json").write_text(json.dumps({"state": "ready", "source": "bucket"}), encoding="utf-8")
+    print("ccrd_bucket_index=ready")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())