fix: Move the retrieval index out of git and onto the Hub

The index grew 8x with the corpus bridge and no longer fits: chunks.json is
187MB, embeddings.npz 144MB, bm25.pkl 124MB. GitHub rejects any file past 100MB,
so `git push` would fail outright. chunks.json also holds the full extracted text
of 666 documents, commercial textbooks among them -- fine to keep on your own
machine, wrong to redistribute.

All three now live in the private Hub dataset repo atakankahya/controlai-rag-index.
controlai_rag/fetch_index.py (./run.sh --fetch-index) pulls them with an HF token.
It copies out of the Hub cache rather than symlinking into it, because the index
is mutated in place by uploads via HybridRetriever.add_chunks and by
scripts/repair_*.py. A clone without access has no index and builds its own from
data/user_docs/ with --build-index.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.gitignore

@@ -24,9 +24,12 @@ data/*
 # zero retrieved chunks (observed directly: /api/status reported
 # indexed_chunks: 0 with this excluded, since data/ is never pushed to the
 # Space's git repo).
-!data/rag_index/
-!data/rag_index/chunks.json
-!data/rag_index/bm25.pkl
+# The retrieval index is NOT in git. The three artefacts total ~455MB (187MB
+# chunks.json, 144MB embeddings.npz, 124MB bm25.pkl), well past GitHub's 100MB
+# per-file limit, and chunks.json holds the full text of commercial textbooks,
+# which is fine to keep locally and wrong to redistribute. It lives in a private
+# Hub dataset repo instead -- `./run.sh --fetch-index`, or `--build-index` to
+# make your own from data/user_docs/.
 
 # Model checkpoints and fine-tuned weights
 adapters/

controlai_rag/fetch_index.py

@@ -0,0 +1,78 @@
+"""Download the prebuilt retrieval index from the Hugging Face Hub.
+
+The three index artefacts total ~455 MB -- far past GitHub's 100 MB per-file
+limit -- so they live in a Hub dataset repo instead of in git. The repo is
+private: the index carries the full extracted text of commercial textbooks,
+which is fine to hold locally and wrong to redistribute. Set HF_TOKEN (or run
+`huggingface-cli login`) with an account that can read it.
+
+Without the index the retriever has nothing to search. Building one from your
+own documents instead is `./run.sh --build-index`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from controlai_rag.index import INDEX_DIR
+
+REPO_ID = os.environ.get("CONTROLAI_INDEX_REPO", "atakankahya/controlai-rag-index")
+
+# chunks.json and bm25.pkl are the lexical side; embeddings.npz is the dense
+# side. Missing the last one degrades retrieval to BM25 silently, so it counts
+# as required here rather than optional.
+FILES = ("chunks.json", "bm25.pkl", "embeddings.npz")
+
+
+def fetch(index_dir: Path = INDEX_DIR, force: bool = False) -> Path:
+    """Place the index files in `index_dir`, downloading what is missing."""
+    from huggingface_hub import hf_hub_download
+
+    index_dir.mkdir(parents=True, exist_ok=True)
+    for name in FILES:
+        target = index_dir / name
+        if target.exists() and not force:
+            print(f"  {name}: already present, skipping")
+            continue
+        print(f"  {name}: downloading...", flush=True)
+        cached = hf_hub_download(
+            repo_id=REPO_ID,
+            filename=name,
+            repo_type="dataset",
+            token=os.environ.get("HF_TOKEN"),
+        )
+        # Copy rather than symlink into the cache: the index is mutated in place
+        # by uploads (HybridRetriever.add_chunks) and by scripts/repair_*.py.
+        target.write_bytes(Path(cached).read_bytes())
+        print(f"  {name}: {target.stat().st_size / 1e6:.0f} MB")
+    return index_dir
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--force", action="store_true", help="re-download files that already exist"
+    )
+    args = parser.parse_args()
+
+    print(f"Fetching retrieval index from {REPO_ID}")
+    try:
+        fetch(force=args.force)
+    except Exception as exc:  # noqa: BLE001 - the message matters more than the type
+        print(f"\nCould not fetch the index: {exc}", file=sys.stderr)
+        print(
+            "\nThe dataset repo is private. Authenticate with an account that can\n"
+            "read it (`huggingface-cli login`, or set HF_TOKEN), or build your own\n"
+            "index from data/user_docs/ with `./run.sh --build-index`.",
+            file=sys.stderr,
+        )
+        return 1
+    print("Index ready in", INDEX_DIR)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data/rag_index/bm25.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aec368fab52a30b88e2bc1633add98c1f6a0a8f5260ad50d73d965381c741210
-size 9492413

data/rag_index/chunks.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:939cfc3169818185b3dd60ba12b2e4799fd10755fd0fcbbff8f390567761c091
-size 12948944

run.sh

@@ -1,26 +1,48 @@
 #!/usr/bin/env bash
-# ControlAI Universal Launcher
-set -e
+# ControlAI launcher.
+set -euo pipefail
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 cd "$DIR"
+[ -d ".venv" ] && source .venv/bin/activate
 
-# Activate virtual environment if present
-if [ -d ".venv" ]; then
-    source .venv/bin/activate
-fi
-
-# Check flags
-if [ "$1" == "--cli" ] || [ "$1" == "-c" ]; then
+case "${1:-}" in
+  --cli|-c)
     shift
     exec python cli.py "$@"
-elif [ "$1" == "--help" ] || [ "$1" == "-h" ]; then
-    echo "ControlAI Launcher Usage:"
-    echo "  ./run.sh            # Launch Web UI & automatically open browser (http://127.0.0.1:8000)"
-    echo "  ./run.sh --cli      # Launch interactive terminal assistant"
-    echo "  ./run.sh --cli \"<query>\" # Run single one-shot query in terminal"
-    exit 0
-else
-    # Default: Launch Web UI & automatically open browser at http://127.0.0.1:8000
+    ;;
+  --build-index)
+    exec python -m controlai_rag.retriever --build
+    ;;
+  --fetch-index)
+    exec python -m controlai_rag.fetch_index "${@:2}"
+    ;;
+  --ingest-corpus)
+    exec python scripts/ingest_processed_corpus.py "${@:2}"
+    ;;
+  --calibrate)
+    exec python scripts/calibrate_retrieval.py "${@:2}"
+    ;;
+  --help|-h)
+    cat <<'USAGE'
+ControlAI
+
+  ./run.sh                     web console at http://127.0.0.1:8000
+  ./run.sh --cli               interactive terminal chat
+  ./run.sh --cli "question"    one-shot question
+  ./run.sh --fetch-index       download the prebuilt index from the HF Hub
+  ./run.sh --build-index       rebuild the dense retrieval index
+  ./run.sh --ingest-corpus     merge data/processed/ chunks into the index
+  ./run.sh --calibrate         re-measure MIN_COSINE against the current corpus
+
+Environment:
+  CONTROLAI_MODEL          MLX model id       (default mlx-community/Qwen3-14B-4bit)
+  CONTROLAI_ADAPTER        LoRA adapter path  (default none)
+  CONTROLAI_THINKING       off | auto | on    (default auto)
+  CONTROLAI_THINK_BUDGET   max reasoning tokens (default 512)
+USAGE
+    ;;
+  *)
     exec python app.py
-fi
+    ;;
+esac