M4: non-indexing Gradio app + deps refresh

app.py: rewrite boot to be non-indexing — load corpus, attach slim BM25 index (full mode) or build in-memory (smoke), connect Qdrant and refuse to embed at boot. Four modes (BM25/Dense/Hybrid/Hybrid+Rerank, pool=20) in a selectable two-column compare UI with per-column latency and per-mode error isolation. requirements.txt: drop dead rank-bm25, add bm25s; mirror pyproject. pyproject/uv.lock: prune rank-bm25 (superseded by bm25s in M1).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app.py

@@ -1,94 +1,166 @@
 """
-M0 Hello World — Gradio UI
+CodeSearch — Gradio UI
 
-Shows BM25 and dense results side-by-side for the same query.
-Uses SMOKE_TEST_SIZE docs (default: 100).
+Side-by-side comparison of four retrieval modes over CodeSearchNet Python:
+    BM25 · Dense (MiniLM) · Hybrid (RRF) · Hybrid + cross-encoder rerank
 
-To run locally:
-    python app.py
+Boot is NON-INDEXING by design:
+    - Dense vectors live in Qdrant, built offline via scripts/index_corpus.py.
+    - BM25 loads a prebuilt slim index (.cache/bm25-slim) in full-corpus mode.
+    - The app refuses to embed the corpus at boot (see the guard below).
 
-To deploy to HF Spaces:
-    - Push this repo to a HF Space (Gradio SDK)
-    - Set QDRANT_URL, QDRANT_API_KEY as Space secrets
-    - Set SMOKE_TEST_SIZE=100 for M0, -1 for full corpus (M4)
+Modes:
+    SMOKE_TEST_SIZE=-1  → full corpus; BM25 from the slim index. (HF Space secret)
+    SMOKE_TEST_SIZE=100 → local pipeline check; BM25 built in-memory. Dense/hybrid
+                          modes return full-Qdrant ids not in the small corpus, so
+                          only BM25 is meaningful in smoke mode.
 """
 
-import sys
 import os
+import sys
+import time
+import traceback
+
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "src"))
 
 import gradio as gr
 
+from codesearch.config import SMOKE_TEST_SIZE, TOP_K
 from codesearch.data import load_codesearch
 from codesearch.retrievers.bm25 import BM25Retriever
+from codesearch.retrievers.bm25_index import BM25Index
 from codesearch.retrievers.dense import DenseRetriever
-from codesearch.config import TOP_K
+from codesearch.retrievers.hybrid import HybridRetriever
+from codesearch.retrievers.reranker import CrossEncoderReranker
 
-# ---------------------------------------------------------------------------
-# Boot: load data + build indexes once at startup
-# ---------------------------------------------------------------------------
+_HERE = os.path.dirname(os.path.abspath(__file__))
+BM25_SLIM_DIR = os.path.join(_HERE, ".cache", "bm25-slim")
 
-print("=== CodeSearch M0 Boot ===")
-corpus, queries = load_codesearch()
+# Candidate pool for hybrid fusion and reranking in the UI. Small on purpose:
+# reranking is inference-bound (~20 pairs/query on CPU), and K=20 keeps
+# hybrid+rerank under the 3s latency budget. Eval uses K=100 for Recall; the UI
+# trades a little recall for responsiveness. See M3.2 design notes.
+UI_POOL = 20
 
-print("Building BM25 index...")
-bm25 = BM25Retriever(corpus)
+# ---------------------------------------------------------------------------
+# Boot: load corpus + wire retrievers once. No indexing happens here.
+# ---------------------------------------------------------------------------
 
-print("Initializing dense retriever...")
-dense = DenseRetriever(recreate_collection=False)
-dense.index_corpus(corpus)
+print("=== CodeSearch Boot ===")
+corpus, _ = load_codesearch()  # queries unused — the app takes live user input
 
-print("Ready.")
+# Two id-keyed views over the corpus:
+#   corpus_by_id  — for rendering result cards (dense/hybrid hits carry id+score only)
+#   corpus_lookup — reranker candidate text (code_tokens, already space-joined)
+corpus_by_id = {d["id"]: d for d in corpus}
+corpus_lookup = {d["id"]: d["code_tokens"] for d in corpus}
 
-# ---------------------------------------------------------------------------
-# Search function
-# ---------------------------------------------------------------------------
+if SMOKE_TEST_SIZE == -1:
+    print(f"Loading slim BM25 index from {BM25_SLIM_DIR} ...")
+    bm25 = BM25Retriever(BM25Index.load_index_only(BM25_SLIM_DIR, corpus))
+else:
+    print(f"Smoke mode (n={SMOKE_TEST_SIZE}): building in-memory BM25 index...")
+    bm25 = BM25Retriever(corpus)
 
-def search(query: str) -> tuple[str, str]:
-    """Run BM25 and dense retrieval, return formatted results for both columns."""
-    if not query.strip():
-        return "Enter a query above.", "Enter a query above."
+dense = DenseRetriever(recreate_collection=False)
+if not dense.collection_exists_and_populated():
+    raise RuntimeError(
+        "Qdrant collection is empty. Build it offline with scripts/index_corpus.py — "
+        "the app will not embed the full corpus at boot."
+    )
 
-    bm25_results = bm25.retrieve(query, top_k=TOP_K)
-    dense_results = dense.retrieve(query, top_k=TOP_K)
+hybrid = HybridRetriever(bm25, dense, pool_size=UI_POOL)
+reranker = CrossEncoderReranker(hybrid, corpus_lookup, pool_size=UI_POOL)
 
-    return _format_results(bm25_results), _format_results(dense_results)
+RETRIEVERS = {
+    "BM25": bm25,
+    "Dense (MiniLM)": dense,
+    "Hybrid (RRF)": hybrid,
+    "Hybrid + Rerank": reranker,
+}
+MODES = list(RETRIEVERS.keys())
 
+print(f"Ready. corpus={len(corpus):,} docs · modes={MODES}")
 
-def _format_results(results: list[dict]) -> str:
-    if not results:
-        return "No results."
+# ---------------------------------------------------------------------------
+# Search + formatting
+# ---------------------------------------------------------------------------
 
-    lines = []
-    for i, r in enumerate(results, 1):
-        score = r.get("score", 0.0)
-        docstring = (r["docstring"] or "").strip()[:200]
-        code_preview = (r["code"] or "").strip()[:300]
-        url = r.get("url", "")
 
-        lines.append(
-            f"**#{i} — score: {score:.4f}**\n"
-            f"**Docstring:** {docstring}{'…' if len(r.get('docstring','')) > 200 else ''}\n\n"
-            f"```python\n{code_preview}{'…' if len(r.get('code','')) > 300 else ''}\n```\n"
+def _fmt_latency(dt: float) -> str:
+    return f"⏱ **{dt:.2f} s**" if dt >= 1.0 else f"⏱ **{dt * 1000:.0f} ms**"
+
+
+def _format(hits: list[dict], mode: str) -> str:
+    if not hits:
+        return "_No results._"
+
+    cards = []
+    for i, h in enumerate(hits, 1):
+        doc = corpus_by_id.get(h["id"])
+        if doc is None:
+            cards.append(
+                f"**#{i}** — `{h['id']}` not in local corpus "
+                f"(expected in smoke mode; full corpus on the Space)."
+            )
+            continue
+
+        docstring = (doc.get("docstring") or "").strip()[:200]
+        code = (doc.get("code") or "").strip()[:400]
+        url = doc.get("url", "")
+        score = h.get("score", 0.0)
+
+        # Provenance line: shows WHY a doc ranked where it did, per mode.
+        prov = f"score **{score:.4f}**"
+        if h.get("bm25_rank") is not None or h.get("dense_rank") is not None:
+            prov += f" · bm25 #{h.get('bm25_rank')} · dense #{h.get('dense_rank')}"
+        if "pre_rerank_score" in h:
+            prov += f" · pre-rerank {h['pre_rerank_score']:.4f}"
+
+        cards.append(
+            f"**#{i}** — {prov}\n\n"
+            f"{docstring}{'…' if docstring else ''}\n\n"
+            f"```python\n{code}{'…' if code else ''}\n```\n"
             + (f"[source]({url})\n" if url else "")
             + "\n---"
         )
+    return "\n".join(cards)
+
+
+def _run(mode: str, query: str) -> tuple[str, str]:
+    retr = RETRIEVERS[mode]
+    t0 = time.perf_counter()
+    try:
+        hits = retr.retrieve(query, top_k=TOP_K)
+    except Exception as e:  # one mode failing must not take down the search
+        traceback.print_exc()
+        return f"⚠️ **{mode} failed:** `{type(e).__name__}: {e}`", "—"
+    dt = time.perf_counter() - t0
+    return _format(hits, mode), _fmt_latency(dt)
+
 
-    return "\n".join(lines)
+def search(query: str, left_mode: str, right_mode: str):
+    """Run both columns and return (left_md, left_latency, right_md, right_latency)."""
+    if not query.strip():
+        msg = "_Enter a query above._"
+        return msg, "", msg, ""
+    left_md, left_lat = _run(left_mode, query)
+    right_md, right_lat = _run(right_mode, query)
+    return left_md, left_lat, right_md, right_lat
 
 
 # ---------------------------------------------------------------------------
-# Gradio UI
+# UI — selectable two-column compare
 # ---------------------------------------------------------------------------
 
-with gr.Blocks(title="CodeSearch M0") as demo:
+with gr.Blocks(title="CodeSearch") as demo:
     gr.Markdown(
-        """
-# 🔍 CodeSearch — M0 Hello World
-**Dataset:** CodeSearchNet Python ({n} docs)  
-**Retrieval modes:** BM25 (left) vs Dense / HNSW (right)  
-Notice where they agree and where they diverge — that gap is what this project is about.
-""".format(n=len(corpus))
+        f"""
+# 🔍 CodeSearch
+Natural-language code search over **CodeSearchNet Python** ({len(corpus):,} functions).
+Pick a retriever per column and compare — watch where lexical (BM25), semantic
+(dense), fusion (RRF), and reranking disagree.
+"""
     )
 
     with gr.Row():
@@ -100,17 +172,21 @@ Notice where they agree and where they diverge — that gap is what this project
         search_btn = gr.Button("Search", variant="primary", scale=1)
 
     with gr.Row():
-        bm25_out = gr.Markdown(label="BM25 results")
-        dense_out = gr.Markdown(label="Dense (MiniLM) results")
-
-    with gr.Row():
-        gr.Markdown(
-            "_Tip: try a query like 'convert string to datetime' — "
-            "BM25 needs those exact words, dense finds semantically similar code._"
-        )
-
-    search_btn.click(fn=search, inputs=query_box, outputs=[bm25_out, dense_out])
-    query_box.submit(fn=search, inputs=query_box, outputs=[bm25_out, dense_out])
+        with gr.Column():
+            left_mode = gr.Dropdown(MODES, value="BM25", label="Left retriever")
+            left_lat = gr.Markdown()
+            left_out = gr.Markdown()
+        with gr.Column():
+            right_mode = gr.Dropdown(
+                MODES, value="Hybrid + Rerank", label="Right retriever"
+            )
+            right_lat = gr.Markdown()
+            right_out = gr.Markdown()
+
+    inputs = [query_box, left_mode, right_mode]
+    outputs = [left_out, left_lat, right_out, right_lat]
+    search_btn.click(fn=search, inputs=inputs, outputs=outputs)
+    query_box.submit(fn=search, inputs=inputs, outputs=outputs)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -8,7 +8,6 @@ version = "0.1.0"
 description = "Semantic code search: BM25 vs dense vs hybrid — with rigorous eval"
 requires-python = ">=3.11"
 dependencies = [
-    "rank-bm25>=0.2.2",
     "sentence-transformers>=3.0.0",
     "qdrant-client>=1.9.0",
     "gradio>=4.36.0",

requirements.txt

@@ -1,4 +1,7 @@
-rank-bm25>=0.2.2
+# HF Spaces build deps. Hand-maintained to mirror pyproject.toml runtime
+# dependencies (top-level only; pip resolves transitives — incl. torch via
+# sentence-transformers — against HF's base image). Keep in sync with pyproject.
+bm25s>=0.3.4
 sentence-transformers>=3.0.0
 qdrant-client>=1.9.0
 gradio>=4.36.0

uv.lock

@@ -417,7 +417,6 @@ dependencies = [
     { name = "numpy" },
     { name = "python-dotenv" },
     { name = "qdrant-client" },
-    { name = "rank-bm25" },
     { name = "sentence-transformers" },
     { name = "tqdm" },
 ]
@@ -437,7 +436,6 @@ requires-dist = [
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
     { name = "python-dotenv", specifier = ">=1.0.0" },
     { name = "qdrant-client", specifier = ">=1.9.0" },
-    { name = "rank-bm25", specifier = ">=0.2.2" },
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" },
     { name = "sentence-transformers", specifier = ">=3.0.0" },
     { name = "tqdm", specifier = ">=4.66.0" },
@@ -2163,18 +2161,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/68/69/77d1a971c4b933e8c79403e99bcbb790463da5e48333cc4fd5d412c63c98/qdrant_client-1.17.1-py3-none-any.whl", hash = "sha256:6cda4064adfeaf211c751f3fbc00edbbdb499850918c7aff4855a9a759d56cbd", size = 389947, upload-time = "2026-03-13T17:13:43.156Z" },
 ]
 
-[[package]]
-name = "rank-bm25"
-version = "0.2.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/fc/0a/f9579384aa017d8b4c15613f86954b92a95a93d641cc849182467cf0bb3b/rank_bm25-0.2.2.tar.gz", hash = "sha256:096ccef76f8188563419aaf384a02f0ea459503fdf77901378d4fd9d87e5e51d", size = 8347, upload-time = "2022-02-16T12:10:52.196Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2a/21/f691fb2613100a62b3fa91e9988c991e9ca5b89ea31c0d3152a3210344f9/rank_bm25-0.2.2-py3-none-any.whl", hash = "sha256:7bd4a95571adadfc271746fa146a4bcfd89c0cf731e49c3d1ad863290adbe8ae", size = 8584, upload-time = "2022-02-16T12:10:50.626Z" },
-]
-
 [[package]]
 name = "regex"
 version = "2026.4.4"