Add retrieval comparison demo (BM25/Dense/Hybrid) + precomputed artifacts

.gitignore

@@ -0,0 +1,4 @@
+.venv/
+__pycache__/
+*.pyc
+prompt.md

README.md

@@ -1,14 +1,61 @@
 ---
-title: Vector Demo
-emoji: 🐢
-colorFrom: red
-colorTo: yellow
+title: RAG Retrieval Compare
+emoji: 🔍
+colorFrom: indigo
+colorTo: green
 sdk: gradio
 sdk_version: 6.17.3
-python_version: '3.12'
 app_file: app.py
 pinned: false
-short_description: Demo of vector database
+short_description: BM25 vs Dense vs Hybrid retrieval, side by side, on ZeroGPU
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🔍 Retrieval methods, side by side
+
+A teaching demo for technical researchers learning vector-database fundamentals for RAG.
+Given a query, it retrieves the top-k results from **three methods** and shows them
+side by side over a corpus of ~4,000 Europe PMC abstracts (microRNA & disease):
+
+1. **BM25** (lexical) — `rank-bm25`
+2. **Dense** (vector) — exact cosine similarity over `all-MiniLM-L6-v2` embeddings
+3. **Hybrid** — Reciprocal Rank Fusion (RRF, k=60) over the BM25 and dense rankings
+
+Plus a **metadata filter** (year / journal) that restricts the candidate pool *before*
+retrieval — the distinctly vector-DB feature — and a **PCA scatter plot** for spatial
+intuition, with connector lines from the query to the documents each method retrieved.
+
+## How it's built (read this before the talk)
+
+- **Everything is precomputed offline** by `build_index.py`, which fetches the abstracts,
+  embeds the corpus, fits the PCA, and tokenises for BM25. The artifacts in `./data/` are
+  committed. The app **never** embeds the corpus or calls an external API at startup —
+  it only embeds your *live query*.
+- **ZeroGPU:** GPU is allocated on demand only inside the `@spaces.GPU`-decorated query
+  embedding functions. The model is loaded on CPU at import; nothing touches CUDA at
+  startup. ⚠️ **The first GPU call after idle has a cold start of a few seconds — pre-warm
+  with a dummy query before presenting.**
+- **Swappable model:** `EMBEDDING_MODEL` in `config.py` is the single source of truth for
+  both the offline build and the live query. Swap the one line (a biomedical
+  PubMedBERT sentence model is included as a comment) and re-run `build_index.py`.
+
+## Exact search, honestly
+
+The corpus is tiny, so dense retrieval uses **exact** cosine similarity (a plain NumPy
+dot product) — instant and correct. At production scale you'd reach for an **ANN index**
+(e.g. HNSW) to keep search fast over millions of vectors. We deliberately don't show a
+fake HNSW timing comparison here: at this scale the difference is invisible, so it would
+mislead rather than teach.
+
+## Teaching caveat on the plot
+
+The 2D PCA projection distorts true high-dimensional distances, so the retrieved points
+may not be the visually-closest dots. **The ranked lists are authoritative; the plot is
+only for intuition.**
+
+## Rebuild the index locally
+
+```bash
+uv venv
+uv pip install -r requirements.txt
+uv run python build_index.py   # writes ./data/
+```

app.py

@@ -1,7 +1,322 @@
+"""Retrieval-methods comparison demo (BM25 vs Dense vs Hybrid/RRF).
+
+A teaching app for EMBL-EBI researchers. Reads pre-built artifacts from ./data/ (made by
+build_index.py) and only embeds the user's LIVE query at runtime. Read top-to-bottom:
+
+    artifacts  ->  embed query (GPU)  ->  filter  ->  3 rankings  ->  RRF  ->  plot
+
+ZeroGPU note: the embedding model is loaded on CPU at import. GPU is touched ONLY inside
+the @spaces.GPU functions. Do not move the model to CUDA anywhere else.
+"""
+
+import json
+import os
+
 import gradio as gr
+import joblib
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import spaces
+import torch
+from rank_bm25 import BM25Okapi
+from sentence_transformers import SentenceTransformer
+
+import config
+from text_utils import tokenize
+
+# ====================================================================================
+# Load pre-built artifacts (fast; no embedding, no network)
+# ====================================================================================
+D = config.DATA_DIR
+EMBEDDINGS = np.load(os.path.join(D, "embeddings.npy"))      # (N, dim), L2-normalised
+PCA_COORDS = np.load(os.path.join(D, "pca_coords.npy"))      # (N, 2)
+PCA = joblib.load(os.path.join(D, "pca.joblib"))
+META = pd.read_parquet(os.path.join(D, "metadata.parquet"))
+with open(os.path.join(D, "bm25_tokens.json")) as f:
+    BM25 = BM25Okapi(json.load(f))
+with open(os.path.join(D, "config.json")) as f:
+    DATA_CFG = json.load(f)
+
+TITLES = META["title"].tolist()
+ABSTRACTS = META["abstract"].tolist()
+YEARS = META["year"].fillna(0).astype(int).to_numpy()
+JOURNALS = META["journal"].astype(str).to_numpy()
+
+YEAR_MIN = int(YEARS[YEARS > 0].min())
+YEAR_MAX = int(YEARS.max())
+# 1300+ distinct journals — show only the most common ones in the dropdown.
+TOP_JOURNALS = META["journal"].value_counts().head(30).index.tolist()
+
+METHOD_COLORS = {"BM25": "#ff7f0e", "Dense": "#1f77b4", "Hybrid": "#2ca02c"}
+
+# ====================================================================================
+# Embedding model — loaded on CPU. GPU is used ONLY inside @spaces.GPU below.
+# ====================================================================================
+MODEL = SentenceTransformer(config.EMBEDDING_MODEL, device="cpu")
+assert MODEL.get_sentence_embedding_dimension() == EMBEDDINGS.shape[1], (
+    "Model dim doesn't match committed embeddings — did you swap EMBEDDING_MODEL "
+    "without re-running build_index.py?"
+)
+
+
+def _encode(text: str) -> np.ndarray:
+    """Embed one string -> (dim,) float32, L2-normalised. Uses CUDA when available."""
+    device = "cuda" if torch.cuda.is_available() else "cpu"  # safe: only called under @spaces.GPU
+    MODEL.to(device)
+    vec = MODEL.encode(
+        [text], normalize_embeddings=True, convert_to_numpy=True, device=device
+    )
+    return vec[0].astype(np.float32)
+
+
+@spaces.GPU
+def embed_query(text: str) -> np.ndarray:
+    """GPU-backed query embedding for retrieval."""
+    return _encode(text)
+
+
+@spaces.GPU
+def embed_text(text: str) -> np.ndarray:
+    """GPU-backed embedding for the 'embed your own text' feature."""
+    return _encode(text)
+
+
+# ====================================================================================
+# Retrieval
+# ====================================================================================
+def candidate_indices(year_lo: int, year_hi: int, journal: str) -> np.ndarray:
+    """Apply the metadata filter BEFORE retrieval. Returns indices of allowed docs.
+
+    This pre-filtering of the candidate pool is the distinctly vector-DB feature: all
+    three methods then search only within these documents.
+    """
+    mask = (YEARS >= int(year_lo)) & (YEARS <= int(year_hi))
+    if journal and journal != "All journals":
+        mask &= JOURNALS == journal
+    return np.flatnonzero(mask)
+
+
+def top_k(scores: np.ndarray, cand: np.ndarray, k: int) -> list[tuple[int, float]]:
+    """Top-k (doc_index, score) among candidates, ranked by score descending."""
+    cand_scores = scores[cand]
+    order = np.argsort(-cand_scores)[:k]
+    return [(int(cand[i]), float(cand_scores[i])) for i in order]
+
+
+def rrf_fuse(bm25: np.ndarray, dense: np.ndarray, cand: np.ndarray, k: int):
+    """Reciprocal Rank Fusion over the FULL candidate rankings, then take top-k.
+
+    Rank every candidate by each method, sum 1/(RRF_K + rank) across methods, and only
+    then truncate. (Truncating each method first would drop docs ranked low by one method
+    but high by the other.) No score normalisation.
+    """
+    def ranks(scores):
+        order = np.argsort(-scores[cand])           # candidate positions, best first
+        r = np.empty(len(cand), dtype=int)
+        r[order] = np.arange(len(cand))             # 0-based rank per candidate position
+        return r
+
+    rb, rd = ranks(bm25), ranks(dense)
+    fused = 1.0 / (config.RRF_K + rb) + 1.0 / (config.RRF_K + rd)
+    order = np.argsort(-fused)[:k]
+    return [(int(cand[i]), float(fused[i])) for i in order]
+
+
+def format_results(method: str, results: list[tuple[int, float]], score_label: str) -> str:
+    """Render a ranked list as Markdown. This list is the SOURCE OF TRUTH for retrieval."""
+    if not results:
+        return f"### {method}\n\n_No results._"
+    lines = [f"### {method}"]
+    for rank, (doc, score) in enumerate(results, start=1):
+        snippet = ABSTRACTS[doc][:200].rsplit(" ", 1)[0] + "…"
+        lines.append(
+            f"**{rank}. {TITLES[doc]}**  \n"
+            f"`{score_label}={score:.3f}` · {int(YEARS[doc])} · *{JOURNALS[doc]}*  \n"
+            f"{snippet}\n"
+        )
+    return "\n".join(lines)
+
+
+def make_plot(query_coord=None, retrieved=None, extra_point=None) -> go.Figure:
+    """PCA scatter of the whole corpus, plus query point + connector lines to hits."""
+    fig = go.Figure()
+    # Whole corpus as grey background — "the space the database searches through".
+    fig.add_trace(
+        go.Scatter(
+            x=PCA_COORDS[:, 0], y=PCA_COORDS[:, 1], mode="markers",
+            marker=dict(size=4, color="lightgrey"), text=TITLES, hoverinfo="text",
+            name="corpus",
+        )
+    )
+    if retrieved and query_coord is not None:
+        for method, results in retrieved.items():
+            color = METHOD_COLORS[method]
+            xs, ys = [], []
+            for doc, _ in results:  # connector lines query -> each hit
+                xs += [query_coord[0], PCA_COORDS[doc, 0], None]
+                ys += [query_coord[1], PCA_COORDS[doc, 1], None]
+            fig.add_trace(
+                go.Scatter(x=xs, y=ys, mode="lines", line=dict(color=color, width=1),
+                           opacity=0.5, name=f"{method} links", hoverinfo="skip")
+            )
+            fig.add_trace(
+                go.Scatter(
+                    x=[PCA_COORDS[d, 0] for d, _ in results],
+                    y=[PCA_COORDS[d, 1] for d, _ in results],
+                    mode="markers",
+                    marker=dict(size=11, color=color, symbol="circle-open", line=dict(width=2)),
+                    text=[TITLES[d] for d, _ in results], hoverinfo="text", name=f"{method} hits",
+                )
+            )
+    if query_coord is not None:
+        fig.add_trace(
+            go.Scatter(x=[query_coord[0]], y=[query_coord[1]], mode="markers",
+                       marker=dict(size=18, color="red", symbol="star"),
+                       text=["your query"], hoverinfo="text", name="query")
+        )
+    if extra_point is not None:
+        coord, label = extra_point
+        fig.add_trace(
+            go.Scatter(x=[coord[0]], y=[coord[1]], mode="markers",
+                       marker=dict(size=15, color="purple", symbol="diamond"),
+                       text=[label], hoverinfo="text", name="your text")
+        )
+    fig.update_layout(
+        margin=dict(l=10, r=10, t=30, b=10), height=520,
+        xaxis_title="PC 1", yaxis_title="PC 2",
+        legend=dict(orientation="h", yanchor="bottom", y=1.0),
+    )
+    return fig
+
+
+# ====================================================================================
+# Gradio handlers
+# ====================================================================================
+def run_search(query, k, year_lo, year_hi, journal):
+    query = (query or "").strip()
+    if not query:
+        return "Enter a query.", "", "", make_plot(), "—"
+
+    cand = candidate_indices(year_lo, year_hi, journal)
+    info = f"**Candidate pool: {len(cand)} / {len(META)} abstracts** after filtering."
+    if len(cand) == 0:
+        return "No documents match the filter.", "", "", make_plot(), info
+
+    k = int(k)
+    qvec = embed_query(query)                       # <-- the only GPU work in search
+    dense_scores = EMBEDDINGS @ qvec                # exact cosine (vectors are normalised)
+    bm25_scores = np.asarray(BM25.get_scores(tokenize(query)))
+
+    bm25_top = top_k(bm25_scores, cand, k)
+    dense_top = top_k(dense_scores, cand, k)
+    hybrid_top = rrf_fuse(bm25_scores, dense_scores, cand, k)
+
+    query_coord = PCA.transform(qvec.reshape(1, -1))[0]
+    fig = make_plot(query_coord, {"BM25": bm25_top, "Dense": dense_top, "Hybrid": hybrid_top})
+
+    return (
+        format_results("BM25", bm25_top, "score"),
+        format_results("Dense", dense_top, "cosine"),
+        format_results("Hybrid", hybrid_top, "RRF"),
+        fig,
+        info,
+    )
+
+
+def run_embed_own(text):
+    text = (text or "").strip()
+    if not text:
+        return make_plot()
+    vec = embed_text(text)                           # <-- GPU work
+    coord = PCA.transform(vec.reshape(1, -1))[0]
+    return make_plot(extra_point=(coord, text[:80]))
+
+
+# ====================================================================================
+# Example queries — TODO: the presenter fills these in.
+# Each placeholder is meant to make ONE method visibly win; pick real strings against the
+# current microRNA/disease corpus before the talk.
+# ====================================================================================
+# BM25 should win: a precise token/ID that appears VERBATIM in abstracts.
+EXACT_ID_QUERY = "TODO_EXACT_ID: e.g. a specific miRNA like 'miR-21' or a gene symbol"
+# Dense should win: a conceptual paraphrase using NONE of the corpus's exact words.
+PARAPHRASE_QUERY = "TODO_PARAPHRASE: e.g. 'small RNAs that switch genes off in tumours'"
+# Lexical vs semantic gap: an acronym whose expansion is what's written in the text.
+ACRONYM_QUERY = "TODO_ACRONYM: e.g. an acronym vs its full form"
+# BM25 strong on rare exact tokens.
+RARE_TERM_QUERY = "TODO_RARE_TERM: e.g. a rare, very specific technical term"
+# Hybrid should win: a broad topic where fusing lexical + semantic beats either alone.
+BROAD_CONCEPT_QUERY = "TODO_BROAD: e.g. 'microRNA biomarkers for early cancer detection'"
+
+EXAMPLES = [
+    ("Exact ID (BM25)", EXACT_ID_QUERY),
+    ("Paraphrase (Dense)", PARAPHRASE_QUERY),
+    ("Acronym", ACRONYM_QUERY),
+    ("Rare term (BM25)", RARE_TERM_QUERY),
+    ("Broad concept (Hybrid)", BROAD_CONCEPT_QUERY),
+]
+
+PLOT_CAVEAT = (
+    "⚠️ **The 2D projection distorts true distances.** This PCA view captures only a "
+    "sliver of the 384-dimensional space, so retrieved points may *not* be the "
+    "visually-closest dots. **The ranked lists above are authoritative** — the plot is "
+    "only for spatial intuition."
+)
+
+# ====================================================================================
+# UI
+# ====================================================================================
+with gr.Blocks(title="RAG retrieval: BM25 vs Dense vs Hybrid") as demo:
+    gr.Markdown(
+        "# 🔍 Retrieval methods, side by side\n"
+        "Compare **BM25** (lexical), **Dense** (vector cosine), and **Hybrid** "
+        "(Reciprocal Rank Fusion) over a corpus of biomedical abstracts. "
+        f"Corpus: {len(META):,} Europe PMC abstracts on microRNA & disease."
+    )
+
+    with gr.Row():
+        query = gr.Textbox(label="Query", placeholder="Type a search query…", scale=4)
+        k = gr.Slider(1, 10, value=5, step=1, label="Top-k", scale=1)
+
+    with gr.Row():
+        for label, text in EXAMPLES:
+            btn = gr.Button(label, size="sm")
+            btn.click(lambda t=text: t, outputs=query)
+
+    with gr.Group():
+        gr.Markdown("**Metadata filter** — restricts the candidate pool *before* retrieval (all 3 methods).")
+        with gr.Row():
+            year_lo = gr.Slider(YEAR_MIN, YEAR_MAX, value=YEAR_MIN, step=1, label="Year from")
+            year_hi = gr.Slider(YEAR_MIN, YEAR_MAX, value=YEAR_MAX, step=1, label="Year to")
+            journal = gr.Dropdown(["All journals"] + TOP_JOURNALS, value="All journals", label="Journal")
+
+    search_btn = gr.Button("Search", variant="primary")
+    filter_info = gr.Markdown("—")
+
+    with gr.Row():
+        bm25_out = gr.Markdown(label="BM25")
+        dense_out = gr.Markdown(label="Dense")
+        hybrid_out = gr.Markdown(label="Hybrid")
+
+    gr.Markdown("## Vector space (PCA projection)")
+    plot = gr.Plot(value=make_plot())
+    gr.Markdown(PLOT_CAVEAT)
+
+    with gr.Accordion("Embed your own text", open=False):
+        gr.Markdown(
+            "Type anything — it gets embedded with the same model and dropped onto the "
+            "map. This *is* the space the database searches through."
+        )
+        own_text = gr.Textbox(label="Your text", placeholder="e.g. a sentence about gene regulation…")
+        own_btn = gr.Button("Embed & plot")
+        own_btn.click(run_embed_own, inputs=own_text, outputs=plot)
+
+    inputs = [query, k, year_lo, year_hi, journal]
+    outputs = [bm25_out, dense_out, hybrid_out, plot, filter_info]
+    search_btn.click(run_search, inputs=inputs, outputs=outputs)
+    query.submit(run_search, inputs=inputs, outputs=outputs)
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()

build_index.py

@@ -0,0 +1,140 @@
+"""Offline index builder — runs LOCALLY, never on the HF Space.
+
+Fetches abstracts from Europe PMC, embeds them, fits a 2D PCA, tokenises for BM25, and
+writes everything to ./data/. The Space then just loads these artifacts at startup; it
+never embeds the corpus or calls an external API at runtime.
+
+Run:  uv run python build_index.py
+"""
+
+import json
+import os
+import time
+
+import joblib
+import numpy as np
+import pandas as pd
+import requests
+from sentence_transformers import SentenceTransformer
+from sklearn.decomposition import PCA
+
+import config
+from text_utils import tokenize
+
+EUROPE_PMC_URL = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
+
+
+def fetch_abstracts(query: str, target_n: int) -> list[dict]:
+    """Page through the Europe PMC REST API (cursorMark) collecting records with abstracts."""
+    records = []
+    cursor = "*"
+    session = requests.Session()
+    while len(records) < target_n:
+        params = {
+            "query": query,
+            "format": "json",
+            "resultType": "core",  # includes abstractText, journalInfo, pubYear
+            "pageSize": 1000,
+            "cursorMark": cursor,
+        }
+        resp = session.get(EUROPE_PMC_URL, params=params, timeout=60)
+        resp.raise_for_status()
+        data = resp.json()
+
+        results = data.get("resultList", {}).get("result", [])
+        if not results:
+            print("  no more results from Europe PMC; stopping early")
+            break
+
+        for r in results:
+            abstract = r.get("abstractText")
+            title = r.get("title")
+            if not abstract or not title:
+                continue  # skip records missing the text we need
+            records.append(
+                {
+                    "id": r.get("id") or r.get("pmid") or "",
+                    "title": title.strip(),
+                    "abstract": abstract.strip(),
+                    "year": int(r["pubYear"]) if r.get("pubYear") else None,
+                    "journal": (
+                        r.get("journalInfo", {}).get("journal", {}).get("title")
+                        or "Unknown"
+                    ),
+                }
+            )
+            if len(records) >= target_n:
+                break
+
+        next_cursor = data.get("nextCursorMark")
+        print(f"  fetched {len(records)} / {target_n} abstracts...")
+        if not next_cursor or next_cursor == cursor:
+            break  # reached the end of the result set
+        cursor = next_cursor
+        time.sleep(0.2)  # be polite to the API
+
+    return records
+
+
+def main():
+    os.makedirs(config.DATA_DIR, exist_ok=True)
+
+    # 1. Fetch ----------------------------------------------------------------------
+    print(f"Fetching abstracts for query:\n  {config.EUROPE_PMC_QUERY}")
+    records = fetch_abstracts(config.EUROPE_PMC_QUERY, config.TARGET_N)
+    if not records:
+        raise SystemExit("No records fetched — check the query / network.")
+    df = pd.DataFrame(records)
+    # Drop the occasional exact-duplicate abstract.
+    df = df.drop_duplicates(subset="abstract").reset_index(drop=True)
+    print(f"Kept {len(df)} unique abstracts.")
+
+    # The text we both embed AND tokenise for BM25 — keep them identical for a fair fight.
+    docs = (df["title"] + ". " + df["abstract"]).tolist()
+
+    # 2. Embed ----------------------------------------------------------------------
+    print(f"Loading embedding model: {config.EMBEDDING_MODEL}")
+    model = SentenceTransformer(config.EMBEDDING_MODEL)
+    print("Embedding corpus (this is the slow part)...")
+    embeddings = model.encode(
+        docs,
+        batch_size=64,
+        show_progress_bar=True,
+        normalize_embeddings=True,  # so cosine similarity == dot product
+        convert_to_numpy=True,
+    ).astype(np.float32)
+    np.save(os.path.join(config.DATA_DIR, "embeddings.npy"), embeddings)
+    print(f"Saved embeddings: {embeddings.shape}")
+
+    # 3. PCA ------------------------------------------------------------------------
+    print("Fitting 2D PCA...")
+    pca = PCA(n_components=2, random_state=0)
+    coords = pca.fit_transform(embeddings).astype(np.float32)
+    joblib.dump(pca, os.path.join(config.DATA_DIR, "pca.joblib"))
+    np.save(os.path.join(config.DATA_DIR, "pca_coords.npy"), coords)
+    print(f"Saved PCA + corpus coords: {coords.shape}")
+
+    # 4. BM25 tokens ----------------------------------------------------------------
+    print("Tokenising for BM25...")
+    tokens = [tokenize(doc) for doc in docs]
+    with open(os.path.join(config.DATA_DIR, "bm25_tokens.json"), "w") as f:
+        json.dump(tokens, f)
+    print(f"Saved {len(tokens)} token lists.")
+
+    # 5. Metadata -------------------------------------------------------------------
+    df.to_parquet(os.path.join(config.DATA_DIR, "metadata.parquet"), index=False)
+    print(f"Saved metadata.parquet: {df.shape}")
+
+    # 6. Config record (startup sanity check in app.py) -----------------------------
+    with open(os.path.join(config.DATA_DIR, "config.json"), "w") as f:
+        json.dump(
+            {"embedding_model": config.EMBEDDING_MODEL, "dim": int(embeddings.shape[1])},
+            f,
+            indent=2,
+        )
+
+    print("\nDone. Artifacts written to", config.DATA_DIR)
+
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,27 @@
+"""Shared configuration — the SINGLE SOURCE OF TRUTH for both build_index.py and app.py.
+
+Keeping these constants in one module guarantees the offline corpus and the live query
+are embedded with the *same* model. If they ever diverge, the query vector and the corpus
+vectors live in different spaces and every retrieval result is silently wrong.
+"""
+
+# --- Embedding model -----------------------------------------------------------------
+# Small, fast, 384-dim. The default for this demo.
+EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+# Biomedical alternative (768-dim) — swap the line above for this one and rebuild to use a
+# domain model. ZeroGPU gives us the headroom for it.
+# EMBEDDING_MODEL = "pritamdeka/S-PubMedBert-MS-MARCO"
+
+# --- Corpus (offline build only) -----------------------------------------------------
+# Europe PMC search query. See https://europepmc.org/Help for the query syntax.
+EUROPE_PMC_QUERY = (
+    "microRNA AND disease AND HAS_ABSTRACT:Y AND LANG:eng "
+    "AND (FIRST_PDATE:[2018-01-01 TO 2025-12-31])"
+)
+TARGET_N = 4000  # number of abstracts to fetch & index
+
+# --- Retrieval -----------------------------------------------------------------------
+RRF_K = 60  # Reciprocal Rank Fusion constant (standard default)
+
+# --- Paths ---------------------------------------------------------------------------
+DATA_DIR = "./data"

data/config.json

@@ -0,0 +1,4 @@
+{
+  "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
+  "dim": 384
+}

data/embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f81a7b345de3c3173cd6b8219789968f061980d7d5d6dffe8445e82d84298c6
+size 6133376

data/metadata.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d002fef7b9475d2794670200a5ec53092d7816d769583330c6ec7dedfabc24a1
+size 3825335

data/pca.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13f7a85bd6fe24bf75709961bcdefb4e2ca4587e3e377cf67f29a7b765baefd
+size 5583

data/pca_coords.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca78a6240daa3c539a57f76faf6efdaf2419b7031d683f780adc112e5eef9746
+size 32072

requirements.txt

@@ -0,0 +1,11 @@
+gradio
+spaces
+sentence-transformers
+rank-bm25
+scikit-learn
+numpy
+pandas
+pyarrow
+plotly
+joblib
+requests

text_utils.py

@@ -0,0 +1,15 @@
+"""Tiny shared text helper.
+
+Lives in its own module so the offline BM25 index (build_index.py) and the live query
+tokenisation (app.py) use the EXACT same tokeniser. If these drift, BM25 scoring becomes
+inconsistent between corpus and query.
+"""
+
+import re
+
+_WORD_RE = re.compile(r"\w+")
+
+
+def tokenize(text: str) -> list[str]:
+    """Lowercase + split on word characters. Simple and good enough for a teaching demo."""
+    return _WORD_RE.findall(text.lower())