Add Space app files

.gitignore

@@ -0,0 +1,46 @@
+# Python
+__pycache__/
+*.py[cod]
+*.so
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+
+# Virtual environments
+venv/
+.venv/
+ENV/
+env/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Local runtime storage (vector DB + sqlite doc store)
+storage/
+*.db
+*.sqlite
+*.sqlite3
+
+# Dataset caches / artifacts
+hf_dataset_prepared/
+.cache/
+.huggingface/
+
+# Raw data (private; never publish to Spaces)
+old_related_files/
+private_scripts/
+
+# Logs
+*.log
+logs/
+
+# Env files
+.env
+.env.local
+
+

README.md

@@ -11,4 +11,86 @@ license: mit
 short_description: 'Ask questions about thoracic radiology and get answers with '
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+## Overview
+
+This repository contains a **Hugging Face Spaces-ready** RAG (Retrieval-Augmented Generation) demo for thoracic radiology Q&A.
+
+- **Default index (prebuilt)**: `ZhangNy/radiology-index-qwen3-embedding-0.6b`
+- **Raw public dataset**: `ZhangNy/radiology-dataset`
+- **No image rendering in UI**: references link to original pages where images can be viewed.
+
+The Space uses **external APIs** for Embeddings / Reranker / LLM via **Secrets**.
+
+## Run (local)
+
+```bash
+cd LangGraphAgent/rebuild_1219
+pip install -r requirements.txt
+
+export EMBED_API_KEY="..."
+export LLM_API_KEY="..."
+# optional:
+export RERANK_API_KEY="..."
+
+python app.py --config config/default_config.yaml --host 0.0.0.0 --port 7860
+```
+
+Open `http://localhost:7860`.
+
+## Required Hugging Face Space Secrets
+
+### Required
+
+- **`EMBED_API_KEY`**: embedding API key (OpenAI-compatible)
+- **`LLM_API_KEY`**: LLM API key (OpenAI-compatible)
+
+### Recommended
+
+- **`RERANK_API_KEY`**: reranker API key (OpenAI-compatible `/rerank` endpoint)
+
+### Optional (override defaults)
+
+- **`EMBED_API_BASE_URL`**, **`EMBED_MODEL_NAME`**
+- **`RERANK_API_BASE_URL`**, **`RERANK_MODEL_NAME`**
+- **`LLM_BASE_URL`**, **`LLM_MODEL_NAME`**
+- **`RAG_INDEX_REPO_ID`** (default: `ZhangNy/radiology-index-qwen3-embedding-0.6b`)
+- **`RAG_STORAGE_DIR`** (default: `/data/radiology_rag` if `/data` exists, else `./storage`)
+
+## Advanced: rebuild your own index (offline)
+
+Install dev deps:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
+The `scripts/` folder (to be used locally) will support:
+- Downloading `ZhangNy/radiology-dataset` to `./hf_dataset_prepared`
+- Building a new index with a different embedding model
+- Publishing that index as a Hugging Face dataset repo
+
+### Fast path (no rebuild): publish your existing local index
+
+If you already have a built index locally (e.g. `rebuild_1217/storage` contains `chroma_db/` + `doc_store.db`),
+you can **package it without images** and upload it:
+
+```bash
+python scripts/package_existing_storage.py \
+  --storage /home/zny/codes/radioagent_prepare/LangGraphAgent/rebuild_1217/storage \
+  --output-dir ./index_out \
+  --overwrite
+
+python scripts/publish_index_to_hf.py \
+  --repo ZhangNy/radiology-index-qwen3-embedding-0.6b \
+  --folder ./index_out \
+  --token $HF_TOKEN
+```
+
+## Notes
+
+- **Do not commit API keys**. This repo is configured to read them from environment variables / Space Secrets.
+- **Index compatibility**: query-time embedding model should match the index embedding model for best retrieval quality.
+
+
+

app.py

@@ -0,0 +1,66 @@
+"""
+Hugging Face Spaces entry point (Gradio).
+
+Run locally:
+  python app.py --config config/default_config.yaml --host 0.0.0.0 --port 7860
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+from pathlib import Path
+
+from radiology_rag.gradio_compat import patch_gradio_predict_body_for_pydantic_v2
+from radiology_rag.ui import RadiologyRAGApp
+
+
+def _configure_logging() -> None:
+    level = os.getenv("LOG_LEVEL", "INFO").upper()
+    logging.basicConfig(
+        level=getattr(logging, level, logging.INFO),
+        format="%(asctime)s - %(levelname)s - %(message)s",
+    )
+
+
+def _default_storage_dir() -> str:
+    # Prefer /data on Spaces if persistent storage is enabled.
+    if Path("/data").exists():
+        return "/data/radiology_rag"
+    return "./storage"
+
+
+def main() -> int:
+    _configure_logging()
+
+    parser = argparse.ArgumentParser(description="Radiology RAG (Spaces-ready)")
+    parser.add_argument("--config", type=str, default="config/default_config.yaml", help="Path to config YAML")
+    parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host")
+    parser.add_argument("--port", type=int, default=int(os.getenv("PORT", "7860")), help="Server port")
+    args = parser.parse_args()
+
+    # Ensure storage dir env is set early so config interpolation uses it.
+    if not os.getenv("RAG_STORAGE_DIR"):
+        os.environ["RAG_STORAGE_DIR"] = _default_storage_dir()
+
+    # Optional compatibility patch for Gradio 4.16 + Pydantic v2.
+    if patch_gradio_predict_body_for_pydantic_v2():
+        logging.getLogger(__name__).info("Applied Gradio/Pydantic v2 compatibility patch")
+
+    app = RadiologyRAGApp(config_path=args.config)
+    demo = app.create_interface()
+
+    demo.launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=False,
+        show_error=True,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

config/default_config.yaml

@@ -0,0 +1,122 @@
+# Default configuration for the Hugging Face Spaces deployment (API-first).
+# IMPORTANT: do NOT put any real API keys in this file. Use Spaces Secrets instead.
+
+# Prebuilt index (vector DB + doc store) stored as a Hugging Face dataset repo.
+index:
+  repo_id: "${RAG_INDEX_REPO_ID:ZhangNy/radiology-index-qwen3-embedding-0.6b}"
+  revision: "${RAG_INDEX_REVISION:main}"
+
+# Embedding configuration (query-time embeddings must match the index embedding model).
+embedding:
+  type: "api"  # Options: "api" or "local" (local requires requirements-dev.txt)
+  api_base_url: "${EMBED_API_BASE_URL:https://api.siliconflow.cn/v1}"
+  api_key: "${EMBED_API_KEY:}"
+  model_name: "${EMBED_MODEL_NAME:Qwen/Qwen3-Embedding-0.6B}"
+  batch_size: 32
+
+# Reranker configuration (recommended; can be disabled if no key is provided).
+reranker:
+  enabled: true
+  type: "api"  # Options: "api" or "local" (local requires requirements-dev.txt)
+  api_base_url: "${RERANK_API_BASE_URL:https://api.siliconflow.cn/v1}"
+  api_key: "${RERANK_API_KEY:}"
+  model_name: "${RERANK_MODEL_NAME:BAAI/bge-reranker-v2-m3}"
+  top_k: 10
+
+# LLM configuration (OpenAI-compatible API).
+llm:
+  base_url: "${LLM_BASE_URL:https://poloai.top/v1}"
+  api_key: "${LLM_API_KEY:}"
+  model_name: "${LLM_MODEL_NAME:gemini-3-flash-preview}"
+  temperature: 0.7
+  max_tokens: 2000
+
+# Storage paths (prefer /data on Spaces; app.py will default RAG_STORAGE_DIR to /data/radiology_rag when available).
+storage:
+  vector_db_path: "${RAG_STORAGE_DIR:./storage}/chroma_db"
+  doc_store_path: "${RAG_STORAGE_DIR:./storage}/doc_store.db"
+
+# Text splitting parameters (used for index build scripts; kept here for transparency).
+processing:
+  chunk_size: 1024
+  chunk_overlap: 200
+  separators:
+    - "\n\n#### "
+    - "\n\n### "
+    - "\n\n## "
+    - "\n\n"
+    - "\n"
+    - " "
+    - ""
+  keep_separator: true
+
+# Retrieval configuration
+retrieval:
+  # Default strategy for this Space:
+  # - balanced_multi_source: includes Wikipedia (encyclopedia) by default
+  strategy: "balanced_multi_source"
+  top_k: 20
+  source_filters:
+    - "article"
+    - "case"
+    - "tutorial"
+    - "encyclopedia"
+
+  search_type: "similarity"  # "similarity" or "mmr"
+  chunk_fetch_multiplier: 3
+
+  # MMR parameters (only if search_type == "mmr")
+  mmr_lambda: 0.5
+  mmr_fetch_k: 50
+
+  # Balanced multi-source retrieval policy
+  multi_source:
+    total_top_k: 8
+    sources_priority: ["article", "case", "encyclopedia", "tutorial"]
+    article:
+      candidate_k: 80
+      max_k: 3
+      min_score: 0.15
+      required: true
+    case:
+      candidate_k: 80
+      max_k: 3
+      min_score: 0.15
+      required: true
+    encyclopedia:
+      candidate_k: 8
+      max_k: 2
+      min_score: 0.15
+      required: true
+    tutorial:
+      candidate_k: 20
+      max_k: 2
+      min_score: 0.50
+      required: false
+
+# Encyclopedia configuration (Wikipedia)
+encyclopedia:
+  wikipedia:
+    language: "en"
+    user_agent: "RadiologyRAG-Space/1.0"
+    timeout_s: 10
+    max_chars_per_doc: 2000
+
+# Citation configuration (no images in this Space; references link to original pages)
+citation:
+  format: "numbered"
+  max_content_length: 900
+
+# Gradio UI configuration
+ui:
+  title: "Thoracic Radiology RAG System"
+  description: "Ask questions about thoracic radiology and get answers with citations (articles, cases, tutorials + Wikipedia)."
+  theme: "soft"
+  show_retrieved_docs: true
+
+# Logging configuration
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(levelname)s - %(message)s"
+
+

radiology_rag/__init__.py

@@ -0,0 +1,12 @@
+"""
+Radiology RAG (Spaces-ready) - minimal, API-first implementation.
+
+This package is designed to be deployed on Hugging Face Spaces and load a
+prebuilt vector index from a public Hugging Face dataset repo.
+"""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"
+
+

radiology_rag/citations.py

@@ -0,0 +1,54 @@
+"""Citation helpers (numbered citations like [1], [2], ...)."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Tuple
+
+
+class CitationManager:
+    def __init__(self, *, max_content_length: int = 900):
+        self.max_content_length = int(max_content_length)
+        self.documents: List[Dict[str, Any]] = []
+        self.doc_id_to_index: Dict[str, int] = {}
+
+    def clear(self) -> None:
+        self.documents = []
+        self.doc_id_to_index = {}
+
+    def add_document(self, document: Dict[str, Any]) -> int:
+        doc_id = document.get("doc_id") or ""
+        if doc_id in self.doc_id_to_index:
+            return self.doc_id_to_index[doc_id]
+        self.documents.append(document)
+        idx = len(self.documents)
+        self.doc_id_to_index[doc_id] = idx
+        return idx
+
+    def add_documents(self, documents: List[Dict[str, Any]]) -> List[int]:
+        return [self.add_document(d) for d in documents]
+
+    @staticmethod
+    def parse_citations_in_text(text: str) -> List[int]:
+        matches = re.findall(r"\[(\d+)\]", text or "")
+        out = []
+        for m in matches:
+            try:
+                out.append(int(m))
+            except Exception:
+                continue
+        return out
+
+    def validate_citations(self, text: str) -> Tuple[bool, List[int]]:
+        cited = self.parse_citations_in_text(text or "")
+        invalid = [i for i in cited if i < 1 or i > len(self.documents)]
+        return (len(invalid) == 0), invalid
+
+    def get_statistics(self) -> Dict[str, Any]:
+        counts: Dict[str, int] = {}
+        for d in self.documents:
+            st = d.get("source_type", "unknown") or "unknown"
+            counts[st] = counts.get(st, 0) + 1
+        return {"total": len(self.documents), "source_type_counts": counts}
+
+

radiology_rag/config.py

@@ -0,0 +1,108 @@
+"""
+Config loader with `${ENV_VAR}` and `${ENV_VAR:default}` interpolation.
+
+We intentionally keep config logic lightweight so it works well in Spaces.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+
+
+class Config:
+    """Load and access YAML config with env var interpolation."""
+
+    def __init__(self, config_path: str):
+        self.config_path = Path(config_path)
+        self._config = self._load_config()
+        self._config = self._recursive_resolve(self._config)
+
+    def _load_config(self) -> Dict[str, Any]:
+        if not self.config_path.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config_path}")
+        with open(self.config_path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        if not isinstance(data, dict):
+            raise ValueError("Config root must be a mapping/dict")
+        return data
+
+    @staticmethod
+    def _resolve_string(value: str) -> str:
+        # Pattern: ${VAR_NAME} or ${VAR_NAME:default_value}
+        # NOTE: default_value may be empty, e.g. `${API_KEY:}`. Use `*` (not `+`) to allow empty.
+        pattern = r"\$\{([^:}]+)(?::([^}]*))?\}"
+
+        def replace(match: re.Match) -> str:
+            var_name = match.group(1)
+            default_value = match.group(2) if match.group(2) is not None else ""
+            return os.getenv(var_name, default_value)
+
+        return re.sub(pattern, replace, value)
+
+    def _recursive_resolve(self, obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: self._recursive_resolve(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [self._recursive_resolve(v) for v in obj]
+        if isinstance(obj, str):
+            return self._resolve_string(obj)
+        return obj
+
+    def get(self, key: str, default: Any = None) -> Any:
+        keys = key.split(".")
+        value: Any = self._config
+        for k in keys:
+            if isinstance(value, dict) and k in value:
+                value = value[k]
+            else:
+                return default
+        return value
+
+    def get_str(self, key: str, default: str = "") -> str:
+        v = self.get(key, default)
+        return default if v is None else str(v)
+
+    def get_int(self, key: str, default: int = 0) -> int:
+        v = self.get(key, default)
+        if v is None:
+            return default
+        if isinstance(v, int):
+            return v
+        try:
+            return int(str(v).strip())
+        except Exception:
+            return default
+
+    def get_float(self, key: str, default: float = 0.0) -> float:
+        v = self.get(key, default)
+        if v is None:
+            return default
+        if isinstance(v, (int, float)):
+            return float(v)
+        try:
+            return float(str(v).strip())
+        except Exception:
+            return default
+
+    def get_bool(self, key: str, default: bool = False) -> bool:
+        v = self.get(key, default)
+        if isinstance(v, bool):
+            return v
+        if v is None:
+            return default
+        s = str(v).strip().lower()
+        if s in {"1", "true", "yes", "y", "on"}:
+            return True
+        if s in {"0", "false", "no", "n", "off"}:
+            return False
+        return default
+
+    def as_dict(self) -> Dict[str, Any]:
+        return dict(self._config)
+
+

radiology_rag/doc_store.py

@@ -0,0 +1,143 @@
+"""
+SQLite-backed document store.
+
+We keep the schema compatible with the previous rebuild_1217 implementation:
+table `documents(doc_id, complete_document, main_content, images, source_type)`.
+
+In this Space we do NOT use images, but we keep the column for compatibility with
+existing indexes and to allow advanced users to extend the system.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sqlite3
+from typing import Any, Iterator, List, Optional, Sequence, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class PersistentDocStore:
+    def __init__(self, db_path: str, *, read_only: bool = False):
+        self.db_path = db_path
+        self.read_only = bool(read_only)
+
+        if not self.read_only:
+            self.init_db()
+
+    def _connect(self) -> sqlite3.Connection:
+        if self.read_only:
+            # Open in read-only mode to avoid accidental writes in Spaces runtime.
+            return sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True)
+        return sqlite3.connect(self.db_path)
+
+    def init_db(self) -> None:
+        db_dir = os.path.dirname(self.db_path) or "."
+        os.makedirs(db_dir, exist_ok=True)
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS documents (
+                    doc_id TEXT PRIMARY KEY,
+                    complete_document TEXT,
+                    main_content TEXT,
+                    images TEXT,
+                    source_type TEXT
+                )
+                """
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def mset(self, key_value_pairs: Sequence[Tuple[str, Any]]) -> None:
+        if self.read_only:
+            raise RuntimeError("DocStore is read-only")
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            for doc_id, content in key_value_pairs:
+                cursor.execute(
+                    """
+                    INSERT OR REPLACE INTO documents
+                    (doc_id, complete_document, main_content, images, source_type)
+                    VALUES (?, ?, ?, ?, ?)
+                    """,
+                    (
+                        doc_id,
+                        json.dumps(content.get("complete_document", {}), ensure_ascii=False),
+                        content.get("main_content", "") or "",
+                        json.dumps(content.get("images", []), ensure_ascii=False),
+                        content.get("source_type", "") or "",
+                    ),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def mget(self, keys: Sequence[str]) -> List[Optional[Any]]:
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            out: List[Optional[Any]] = []
+            for doc_id in keys:
+                cursor.execute(
+                    "SELECT complete_document, main_content, images, source_type FROM documents WHERE doc_id = ?",
+                    (doc_id,),
+                )
+                row = cursor.fetchone()
+                if not row:
+                    out.append(None)
+                    continue
+                complete_document, main_content, images, source_type = row
+                out.append(
+                    {
+                        "complete_document": json.loads(complete_document or "{}"),
+                        "main_content": main_content or "",
+                        "images": json.loads(images or "[]"),
+                        "source_type": source_type or "",
+                    }
+                )
+            return out
+        finally:
+            conn.close()
+
+    def yield_keys(self, *, prefix: Optional[str] = None) -> Iterator[str]:
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            if prefix:
+                cursor.execute("SELECT doc_id FROM documents WHERE doc_id LIKE ?", (f"{prefix}%",))
+            else:
+                cursor.execute("SELECT doc_id FROM documents")
+            for (doc_id,) in cursor.fetchall():
+                yield str(doc_id)
+        finally:
+            conn.close()
+
+    def count(self) -> int:
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM documents")
+            return int(cursor.fetchone()[0])
+        finally:
+            conn.close()
+
+    def count_by_source_type(self) -> dict:
+        conn = self._connect()
+        try:
+            cursor = conn.cursor()
+            cursor.execute("SELECT source_type, COUNT(*) FROM documents GROUP BY source_type")
+            counts = {}
+            for source_type, count in cursor.fetchall():
+                counts[str(source_type)] = int(count)
+            return counts
+        finally:
+            conn.close()
+
+

radiology_rag/embedding.py

@@ -0,0 +1,41 @@
+"""Embedding utilities (OpenAI-compatible, API-first)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from langchain_openai import OpenAIEmbeddings
+
+
+@dataclass(frozen=True)
+class EmbeddingConfig:
+    base_url: str
+    api_key: str
+    model_name: str
+    batch_size: int = 32
+
+
+class EmbeddingClient:
+    """Thin wrapper over LangChain OpenAIEmbeddings."""
+
+    def __init__(self, cfg: EmbeddingConfig):
+        self.cfg = cfg
+        self._emb = OpenAIEmbeddings(
+            base_url=cfg.base_url,
+            api_key=cfg.api_key,
+            model=cfg.model_name,
+            chunk_size=int(cfg.batch_size or 32),
+        )
+
+    def embed_query(self, text: str) -> list[float]:
+        return self._emb.embed_query(text)
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        return self._emb.embed_documents(texts)
+
+    @property
+    def langchain_embeddings(self) -> OpenAIEmbeddings:
+        """Expose the underlying LangChain embeddings for Chroma."""
+        return self._emb
+
+

radiology_rag/encyclopedia.py

@@ -0,0 +1,194 @@
+"""Wikipedia encyclopedia retrieval (MediaWiki API)."""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class WikipediaConfig:
+    language: str = "en"
+    user_agent: str = "RadiologyRAG-Space/1.0"
+    timeout_s: int = 15
+    max_chars_per_doc: int = 2000
+
+
+class WikipediaEncyclopediaService:
+    def __init__(self, config: Optional[WikipediaConfig] = None):
+        self.config = config or WikipediaConfig()
+        self._session = requests.Session()
+        self._session.headers.update({"User-Agent": self.config.user_agent})
+
+    @property
+    def api_base(self) -> str:
+        return f"https://{self.config.language}.wikipedia.org/w/api.php"
+
+    @staticmethod
+    def _derive_search_query(user_query: str) -> str:
+        q = (user_query or "").strip()
+        if not q:
+            return ""
+
+        tokens = re.findall(r"[A-Za-z][A-Za-z'\\-]*", q.lower())
+        if not tokens:
+            return q
+
+        stop = {
+            "what",
+            "which",
+            "who",
+            "whom",
+            "whose",
+            "when",
+            "where",
+            "why",
+            "how",
+            "is",
+            "are",
+            "was",
+            "were",
+            "be",
+            "been",
+            "being",
+            "do",
+            "does",
+            "did",
+            "can",
+            "could",
+            "should",
+            "would",
+            "may",
+            "might",
+            "will",
+            "shall",
+            "a",
+            "an",
+            "the",
+            "and",
+            "or",
+            "but",
+            "to",
+            "of",
+            "for",
+            "with",
+            "without",
+            "in",
+            "on",
+            "at",
+            "by",
+            "from",
+            "as",
+            "it",
+            "its",
+            "this",
+            "that",
+            "these",
+            "those",
+            "your",
+            "my",
+            "their",
+            "our",
+            "about",
+        }
+
+        keep_short = {"ct", "mr", "mri", "pet", "us", "cxr"}
+        keywords: List[str] = []
+        seen = set()
+        for t in tokens:
+            if t in stop:
+                continue
+            if len(t) < 3 and t not in keep_short:
+                continue
+            if t in seen:
+                continue
+            seen.add(t)
+            keywords.append(t)
+
+        return " ".join(keywords[:8]) if keywords else q
+
+    def retrieve(self, query: str, top_k: int = 5, max_chars_per_doc: Optional[int] = None) -> List[Dict[str, Any]]:
+        q = (query or "").strip()
+        if not q:
+            return []
+
+        search_q = self._derive_search_query(q)
+        if not search_q:
+            return []
+
+        max_chars = int(max_chars_per_doc or self.config.max_chars_per_doc)
+        try:
+            search_params = {
+                "action": "query",
+                "list": "search",
+                "srsearch": search_q,
+                "srlimit": max(1, min(int(top_k), 20)),
+                "format": "json",
+            }
+            resp = self._session.get(self.api_base, params=search_params, timeout=self.config.timeout_s)
+            resp.raise_for_status()
+            data = resp.json() or {}
+            hits = (data.get("query", {}) or {}).get("search", []) or []
+
+            # Fallback to raw query if rewrite yields no hits
+            if not hits and search_q != q:
+                search_params["srsearch"] = q
+                resp = self._session.get(self.api_base, params=search_params, timeout=self.config.timeout_s)
+                resp.raise_for_status()
+                data = resp.json() or {}
+                hits = (data.get("query", {}) or {}).get("search", []) or []
+
+            if not hits:
+                return []
+
+            pageids = [str(h.get("pageid")) for h in hits if h.get("pageid") is not None]
+            if not pageids:
+                return []
+
+            pages_params = {
+                "action": "query",
+                "pageids": "|".join(pageids),
+                "prop": "extracts|info",
+                "explaintext": 1,
+                "exintro": 1,
+                "exchars": max_chars,
+                "inprop": "url",
+                "format": "json",
+            }
+            resp2 = self._session.get(self.api_base, params=pages_params, timeout=self.config.timeout_s)
+            resp2.raise_for_status()
+            pages_data = resp2.json() or {}
+            pages = (pages_data.get("query", {}) or {}).get("pages", {}) or {}
+
+            docs: List[Dict[str, Any]] = []
+            for pid in pageids:
+                page = pages.get(pid) or {}
+                title = page.get("title") or ""
+                extract = (page.get("extract") or "").strip()
+                url = page.get("fullurl") or ""
+                if not title or not extract:
+                    continue
+                docs.append(
+                    {
+                        "doc_id": f"encyclopedia_{pid}",
+                        "source_type": "encyclopedia",
+                        "title": title,
+                        "content": extract,
+                        "url": url,
+                        "metadata": {"provider": "wikipedia", "pageid": pid},
+                        "score": 0.0,
+                    }
+                )
+
+            return docs[: int(top_k)]
+        except Exception as e:
+            logger.warning(f"Wikipedia retrieval failed: {e}")
+            return []
+
+

radiology_rag/gradio_compat.py

@@ -0,0 +1,60 @@
+"""
+Compatibility patches for Gradio when running with newer FastAPI/Pydantic versions.
+
+Background:
+- Gradio 4.16 defines `gradio.data_classes.PredictBody.request: Optional[fastapi.Request]`.
+- Under Pydantic v2, `fastapi/starlette Request` cannot be converted into a JSON schema,
+  which can crash FastAPI request parsing for Gradio's `/run/{api_name}` endpoint.
+
+This module applies a targeted runtime patch that replaces that field with `Any`.
+It is intentionally narrow and only runs when we detect the problematic combination.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def patch_gradio_predict_body_for_pydantic_v2() -> bool:
+    """Return True if a patch was applied."""
+    try:
+        import pydantic
+
+        major = int(str(pydantic.__version__).split(".", 1)[0])
+        if major < 2:
+            return False
+
+        import gradio.data_classes as gr_data_classes
+        import gradio.routes as gr_routes
+        from pydantic import BaseModel, ConfigDict, create_model
+        from typing import List, Optional
+
+        PredictBody = getattr(gr_data_classes, "PredictBody", None)
+        if PredictBody is None:
+            return False
+
+        ann = getattr(PredictBody, "__annotations__", {}) or {}
+        if "request" not in ann:
+            return False
+
+        PatchedPredictBody = create_model(  # type: ignore[call-arg]
+            "PredictBody",
+            __base__=BaseModel,
+            __config__=ConfigDict(arbitrary_types_allowed=True),
+            session_hash=(Optional[str], None),
+            event_id=(Optional[str], None),
+            data=(List[Any], ...),
+            event_data=(Optional[Any], None),
+            fn_index=(Optional[int], None),
+            trigger_id=(Optional[int], None),
+            batched=(Optional[bool], False),
+            request=(Optional[Any], None),
+        )
+
+        gr_data_classes.PredictBody = PatchedPredictBody  # type: ignore[attr-defined]
+        gr_routes.PredictBody = PatchedPredictBody  # type: ignore[attr-defined]
+        return True
+    except Exception:
+        return False
+
+

radiology_rag/index_bootstrap.py

@@ -0,0 +1,160 @@
+"""
+Index bootstrap utilities for Hugging Face Spaces.
+
+This Space relies on a prebuilt index stored on Hugging Face Datasets:
+- ChromaDB persist directory (vector store)
+- SQLite doc store (parent documents)
+
+At startup we download (once) and place the index into a writable storage dir
+(prefer /data on Spaces when persistent storage is enabled).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+from huggingface_hub import snapshot_download
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_INDEX_REPO_ID = "ZhangNy/radiology-index-qwen3-embedding-0.6b"
+
+
+@dataclass(frozen=True)
+class IndexPaths:
+    vector_db_path: Path
+    doc_store_path: Path
+    manifest_path: Optional[Path]
+    snapshot_dir: Optional[Path]
+
+
+def resolve_default_storage_dir() -> Path:
+    """
+    Determine a good default storage directory for Spaces.
+
+    Priority:
+    - $RAG_STORAGE_DIR (user override)
+    - /data/radiology_rag (Spaces persistent storage)
+    - ./storage (local)
+    """
+    env = (os.getenv("RAG_STORAGE_DIR") or "").strip()
+    if env:
+        return Path(env)
+    if Path("/data").exists():
+        return Path("/data") / "radiology_rag"
+    return Path("./storage")
+
+
+def _find_index_artifacts(snapshot_dir: Path) -> Tuple[Path, Path, Optional[Path]]:
+    """
+    Find (chroma_db_dir, doc_store_db, manifest_json) inside a HF snapshot.
+
+    We support either:
+    - chroma_db/, doc_store.db, manifest.json
+    - storage/chroma_db/, storage/doc_store.db, storage/manifest.json
+    """
+    candidates = [
+        (snapshot_dir / "chroma_db", snapshot_dir / "doc_store.db", snapshot_dir / "manifest.json"),
+        (snapshot_dir / "storage" / "chroma_db", snapshot_dir / "storage" / "doc_store.db", snapshot_dir / "storage" / "manifest.json"),
+    ]
+    for chroma_dir, doc_db, manifest in candidates:
+        if chroma_dir.exists() and chroma_dir.is_dir() and doc_db.exists() and doc_db.is_file():
+            return chroma_dir, doc_db, (manifest if manifest.exists() else None)
+
+    raise FileNotFoundError(
+        "Could not locate index artifacts inside snapshot. "
+        "Expected either {chroma_db/, doc_store.db} or {storage/chroma_db/, storage/doc_store.db}."
+    )
+
+
+def read_manifest(manifest_path: Optional[Path]) -> Optional[Dict[str, Any]]:
+    if not manifest_path or not manifest_path.exists():
+        return None
+    try:
+        with open(manifest_path, "r", encoding="utf-8") as f:
+            return json.load(f) or {}
+    except Exception as e:
+        logger.warning(f"Failed to read manifest.json: {e}")
+        return None
+
+
+def ensure_index(
+    *,
+    repo_id: str = DEFAULT_INDEX_REPO_ID,
+    revision: Optional[str] = None,
+    target_vector_db_path: Optional[str] = None,
+    target_doc_store_path: Optional[str] = None,
+    storage_dir: Optional[str] = None,
+    force_download: bool = False,
+) -> IndexPaths:
+    """
+    Ensure the index exists locally at the configured storage paths.
+
+    Returns resolved IndexPaths; raises on unrecoverable errors.
+    """
+    # Resolve target paths
+    if storage_dir:
+        base_dir = Path(storage_dir)
+    else:
+        base_dir = resolve_default_storage_dir()
+    base_dir.mkdir(parents=True, exist_ok=True)
+
+    vector_db_path = Path(target_vector_db_path) if target_vector_db_path else (base_dir / "chroma_db")
+    doc_store_path = Path(target_doc_store_path) if target_doc_store_path else (base_dir / "doc_store.db")
+
+    # Fast path: already present
+    if (
+        not force_download
+        and vector_db_path.exists()
+        and vector_db_path.is_dir()
+        and doc_store_path.exists()
+        and doc_store_path.is_file()
+    ):
+        logger.info(f"Index already present: vector_db={vector_db_path} doc_store={doc_store_path}")
+        manifest_path = (base_dir / "manifest.json") if (base_dir / "manifest.json").exists() else None
+        return IndexPaths(vector_db_path=vector_db_path, doc_store_path=doc_store_path, manifest_path=manifest_path, snapshot_dir=None)
+
+    # Download snapshot
+    repo_id = (repo_id or "").strip() or DEFAULT_INDEX_REPO_ID
+    logger.info(f"Downloading index snapshot from HF dataset repo: {repo_id} (revision={revision or 'main'})")
+    snapshot_dir = Path(
+        snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            revision=revision or None,
+            local_files_only=False,
+        )
+    )
+
+    src_chroma_dir, src_doc_db, src_manifest = _find_index_artifacts(snapshot_dir)
+    logger.info(f"Found index artifacts in snapshot: chroma={src_chroma_dir} doc_store={src_doc_db}")
+
+    # Copy to writable target locations
+    if vector_db_path.exists():
+        shutil.rmtree(vector_db_path, ignore_errors=True)
+    vector_db_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(src_chroma_dir, vector_db_path, dirs_exist_ok=False)
+
+    doc_store_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src_doc_db, doc_store_path)
+
+    manifest_path: Optional[Path] = None
+    if src_manifest and src_manifest.exists():
+        manifest_path = doc_store_path.parent / "manifest.json"
+        try:
+            shutil.copy2(src_manifest, manifest_path)
+        except Exception as e:
+            logger.warning(f"Failed to copy manifest.json: {e}")
+            manifest_path = None
+
+    logger.info(f"Index ready: vector_db={vector_db_path} doc_store={doc_store_path}")
+    return IndexPaths(vector_db_path=vector_db_path, doc_store_path=doc_store_path, manifest_path=manifest_path, snapshot_dir=snapshot_dir)
+
+

radiology_rag/rag.py

@@ -0,0 +1,257 @@
+"""
+RAG engine (Spaces-ready, API-first).
+
+Pipeline:
+1) Retrieve documents from prebuilt Chroma+SQLite index (local) + Wikipedia (optional)
+2) Rerank (API; auto-disables if missing key)
+3) Build a prompt with numbered citations [1], [2], ...
+4) Call LLM (OpenAI-compatible) and stream answer
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from langchain_core.prompts import PromptTemplate
+from langchain_openai import ChatOpenAI
+
+from radiology_rag.config import Config
+from radiology_rag.citations import CitationManager
+from radiology_rag.retrieval import MultiSourceRetrievalService, RetrievalService
+from radiology_rag.reranker import RerankerConfig, RerankerService
+
+logger = logging.getLogger(__name__)
+
+
+class RAGEngine:
+    """Retrieval-Augmented Generation engine for radiology queries."""
+
+    RAG_PROMPT_TEMPLATE = """You are a helpful radiology assistant with access to medical literature.
+
+Answer the user's question based on the provided context.
+
+**Rules**
+1. Use the context as primary evidence.
+2. Add numbered citations like [1], [2], ... immediately after the relevant sentences.
+3. Do NOT invent citations. Only cite sources that appear in the context.
+4. If the context does not contain enough information, say so and provide the best general explanation you can.
+
+**Context**
+{context}
+
+**User Question**
+{question}
+
+**Answer (with citations)**
+"""
+
+    def __init__(self, config: Config):
+        self.config = config
+
+        # Retrieval + rerank
+        self.retrieval_service = RetrievalService(config)
+
+        rr_cfg = RerankerConfig(
+            enabled=config.get_bool("reranker.enabled", True),
+            base_url=config.get_str("reranker.api_base_url"),
+            api_key=config.get_str("reranker.api_key"),
+            model_name=config.get_str("reranker.model_name"),
+            top_k=config.get_int("reranker.top_k", 10),
+        )
+        self.reranker_service = RerankerService(rr_cfg)
+
+        self.multi_source_retrieval_service = MultiSourceRetrievalService(
+            config=config,
+            retrieval_service=self.retrieval_service,
+            reranker_service=self.reranker_service,
+        )
+
+        self.citation_manager = CitationManager(max_content_length=config.get_int("citation.max_content_length", 900))
+
+        # LLM (OpenAI-compatible)
+        self.llm = ChatOpenAI(
+            base_url=config.get_str("llm.base_url"),
+            api_key=config.get_str("llm.api_key"),
+            model=config.get_str("llm.model_name"),
+            temperature=config.get_float("llm.temperature", 0.7),
+            max_tokens=config.get_int("llm.max_tokens", 2000),
+        )
+
+        self.prompt = PromptTemplate(template=self.RAG_PROMPT_TEMPLATE, input_variables=["context", "question"])
+
+    @staticmethod
+    def _normalize_retrieval_strategy(strategy: Optional[str]) -> str:
+        s = (strategy or "").strip() or "default"
+        if s not in {"default", "balanced_multi_source"}:
+            logger.warning(f"Unknown retrieval strategy '{s}', falling back to 'default'")
+            return "default"
+        return s
+
+    @staticmethod
+    def _format_context(documents: List[Dict[str, Any]], citation_indices: List[int]) -> str:
+        parts: List[str] = []
+        for doc, idx in zip(documents, citation_indices):
+            source_type = (doc.get("source_type") or "").upper()
+            title = doc.get("title") or "Untitled"
+            content = doc.get("content") or ""
+            url = doc.get("url") or ""
+
+            block = f"[{idx}] **{source_type}: {title}**\n{content}"
+            if url:
+                block += f"\nURL: {url}"
+            parts.append(block)
+
+        return "\n\n---\n\n".join(parts)
+
+    def _retrieve(self, *, question: str, top_k: Optional[int], source_filters: Optional[List[str]], retrieval_strategy: str):
+        if retrieval_strategy == "balanced_multi_source":
+            final_k = int(top_k or self.config.get_int("retrieval.multi_source.total_top_k", 8))
+            return self.multi_source_retrieval_service.retrieve(
+                query=question,
+                total_top_k=final_k,
+                source_filters=source_filters,
+                return_debug=True,
+            )
+
+        k = int(top_k or self.config.get_int("retrieval.top_k", 20))
+        docs = self.retrieval_service.retrieve(query=question, top_k=k, source_filters=source_filters)
+        return docs, None
+
+    def query_stream(
+        self,
+        *,
+        question: str,
+        top_k: Optional[int] = None,
+        source_filters: Optional[List[str]] = None,
+        retrieval_strategy: Optional[str] = None,
+        stream_yield_interval_s: float = 0.15,
+        stream_min_chars: int = 80,
+    ) -> Iterator[Dict[str, Any]]:
+        q = (question or "").strip()
+        if not q:
+            yield {
+                "type": "final",
+                "answer": "Please enter a question.",
+                "references": [],
+                "metadata": {"num_retrieved": 0, "num_reranked": 0, "retrieval_strategy": "default"},
+            }
+            return
+
+        self.citation_manager.clear()
+        retrieval_strategy_n = self._normalize_retrieval_strategy(
+            retrieval_strategy or self.config.get_str("retrieval.strategy", "default")
+        )
+
+        start_time = time.time()
+
+        # Step 1: retrieve
+        docs, retrieval_debug = self._retrieve(
+            question=q, top_k=top_k, source_filters=source_filters, retrieval_strategy=retrieval_strategy_n
+        )
+        if not docs:
+            yield {
+                "type": "final",
+                "answer": "I couldn't find any relevant information to answer your question.",
+                "references": [],
+                "metadata": {"num_retrieved": 0, "num_reranked": 0, "retrieval_strategy": retrieval_strategy_n},
+            }
+            return
+
+        # Step 2: rerank (default strategy only; balanced already reranks per-source)
+        if retrieval_strategy_n == "balanced_multi_source":
+            reranked_docs = sorted(docs, key=lambda d: float(d.get("score", 0.0)), reverse=True)
+            num_retrieved = int(sum((retrieval_debug.get("candidate_counts") or {}).values())) if retrieval_debug else len(docs)
+            retrieved_label = "Recalled (candidates)"
+            reranked_label = "Selected (after rerank)"
+        else:
+            reranked_docs = self.reranker_service.rerank(
+                query=q, documents=docs, top_k=self.config.get_int("reranker.top_k", 10)
+            )
+            num_retrieved = len(docs)
+            retrieved_label = "Retrieved"
+            reranked_label = "After Reranking"
+
+        # Step 3: citations + context + prompt
+        citation_indices = self.citation_manager.add_documents(reranked_docs)
+        context = self._format_context(reranked_docs, citation_indices)
+        prompt_text = self.prompt.format(context=context, question=q)
+
+        # Step 4: stream LLM
+        answer_parts: List[str] = []
+        buffered = ""
+        last_yield = time.monotonic()
+
+        try:
+            for chunk in self.llm.stream(prompt_text):
+                delta = getattr(chunk, "content", "") or ""
+                if not delta:
+                    continue
+                answer_parts.append(delta)
+                buffered += delta
+                now = time.monotonic()
+                if (now - last_yield) >= float(stream_yield_interval_s) or len(buffered) >= int(stream_min_chars):
+                    yield {"type": "answer", "answer": "".join(answer_parts)}
+                    buffered = ""
+                    last_yield = now
+
+            answer = "".join(answer_parts).strip()
+            if not answer:
+                response = self.llm.invoke(prompt_text)
+                answer = (response.content or "").strip()
+        except Exception as e:
+            logger.error(f"LLM streaming failed: {e}", exc_info=True)
+            try:
+                response = self.llm.invoke(prompt_text)
+                answer = (response.content or "").strip()
+            except Exception:
+                answer = "An error occurred while generating the answer. Please try again."
+
+        is_valid, invalid = self.citation_manager.validate_citations(answer)
+        elapsed = time.time() - start_time
+        source_dist = self.citation_manager.get_statistics().get("source_type_counts", {})
+
+        yield {
+            "type": "final",
+            "answer": answer,
+            "references": reranked_docs,
+            "metadata": {
+                "retrieval_strategy": retrieval_strategy_n,
+                "num_retrieved": num_retrieved,
+                "num_reranked": len(reranked_docs),
+                "retrieved_label": retrieved_label,
+                "reranked_label": reranked_label,
+                "citations_valid": is_valid,
+                "invalid_citations": invalid,
+                "source_type_distribution": source_dist,
+                "elapsed_time": elapsed,
+                "candidate_counts": (retrieval_debug.get("candidate_counts") if retrieval_debug else None),
+                "gated_counts": (retrieval_debug.get("gated_counts") if retrieval_debug else None),
+                "selected_counts": (retrieval_debug.get("selected_counts") if retrieval_debug else None),
+            },
+        }
+
+    def query(
+        self,
+        *,
+        question: str,
+        top_k: Optional[int] = None,
+        source_filters: Optional[List[str]] = None,
+        retrieval_strategy: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Non-streaming convenience wrapper."""
+        final: Dict[str, Any] = {}
+        for event in self.query_stream(
+            question=question,
+            top_k=top_k,
+            source_filters=source_filters,
+            retrieval_strategy=retrieval_strategy,
+            stream_yield_interval_s=999.0,
+            stream_min_chars=10**9,
+        ):
+            if event.get("type") == "final":
+                final = event
+        return final
+
+

radiology_rag/reranker.py

@@ -0,0 +1,143 @@
+"""
+Reranker client (OpenAI-compatible API).
+
+Expected endpoint:
+POST {base_url}/rerank
+{
+  "model": "...",
+  "query": "...",
+  "documents": ["...", "..."],
+  "top_n": 10
+}
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class RerankerConfig:
+    enabled: bool
+    base_url: str
+    api_key: str
+    model_name: str
+    top_k: int = 10
+
+
+class NoOpReranker:
+    def rerank(self, query: str, documents: List[Dict[str, Any]], top_k: Optional[int] = None) -> List[Dict[str, Any]]:
+        k = int(top_k or len(documents))
+        out = []
+        for d in documents[:k]:
+            dc = dict(d)
+            dc.setdefault("score", 0.0)
+            out.append(dc)
+        return out
+
+
+class APIReranker:
+    def __init__(self, cfg: RerankerConfig):
+        self.cfg = cfg
+
+    def rerank(self, query: str, documents: List[Dict[str, Any]], top_k: Optional[int] = None) -> List[Dict[str, Any]]:
+        if not documents:
+            return []
+
+        texts = [doc.get("content", "") for doc in documents]
+        k = int(top_k or self.cfg.top_k or len(documents))
+        k = max(1, min(k, len(documents)))
+
+        max_docs_per_request = int(os.getenv("RERANK_MAX_DOCS_PER_REQUEST", "64"))
+        max_docs_per_request = max(1, max_docs_per_request)
+
+        def _call_once(doc_texts: List[str], top_n: int) -> dict:
+            last_err: Optional[Exception] = None
+            for attempt in range(3):
+                try:
+                    resp = requests.post(
+                        f"{self.cfg.base_url.rstrip('/')}/rerank",
+                        json={
+                            "model": self.cfg.model_name,
+                            "query": query,
+                            "documents": doc_texts,
+                            "top_n": int(top_n),
+                        },
+                        headers={"Authorization": f"Bearer {self.cfg.api_key}"},
+                        timeout=30,
+                    )
+                    resp.raise_for_status()
+                    return resp.json() or {}
+                except Exception as e:
+                    last_err = e
+                    if attempt < 2:
+                        time.sleep(0.5 * (attempt + 1))
+            raise last_err or RuntimeError("Unknown reranker API error")
+
+        try:
+            if len(texts) <= max_docs_per_request:
+                result = _call_once(texts, top_n=k)
+                reranked_docs: List[Dict[str, Any]] = []
+                for item in result.get("results", []) or []:
+                    idx = item.get("index")
+                    score = float(item.get("relevance_score", 0.0) or 0.0)
+                    if idx is None:
+                        continue
+                    idx = int(idx)
+                    if idx < 0 or idx >= len(documents):
+                        continue
+                    dc = dict(documents[idx])
+                    dc["score"] = score
+                    reranked_docs.append(dc)
+                return reranked_docs
+
+            # Chunked: score all docs per chunk then globally sort.
+            scored: List[Dict[str, Any]] = []
+            for offset in range(0, len(texts), max_docs_per_request):
+                chunk_texts = texts[offset : offset + max_docs_per_request]
+                result = _call_once(chunk_texts, top_n=len(chunk_texts))
+                for item in result.get("results", []) or []:
+                    idx = item.get("index")
+                    if idx is None:
+                        continue
+                    global_idx = offset + int(idx)
+                    if global_idx < 0 or global_idx >= len(documents):
+                        continue
+                    score = float(item.get("relevance_score", 0.0) or 0.0)
+                    dc = dict(documents[global_idx])
+                    dc["score"] = score
+                    scored.append(dc)
+
+            scored.sort(key=lambda x: float(x.get("score", 0.0)), reverse=True)
+            return scored[:k]
+        except Exception as e:
+            logger.warning(f"Reranker API failed; falling back to no-op ordering. Error: {e}")
+            return NoOpReranker().rerank(query, documents, top_k=k)
+
+
+class RerankerService:
+    """High-level reranker wrapper (auto-disables if misconfigured)."""
+
+    def __init__(self, cfg: RerankerConfig):
+        self.cfg = cfg
+        if not cfg.enabled:
+            self._impl = NoOpReranker()
+            return
+        if not (cfg.api_key or "").strip():
+            logger.warning("Reranker enabled but RERANK_API_KEY is empty; disabling reranker.")
+            self._impl = NoOpReranker()
+            return
+        self._impl = APIReranker(cfg)
+
+    def rerank(self, query: str, documents: List[Dict[str, Any]], top_k: Optional[int] = None) -> List[Dict[str, Any]]:
+        return self._impl.rerank(query, documents, top_k=top_k)
+
+

radiology_rag/retrieval.py

@@ -0,0 +1,403 @@
+"""Vector retrieval + balanced multi-source retrieval (with Wikipedia)."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from langchain_chroma import Chroma
+
+from radiology_rag.config import Config
+from radiology_rag.doc_store import PersistentDocStore
+from radiology_rag.embedding import EmbeddingClient, EmbeddingConfig
+from radiology_rag.encyclopedia import WikipediaConfig, WikipediaEncyclopediaService
+from radiology_rag.reranker import RerankerService, RerankerConfig
+
+logger = logging.getLogger(__name__)
+
+
+def _dedupe_by_doc_id(docs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    seen = set()
+    out = []
+    for d in docs:
+        did = d.get("doc_id")
+        if not did or did in seen:
+            continue
+        seen.add(did)
+        out.append(d)
+    return out
+
+
+class RetrievalService:
+    """Retrieve parent documents from Chroma + SQLite doc store."""
+
+    def __init__(self, config: Config):
+        self.config = config
+
+        vector_db_path = config.get_str("storage.vector_db_path")
+        doc_store_path = config.get_str("storage.doc_store_path")
+
+        self.embedding_client = EmbeddingClient(
+            EmbeddingConfig(
+                base_url=config.get_str("embedding.api_base_url"),
+                api_key=config.get_str("embedding.api_key"),
+                model_name=config.get_str("embedding.model_name"),
+                batch_size=config.get_int("embedding.batch_size", 32),
+            )
+        )
+
+        self.doc_store = PersistentDocStore(doc_store_path, read_only=True)
+        self.vectorstore = Chroma(
+            collection_name="radiology_docs",
+            embedding_function=self.embedding_client.langchain_embeddings,
+            persist_directory=vector_db_path,
+        )
+
+        self.search_type = config.get_str("retrieval.search_type", "similarity")
+        self.chunk_fetch_multiplier = config.get_int("retrieval.chunk_fetch_multiplier", 6)
+
+    def retrieve_candidates_by_vector(
+        self,
+        *,
+        query_embedding: List[float],
+        candidate_k: int,
+        source_type_filter: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        k = max(1, int(candidate_k))
+        chunk_k = max(k * int(self.chunk_fetch_multiplier), k)
+
+        chunk_filter = {"source_type": source_type_filter} if source_type_filter else None
+
+        # Fetch chunks from vector store using the embedding vector
+        if self.search_type == "mmr":
+            mmr_lambda = self.config.get_float("retrieval.mmr_lambda", 0.5)
+            fetch_k = max(self.config.get_int("retrieval.mmr_fetch_k", 50), chunk_k)
+            try:
+                chunk_docs = self.vectorstore.max_marginal_relevance_search_by_vector(
+                    query_embedding,
+                    k=chunk_k,
+                    fetch_k=fetch_k,
+                    lambda_mult=mmr_lambda,
+                    filter=chunk_filter,
+                )
+            except TypeError:
+                chunk_docs = self.vectorstore.max_marginal_relevance_search_by_vector(
+                    query_embedding,
+                    k=chunk_k,
+                    fetch_k=fetch_k,
+                    lambda_mult=mmr_lambda,
+                )
+        else:
+            try:
+                chunk_docs = self.vectorstore.similarity_search_by_vector(
+                    query_embedding,
+                    k=chunk_k,
+                    filter=chunk_filter,
+                )
+            except TypeError:
+                chunk_docs = self.vectorstore.similarity_search_by_vector(
+                    query_embedding,
+                    k=chunk_k,
+                )
+
+        # Unique parent IDs
+        parent_ids: List[str] = []
+        seen = set()
+        for doc in chunk_docs:
+            parent_id = (doc.metadata or {}).get("parent_id")
+            if not parent_id or parent_id in seen:
+                continue
+            seen.add(parent_id)
+            parent_ids.append(parent_id)
+            if len(parent_ids) >= k:
+                break
+
+        # Hydrate parents from doc store
+        parent_docs = self.doc_store.mget(parent_ids)
+        results: List[Dict[str, Any]] = []
+        for doc_id, doc_content in zip(parent_ids, parent_docs):
+            if doc_content is None:
+                continue
+            complete = doc_content.get("complete_document", {}) or {}
+            results.append(
+                {
+                    "doc_id": doc_id,
+                    "source_type": doc_content.get("source_type", "") or "",
+                    "title": complete.get("title", "") or "",
+                    "content": doc_content.get("main_content", "") or "",
+                    "url": complete.get("url", "") or "",
+                    "metadata": complete.get("metadata", {}) or {},
+                    "score": 0.0,
+                }
+            )
+
+        logger.info(
+            f"Retrieved {len(results)} candidate parents (candidate_k={k}, source_type_filter={source_type_filter or 'ALL'})"
+        )
+        return results
+
+    def retrieve(
+        self,
+        *,
+        query: str,
+        top_k: int,
+        source_filters: Optional[List[str]] = None,
+    ) -> List[Dict[str, Any]]:
+        q = (query or "").strip()
+        if not q:
+            return []
+
+        query_embedding = self.embedding_client.embed_query(q)
+        k = max(1, int(top_k))
+
+        if not source_filters:
+            return self.retrieve_candidates_by_vector(query_embedding=query_embedding, candidate_k=k)
+
+        allowed = [s for s in source_filters if s in {"article", "case", "tutorial"}]
+        if not allowed:
+            # If only encyclopedia is selected, local retrieval yields none.
+            return []
+
+        if len(allowed) == 1:
+            return self.retrieve_candidates_by_vector(query_embedding=query_embedding, candidate_k=k, source_type_filter=allowed[0])
+
+        merged: List[Dict[str, Any]] = []
+        for st in allowed:
+            merged.extend(
+                self.retrieve_candidates_by_vector(query_embedding=query_embedding, candidate_k=k, source_type_filter=st)
+            )
+        merged = _dedupe_by_doc_id(merged)
+        return merged[:k]
+
+    def get_document_by_id(self, doc_id: str) -> Optional[Dict[str, Any]]:
+        doc_id = (doc_id or "").strip()
+        if not doc_id:
+            return None
+        docs = self.doc_store.mget([doc_id])
+        if not docs or not docs[0]:
+            return None
+        doc_content = docs[0]
+        complete = doc_content.get("complete_document", {}) or {}
+        return {
+            "doc_id": doc_id,
+            "source_type": doc_content.get("source_type", "") or "",
+            "title": complete.get("title", "") or "",
+            "content": doc_content.get("main_content", "") or "",
+            "url": complete.get("url", "") or "",
+            "metadata": complete.get("metadata", {}) or {},
+        }
+
+
+@dataclass
+class PerSourcePolicy:
+    candidate_k: int
+    max_k: int
+    min_score: float
+    required: bool = False
+
+
+@dataclass
+class BalancedMultiSourcePolicy:
+    total_top_k: int = 8
+    sources_priority: Sequence[str] = ("article", "case", "encyclopedia", "tutorial")
+    article: PerSourcePolicy = field(
+        default_factory=lambda: PerSourcePolicy(candidate_k=200, max_k=3, min_score=0.15, required=True)
+    )
+    case: PerSourcePolicy = field(
+        default_factory=lambda: PerSourcePolicy(candidate_k=200, max_k=3, min_score=0.15, required=True)
+    )
+    encyclopedia: PerSourcePolicy = field(
+        default_factory=lambda: PerSourcePolicy(candidate_k=8, max_k=2, min_score=0.15, required=True)
+    )
+    tutorial: PerSourcePolicy = field(
+        default_factory=lambda: PerSourcePolicy(candidate_k=20, max_k=2, min_score=0.50, required=False)
+    )
+
+
+class MultiSourceRetrievalService:
+    """Per-source recall + per-source rerank + gating + merge (includes Wikipedia)."""
+
+    def __init__(
+        self,
+        config: Config,
+        retrieval_service: Optional[RetrievalService] = None,
+        reranker_service: Optional[RerankerService] = None,
+        encyclopedia_service: Optional[WikipediaEncyclopediaService] = None,
+    ):
+        self.config = config
+        self.retrieval_service = retrieval_service or RetrievalService(config)
+
+        rr_cfg = RerankerConfig(
+            enabled=config.get_bool("reranker.enabled", True),
+            base_url=config.get_str("reranker.api_base_url"),
+            api_key=config.get_str("reranker.api_key"),
+            model_name=config.get_str("reranker.model_name"),
+            top_k=config.get_int("reranker.top_k", 10),
+        )
+        self.reranker_service = reranker_service or RerankerService(rr_cfg)
+
+        wiki_cfg = WikipediaConfig(
+            language=config.get_str("encyclopedia.wikipedia.language", "en"),
+            user_agent=config.get_str("encyclopedia.wikipedia.user_agent", "RadiologyRAG-Space/1.0"),
+            timeout_s=config.get_int("encyclopedia.wikipedia.timeout_s", 15),
+            max_chars_per_doc=config.get_int("encyclopedia.wikipedia.max_chars_per_doc", 2000),
+        )
+        self.encyclopedia_service = encyclopedia_service or WikipediaEncyclopediaService(wiki_cfg)
+
+    def _load_policy(self, total_top_k: Optional[int] = None) -> BalancedMultiSourcePolicy:
+        total = int(total_top_k or self.config.get_int("retrieval.multi_source.total_top_k", 8))
+        total = max(1, total)
+
+        def pol(name: str, default: PerSourcePolicy) -> PerSourcePolicy:
+            base = f"retrieval.multi_source.{name}"
+            return PerSourcePolicy(
+                candidate_k=self.config.get_int(f"{base}.candidate_k", default.candidate_k),
+                max_k=self.config.get_int(f"{base}.max_k", default.max_k),
+                min_score=self.config.get_float(f"{base}.min_score", default.min_score),
+                required=self.config.get_bool(f"{base}.required", default.required),
+            )
+
+        defaults = BalancedMultiSourcePolicy(total_top_k=total)
+        sources_priority = self.config.get("retrieval.multi_source.sources_priority", defaults.sources_priority)
+        sources_priority = tuple(sources_priority) if isinstance(sources_priority, (list, tuple)) else defaults.sources_priority
+
+        return BalancedMultiSourcePolicy(
+            total_top_k=total,
+            sources_priority=sources_priority,
+            article=pol("article", defaults.article),
+            case=pol("case", defaults.case),
+            encyclopedia=pol("encyclopedia", defaults.encyclopedia),
+            tutorial=pol("tutorial", defaults.tutorial),
+        )
+
+    @staticmethod
+    def _gate_and_trim(docs: List[Dict[str, Any]], policy: PerSourcePolicy) -> List[Dict[str, Any]]:
+        filtered = [d for d in docs if float(d.get("score", 0.0)) >= float(policy.min_score)]
+        return filtered[: max(0, int(policy.max_k))]
+
+    def retrieve(
+        self,
+        *,
+        query: str,
+        total_top_k: Optional[int] = None,
+        source_filters: Optional[List[str]] = None,
+        return_debug: bool = False,
+    ) -> Any:
+        q = (query or "").strip()
+        if not q:
+            return ([], {}) if return_debug else []
+
+        policy = self._load_policy(total_top_k=total_top_k)
+        allowed = set(source_filters) if source_filters else set(policy.sources_priority)
+        allowed = {s for s in allowed if s in {"article", "case", "tutorial", "encyclopedia"}}
+        if not allowed:
+            allowed = set(policy.sources_priority)
+
+        # Compute query embedding once for local sources
+        needs_local = any(s in allowed for s in {"article", "case", "tutorial"})
+        query_embedding = self.retrieval_service.embedding_client.embed_query(q) if needs_local else None
+
+        # 1) Recall candidates
+        candidates: Dict[str, List[Dict[str, Any]]] = {}
+        if "article" in allowed and query_embedding is not None:
+            candidates["article"] = self.retrieval_service.retrieve_candidates_by_vector(
+                query_embedding=query_embedding, candidate_k=policy.article.candidate_k, source_type_filter="article"
+            )
+        if "case" in allowed and query_embedding is not None:
+            candidates["case"] = self.retrieval_service.retrieve_candidates_by_vector(
+                query_embedding=query_embedding, candidate_k=policy.case.candidate_k, source_type_filter="case"
+            )
+        if "tutorial" in allowed and query_embedding is not None:
+            candidates["tutorial"] = self.retrieval_service.retrieve_candidates_by_vector(
+                query_embedding=query_embedding, candidate_k=policy.tutorial.candidate_k, source_type_filter="tutorial"
+            )
+        if "encyclopedia" in allowed:
+            candidates["encyclopedia"] = self.encyclopedia_service.retrieve(
+                q,
+                top_k=policy.encyclopedia.candidate_k,
+                max_chars_per_doc=self.config.get_int("encyclopedia.wikipedia.max_chars_per_doc", 2000),
+            )
+
+        # 2) Rerank per-source (full ordering; trim later)
+        reranked: Dict[str, List[Dict[str, Any]]] = {}
+        for src, docs in candidates.items():
+            if not docs:
+                reranked[src] = []
+                continue
+            rr = self.reranker_service.rerank(query=q, documents=docs, top_k=len(docs))
+            reranked[src] = rr
+
+        # 3) Gating
+        gated: Dict[str, List[Dict[str, Any]]] = {}
+        if "article" in reranked:
+            gated["article"] = self._gate_and_trim(reranked["article"], policy.article)
+        if "case" in reranked:
+            gated["case"] = self._gate_and_trim(reranked["case"], policy.case)
+        if "encyclopedia" in reranked:
+            gated["encyclopedia"] = self._gate_and_trim(reranked["encyclopedia"], policy.encyclopedia)
+
+        if "tutorial" in reranked:
+            tdocs = reranked["tutorial"]
+            best = float(tdocs[0].get("score", 0.0)) if tdocs else 0.0
+            if best >= policy.tutorial.min_score:
+                gated["tutorial"] = tdocs[: max(0, int(policy.tutorial.max_k))]
+            else:
+                gated["tutorial"] = []
+
+        # 4) Merge with global budget
+        selected: List[Dict[str, Any]] = []
+        selected_ids = set()
+
+        def _add(doc: Dict[str, Any]) -> None:
+            did = doc.get("doc_id")
+            if not did or did in selected_ids:
+                return
+            selected.append(doc)
+            selected_ids.add(did)
+
+        src_to_pol = {
+            "article": policy.article,
+            "case": policy.case,
+            "encyclopedia": policy.encyclopedia,
+            "tutorial": policy.tutorial,
+        }
+
+        # required pass
+        for src in policy.sources_priority:
+            if src not in allowed:
+                continue
+            p = src_to_pol[src]
+            if not p.required:
+                continue
+            if gated.get(src):
+                _add(gated[src][0])
+
+        # fill by global score
+        remaining_pool: List[Dict[str, Any]] = []
+        for docs in gated.values():
+            for d in docs:
+                if d.get("doc_id") not in selected_ids:
+                    remaining_pool.append(d)
+        remaining_pool.sort(key=lambda x: float(x.get("score", 0.0)), reverse=True)
+
+        for d in remaining_pool:
+            if len(selected) >= policy.total_top_k:
+                break
+            _add(d)
+
+        debug = {
+            "allowed_sources": sorted(list(allowed)),
+            "candidate_counts": {k: len(v) for k, v in candidates.items()},
+            "gated_counts": {k: len(v) for k, v in gated.items()},
+            "selected_counts": {},
+        }
+        for d in selected:
+            st = d.get("source_type", "unknown")
+            debug["selected_counts"][st] = debug["selected_counts"].get(st, 0) + 1
+
+        if return_debug:
+            return selected, debug
+        return selected
+
+

radiology_rag/ui.py

@@ -0,0 +1,423 @@
+"""Gradio UI for the Radiology RAG Space."""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import time
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+import gradio as gr
+
+from radiology_rag.config import Config
+from radiology_rag.index_bootstrap import ensure_index, read_manifest
+from radiology_rag.rag import RAGEngine
+
+logger = logging.getLogger(__name__)
+
+
+def _truncate(text: str, max_len: int) -> str:
+    s = (text or "").strip()
+    if len(s) <= max_len:
+        return s
+    return s[: max(0, max_len - 3)] + "..."
+
+
+def format_error_message(error: str) -> str:
+    return f"**⚠️ Error**\n\n{error}"
+
+
+def format_loading_message() -> str:
+    return "**🔄 Processing your query...**\n\nRetrieving relevant sources and generating an answer with citations."
+
+
+def format_reference_card(doc: Dict[str, Any], index: int) -> str:
+    title = doc.get("title", "Untitled") or "Untitled"
+    source_type = (doc.get("source_type") or "").upper()
+    url = doc.get("url", "") or ""
+    content = doc.get("content", "") or ""
+    score = float(doc.get("score", 0.0) or 0.0)
+
+    max_preview_length = 350
+    preview = _truncate(content, max_preview_length).replace("\n", " ")
+
+    type_colors = {
+        "ARTICLE": "#3b82f6",
+        "CASE": "#10b981",
+        "TUTORIAL": "#f59e0b",
+        "ENCYCLOPEDIA": "#8b5cf6",
+    }
+    color = type_colors.get(source_type, "#6b7280")
+
+    score_html = f"<span style='color:#6b7280;font-size:12px;'>Score: {score:.3f}</span>" if score > 0 else ""
+    url_html = (
+        f"<p style='margin:0 0 8px 0;font-size:12px;'><a href='{url}' target='_blank' "
+        f"style='color:#3b82f6;text-decoration:none;'>🔗 View Source</a></p>"
+        if url
+        else ""
+    )
+
+    return f"""
+    <div id="ref-{index}" style="border:1px solid #e5e7eb;border-radius:8px;padding:16px;margin-bottom:16px;background:white;scroll-margin-top:90px;">
+      <div style="display:flex;align-items:center;gap:8px;margin-bottom:12px;flex-wrap:wrap;">
+        <span style="background:{color};color:white;padding:4px 12px;border-radius:12px;font-size:12px;font-weight:600;">
+          {source_type or "SOURCE"}
+        </span>
+        <span style="background:#f3f4f6;color:#374151;padding:4px 12px;border-radius:12px;font-size:12px;font-weight:600;">
+          [{index}]
+        </span>
+        {score_html}
+      </div>
+      <h3 style="margin:0 0 8px 0;color:#111827;font-size:18px;">{title}</h3>
+      {url_html}
+      <p style="margin:0;color:#4b5563;font-size:14px;line-height:1.5;">{preview}</p>
+    </div>
+    """
+
+
+def format_reference_panel(references: List[Dict[str, Any]]) -> str:
+    if not references:
+        return "<p style='color:#6b7280;text-align:center;padding:20px;'>No references available</p>"
+    html_parts = ['<div style="max-height: 600px; overflow-y: auto;">']
+    for i, doc in enumerate(references, 1):
+        html_parts.append(format_reference_card(doc, i))
+    html_parts.append("</div>")
+    return "".join(html_parts)
+
+
+def format_statistics(metadata: Dict[str, Any]) -> str:
+    num_retrieved = int(metadata.get("num_retrieved", 0) or 0)
+    num_reranked = int(metadata.get("num_reranked", 0) or 0)
+    source_dist = metadata.get("source_type_distribution", {}) or {}
+    retrieved_label = metadata.get("retrieved_label", "Retrieved")
+    reranked_label = metadata.get("reranked_label", "After Reranking")
+    elapsed = float(metadata.get("elapsed_time", 0.0) or 0.0)
+    strategy = metadata.get("retrieval_strategy", "")
+
+    chips = "".join(
+        [
+            f"<span style='display:inline-block;background:#e5e7eb;color:#111827;padding:4px 8px;border-radius:4px;margin-right:8px;font-size:12px;line-height:1.2;'>{k}: {v}</span>"
+            for k, v in source_dist.items()
+        ]
+    )
+
+    return f"""
+    <div style="background:#f9fafb;padding:16px;border-radius:8px;margin-top:16px;">
+      <h4 style="margin:0 0 12px 0;color:#374151;font-size:14px;">📊 Query Statistics</h4>
+      <div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;">
+        <div>
+          <p style="margin:0;color:#6b7280;font-size:12px;">{retrieved_label}</p>
+          <p style="margin:0;color:#111827;font-size:20px;font-weight:600;">{num_retrieved}</p>
+        </div>
+        <div>
+          <p style="margin:0;color:#6b7280;font-size:12px;">{reranked_label}</p>
+          <p style="margin:0;color:#111827;font-size:20px;font-weight:600;">{num_reranked}</p>
+        </div>
+        <div>
+          <p style="margin:0;color:#6b7280;font-size:12px;">Elapsed</p>
+          <p style="margin:0;color:#111827;font-size:20px;font-weight:600;">{elapsed:.2f}s</p>
+        </div>
+      </div>
+      <div style="margin-top:12px;">
+        <p style="margin:0 0 6px 0;color:#6b7280;font-size:12px;">Retrieval Strategy: <code>{strategy}</code></p>
+        <p style="margin:0 0 4px 0;color:#6b7280;font-size:12px;">Source Distribution:</p>
+        {chips if chips else "<span style='color:#6b7280;font-size:12px;'>N/A</span>"}
+      </div>
+    </div>
+    """
+
+
+def create_settings_accordion(
+    *,
+    default_strategy: str,
+    default_temperature: float,
+    default_sources: List[str],
+) -> Tuple[gr.Radio, gr.Slider, gr.CheckboxGroup]:
+    with gr.Accordion("⚙️ Advanced Settings", open=False):
+        gr.Markdown(
+            "#### Retrieval Strategy\n"
+            "- **default**: one mixed retrieval + single rerank (fast)\n"
+            "- **balanced_multi_source**: per-source recall + per-source rerank + Wikipedia (more diverse)\n"
+        )
+
+        retrieval_strategy = gr.Radio(
+            choices=["default", "balanced_multi_source"],
+            value=default_strategy,
+            label="Retrieval Strategy",
+        )
+
+        temperature_slider = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=float(default_temperature),
+            step=0.1,
+            label="LLM Temperature",
+        )
+
+        source_filter = gr.CheckboxGroup(
+            choices=["article", "case", "tutorial", "encyclopedia"],
+            value=default_sources,
+            label="Filter by Source Type",
+        )
+
+    return retrieval_strategy, temperature_slider, source_filter
+
+
+class RadiologyRAGApp:
+    def __init__(self, config_path: str):
+        self.config = Config(config_path)
+        self.startup_error: Optional[str] = None
+        self.startup_warnings: List[str] = []
+        self.index_manifest: Optional[Dict[str, Any]] = None
+        self.rag_engine: Optional[RAGEngine] = None
+
+        # Validate required secrets
+        missing: List[str] = []
+        if not self.config.get_str("embedding.api_key"):
+            missing.append("EMBED_API_KEY")
+        if not self.config.get_str("llm.api_key"):
+            missing.append("LLM_API_KEY")
+        if missing:
+            self.startup_error = (
+                "Missing required Hugging Face Space Secrets: "
+                + ", ".join([f"`{m}`" for m in missing])
+                + ".\n\nPlease set them in the Space **Settings → Secrets** and restart the Space."
+            )
+            return
+
+        # Reranker is optional; warn if enabled but missing key
+        if self.config.get_bool("reranker.enabled", True) and not self.config.get_str("reranker.api_key"):
+            self.startup_warnings.append(
+                "Reranker is enabled but `RERANK_API_KEY` is missing. Reranking will be disabled (fallback to no-op)."
+            )
+
+        # Ensure index exists locally (download if needed)
+        try:
+            idx = ensure_index(
+                repo_id=self.config.get_str("index.repo_id"),
+                revision=self.config.get_str("index.revision", "main") or None,
+                target_vector_db_path=self.config.get_str("storage.vector_db_path"),
+                target_doc_store_path=self.config.get_str("storage.doc_store_path"),
+                storage_dir=str(Path(self.config.get_str("storage.doc_store_path")).parent),
+            )
+            self.index_manifest = read_manifest(idx.manifest_path)
+
+            # Optional: warn if embedding model differs
+            if self.index_manifest:
+                idx_model = (
+                    (self.index_manifest.get("embedding") or {}).get("model_name")
+                    or self.index_manifest.get("embedding_model")
+                    or ""
+                )
+                cfg_model = self.config.get_str("embedding.model_name")
+                if idx_model and cfg_model and idx_model != cfg_model:
+                    self.startup_warnings.append(
+                        f"Index embedding model mismatch: index='{idx_model}' vs config='{cfg_model}'. "
+                        "For best results, rebuild the index with the same embedding model."
+                    )
+
+        except Exception as e:
+            # Try to provide actionable guidance for common HF Hub errors.
+            repo_id = self.config.get_str("index.repo_id")
+            try:
+                from huggingface_hub.utils import (  # type: ignore
+                    GatedRepoError,
+                    HfHubHTTPError,
+                    RepositoryNotFoundError,
+                )
+
+                if isinstance(e, RepositoryNotFoundError):
+                    self.startup_error = (
+                        f"Index dataset repo not found: `{repo_id}`.\n\n"
+                        "If you haven't uploaded the prebuilt index yet, build and publish it locally:\n"
+                        "1) `pip install -r requirements-dev.txt`\n"
+                        "2) `python scripts/build_vector_db.py --config config/default_config.yaml --source huggingface --dataset ZhangNy/radiology-dataset --output-dir ./index_out`\n"
+                        f"3) `python scripts/publish_index_to_hf.py --repo {repo_id} --folder ./index_out --token $HF_TOKEN`\n\n"
+                        "Or set `RAG_INDEX_REPO_ID` to an existing index repo."
+                    )
+                    return
+                if isinstance(e, GatedRepoError):
+                    self.startup_error = (
+                        f"Index dataset repo is gated/private: `{repo_id}`.\n\n"
+                        "Make sure the repo is public, or provide authentication (HF token) in the environment."
+                    )
+                    return
+                if isinstance(e, HfHubHTTPError):
+                    self.startup_error = (
+                        f"Failed to download index from `{repo_id}`.\n\n"
+                        f"HF Hub error: {e}"
+                    )
+                    return
+            except Exception:
+                # If importing HF-specific exceptions fails, fall back to generic message.
+                pass
+
+            self.startup_error = (
+                f"Failed to prepare index from `{repo_id}`.\n\n"
+                f"Error: {e}"
+            )
+            return
+
+        # Build RAG engine
+        try:
+            self.rag_engine = RAGEngine(self.config)
+        except Exception as e:
+            self.startup_error = f"Failed to initialize RAG engine: {e}"
+            return
+
+    def process_query(
+        self,
+        question: str,
+        temperature: float,
+        source_filters: List[str],
+        retrieval_strategy: str,
+    ) -> Iterator[Tuple[str, str, str]]:
+        if self.startup_error:
+            yield format_error_message(self.startup_error), "", ""
+            return
+        if self.rag_engine is None:
+            yield format_error_message("RAG engine not initialized."), "", ""
+            return
+
+        q = (question or "").strip()
+        if not q:
+            yield format_error_message("Please enter a question."), "", ""
+            return
+
+        # Update LLM temperature on the fly
+        try:
+            self.rag_engine.llm.temperature = float(temperature)
+        except Exception:
+            pass
+
+        sources = source_filters or []
+        loading_md = (
+            f"{format_loading_message()}\n\n"
+            f"**Retrieval Strategy**: `{retrieval_strategy}`\n\n"
+            f"**Sources**: `{', '.join(sources) if sources else 'ALL'}`"
+        )
+        loading_refs = "<p style='color:#6b7280;text-align:center;padding:20px;'>Retrieving & reranking...</p>"
+        loading_stats = "<p style='color:#6b7280;padding:10px;'>Working...</p>"
+        yield loading_md, loading_refs, loading_stats
+
+        start_time = time.time()
+        last_partial = ""
+
+        try:
+            for event in self.rag_engine.query_stream(
+                question=q,
+                source_filters=sources if sources else None,
+                retrieval_strategy=retrieval_strategy,
+            ):
+                etype = (event or {}).get("type")
+                if etype == "answer":
+                    partial = (event.get("answer") or "")
+                    if partial and partial != last_partial:
+                        # Make citations clickable: [1] -> [1](#ref-1)
+                        answer_md = re.sub(r"\[(\d+)\](?!\()", r"[\1](#ref-\1)", partial)
+                        last_partial = partial
+                        yield answer_md, loading_refs, loading_stats
+                elif etype == "final":
+                    meta = event.get("metadata") or {}
+                    # If engine didn't populate elapsed_time (it does), we fill it.
+                    meta.setdefault("elapsed_time", time.time() - start_time)
+
+                    final_answer = (event.get("answer") or "")
+                    answer_md = re.sub(r"\[(\d+)\](?!\()", r"[\1](#ref-\1)", final_answer)
+                    references_html = format_reference_panel(event.get("references") or [])
+                    stats_html = format_statistics(meta)
+                    yield answer_md, references_html, stats_html
+                    return
+
+            yield format_error_message("No response was generated. Please try again."), "", ""
+        except Exception as e:
+            logger.error(f"Error processing query: {e}", exc_info=True)
+            yield format_error_message(f"An error occurred: {e}"), "", ""
+
+    def create_interface(self) -> gr.Blocks:
+        title = self.config.get_str("ui.title", "Radiology RAG")
+        description = self.config.get_str("ui.description", "")
+        theme = self.config.get_str("ui.theme", "soft")
+        default_strategy = self.config.get_str("retrieval.strategy", "balanced_multi_source")
+        default_sources = self.config.get("retrieval.source_filters", ["article", "case", "tutorial", "encyclopedia"])
+        if not isinstance(default_sources, list):
+            default_sources = ["article", "case", "tutorial", "encyclopedia"]
+        default_temp = self.config.get_float("llm.temperature", 0.7)
+
+        with gr.Blocks(title=title, theme=theme) as interface:
+            gr.Markdown(f"# {title}")
+            if description:
+                gr.Markdown(description)
+
+            if self.startup_error:
+                gr.Markdown(format_error_message(self.startup_error))
+                gr.Markdown(
+                    "### Required Secrets\n"
+                    "- `EMBED_API_KEY`\n"
+                    "- `LLM_API_KEY`\n\n"
+                    "Optional (recommended):\n"
+                    "- `RERANK_API_KEY`\n"
+                )
+                return interface
+
+            if self.startup_warnings:
+                gr.Markdown("### ⚠️ Startup Warnings")
+                gr.Markdown("\n".join([f"- {w}" for w in self.startup_warnings]))
+
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.Markdown("### Ask a Question")
+                    question_input = gr.Textbox(
+                        label="Your Question",
+                        placeholder="e.g., What is achalasia and how is it diagnosed?",
+                        lines=3,
+                    )
+
+                    retrieval_strategy, temperature_slider, source_filter = create_settings_accordion(
+                        default_strategy=default_strategy,
+                        default_temperature=default_temp,
+                        default_sources=default_sources,
+                    )
+
+                    submit_btn = gr.Button("Search & Answer", variant="primary", size="lg")
+
+                    gr.Markdown("### Example Questions")
+                    gr.Examples(
+                        examples=[
+                            ["What is achalasia and how is it diagnosed on imaging?"],
+                            ["Explain the imaging findings in Barrett's esophagus"],
+                            ["What are the characteristics of a Zenker's diverticulum?"],
+                            ["Describe the CT findings of esophageal cancer"],
+                        ],
+                        inputs=[question_input],
+                        label="Click an example to try it",
+                    )
+
+                with gr.Column(scale=2):
+                    gr.Markdown("### Answer (with citations)")
+                    answer_output = gr.Markdown(value="*Your answer will appear here...*")
+                    stats_output = gr.HTML(label="Statistics")
+                    gr.Markdown("### Retrieved References")
+                    references_output = gr.HTML(
+                        value="<p style='color:#6b7280;text-align:center;padding:20px;'>References will appear here...</p>"
+                    )
+
+            submit_btn.click(
+                fn=self.process_query,
+                inputs=[question_input, temperature_slider, source_filter, retrieval_strategy],
+                outputs=[answer_output, references_output, stats_output],
+            )
+
+            gr.Markdown("---")
+            with gr.Accordion("About", open=False):
+                gr.Markdown(
+                    "This Space demonstrates a radiology RAG system using a prebuilt vector index "
+                    f"(`{self.config.get_str('index.repo_id')}`) and external APIs for embeddings/LLM.\n\n"
+                    "**Disclaimer**: Educational use only. Always consult qualified professionals for clinical decisions."
+                )
+
+        return interface
+
+

requirements-dev.txt

@@ -0,0 +1,19 @@
+# Dev / offline build dependencies (NOT used by Spaces runtime).
+# Includes dataset loading + optional local embedding/rerank tooling.
+
+-r requirements.txt
+
+datasets>=2.16.0
+tqdm>=4.66.0
+pillow>=10.0.0
+
+# Needed for index build (text splitting + Document objects)
+langchain>=0.1.0
+langchain-text-splitters>=0.0.1
+
+# Optional: local models for advanced users (large)
+sentence-transformers>=2.3.0
+torch>=2.0.0
+transformers>=4.36.0
+
+

requirements.txt

@@ -0,0 +1,21 @@
+# Runtime dependencies for Hugging Face Spaces (CPU-friendly, API-first).
+# Keep this minimal to improve build stability.
+
+# Web UI
+gradio==4.16.0
+gradio_client==0.8.1
+
+# RAG core
+langchain-core>=0.1.0
+langchain-openai>=0.0.5
+langchain-chroma>=0.1.0
+chromadb>=0.4.22
+
+# Index download (from HF Hub)
+huggingface-hub>=0.20.0
+
+# Utilities
+pyyaml>=6.0
+requests>=2.31.0
+
+

scripts/build_vector_db.py

@@ -0,0 +1,219 @@
+"""
+Build a Chroma + SQLite index for this RAG system (offline / advanced users).
+
+The index output folder is compatible with the Space runtime bootstrap:
+  <output_dir>/
+    chroma_db/
+    doc_store.db
+    manifest.json
+
+Examples:
+
+1) Build from HF dataset directly (streaming is not supported for save_to_disk-based build):
+  python scripts/build_vector_db.py \
+    --config config/default_config.yaml \
+    --source huggingface \
+    --dataset ZhangNy/radiology-dataset \
+    --output-dir ./index_out
+
+2) Build from local saved dataset:
+  python scripts/build_vector_db.py \
+    --config config/default_config.yaml \
+    --source local \
+    --local-path ./hf_dataset_prepared \
+    --output-dir ./index_out
+
+Notes:
+- Embedding model used at build time must match query-time embeddings used in the Space,
+  otherwise retrieval quality will degrade.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import shutil
+import time
+from collections import Counter
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+# Allow running as `python scripts/*.py` without installing the package.
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+
+def _clean_text(text: str) -> str:
+    # Remove markdown hyperlinks [text](url) -> text
+    import re
+
+    t = re.sub(r"\[(.*?)\]\(.*?\)", r"\1", text or "")
+    return t.replace("\xa0", " ")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Build vector index (Chroma + SQLite doc store)")
+    parser.add_argument("--config", type=str, default="config/default_config.yaml", help="Config YAML path")
+    parser.add_argument("--source", choices=["local", "huggingface"], default="huggingface")
+    parser.add_argument("--local-path", type=str, default=None, help="Path to dataset saved via save_to_disk()")
+    parser.add_argument("--dataset", type=str, default="ZhangNy/radiology-dataset", help="HF dataset repo id")
+    parser.add_argument("--split", type=str, default="train")
+    parser.add_argument("--limit", type=int, default=None, help="Limit number of documents (debug)")
+    parser.add_argument("--output-dir", type=str, default="./index_out", help="Output directory for index artifacts")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite output dir if exists")
+    args = parser.parse_args()
+
+    from datasets import load_dataset, load_from_disk
+    from langchain_chroma import Chroma
+    from langchain_core.documents import Document
+    from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+    from radiology_rag.config import Config
+    from radiology_rag.doc_store import PersistentDocStore
+    from radiology_rag.embedding import EmbeddingClient, EmbeddingConfig
+
+    cfg = Config(args.config)
+
+    out_dir = Path(args.output_dir)
+    chroma_dir = out_dir / "chroma_db"
+    doc_db = out_dir / "doc_store.db"
+    manifest_path = out_dir / "manifest.json"
+
+    if out_dir.exists() and args.overwrite:
+        shutil.rmtree(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    if chroma_dir.exists() or doc_db.exists():
+        if not args.overwrite:
+            raise SystemExit(f"Output dir already has index artifacts. Use --overwrite. ({out_dir})")
+
+    # Load dataset
+    if args.source == "local":
+        if not args.local_path:
+            raise SystemExit("--local-path is required when --source local")
+        dataset = load_from_disk(args.local_path)
+    else:
+        dataset = load_dataset(args.dataset, split=args.split)
+
+    if args.limit:
+        dataset = dataset.select(range(min(int(args.limit), len(dataset))))
+
+    # Splitter
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=cfg.get_int("processing.chunk_size", 1024),
+        chunk_overlap=cfg.get_int("processing.chunk_overlap", 200),
+        separators=cfg.get("processing.separators", ["\n\n", "\n", " "]),
+        keep_separator=cfg.get_bool("processing.keep_separator", True),
+    )
+
+    # Embeddings
+    emb = EmbeddingClient(
+        EmbeddingConfig(
+            base_url=cfg.get_str("embedding.api_base_url"),
+            api_key=cfg.get_str("embedding.api_key"),
+            model_name=cfg.get_str("embedding.model_name"),
+            batch_size=cfg.get_int("embedding.batch_size", 32),
+        )
+    )
+
+    # Storage
+    doc_store = PersistentDocStore(str(doc_db), read_only=False)
+    vectorstore = Chroma(
+        collection_name="radiology_docs",
+        embedding_function=emb.langchain_embeddings,
+        persist_directory=str(chroma_dir),
+    )
+
+    # Build
+    start = time.time()
+    parent_pairs: List[Tuple[str, Dict[str, Any]]] = []
+    child_docs: List[Document] = []
+    counts = Counter()
+
+    for item in dataset:
+        doc_id = (item.get("doc_id") or "").strip()
+        if not doc_id:
+            continue
+        source_type = (item.get("source_type") or "").strip()
+        title = (item.get("title") or "").strip()
+        content = _clean_text(item.get("content") or "")
+        url = (item.get("url") or "").strip()
+        metadata = item.get("metadata") or {}
+
+        counts[source_type or "unknown"] += 1
+
+        # Parent document record
+        parent_pairs.append(
+            (
+                doc_id,
+                {
+                    "complete_document": {
+                        "doc_id": doc_id,
+                        "title": title,
+                        "content": content,
+                        "url": url,
+                        "metadata": metadata,
+                    },
+                    "main_content": content,
+                    "images": [],  # not used in this Space
+                    "source_type": source_type,
+                },
+            )
+        )
+
+        # Child chunks for vector store
+        chunks = splitter.split_text(content)
+        total = len(chunks)
+        for i, chunk in enumerate(chunks):
+            child_docs.append(
+                Document(
+                    page_content=chunk,
+                    metadata={
+                        "doc_id": f"{doc_id}_chunk_{i}",
+                        "parent_id": doc_id,
+                        "source_type": source_type,
+                        "title": title,
+                        "chunk_index": i,
+                        "total_chunks": total,
+                    },
+                )
+            )
+
+    # Persist parent docs
+    doc_store.mset(parent_pairs)
+
+    # Add chunks in batches
+    batch_size = int(cfg.get_int("processing.batch_size", 32))
+    for i in range(0, len(child_docs), batch_size):
+        vectorstore.add_documents(child_docs[i : i + batch_size])
+
+    elapsed = time.time() - start
+
+    # Manifest
+    manifest = {
+        "built_at": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()),
+        "seconds": elapsed,
+        "dataset": {"source": args.source, "dataset": args.dataset, "split": args.split, "limit": args.limit},
+        "embedding": {"type": "api", "model_name": cfg.get_str("embedding.model_name"), "base_url": cfg.get_str("embedding.api_base_url")},
+        "processing": {
+            "chunk_size": cfg.get_int("processing.chunk_size", 1024),
+            "chunk_overlap": cfg.get_int("processing.chunk_overlap", 200),
+        },
+        "counts_by_source_type": dict(counts),
+        "artifacts": {"chroma_dir": "chroma_db", "doc_store": "doc_store.db"},
+    }
+    with open(manifest_path, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, ensure_ascii=False, indent=2)
+
+    print(f"✓ Index built at: {out_dir}")
+    print(f"  - documents: {sum(counts.values())}  (by type: {dict(counts)})")
+    print(f"  - chunks: {len(child_docs)}")
+    print(f"  - elapsed: {elapsed:.1f}s")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

scripts/download_hf_dataset.py

@@ -0,0 +1,43 @@
+"""
+Download the public dataset from Hugging Face and save it to disk.
+
+Example:
+  python scripts/download_hf_dataset.py \
+    --dataset ZhangNy/radiology-dataset \
+    --split train \
+    --output ./hf_dataset_prepared
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Allow running as `python scripts/*.py` without installing the package.
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Download HF dataset to local disk")
+    parser.add_argument("--dataset", type=str, default="ZhangNy/radiology-dataset", help="HF dataset repo id")
+    parser.add_argument("--split", type=str, default="train", help="Dataset split")
+    parser.add_argument("--output", type=str, default="./hf_dataset_prepared", help="Output directory (save_to_disk)")
+    parser.add_argument("--cache-dir", type=str, default=None, help="Optional datasets cache dir")
+    args = parser.parse_args()
+
+    from datasets import load_dataset
+
+    out_dir = Path(args.output)
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+
+    ds = load_dataset(args.dataset, split=args.split, cache_dir=args.cache_dir)
+    ds.save_to_disk(str(out_dir))
+    print(f"✓ Saved dataset to: {out_dir}  (rows={len(ds)})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

scripts/package_existing_storage.py

@@ -0,0 +1,110 @@
+"""
+Package an existing local index folder (e.g. rebuild_1217/storage) into a clean index folder.
+
+This is the fastest path if you already built the index locally and want to publish it
+to Hugging Face without rebuilding embeddings.
+
+Input (example):
+  /path/to/storage/
+    chroma_db/
+    doc_store.db
+    images/        # optional (ignored)
+
+Output:
+  ./index_out/
+    chroma_db/
+    doc_store.db
+    manifest.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import shutil
+import sqlite3
+import time
+from pathlib import Path
+from typing import Dict
+
+# Allow running as `python scripts/*.py` without installing the package.
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+
+def _count_by_source_type(doc_store_db: Path) -> Dict[str, int]:
+    counts: Dict[str, int] = {}
+    conn = sqlite3.connect(str(doc_store_db))
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT source_type, COUNT(*) FROM documents GROUP BY source_type")
+        for source_type, count in cur.fetchall():
+            counts[str(source_type)] = int(count)
+    finally:
+        conn.close()
+    return counts
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Package existing index storage into index_out (no images)")
+    parser.add_argument("--storage", type=str, required=True, help="Existing storage dir containing chroma_db/ + doc_store.db")
+    parser.add_argument("--output-dir", type=str, default="./index_out", help="Output folder")
+    parser.add_argument("--config", type=str, default="config/default_config.yaml", help="Config YAML (for embedding metadata)")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite output dir if exists")
+    args = parser.parse_args()
+
+    from radiology_rag.config import Config
+
+    storage = Path(args.storage)
+    src_chroma = storage / "chroma_db"
+    src_doc = storage / "doc_store.db"
+    if not src_chroma.exists() or not src_doc.exists():
+        raise SystemExit(f"Storage missing required files: {src_chroma} / {src_doc}")
+
+    out_dir = Path(args.output_dir)
+    out_chroma = out_dir / "chroma_db"
+    out_doc = out_dir / "doc_store.db"
+    out_manifest = out_dir / "manifest.json"
+
+    if out_dir.exists() and args.overwrite:
+        shutil.rmtree(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    if out_chroma.exists() or out_doc.exists():
+        if not args.overwrite:
+            raise SystemExit(f"Output already exists. Use --overwrite. ({out_dir})")
+
+    # Copy artifacts (exclude images/)
+    if out_chroma.exists():
+        shutil.rmtree(out_chroma, ignore_errors=True)
+    shutil.copytree(src_chroma, out_chroma, dirs_exist_ok=False)
+    shutil.copy2(src_doc, out_doc)
+
+    cfg = Config(args.config)
+    counts = _count_by_source_type(out_doc)
+    manifest = {
+        "packaged_at": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()),
+        "source_storage": str(storage),
+        "embedding": {"model_name": cfg.get_str("embedding.model_name"), "type": cfg.get_str("embedding.type", "api")},
+        "processing": {
+            "chunk_size": cfg.get_int("processing.chunk_size", 1024),
+            "chunk_overlap": cfg.get_int("processing.chunk_overlap", 200),
+        },
+        "counts_by_source_type": counts,
+        "artifacts": {"chroma_dir": "chroma_db", "doc_store": "doc_store.db"},
+        "images_included": False,
+    }
+    with open(out_manifest, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, ensure_ascii=False, indent=2)
+
+    print(f"✓ Packaged index to: {out_dir}")
+    print(f"  - chroma_db: {out_chroma}")
+    print(f"  - doc_store: {out_doc}")
+    print(f"  - manifest:  {out_manifest}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

scripts/publish_index_to_hf.py

@@ -0,0 +1,78 @@
+"""
+Publish a built index folder to Hugging Face Datasets.
+
+Example:
+  python scripts/publish_index_to_hf.py \
+    --repo ZhangNy/radiology-index-qwen3-embedding-0.6b \
+    --folder ./index_out \
+    --token $HF_TOKEN
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Allow running as `python scripts/*.py` without installing the package.
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Upload index artifacts to HF datasets repo")
+    parser.add_argument("--repo", type=str, required=True, help="HF dataset repo id, e.g. user/my-index")
+    parser.add_argument("--folder", type=str, required=True, help="Local folder containing chroma_db/ + doc_store.db")
+    parser.add_argument("--token", type=str, default=None, help="HF token (or set HF_TOKEN env)")
+    parser.add_argument("--private", action="store_true", help="Create repo as private")
+    parser.add_argument("--revision", type=str, default="main", help="Target revision/branch")
+    parser.add_argument(
+        "--ignore",
+        type=str,
+        default="",
+        help="Comma-separated ignore patterns for upload_folder (e.g. 'images/**,**/images/**')",
+    )
+    args = parser.parse_args()
+
+    from huggingface_hub import HfApi
+
+    token = args.token or None
+    if token is None:
+        import os
+
+        token = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_TOKEN")
+
+    if not token:
+        raise SystemExit("Missing token. Provide --token or set HF_TOKEN.")
+
+    folder = Path(args.folder)
+    if not folder.exists():
+        raise SystemExit(f"Folder not found: {folder}")
+
+    api = HfApi()
+    api.create_repo(
+        repo_id=args.repo,
+        repo_type="dataset",
+        private=bool(args.private),
+        exist_ok=True,
+        token=token,
+    )
+
+    api.upload_folder(
+        repo_id=args.repo,
+        repo_type="dataset",
+        folder_path=str(folder),
+        path_in_repo="",
+        token=token,
+        revision=args.revision,
+        commit_message="Upload prebuilt radiology RAG index",
+        ignore_patterns=[p.strip() for p in (args.ignore or "").split(",") if p.strip()] or None,
+    )
+
+    print(f"✓ Uploaded index folder to HF dataset repo: {args.repo}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+