Delete app.py

app.py

@@ -1,845 +0,0 @@
-"""
-RAG (Retrieval-Augmented Generation) Application
-==================================================
-A full-featured RAG system with:
-- Document processing (PDF, HTML, DOCX, TXT, MD)
-- Vector database (ChromaDB with persistent storage)
-- Hybrid search (semantic + BM25 keyword search)
-- Conversation memory (last 10 exchanges)
-- Streaming LLM responses with source citations
-- Gradio-based conversational UI
-
-Requirements (install via pip):
-    pip install chromadb sentence-transformers gradio openai pymupdf python-docx \
-                beautifulsoup4 rank_bm25 nltk tiktoken numpy
-
-Usage:
-    1. Place 50+ documents in a ./documents/ folder (PDF, HTML, DOCX, TXT, MD)
-    2. Set your OpenAI API key:  export OPENAI_API_KEY="sk-..."
-    3. Run:  python rag_app.py
-    4. Open the Gradio URL in your browser
-"""
-
-import os
-import re
-import json
-import hashlib
-import logging
-import textwrap
-from pathlib import Path
-from typing import Optional
-from dataclasses import dataclass, field
-from collections import defaultdict
-
-import numpy as np
-
-# -- Document parsing --
-import fitz  # PyMuPDF
-from docx import Document as DocxDocument
-from bs4 import BeautifulSoup
-
-# -- NLP / chunking --
-import nltk
-from nltk.tokenize import sent_tokenize
-
-# -- Embeddings & vector DB --
-from sentence_transformers import SentenceTransformer
-import chromadb
-from chromadb.config import Settings
-
-# -- BM25 keyword search --
-from rank_bm25 import BM25Okapi
-
-# -- LLM --
-import openai
-
-# -- UI --
-import gradio as gr
-
-# ---------------------------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------------------------
-
-@dataclass
-class Config:
-    """Central configuration for the RAG pipeline."""
-
-    # Paths
-    documents_dir: str = "./documents"
-    chroma_persist_dir: str = "./chroma_db"
-
-    # Chunking
-    chunk_size: int = 512          # target tokens per chunk (sentence-based)
-    chunk_overlap: int = 64        # overlap tokens between consecutive chunks
-    min_chunk_length: int = 40     # discard chunks shorter than this (chars)
-
-    # Embedding model (runs locally via sentence-transformers)
-    embedding_model: str = "all-MiniLM-L6-v2"
-    chroma_collection: str = "rag_docs"
-
-    # Retrieval
-    top_k_semantic: int = 20       # initial semantic retrieval
-    top_k_bm25: int = 20           # initial BM25 retrieval
-    top_k_final: int = 5           # after hybrid merge / re-rank
-
-    # Hybrid search weight  (0 = pure BM25, 1 = pure semantic)
-    semantic_weight: float = 0.6
-
-    # LLM
-    openai_model: str = "gpt-4o-mini"
-    temperature: float = 0.2
-    max_context_tokens: int = 6000
-    system_prompt: str = textwrap.dedent("""\
-        You are a knowledgeable assistant. Answer the user's question using ONLY
-        the provided context passages. If the context does not contain enough
-        information, say so honestly.
-
-        Rules:
-        - Cite sources using [Source N] notation after each claim.
-        - Be concise but thorough.
-        - If multiple sources agree, prefer the most specific one.
-        - For follow-up questions, use conversation history for context.
-    """)
-
-    # Conversation memory
-    memory_length: int = 10  # number of past exchanges to keep
-
-    # Server
-    server_port: int = 7860
-    share: bool = True  # set True for public URL via Gradio
-
-
-CFG = Config()
-
-# ---------------------------------------------------------------------------
-# Logging
-# ---------------------------------------------------------------------------
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s | %(levelname)-7s | %(message)s",
-    datefmt="%H:%M:%S",
-)
-log = logging.getLogger("rag")
-
-# ---------------------------------------------------------------------------
-# 1. Document Processing
-# ---------------------------------------------------------------------------
-
-@dataclass
-class RawDocument:
-    """A single extracted document before chunking."""
-    text: str
-    metadata: dict = field(default_factory=dict)
-
-
-def extract_pdf(path: str) -> RawDocument:
-    """Extract text and metadata from a PDF using PyMuPDF."""
-    doc = fitz.open(path)
-    pages = []
-    for page in doc:
-        pages.append(page.get_text("text"))
-    meta = doc.metadata or {}
-    return RawDocument(
-        text="\n\n".join(pages),
-        metadata={
-            "source": os.path.basename(path),
-            "path": path,
-            "type": "pdf",
-            "title": meta.get("title", ""),
-            "author": meta.get("author", ""),
-            "pages": len(doc),
-        },
-    )
-
-
-def extract_docx(path: str) -> RawDocument:
-    """Extract text from a DOCX file."""
-    doc = DocxDocument(path)
-    paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
-    core = doc.core_properties
-    return RawDocument(
-        text="\n\n".join(paragraphs),
-        metadata={
-            "source": os.path.basename(path),
-            "path": path,
-            "type": "docx",
-            "title": core.title or "",
-            "author": core.author or "",
-        },
-    )
-
-
-def extract_html(path: str) -> RawDocument:
-    """Extract text from an HTML file."""
-    with open(path, "r", encoding="utf-8", errors="replace") as f:
-        soup = BeautifulSoup(f.read(), "html.parser")
-    # Remove script and style elements
-    for tag in soup(["script", "style", "nav", "footer", "header"]):
-        tag.decompose()
-    title = soup.title.string if soup.title else ""
-    text = soup.get_text(separator="\n", strip=True)
-    return RawDocument(
-        text=text,
-        metadata={
-            "source": os.path.basename(path),
-            "path": path,
-            "type": "html",
-            "title": title,
-        },
-    )
-
-
-def extract_text(path: str) -> RawDocument:
-    """Extract text from a plain text or markdown file."""
-    with open(path, "r", encoding="utf-8", errors="replace") as f:
-        text = f.read()
-    return RawDocument(
-        text=text,
-        metadata={
-            "source": os.path.basename(path),
-            "path": path,
-            "type": "text",
-        },
-    )
-
-
-EXTRACTORS = {
-    ".pdf": extract_pdf,
-    ".docx": extract_docx,
-    ".html": extract_html,
-    ".htm": extract_html,
-    ".txt": extract_text,
-    ".md": extract_text,
-}
-
-
-def load_documents(directory: str) -> list[RawDocument]:
-    """Recursively load all supported documents from a directory."""
-    docs = []
-    directory = Path(directory)
-    if not directory.exists():
-        log.warning(f"Documents directory not found: {directory}")
-        return docs
-
-    for fpath in sorted(directory.rglob("*")):
-        ext = fpath.suffix.lower()
-        if ext in EXTRACTORS:
-            try:
-                doc = EXTRACTORS[ext](str(fpath))
-                if len(doc.text.strip()) > 50:
-                    docs.append(doc)
-                    log.info(f"  Loaded: {fpath.name} ({len(doc.text):,} chars)")
-            except Exception as e:
-                log.error(f"  Failed: {fpath.name} -> {e}")
-    log.info(f"Total documents loaded: {len(docs)}")
-    return docs
-
-# ---------------------------------------------------------------------------
-# 2. Smart Chunking (Sentence-Based with Overlap)
-# ---------------------------------------------------------------------------
-
-@dataclass
-class Chunk:
-    """A text chunk ready for embedding."""
-    text: str
-    metadata: dict
-    chunk_id: str
-
-
-def _approx_token_count(text: str) -> int:
-    """Rough token count (≈ 4 chars per token for English)."""
-    return len(text) // 4
-
-
-def sentence_chunk(doc: RawDocument, chunk_size: int = 512, overlap: int = 64) -> list[Chunk]:
-    """
-    Sentence-based chunking strategy:
-    - Split text into sentences.
-    - Accumulate sentences until chunk_size tokens is reached.
-    - Overlap by re-including trailing sentences from previous chunk.
-    """
-    try:
-        sentences = sent_tokenize(doc.text)
-    except Exception:
-        nltk.download("punkt_tab", quiet=True)
-        sentences = sent_tokenize(doc.text)
-
-    if not sentences:
-        return []
-
-    chunks: list[Chunk] = []
-    current_sentences: list[str] = []
-    current_tokens = 0
-
-    def _flush(sents: list[str], idx: int):
-        text = " ".join(sents).strip()
-        if len(text) < CFG.min_chunk_length:
-            return
-        chunk_id = hashlib.md5(
-            f"{doc.metadata.get('source', '')}:{idx}:{text[:80]}".encode()
-        ).hexdigest()[:12]
-        chunks.append(Chunk(
-            text=text,
-            metadata={**doc.metadata, "chunk_index": idx},
-            chunk_id=chunk_id,
-        ))
-
-    chunk_idx = 0
-    for sent in sentences:
-        sent_tokens = _approx_token_count(sent)
-        if current_tokens + sent_tokens > chunk_size and current_sentences:
-            _flush(current_sentences, chunk_idx)
-            chunk_idx += 1
-            # Keep overlap sentences from the tail
-            overlap_sents: list[str] = []
-            overlap_tok = 0
-            for s in reversed(current_sentences):
-                t = _approx_token_count(s)
-                if overlap_tok + t > overlap:
-                    break
-                overlap_sents.insert(0, s)
-                overlap_tok += t
-            current_sentences = overlap_sents
-            current_tokens = overlap_tok
-        current_sentences.append(sent)
-        current_tokens += sent_tokens
-
-    if current_sentences:
-        _flush(current_sentences, chunk_idx)
-
-    return chunks
-
-
-def chunk_all_documents(docs: list[RawDocument]) -> list[Chunk]:
-    """Chunk every loaded document."""
-    all_chunks = []
-    for doc in docs:
-        doc_chunks = sentence_chunk(doc, CFG.chunk_size, CFG.chunk_overlap)
-        all_chunks.extend(doc_chunks)
-    log.info(f"Total chunks created: {len(all_chunks)}")
-    return all_chunks
-
-# ---------------------------------------------------------------------------
-# 3. Vector Database (ChromaDB with Persistent Storage)
-# ---------------------------------------------------------------------------
-
-class VectorStore:
-    """Manages ChromaDB collection and embedding model."""
-
-    def __init__(self, config: Config):
-        self.config = config
-        log.info(f"Loading embedding model: {config.embedding_model}")
-        self.embedder = SentenceTransformer(config.embedding_model)
-
-        self.client = chromadb.Client(Settings(
-            persist_directory=config.chroma_persist_dir,
-            anonymized_telemetry=False,
-            is_persistent=True,
-        ))
-        self.collection = self.client.get_or_create_collection(
-            name=config.chroma_collection,
-            metadata={"hnsw:space": "cosine"},
-        )
-        log.info(
-            f"ChromaDB collection '{config.chroma_collection}' "
-            f"has {self.collection.count()} vectors"
-        )
-
-    def embed_text(self, texts: list[str]) -> list[list[float]]:
-        """Generate embeddings for a list of texts."""
-        return self.embedder.encode(texts, show_progress_bar=False).tolist()
-
-    def embed_single(self, text: str) -> list[float]:
-        """Embed a single query string."""
-        return self.embedder.encode(text).tolist()
-
-    def add_chunks(self, chunks: list[Chunk], batch_size: int = 256):
-        """Insert chunks into ChromaDB (skip duplicates by ID)."""
-        existing = set(self.collection.get()["ids"]) if self.collection.count() > 0 else set()
-        new_chunks = [c for c in chunks if c.chunk_id not in existing]
-        if not new_chunks:
-            log.info("No new chunks to add (all already indexed).")
-            return
-
-        for i in range(0, len(new_chunks), batch_size):
-            batch = new_chunks[i : i + batch_size]
-            ids = [c.chunk_id for c in batch]
-            texts = [c.text for c in batch]
-            metas = [c.metadata for c in batch]
-            embeddings = self.embed_text(texts)
-            self.collection.add(
-                ids=ids,
-                documents=texts,
-                metadatas=metas,
-                embeddings=embeddings,
-            )
-            log.info(f"  Indexed batch {i // batch_size + 1} ({len(batch)} chunks)")
-
-        log.info(f"Total vectors in DB: {self.collection.count()}")
-
-    def semantic_search(self, query: str, k: int = 20) -> list[dict]:
-        """Return top-k results by cosine similarity."""
-        embedding = self.embed_single(query)
-        results = self.collection.query(
-            query_embeddings=[embedding],
-            n_results=min(k, self.collection.count()),
-            include=["documents", "metadatas", "distances"],
-        )
-        hits = []
-        for doc, meta, dist in zip(
-            results["documents"][0],
-            results["metadatas"][0],
-            results["distances"][0],
-        ):
-            hits.append({
-                "text": doc,
-                "metadata": meta,
-                "score": 1 - dist,  # cosine distance -> similarity
-            })
-        return hits
-
-# ---------------------------------------------------------------------------
-# 4. BM25 Keyword Search (for Hybrid Retrieval)
-# ---------------------------------------------------------------------------
-
-class BM25Index:
-    """Maintains a BM25 index over all chunk texts."""
-
-    def __init__(self):
-        self.corpus: list[str] = []
-        self.metadata: list[dict] = []
-        self.bm25: Optional[BM25Okapi] = None
-
-    def build(self, chunks: list[Chunk]):
-        """Build BM25 index from chunks."""
-        self.corpus = [c.text for c in chunks]
-        self.metadata = [c.metadata for c in chunks]
-        tokenized = [self._tokenize(t) for t in self.corpus]
-        self.bm25 = BM25Okapi(tokenized)
-        log.info(f"BM25 index built over {len(self.corpus)} chunks")
-
-    @staticmethod
-    def _tokenize(text: str) -> list[str]:
-        return re.findall(r"\w+", text.lower())
-
-    def search(self, query: str, k: int = 20) -> list[dict]:
-        """Return top-k BM25 results."""
-        if self.bm25 is None:
-            return []
-        tokens = self._tokenize(query)
-        scores = self.bm25.get_scores(tokens)
-        top_idx = np.argsort(scores)[::-1][:k]
-        results = []
-        for idx in top_idx:
-            if scores[idx] > 0:
-                results.append({
-                    "text": self.corpus[idx],
-                    "metadata": self.metadata[idx],
-                    "score": float(scores[idx]),
-                })
-        return results
-
-# ---------------------------------------------------------------------------
-# 5. Hybrid Search: Merge Semantic + BM25 with RRF
-# ---------------------------------------------------------------------------
-
-def reciprocal_rank_fusion(
-    semantic_hits: list[dict],
-    bm25_hits: list[dict],
-    semantic_weight: float = 0.6,
-    k_constant: int = 60,
-    top_k: int = 5,
-) -> list[dict]:
-    """
-    Reciprocal Rank Fusion (RRF) to merge two ranked lists.
-    score(doc) = w_s / (k + rank_semantic) + w_b / (k + rank_bm25)
-    """
-    scores: dict[str, float] = defaultdict(float)
-    doc_map: dict[str, dict] = {}
-
-    bm25_weight = 1.0 - semantic_weight
-
-    for rank, hit in enumerate(semantic_hits, start=1):
-        key = hit["text"][:200]  # use text prefix as dedup key
-        scores[key] += semantic_weight / (k_constant + rank)
-        doc_map[key] = hit
-
-    for rank, hit in enumerate(bm25_hits, start=1):
-        key = hit["text"][:200]
-        scores[key] += bm25_weight / (k_constant + rank)
-        if key not in doc_map:
-            doc_map[key] = hit
-
-    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
-    results = []
-    for key, score in ranked:
-        entry = doc_map[key].copy()
-        entry["hybrid_score"] = score
-        results.append(entry)
-    return results
-
-# ---------------------------------------------------------------------------
-# 6. Conversation Memory
-# ---------------------------------------------------------------------------
-
-class ConversationMemory:
-    """Tracks the last N exchanges for multi-turn support."""
-
-    def __init__(self, max_turns: int = 10):
-        self.max_turns = max_turns
-        self.history: list[dict] = []  # [{"role": "user"/"assistant", "content": ...}]
-
-    def add_user(self, message: str):
-        self.history.append({"role": "user", "content": message})
-        self._trim()
-
-    def add_assistant(self, message: str):
-        self.history.append({"role": "assistant", "content": message})
-        self._trim()
-
-    def _trim(self):
-        # Keep last N *exchanges* (each exchange = 2 messages)
-        max_messages = self.max_turns * 2
-        if len(self.history) > max_messages:
-            self.history = self.history[-max_messages:]
-
-    def get_messages(self) -> list[dict]:
-        return list(self.history)
-
-    def get_context_summary(self) -> str:
-        """Produce a short summary for query rewriting."""
-        if not self.history:
-            return ""
-        recent = self.history[-6:]  # last 3 exchanges
-        lines = []
-        for msg in recent:
-            role = "User" if msg["role"] == "user" else "Assistant"
-            # Truncate long assistant replies
-            content = msg["content"][:300]
-            lines.append(f"{role}: {content}")
-        return "\n".join(lines)
-
-    def clear(self):
-        self.history.clear()
-
-# ---------------------------------------------------------------------------
-# 7. RAG Pipeline (Query → Retrieve → Generate)
-# ---------------------------------------------------------------------------
-
-class RAGPipeline:
-    """Orchestrates the full RAG pipeline."""
-
-    def __init__(self, config: Config):
-        self.config = config
-        self.vector_store = VectorStore(config)
-        self.bm25_index = BM25Index()
-        self.memory = ConversationMemory(max_turns=config.memory_length)
-        self.openai_client = openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY", ""))
-
-    # -- Indexing ----------------------------------------------------------
-
-    def index_documents(self, docs_dir: Optional[str] = None):
-        """Load, chunk, and index all documents."""
-        directory = docs_dir or self.config.documents_dir
-        raw_docs = load_documents(directory)
-        if not raw_docs:
-            log.warning("No documents found. Please add files to the documents folder.")
-            return
-
-        chunks = chunk_all_documents(raw_docs)
-        self.vector_store.add_chunks(chunks)
-        self.bm25_index.build(chunks)
-        log.info("Indexing complete.")
-
-    # -- Query rewriting for follow-ups ------------------------------------
-
-    def _rewrite_query(self, user_query: str) -> str:
-        """Use conversation context to make follow-up queries self-contained."""
-        context = self.memory.get_context_summary()
-        if not context:
-            return user_query
-
-        try:
-            response = self.openai_client.chat.completions.create(
-                model=self.config.openai_model,
-                temperature=0,
-                max_tokens=200,
-                messages=[
-                    {
-                        "role": "system",
-                        "content": (
-                            "Rewrite the user's latest question so it is self-contained, "
-                            "incorporating any necessary context from the conversation. "
-                            "Output ONLY the rewritten question, nothing else."
-                        ),
-                    },
-                    {
-                        "role": "user",
-                        "content": f"Conversation:\n{context}\n\nLatest question: {user_query}",
-                    },
-                ],
-            )
-            rewritten = response.choices[0].message.content.strip()
-            if rewritten:
-                log.info(f"Rewritten query: {rewritten}")
-                return rewritten
-        except Exception as e:
-            log.warning(f"Query rewriting failed: {e}")
-        return user_query
-
-    # -- Retrieval ---------------------------------------------------------
-
-    def retrieve(self, query: str) -> list[dict]:
-        """Hybrid retrieval: semantic + BM25 merged via RRF."""
-        semantic_hits = self.vector_store.semantic_search(
-            query, k=self.config.top_k_semantic
-        )
-        bm25_hits = self.bm25_index.search(query, k=self.config.top_k_bm25)
-
-        merged = reciprocal_rank_fusion(
-            semantic_hits,
-            bm25_hits,
-            semantic_weight=self.config.semantic_weight,
-            top_k=self.config.top_k_final,
-        )
-        return merged
-
-    # -- Generation --------------------------------------------------------
-
-    def generate(self, user_query: str, retrieved_chunks: list[dict]) -> str:
-        """Call the LLM with retrieved context and conversation history."""
-
-        # Build context block with source labels
-        context_parts = []
-        for i, chunk in enumerate(retrieved_chunks, 1):
-            source = chunk["metadata"].get("source", "unknown")
-            title = chunk["metadata"].get("title", "")
-            label = f"[Source {i}: {source}"
-            if title:
-                label += f" — {title}"
-            label += "]"
-            context_parts.append(f"{label}\n{chunk['text']}")
-
-        context_block = "\n\n---\n\n".join(context_parts)
-
-        # Assemble messages
-        messages = [{"role": "system", "content": self.config.system_prompt}]
-
-        # Add conversation history
-        messages.extend(self.memory.get_messages())
-
-        # Add current turn with context
-        user_content = (
-            f"Context passages:\n\n{context_block}\n\n"
-            f"---\n\nQuestion: {user_query}"
-        )
-        messages.append({"role": "user", "content": user_content})
-
-        try:
-            response = self.openai_client.chat.completions.create(
-                model=self.config.openai_model,
-                temperature=self.config.temperature,
-                max_tokens=1500,
-                messages=messages,
-            )
-            return response.choices[0].message.content
-        except Exception as e:
-            return f"LLM generation error: {e}"
-
-    # -- Full pipeline -----------------------------------------------------
-
-    def query(self, user_input: str) -> tuple[str, list[dict]]:
-        """
-        Full RAG pipeline:
-        1. Rewrite query using conversation context
-        2. Hybrid retrieve top-K chunks
-        3. Generate answer with citations
-        4. Update memory
-        Returns (answer_text, retrieved_sources)
-        """
-        # Step 1: Rewrite for follow-ups
-        search_query = self._rewrite_query(user_input)
-
-        # Step 2: Retrieve
-        chunks = self.retrieve(search_query)
-        if not chunks:
-            answer = (
-                "I couldn't find any relevant information in the document collection "
-                "to answer your question. Could you rephrase or ask about a different topic?"
-            )
-            self.memory.add_user(user_input)
-            self.memory.add_assistant(answer)
-            return answer, []
-
-        # Step 3: Generate
-        answer = self.generate(user_input, chunks)
-
-        # Step 4: Update memory
-        self.memory.add_user(user_input)
-        self.memory.add_assistant(answer)
-
-        return answer, chunks
-
-    def reset_conversation(self):
-        """Clear conversation history."""
-        self.memory.clear()
-        return "Conversation history cleared."
-
-# ---------------------------------------------------------------------------
-# 8. Gradio Conversational UI
-# ---------------------------------------------------------------------------
-
-def build_ui(pipeline: RAGPipeline) -> gr.Blocks:
-    """Create the Gradio chat interface with source citations."""
-
-    CUSTOM_CSS = """
-    .gradio-container {
-        max-width: 960px !important;
-        margin: auto !important;
-        font-family: 'Segoe UI', system-ui, sans-serif !important;
-    }
-    .source-card {
-        background: #f8f9fa;
-        border-left: 3px solid #4a90d9;
-        padding: 10px 14px;
-        margin: 6px 0;
-        border-radius: 4px;
-        font-size: 0.88em;
-        line-height: 1.5;
-    }
-    .source-card strong { color: #2c5282; }
-    .status-bar {
-        text-align: center;
-        padding: 6px;
-        font-size: 0.85em;
-        color: #718096;
-    }
-    """
-
-    with gr.Blocks(css=CUSTOM_CSS, title="RAG Assistant", theme=gr.themes.Soft()) as demo:
-        gr.Markdown(
-            "# 📚 RAG Document Assistant\n"
-            "Ask questions about the indexed document collection. "
-            "Sources are cited inline and shown below each answer."
-        )
-
-        chatbot = gr.Chatbot(
-            label="Conversation",
-            height=520,
-            show_copy_button=True,
-            bubble_full_width=False,
-            avatar_images=(None, "https://em-content.zobj.net/source/twitter/376/robot_1f916.png"),
-        )
-
-        sources_display = gr.HTML(
-            value='<div class="status-bar">Sources will appear here after each answer.</div>',
-            label="Retrieved Sources",
-        )
-
-        with gr.Row():
-            msg_input = gr.Textbox(
-                placeholder="Ask a question about your documents...",
-                show_label=False,
-                scale=9,
-                container=False,
-            )
-            send_btn = gr.Button("Send", variant="primary", scale=1)
-
-        with gr.Row():
-            clear_btn = gr.Button("🗑 Clear Chat", size="sm")
-            status = gr.Markdown(
-                f"*{pipeline.vector_store.collection.count()} chunks indexed "
-                f"| Hybrid search (semantic + BM25) "
-                f"| Memory: last {pipeline.config.memory_length} exchanges*"
-            )
-
-        # -- Event handlers ------------------------------------------------
-
-        def respond(user_message: str, chat_history: list):
-            if not user_message.strip():
-                return "", chat_history, ""
-
-            answer, sources = pipeline.query(user_message)
-            chat_history = chat_history + [[user_message, answer]]
-
-            # Format sources as HTML cards
-            if sources:
-                cards = []
-                for i, s in enumerate(sources, 1):
-                    src = s["metadata"].get("source", "unknown")
-                    title = s["metadata"].get("title", "")
-                    score = s.get("hybrid_score", s.get("score", 0))
-                    preview = s["text"][:250].replace("\n", " ") + "..."
-                    card = (
-                        f'<div class="source-card">'
-                        f"<strong>[Source {i}]</strong> {src}"
-                        f"{f' — <em>{title}</em>' if title else ''}"
-                        f" (score: {score:.4f})<br>"
-                        f"<span style='color:#555'>{preview}</span>"
-                        f"</div>"
-                    )
-                    cards.append(card)
-                sources_html = "".join(cards)
-            else:
-                sources_html = '<div class="status-bar">No relevant sources found.</div>'
-
-            return "", chat_history, sources_html
-
-        def clear_chat():
-            pipeline.reset_conversation()
-            return [], '<div class="status-bar">Conversation cleared. Sources will appear here.</div>'
-
-        # Wire events
-        msg_input.submit(respond, [msg_input, chatbot], [msg_input, chatbot, sources_display])
-        send_btn.click(respond, [msg_input, chatbot], [msg_input, chatbot, sources_display])
-        clear_btn.click(clear_chat, outputs=[chatbot, sources_display])
-
-    return demo
-
-# ---------------------------------------------------------------------------
-# 9. Main Entry Point
-# ---------------------------------------------------------------------------
-
-def main():
-    """Initialize the pipeline, index documents, and launch the UI."""
-    log.info("=" * 60)
-    log.info("RAG Application Starting")
-    log.info("=" * 60)
-
-    # Ensure NLTK data is available
-    try:
-        sent_tokenize("Hello world.")
-    except LookupError:
-        nltk.download("punkt_tab", quiet=True)
-
-    # Validate API key
-    api_key = os.getenv("OPENAI_API_KEY", "")
-    if not api_key:
-        log.warning(
-            "OPENAI_API_KEY not set. LLM generation will fail. "
-            "Set it with: export OPENAI_API_KEY='sk-...'"
-        )
-
-    # Create documents directory if needed
-    os.makedirs(CFG.documents_dir, exist_ok=True)
-
-    # Initialize pipeline
-    pipeline = RAGPipeline(CFG)
-
-    # Index documents (idempotent — skips already-indexed chunks)
-    pipeline.index_documents()
-
-    # Build and launch UI
-    demo = build_ui(pipeline)
-    log.info(f"Launching Gradio on port {CFG.server_port} (share={CFG.share})")
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=CFG.server_port,
-        share=CFG.share,
-        show_error=True,
-    )
-
-
-if __name__ == "__main__":
-    main()