feat: RAG pipeline complete - ResearchPilot end-to-end working

- Full RAG pipeline: retrieval → prompt building → LLM generation
- Groq API integration: llama-3.3-70b-versatile
- Structured RAGResponse: answer + citations + timing metadata
- Prompt engineering: grounding rules + citation requirements
- Hallucination resistance: correctly refuses out-of-scope questions
- Citations working: paper IDs cited inline + source list populated
- Fixed hybrid retriever: spread all metadata fields from dense results

Test results:
LoRA question: grounded answer with 4 cited sources
MARL question: synthesized from multiple papers with citations
Python history: correctly refused (no relevant context)
Warm query latency: ~8s (cross-encoder on CPU, optimizing)

config/settings.py

@@ -73,16 +73,16 @@ EMBEDDING_DIMENSION  = 768                      # BGE-base output dimension
 # ------------------------------------------
 QDRANT_COLLECTION_NAME = 'research_papers'
 QDRANT_PATH = str(ROOT_DIR / 'data' / 'qdrant_db')  # Local Storage path
-TOP_K_RETRIEVAL = 20                                # Retieve top 20 candidates
+TOP_K_RETRIEVAL = 20                                # Retrieve top 20 candidates
 TOP_K_RERANK = 5                                    # Keep top 5 after reranking
 
 # ------------------------------------------
 # LLM SETTINGS
 # ------------------------------------------
-GROQ_API_KEY = os.getenv('GROQ_API_KEY')    # Loaded from .env
-LLM_MODEL_NAME = 'llama3-8b-8192'           # Groq model ID
-LLM_TEMPERATURE = 0.1                       # Low = More factual/consistent 
-LLM_MAX_TOKENS = 1024                       # Max response tokens
+GROQ_API_KEY = os.getenv('GROQ_API_KEY')            # Loaded from .env
+LLM_MODEL_NAME = 'llama-3.3-70b-versatile'          # Groq model ID
+LLM_TEMPERATURE = 0.1                               # Low = More factual/consistent 
+LLM_MAX_TOKENS = 1024                               # Max response tokens
 
 # ------------------------------------------
 # API SETTINGS

src/rag/llm_client.py

@@ -0,0 +1,103 @@
+"""
+Groq API client for LLM inference.
+
+WHY GROQ:
+    - Free tier: 14,400 requests/day with Llama3
+    - Speed: ~500 tokens/second (vs 10 tokens/second local CPU)
+    - No GPU needed on our machine
+    - Production-quality latency for demos
+
+WHY LLAMA3-8B:
+    - Free on Groq
+    - 8B parameters: strong reasoning for research QA
+    - 8192 token context window: fits our 5 retrieved chunks
+    - Fast: ~1-2 seconds for a full response
+"""
+
+import os
+from groq import Groq
+from src.utils.logger import get_logger
+from config.settings import (
+    GROQ_API_KEY,
+    LLM_MODEL_NAME,
+    LLM_TEMPERATURE,
+    LLM_MAX_TOKENS,
+)
+
+logger = get_logger(__name__)
+
+
+class LLMClient:
+    """
+    Wrapper around Groq API for LLM inference.
+
+    Designed as a simple interface so we can swap
+    to any other LLM provider (OpenAI, Anthropic, local)
+    by changing only this file.
+    """
+
+
+    def __init__(self):
+        if not GROQ_API_KEY:
+            raise ValueError(
+                "GROQ_API_KEY not found. "
+                "Add it to your .env file: GROQ_API_KEY=gsk_..."
+            )
+        self.client = Groq(api_key = GROQ_API_KEY)
+        self.model  = LLM_MODEL_NAME
+        logger.info(f"LLMClient initialized with model: {self.model}")
+
+
+    def generate(
+        self,
+        system_prompt: str,
+        user_prompt:   str,
+        temperature:   float = LLM_TEMPERATURE,
+        max_tokens:    int   = LLM_MAX_TOKENS,
+    ) -> str:
+        """
+        Generate a response from the LLM.
+
+        Args:
+            system_prompt: Instructions for the LLM's behavior
+            user_prompt:   The actual question + context
+            temperature:   0.0 = deterministic, 1.0 = creative
+                          We use 0.1 for factual research QA
+            max_tokens:    Maximum response length
+
+        Returns:
+            Generated text string
+
+        GROQ API STRUCTURE:
+            Uses OpenAI-compatible chat format:
+            [{"role": "system", "content": "..."}, 
+             {"role": "user",   "content": "..."}]
+        """
+
+        try:
+            response = self.client.chat.completions.create(
+                model       = self.model,
+                messages    = [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt}
+                ],
+                temperature = temperature,
+                max_tokens  = max_tokens,
+            )
+
+            answer = response.choices[0].message.content
+
+            # Log token usage for monitoring
+            usage = response.usage
+            logger.debug(
+                f"LLM usage - "
+                f"prompt: {usage.prompt_tokens} tokens, "
+                f"completion: {usage.completion_tokens} tokens, "
+                f"total: {usage.total_tokens} tokens"
+            )
+
+            return answer
+
+        except Exception as e:
+            logger.error(f"LLM generation failed: {e}")
+            raise

src/rag/pipeline.py

@@ -0,0 +1,181 @@
+"""
+The complete RAG pipeline - orchestrates retrieval + generation.
+
+This is the core of ResearchPilot. Every user query goes through this.
+
+PIPELINE FLOW:
+    1. Validate and clean the query
+    2. Retrieve top-5 relevant chunks (Phase 8 pipeline)
+    3. Build prompt with context
+    4. Generate answer via Groq LLM
+    5. Parse and structure the response
+    6. Return answer + citations + metadata
+"""
+
+import time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from src.retrieval.retrieval_pipeline import RetrievalPipeline
+from src.rag.llm_client import LLMClient
+from src.rag.prompt_templates import (
+    SYSTEM_PROMPT,
+    build_rag_prompt,
+    build_citation_list,
+)
+from src.utils.logger import get_logger
+from config.settings import TOP_K_RERANK
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class RAGResponse:
+    """
+    Structured response from the RAG pipeline.
+
+    WHY A DATACLASS INSTEAD OF A DICT:
+        Dicts can have any keys - you never know what's in them.
+        A dataclass defines the exact contract. The FastAPI layer
+        (Phase 11) and frontend (Phase 12) can rely on these
+        fields always being present.
+    """
+    # The generated answer
+    answer:     str
+
+    # Source papers used to generate the answer
+    citations:  list[dict]
+
+    # Raw retrieved chunks (for debugging / evaluation)
+    retrieved_chunks: list[dict]
+
+    # Performance metadata
+    query:      str
+    retrieval_time_ms: float
+    generation_time_ms: float
+    total_time_ms:  float
+
+    # Whether retrieval found retrieval content
+    has_context:    bool
+
+
+    def to_dict(self) -> dict:
+        return {
+            "answer":             self.answer,
+            "citations":          self.citations,
+            "query":              self.query,
+            "retrieval_time_ms":  round(self.retrieval_time_ms, 1),
+            "generation_time_ms": round(self.generation_time_ms, 1),
+            "total_time_ms":      round(self.total_time_ms, 1),
+            "has_context":        self.has_context,
+            "chunks_used":        len(self.retrieved_chunks),
+        }
+
+
+
+
+class RAGPipeline:
+    """
+    End-to-end RAG pipeline: query -> retrieve -> generate -> respond.
+
+    Usage:
+        pipeline = RAGPipeline()
+        response = pipeline.query("How does LoRA reduce training parameters?")
+        print(response.answer)
+        for cite in response.citations:
+            print(cite["title"], cite["arxiv_url"])
+    """
+
+    def __init__(self):
+        logger.info("Initializing RAGPipeline...")
+
+        self.retriever  = RetrievalPipeline()
+        self.llm        = LLMClient()
+
+        logger.info("RAGPipeline ready")
+
+    def query(
+        self,
+        question:        str,
+        top_k:           int = TOP_K_RERANK,
+        filter_category: Optional[str] = None,
+        filter_year_gte: Optional[int] = None,
+    ) -> RAGResponse:
+        """
+        Process a user question through the full RAG pipeline.
+
+        Args:
+            question:        User's natural language question
+            top_k:           Number of chunks to retrieve
+            filter_category: Optional ArXiv category filter
+            filter_year_gte: Optional year filter
+
+        Returns:
+            RAGResponse with answer, citations, and timing metadata
+        """
+        question = question.strip()
+
+        if not question:
+            raise ValueError("Question cannot be empty")
+
+        total_start = time.time()
+
+        # ------------ Stage 1: Retrieval ------------
+        retrieval_start = time.time()
+
+        chunks = self.retriever.retrieve(
+            query           =  question,
+            top_k_final     = top_k,
+            filter_category = filter_category,
+            filter_year_gte = filter_year_gte,
+        )
+
+        retrieval_ms = (time.time() - retrieval_start) * 1000
+
+        logger.info(
+            f"Retrieved: {len(chunks)} chunks in {retrieval_ms:.0f}ms"
+        )
+
+        has_context = len(chunks) > 0
+
+        # ------------ Stage 2: Prompt Construction ------------
+        if has_context:
+            user_prompt = build_rag_prompt(question, chunks)
+        else:
+            # Fallback prompt when no relevant context found
+            user_prompt = (
+                f"The user asked: {question}\n\n"
+                f"No relevant research papers were found in the database. "
+                f"Politely inform the user and suggest they try rephrasing "
+                f"or broadening their query."
+            )
+
+        # ------------ Stage 3: LLM Generation ------------
+        generation_start = time.time()
+
+        answer = self.llm.generate(
+            system_prompt = SYSTEM_PROMPT,
+            user_prompt   = user_prompt,
+        )
+
+        generation_ms   = (time.time() - generation_start) * 1000
+        total_ms        = (time.time() - total_start) * 1000
+
+        logger.info(
+            f"Generated answer in {generation_ms:.0f}ms | "
+            f"Total: {total_ms:.0f}ms"
+        )
+
+        # ------------ Stage 4: Build Citations ------------
+        citations = build_citation_list(chunks)
+
+        return RAGResponse(
+            answer             = answer,
+            citations          = citations,
+            retrieved_chunks   = chunks,
+            query              = question,
+            retrieval_time_ms  = retrieval_ms,
+            generation_time_ms = generation_ms,
+            total_time_ms      = total_ms,
+            has_context        = has_context,
+        )

src/rag/prompt_templates.py

@@ -0,0 +1,121 @@
+"""
+Prompt templates for the ResearchPilot RAG system.
+
+PROMPT ENGINEERING IS NOT OPTIONAL.
+The difference between a good RAG system and a bad one
+is often entirely in the prompt design.
+
+Key principles we apply:
+    1. EXPLICIT GROUNDING: Tell the LLM to ONLY use provided context
+    2. CITATION REQUIREMENT: Force the LLM to cite which paper it used
+    3. UNCERTAINTY ACKNOWLEDGMENT: If context doesn't answer, say so
+    4. STRUCTURED OUTPUT: Consistent format makes parsing reliable
+"""
+
+SYSTEM_PROMPT = """You are ResearchPilot, an expert AI research assistant 
+specialized in machine learning and AI research papers.
+
+Your job is to answer questions based EXCLUSIVELY on the research paper 
+excerpts provided in the context below. 
+
+STRICT RULES:
+    1. Only use information from the provided context excerpts
+    2. Always cite the paper title and ID when using information from it
+    3. If the context does not contain enough information to answer, 
+    say "The provided papers do not contain sufficient information 
+    to answer this question" - do NOT make up information
+    4. Be precise and technical - your users are ML researchers and engineers
+    5. When multiple papers discuss the same topic, synthesize their findings
+    6. Keep answers focused and well-structured
+"""
+
+
+def build_rag_prompt(query: str, context_chunks: list[dict]) -> str:
+    """
+    Build the full prompt for the LLM with retrieved context.
+
+    Args:
+        query:          User's question
+        context_chunks: List of retrieved chunk dicts from RetrievalPipeline
+
+    Returns:
+        Formatted prompt string ready to send to the LLM
+
+    PROMPT STRUCTURE:
+        [System prompt]
+        [Context block - all retrieved chunks with citations]
+        [User question]
+
+    WHY WE FORMAT CONTEXT THIS WAY:
+        Each chunk is labeled with its paper title and ID.
+        This enables the LLM to produce citations like:
+        "According to [2603.12248], LoRA constrains..."
+        
+        Without this labeling, the LLM cannot cite sources
+        even if it wanted to.
+    """
+
+    # Build context block from retrieved chunks
+    context_parts = []
+
+
+    for i, chunk in enumerate(context_chunks, 1):
+        paper_id = chunk.get("paper_id", "unknown")
+        title    = chunk.get("title", "Unknown Paper")
+        date     = chunk.get("published_date", "")
+        text     = chunk.get("text", "")
+
+        context_parts.append(
+            f"[SOURCE {i}]\n"
+            f"Paper ID: {paper_id}\n"
+            f"Title: {title}\n"
+            f"Published: {date}\n"
+            f"Excerpt:\n{text}\n"
+        ) 
+
+
+    context_block = "\n---\n".join(context_parts)
+
+
+    prompt = f"""
+    CONTEXT - Research Paper Excerpts:
+    {context_block}
+
+    ---
+
+    QUESTION: {query}
+
+    INSTRUCTIONS: Answer the question using ONLY the context above. 
+    Cite sources using their Paper ID in brackets, e.g. [2603.12248].
+    If the context is insufficient, say so clearly.
+    """
+
+    return prompt
+
+
+
+def build_citation_list(context_chunks: list[dict]) -> list[dict]:
+    """
+    Build a structured list of cited sources from retrieved chunks.
+
+    Returns deduplicated list of papers used as sources. 
+    """
+    seen_papers = set()
+    citations = []
+
+
+    for chunk in context_chunks:
+        paper_id = chunk.get("paper_id", "")
+        if paper_id and paper_id not in seen_papers:
+            seen_papers.add(paper_id)
+            citations.append(
+                {
+                    "paper_id":         paper_id,
+                    "title":            chunk.get("title", ""),
+                    "authors":          chunk.get("authors", []),
+                    "published_date":   chunk.get("published_date", ""),
+                    "arxiv_url":        chunk.get("arxiv_url", ""),
+                }
+            )
+
+    return citations

src/retrieval/hybrid_retriever.py

@@ -116,14 +116,23 @@ class HybridRetriever:
         # -------------- Step 3: Build chunk_id -> full data lookup --------------
         # Dense results have full payload (text, metadata)
         # Sparse results only have chunk_id and text
+
         chunk_data = {}
+        
+        # -------------------------------------------------------
+        # for r in dense_results:
+        #     if r["chunk_id"] not in chunk_data:
+        #         chunk_data[r["chunk_id"]] = {
+        #             "chunk_id": r["chunk_id"],
+        #             "text":     r["text"],
+        #             "score":    0.0,
+        #         }
+        # -------------------------------------------------------
+
         for r in dense_results:
             if r["chunk_id"] not in chunk_data:
-                chunk_data[r["chunk_id"]] = {
-                    "chunk_id": r["chunk_id"],
-                    "text":     r["text"],
-                    "score":    0.0,
-                }
+                chunk_data[r["chunk_id"]] = {**r}
+
 
         # -------------- Step 4: Compute RRF score --------------
         RRF_scores = {}

src/retrieval/retrieval_pipeline.py

@@ -98,8 +98,8 @@ class RetrievalPipeline:
         # Stage 2: Cross-encoder re-ranking -> top-5
         reranked = self.reranker.rerank(
             query   = query,
-            results = candidates,
-            top_k   = top_k_final * 2,  # Keep extra before diversity filter
+            results = candidates[:10],
+            top_k   = TOP_K_RETRIEVAL * 2,  # Keep extra before diversity filter
         )
 
         # Stage 3: Diversity filter -> max 2 chunks per paper

test_rag.py

@@ -0,0 +1,60 @@
+"""
+End-to-end test of the RAG pipeline.
+This is the most important test in the project.
+"""
+
+from src.utils.logger import setup_logger, get_logger
+from src.rag.pipeline import RAGPipeline
+
+
+setup_logger()
+logger = get_logger(__name__)
+
+
+
+def ask(pipeline: RAGPipeline, question: str, **kwargs):
+    print(f"\n{'='*65}")
+    print(f"Q: {question}")
+    print(f"{'='*65}")
+
+    response = pipeline.query(question, **kwargs)
+
+    print(f"\nANSWER:\n{response.answer}")
+
+    print(f"\nSOURCES ({len(response.citations)}):")
+    for i, cite in enumerate(response.citations, 1):
+        print(f"  [{i}] {cite['paper_id']} — {cite['title'][:60]}...")
+        print(f"       {cite['arxiv_url']}")
+
+    print(f"\nTIMING:")
+    print(f"  Retrieval:  {response.retrieval_time_ms:.0f}ms")
+    print(f"  Generation: {response.generation_time_ms:.0f}ms")
+    print(f"  Total:      {response.total_time_ms:.0f}ms")
+    print(f"  Chunks used: {len(response.retrieved_chunks)}")
+
+
+def main():
+    logger.info("Initializing RAG pipeline...")
+    pipeline = RAGPipeline()
+
+    # Test 1: Specific technical question
+    ask(
+        pipeline,
+        "What is LoRA and how does it reduce the number of trainable parameters?"
+    )
+
+    # Test 2: Comparison question
+    ask(
+        pipeline,
+        "What are the main challenges in multi-agent reinforcement learning?"
+    )
+
+    # Test 3: Question that may not be in corpus
+    ask(
+        pipeline,
+        "What is the history of the Python programming language?"
+    )
+
+
+if __name__ == "__main__":
+    main()