feat: Add full multi-turn conversation memory and context rewriting

frontend-next/app/page.tsx

@@ -430,6 +430,11 @@ export default function App() {
         setSidebarOpen(false);
     };
 
+    const handleClearConversation = () => {
+        if (!activeSessionId) return;
+        setSessions(prev => prev.map(s => s.id === activeSessionId ? { ...s, messages: [] } : s));
+    };
+
     const handleSend = async () => {
         if (!query.trim() || isStreaming) return;
         
@@ -473,12 +478,22 @@ export default function App() {
         setQuery("");
         setIsStreaming(true);
 
+        const history = currentMessages
+            .filter(m => m.role === "user" || m.role === "assistant")
+            .map(m => ({
+                role: m.role,
+                content: m.content,
+                citations: m.citations || []
+            }))
+            .slice(-20);
+
         try {
             const res = await fetch(`${API_URL}/query/stream`, {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
                 body: JSON.stringify({ 
                     question: originalQuery, 
+                    history: history,
                     top_k: topK, 
                     filter_category: category === "All" ? undefined : category,
                     filter_year_gte: filterYear === "All" ? undefined : parseInt(filterYear, 10)
@@ -674,11 +689,15 @@ export default function App() {
                         </motion.button>
                     )}
                 </AnimatePresence>
-                {/* Header API Status */}
                 <div className="top-api-status" style={{ display: 'flex', gap: '12px', alignItems: 'center' }}>
                     <button onClick={() => setShowInfo(true)} className="nav-icon-btn" aria-label="Project Info" style={{ background: 'rgba(255,255,255,0.05)', border: '1px solid rgba(255,255,255,0.1)', padding: '6px', borderRadius: '50%', color: 'var(--text-muted)', cursor: 'pointer', display: 'flex', alignItems: 'center', justifyContent: 'center' }}>
                         <Info size={16} />
                     </button>
+                    {activeSessionId && currentMessages.length > 0 && (
+                        <button onClick={handleClearConversation} className="nav-icon-btn" aria-label="Clear Conversation" title="Clear current conversation context" style={{ background: 'rgba(255,255,255,0.05)', border: '1px solid rgba(255,255,255,0.1)', padding: '6px 12px', borderRadius: '16px', color: 'var(--text-muted)', cursor: 'pointer', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: '6px', fontSize: '0.8rem' }}>
+                            <Trash2 size={14} /> Clear context
+                        </button>
+                    )}
                     <div className="nav-status">
                         <div className={`status-dot ${apiStatus === 'online' ? 'status-online' : 'status-offline'}`} />
                         {apiStatus === 'online' ? 'API Online' : apiStatus === 'connecting' ? 'Connecting...' : 'API Offline'}
@@ -782,6 +801,11 @@ export default function App() {
                                             <span className="model-badge" style={{ background: 'rgba(255,255,255,0.05)', padding: '2px 8px', borderRadius: '4px', border: '1px solid rgba(255,255,255,0.1)', color: 'var(--text-muted)', fontSize: '0.75rem' }}>
                                                 {msg.model_used || "Auto-Detecting..."}
                                             </span>
+                                            {i >= 2 && (
+                                                <span style={{ fontSize: '0.7rem', background: 'rgba(138, 43, 226, 0.15)', border: '1px solid rgba(138, 43, 226, 0.3)', padding: '2px 8px', borderRadius: '4px', color: 'var(--accent-2)', marginLeft: 'auto', display: 'flex', alignItems: 'center', gap: '4px' }}>
+                                                    <Layers size={10} /> Using conversation context
+                                                </span>
+                                            )}
                                         </div>
 
                                         <>

src/api/main.py

@@ -47,7 +47,7 @@ class FeedbackRequest(BaseModel):
     model_used: str
     citations_count: int
     total_time_ms: float
-from src.rag.pipeline import RAGPipeline
+from src.rag.pipeline import RAGPipeline, ConversationTurn
 from src.utils.logger import setup_logger, get_logger
 
 
@@ -187,6 +187,7 @@ async def stream_query_papers(
             try:
                 for chunk in pipeline.stream_query(
                     question        = query_input.question,
+                    history         = [ConversationTurn(role=t.role, content=t.content, citations=t.citations) for t in query_input.history],
                     top_k           = query_input.top_k,
                     filter_category = query_input.filter_category,
                     filter_year_gte = query_input.filter_year_gte,
@@ -265,6 +266,7 @@ async def query_papers(
         response = await asyncio.to_thread(
             pipeline.query,
             query_input.question,
+            [ConversationTurn(role=t.role, content=t.content, citations=t.citations) for t in query_input.history],
             query_input.top_k,
             query_input.filter_category,
             query_input.filter_year_gte,

src/api/schemas.py

@@ -11,7 +11,13 @@ WHY PYDANTIC SCHEMAS IN THE API LAYER:
 """
 
 from pydantic import BaseModel, Field
-from typing import Optional
+from typing import Optional, List
+
+
+class ConversationTurnSchema(BaseModel):
+    role:     str
+    content:  str
+    citations: list = []
 
 
 
@@ -28,6 +34,10 @@ class QueryRequest(BaseModel):
         description     = "Research question to answer",
         examples        = ["How does LoRA reduce trainable parameters?"]
     )
+    history: List[ConversationTurnSchema] = Field(
+        default=[],
+        description="Conversation history for context"
+    )
     top_k: int = Field(
         default     = 5,
         ge          = 1,                 # ge = greater than or equal

src/rag/llm_client.py

@@ -113,6 +113,7 @@ class MultiModelClient:
         system_prompt: str,
         user_prompt: str,
         original_query: str = "",
+        history: list = None,
         temperature: float = LLM_TEMPERATURE,
         max_tokens: int = LLM_MAX_TOKENS,
         stream: bool = False
@@ -124,10 +125,10 @@ class MultiModelClient:
         Otherwise, result is a string.
         """
         models_to_try = self.get_model_for_query(original_query)
-        messages = [
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": user_prompt}
-        ]
+        messages = [{"role": "system", "content": system_prompt}]
+        if history:
+            messages.extend(history)
+        messages.append({"role": "user", "content": user_prompt})
 
         for model in models_to_try:
             try:

src/rag/pipeline.py

@@ -15,7 +15,13 @@ PIPELINE FLOW:
 import time
 import json
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Optional, List
+
+@dataclass
+class ConversationTurn:
+    role: str
+    content: str
+    citations: list = field(default_factory=list)
 
 from src.retrieval.retrieval_pipeline import RetrievalPipeline
 from src.rag.llm_client import MultiModelClient
@@ -63,22 +69,53 @@ class RAGPipeline:
         self.llm        = MultiModelClient()
         logger.info("RAGPipeline ready")
 
+    def _build_retrieval_query(
+        self,
+        question: str,
+        history: list[ConversationTurn]
+    ) -> str:
+        followup_signals = [
+            "it", "that", "this", "they", "them",
+            "more", "example", "explain", "clarify",
+            "simpler", "detail", "elaborate", "again"
+        ]
+        question_lower = question.lower()
+        is_followup = (
+            len(question.split()) < 12 and
+            any(word in question_lower for word in followup_signals)
+        )
+
+        if is_followup and history:
+            last_substantial = ""
+            for turn in reversed(history):
+                if turn.role == "user" and len(turn.content.split()) > 5:
+                    last_substantial = turn.content
+                    break
+            if last_substantial:
+                return f"{last_substantial} {question}"
+
+        return question
+
     def query(
         self,
         question:        str,
+        history:         list[ConversationTurn] = None,
         top_k:           int = TOP_K_RERANK,
         filter_category: Optional[str] = None,
         filter_year_gte: Optional[int] = None,
     ) -> RAGResponse:
         question = question.strip()
+        history = history or []
         if not question:
             raise ValueError("Question cannot be empty")
 
         total_start = time.time()
         retrieval_start = time.time()
 
+        retrieval_query = self._build_retrieval_query(question, history)
+
         chunks = self.retriever.retrieve(
-            query           =  question,
+            query           = retrieval_query,
             top_k_final     = top_k,
             filter_category = filter_category,
             filter_year_gte = filter_year_gte,
@@ -96,11 +133,20 @@ class RAGPipeline:
                 f"or broadening their query."
             )
 
+        history_messages = []
+        if history:
+            for turn in history[-10:]:
+                history_messages.append({
+                    "role": turn.role,
+                    "content": turn.content
+                })
+
         generation_start = time.time()
         answer, model_used = self.llm.generate(
             system_prompt = SYSTEM_PROMPT,
             user_prompt   = user_prompt,
             original_query = question,
+            history = history_messages,
             stream=False
         )
 
@@ -123,18 +169,23 @@ class RAGPipeline:
     def stream_query(
         self,
         question:        str,
+        history:         list[ConversationTurn] = None,
         top_k:           int = TOP_K_RERANK,
         filter_category: Optional[str] = None,
         filter_year_gte: Optional[int] = None,
     ):
         question = question.strip()
+        history = history or []
         if not question:
             raise ValueError("Question cannot be empty")
 
         total_start = time.time()
         retrieval_start = time.time()
+
+        retrieval_query = self._build_retrieval_query(question, history)
+
         chunks = self.retriever.retrieve(
-            query           =  question,
+            query           = retrieval_query,
             top_k_final     = top_k,
             filter_category = filter_category,
             filter_year_gte = filter_year_gte,
@@ -152,11 +203,20 @@ class RAGPipeline:
                 f"or broadening their query."
             )
 
+        history_messages = []
+        if history:
+            for turn in history[-10:]:
+                history_messages.append({
+                    "role": turn.role,
+                    "content": turn.content
+                })
+
         generation_start = time.time()
         generator, model_used = self.llm.generate(
             system_prompt = SYSTEM_PROMPT,
             user_prompt   = user_prompt,
             original_query = question,
+            history = history_messages,
             stream=True
         )
 

src/rag/prompt_templates.py

@@ -36,6 +36,8 @@ FORMATTING RULES:
 7. Use markdown formatting: **bold** for key terms, numbered lists for steps
 8. For algorithm explanations, structure as: Intuition -> Math -> Steps
 9. Write comprehensive, detailed answers — do not truncate explanations
+
+You have access to the conversation history above. Use it to understand follow-up questions, resolve pronouns (like 'it', 'that', 'this method'), and give answers that build on what was already discussed. If the user asks for clarification or says 'explain more' or 'give an example', refer back to your previous answer.
 """