Merge pull request #53 from sohank-17/girish-rag-feature

.gitignore

@@ -3,5 +3,10 @@
 
 # backend
 multi_llm_chatbot_backend/.env
+multi_llm_chatbot_backend/chroma_db/
+
+# frontend
+
+# Ignore all __pycache__ folders
+**/__pycache__/
 
-# frontend

multi_llm_chatbot_backend/app/api/routes.py

@@ -8,6 +8,7 @@ from app.llm.improved_ollama_client import ImprovedOllamaClient
 from app.models.persona import Persona
 from app.core.improved_orchestrator import ImprovedChatOrchestrator
 from app.core.session_manager import get_session_manager
+from app.core.rag_manager import get_rag_manager
 from app.models.default_personas import get_default_personas
 from app.utils.document_extractor import extract_text_from_file
 from app.utils.file_limits import is_within_upload_limit
@@ -182,64 +183,38 @@ async def switch_provider(provider_data: ProviderSwitch):
 # Main chat endpoint (SAME INTERFACE, improved backend)
 @router.post("/chat-sequential")
 async def chat_sequential(message: ChatMessage, request: Request):
-    """
-    SAME INTERFACE AS BEFORE - Generate advisor responses 
-    Now with improved session management behind the scenes
-    """
+    """Generate advisor responses - ENHANCED with RAG"""
     try:
-        # Get session using compatibility layer
         session_id = get_or_create_session_for_request(request, message.session_id)
         
-        # Use the new orchestrator with session management
+        # Process message through improved orchestrator (now with RAG)
         result = await chat_orchestrator.process_message(
             user_input=message.user_input,
             session_id=session_id,
-            response_length=message.response_length
+            response_length=message.response_length or "medium"
         )
         
-        # Convert new format back to old format for backward compatibility
-        if result["type"] == "clarification":
-            return {
-                "type": "orchestrator_question",
-                "responses": [{
-                    "persona": "PhD Advisor Assistant",
-                    "response": result["message"]
-                }],
-                "collected_info": {}
+        # Add RAG information to response
+        if result["type"] == "persona_responses":
+            # Count how many personas used documents
+            personas_with_docs = sum(1 for r in result["responses"] if r.get("used_documents", False))
+            total_chunks_used = sum(r.get("document_chunks_used", 0) for r in result["responses"])
+            
+            result["rag_info"] = {
+                "personas_using_documents": personas_with_docs,
+                "total_document_chunks_used": total_chunks_used,
+                "rag_enabled": True
             }
         
-        elif result["type"] == "persona_responses":
-            # Convert new response format to old format
-            return {
-                "type": "sequential_responses", 
-                "responses": [
-                    {
-                        "persona": resp["persona_name"],
-                        "persona_id": resp["persona_id"],
-                        "response": resp["response"]
-                    }
-                    for resp in result["responses"]
-                ],
-                "collected_info": {}
-            }
+        return result
         
-        else:
-            return {
-                "type": "error",
-                "responses": [{
-                    "persona": "System",
-                    "response": result.get("message", "Please try again.")
-                }]
-            }
-            
     except Exception as e:
-        logger.error(f"Error in chat_sequential: {e}")
+        logger.error(f"Error in chat-sequential: {str(e)}")
         return {
             "type": "error",
-            "responses": [{
-                "persona": "System",
-                "response": "I'm having trouble processing your request. Could you please try again?"
-            }]
+            "message": "I encountered an error processing your request. Please try again.",
+            "error": str(e),
+            "rag_enabled": False
         }
 
 # Individual advisor endpoint (SAME INTERFACE)
@@ -326,13 +301,21 @@ async def reply_to_advisor(reply: ReplyToAdvisor, request: Request):
             "response": "I'm having trouble generating a reply right now. Please try again."
         }
 
-# Document upload (SAME INTERFACE)
 @router.post("/upload-document")
 async def upload_document(file: UploadFile = File(...), request: Request = None):
-    """Upload document - SAME INTERFACE"""
+    """
+    Upload document with RAG integration - ENHANCED VERSION
+    
+    Now documents are:
+    1. Chunked intelligently 
+    2. Embedded and stored in ChromaDB
+    3. Available for semantic retrieval
+    4. NOT stored in session context (saves memory)
+    """
+    # Validate file type
     if file.content_type not in [
         "application/pdf",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 
         "text/plain"
     ]:
         raise HTTPException(status_code=400, detail="Unsupported file type.")
@@ -347,16 +330,49 @@ async def upload_document(file: UploadFile = File(...), request: Request = None)
         # Simple size check
         MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB
         if len(file_bytes) > MAX_FILE_SIZE:
-            raise HTTPException(status_code=400, detail="Upload exceeds session document size limit (10 MB).")
+            raise HTTPException(status_code=400, detail="Upload exceeds file size limit (10 MB).")
 
+        # Extract text content
         content = extract_text_from_file(file_bytes, file.content_type)
         if not content.strip():
             raise HTTPException(status_code=400, detail="Document is empty or unreadable.")
 
-        # Add to session context using new system
-        session.add_uploaded_file(file.filename, content, len(file_bytes))
+        # Get RAG manager
+        rag_manager = get_rag_manager()
+        
+        # Determine file type for metadata
+        file_type_map = {
+            "application/pdf": "pdf",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+            "text/plain": "txt"
+        }
+        file_type = file_type_map.get(file.content_type, "unknown")
+        
+        # Add document to vector database instead of session context
+        rag_result = rag_manager.add_document(
+            content=content,
+            filename=file.filename,
+            session_id=session_id,
+            file_type=file_type
+        )
+        
+        if not rag_result["success"]:
+            raise HTTPException(status_code=500, detail=f"Failed to process document: {rag_result.get('error', 'Unknown error')}")
+        
+        # Add just the filename to session for tracking (not the full content)
+        session.uploaded_files.append(file.filename)
+        session.total_upload_size += len(file_bytes)
+        
+        # Add a brief document reference to session messages (not full content)
+        session.append_message("system", f"Document uploaded: {file.filename} ({rag_result['chunks_created']} chunks, {rag_result['total_tokens']} tokens)")
 
-        return {"message": "Document uploaded and added to context successfully."}
+        return {
+            "message": "Document uploaded and processed successfully.",
+            "filename": file.filename,
+            "chunks_created": rag_result["chunks_created"],
+            "total_tokens": rag_result["total_tokens"],
+            "processing_method": "RAG_vector_storage"
+        }
 
     except HTTPException:
         raise
@@ -364,6 +380,21 @@ async def upload_document(file: UploadFile = File(...), request: Request = None)
         logger.error(f"Error uploading document: {str(e)}")
         raise HTTPException(status_code=500, detail=f"Error processing document: {str(e)}")
 
+# Add new endpoint to get document statistics
+@router.get("/document-stats")
+async def get_document_stats(request: Request):
+    """Get statistics about uploaded documents in vector database"""
+    try:
+        session_id = get_or_create_session_for_request(request)
+        rag_manager = get_rag_manager()
+        
+        stats = rag_manager.get_document_stats(session_id)
+        return stats
+        
+    except Exception as e:
+        logger.error(f"Error getting document stats: {str(e)}")
+        return {"total_chunks": 0, "total_documents": 0, "documents": []}
+
 # Get uploaded files (SAME INTERFACE)
 @router.get("/uploaded-files")
 async def get_uploaded_filenames(request: Request):
@@ -379,31 +410,44 @@ async def get_uploaded_filenames(request: Request):
 # Context endpoint (SAME INTERFACE)
 @router.get("/context")
 async def get_context(request: Request):
-    """Get context - SAME INTERFACE"""
+    """Get context - ENHANCED with RAG information"""
     try:
         session_id = get_or_create_session_for_request(request)
         session = session_manager.get_session(session_id)
-        return session.messages  # Return messages in same format as before
+
+        # Get RAG statistics
+        rag_stats = session.get_rag_stats()
+        
+        return {
+            "messages": session.messages,
+            "rag_info": {
+                "total_documents": rag_stats.get("total_documents", 0),
+                "total_chunks": rag_stats.get("total_chunks", 0),
+                "documents": rag_stats.get("documents", [])
+            }
+        }
     except Exception as e:
         logger.error(f"Error getting context: {str(e)}")
-        return []
+        return {"messages": [], "rag_info": {"total_documents": 0, "total_chunks": 0}}
 
-# Reset session (SAME INTERFACE) 
 @router.post("/reset-session")
 async def reset_session(request: Request):
-    """Reset session - SAME INTERFACE"""
+    """Reset session - ENHANCED with RAG cleanup"""
     try:
         session_id = get_or_create_session_for_request(request)
-        success = chat_orchestrator.reset_session(session_id)
+        
+        # Use the enhanced reset that clears both conversation and vector DB
+        success = session_manager.reset_session_completely(session_id)
         
         if success:
-            return {"status": "reset", "message": "Session reset successfully"}
+            return {"status": "reset", "message": "Session and all documents reset successfully"}
         else:
             return {"status": "error", "message": "Failed to reset session"}
     except Exception as e:
         logger.error(f"Error resetting session: {e}")
         return {"status": "error", "message": "Failed to reset session"}
 
+
 # Legacy model endpoints (SAME INTERFACE)
 @router.post("/switch-model")
 async def switch_model(model_name: str = Body(...)):
@@ -422,30 +466,195 @@ async def get_current_model():
         "provider": current_provider
     }
 
-# Debug endpoint (SAME INTERFACE)
+@router.post("/search-documents")
+async def search_documents(request: Request, query: str = Body(..., embed=True), persona: str = Body("", embed=True)):
+    """
+    Search uploaded documents using RAG
+    
+    This endpoint allows direct document search for debugging/testing
+    """
+    try:
+        session_id = get_or_create_session_for_request(request)
+        rag_manager = get_rag_manager()
+        
+        # Get persona context for search enhancement
+        persona_contexts = {
+            "methodist": "methodology research design analysis",
+            "theorist": "theory theoretical framework conceptual",
+            "pragmatist": "practical application implementation"
+        }
+        persona_context = persona_contexts.get(persona, "")
+        
+        # Search documents
+        results = rag_manager.search_documents(
+            query=query,
+            session_id=session_id,
+            persona_context=persona_context,
+            n_results=5
+        )
+        
+        return {
+            "query": query,
+            "persona_filter": persona,
+            "results_count": len(results),
+            "results": results
+        }
+        
+    except Exception as e:
+        logger.error(f"Error searching documents: {str(e)}")
+        return {"query": query, "results_count": 0, "results": [], "error": str(e)}
+
+@router.get("/session-stats")
+async def get_session_stats(request: Request):
+    """Get comprehensive session statistics including RAG data"""
+    try:
+        session_id = get_or_create_session_for_request(request)
+        stats = session_manager.get_session_stats(session_id)
+        return stats
+    except Exception as e:
+        logger.error(f"Error getting session stats: {str(e)}")
+        return {"error": str(e)}
+
+
 @router.get("/debug/personas")
 async def debug_personas(request: Request):
-    """Debug personas - SAME INTERFACE"""
+    """Debug personas - ENHANCED with RAG information"""
     try:
         session_id = get_or_create_session_for_request(request)
         session = session_manager.get_session(session_id)
         
+        # Get RAG statistics
+        rag_manager = get_rag_manager()
+        rag_stats = rag_manager.get_document_stats(session_id)
+        
         return {
             "personas": {
                 pid: {
                     "name": persona.name,
-                    "prompt": persona.system_prompt[:100] + "..."
+                    "prompt": persona.system_prompt[:100] + "...",
+                    "retrieval_keywords": chat_orchestrator._get_persona_context_keywords(pid)
                 } for pid, persona in chat_orchestrator.personas.items()
             },
-            "context_length": len(session.messages),
-            "current_provider": current_provider
+            "session_info": {
+                "context_length": len(session.messages),
+                "uploaded_files": session.uploaded_files,
+                "rag_stats": rag_stats
+            },
+            "current_provider": current_provider,
+            "rag_enabled": True
         }
     except Exception as e:
         logger.error(f"Error in debug endpoint: {str(e)}")
         return {
             "personas": {},
-            "context_length": 0,
-            "current_provider": current_provider
+            "session_info": {"context_length": 0},
+            "current_provider": current_provider,
+            "rag_enabled": False,
+            "error": str(e)
+        }
+
+@router.post("/chat/{persona_id}")
+async def chat_with_specific_persona(persona_id: str, message: ChatMessage, request: Request):
+    """
+    Chat with a specific persona - Enhanced with RAG debugging
+    
+    This endpoint helps debug RAG integration by testing individual personas
+    """
+    try:
+        session_id = get_or_create_session_for_request(request, message.session_id)
+        
+        # Validate persona exists
+        if persona_id not in chat_orchestrator.personas:
+            available_personas = list(chat_orchestrator.personas.keys())
+            raise HTTPException(
+                status_code=400, 
+                detail=f"Persona '{persona_id}' not found. Available: {available_personas}"
+            )
+        
+        # Use the enhanced orchestrator method
+        result = await chat_orchestrator.chat_with_persona(
+            user_input=message.user_input,
+            persona_id=persona_id,
+            session_id=session_id,
+            response_length=message.response_length or "medium"
+        )
+        
+        # Fix: Handle the response structure properly
+        if result.get("type") == "single_persona_response" and "persona" in result:
+            persona_data = result["persona"]
+            
+            # Add debugging information
+            result["debug_info"] = {
+                "persona_id": persona_id,
+                "session_id": session_id,
+                "query_length": len(message.user_input),
+                "rag_manager_available": True,
+                "used_documents": persona_data.get("used_documents", False),
+                "chunks_used": persona_data.get("document_chunks_used", 0)
+            }
+        
+        return result
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error in individual persona chat: {str(e)}")
+        return {
+            "type": "error",
+            "message": f"Error chatting with {persona_id}: {str(e)}",
+            "persona_id": persona_id
+        }
+
+# Also add a debug endpoint to check RAG status:
+
+@router.get("/debug/rag-status")
+async def debug_rag_status(request: Request):
+    """
+    Debug endpoint to check RAG system status
+    """
+    try:
+        session_id = get_or_create_session_for_request(request)
+        
+        # Get RAG manager
+        rag_manager = get_rag_manager()
+        
+        # Get session stats
+        session_stats = session_manager.get_session_stats(session_id)
+        
+        # Test a simple search
+        test_search = rag_manager.search_documents(
+            query="test methodology research",
+            session_id=session_id,
+            persona_context="",
+            n_results=3
+        )
+        
+        return {
+            "rag_manager_healthy": True,
+            "session_id": session_id,
+            "session_stats": session_stats.get("rag_stats", {}),
+            "test_search_results": len(test_search),
+            "test_search_details": [
+                {
+                    "relevance": chunk.get("relevance_score", 0),
+                    "distance": chunk.get("distance", "unknown"),
+                    "text_length": len(chunk.get("text", "")),
+                    "filename": chunk.get("metadata", {}).get("filename", "unknown")
+                }
+                for chunk in test_search[:3]
+            ],
+            "persona_keywords": {
+                pid: chat_orchestrator._get_persona_context_keywords(pid)
+                for pid in chat_orchestrator.personas.keys()
+            }
+        }
+        
+    except Exception as e:
+        logger.error(f"Error in RAG debug: {str(e)}")
+        return {
+            "rag_manager_healthy": False,
+            "error": str(e),
+            "session_id": session_id if 'session_id' in locals() else "unknown"
         }
 
 # Ask endpoint (SAME INTERFACE)
@@ -476,6 +685,8 @@ async def ask_question(query: PersonaQuery, request: Request):
     except Exception as e:
         logger.error(f"Error in ask endpoint: {str(e)}")
         return {"response": "I encountered an error. Please try again."}
+    
+
 
 # Root endpoint (SAME INTERFACE)
 @router.get("/")

multi_llm_chatbot_backend/app/core/improved_orchestrator.py

@@ -2,6 +2,7 @@ from typing import Dict, List, Optional, Any
 from app.models.persona import Persona
 from app.core.session_manager import ConversationContext, get_session_manager
 from app.core.context_manager import get_context_manager
+from app.core.rag_manager import get_rag_manager
 from app.llm.llm_client import LLMClient
 import logging
 import re
@@ -85,7 +86,7 @@ class ImprovedChatOrchestrator:
                                session_id: Optional[str] = None,
                                response_length: str = "medium") -> Dict[str, Any]:
         """
-        Chat with a specific persona
+        Chat with a specific persona - ENHANCED WITH RAG
         """
         try:
             if persona_id not in self.personas:
@@ -100,7 +101,7 @@ class ImprovedChatOrchestrator:
             # Add user message
             session.append_message("user", user_input)
             
-            # Generate response from specific persona
+            # Generate response from specific persona with RAG
             persona = self.personas[persona_id]
             response = await self._generate_single_persona_response(session, persona, response_length)
             
@@ -111,7 +112,11 @@ class ImprovedChatOrchestrator:
                 "type": "single_persona_response",
                 "session_id": session.session_id,
                 "persona": response,
-                "context_summary": self.context_manager.get_context_summary(session.messages)
+                "context_summary": self.context_manager.get_context_summary(session.messages),
+                "rag_info": {
+                    "used_documents": response.get("used_documents", False),
+                    "document_chunks_used": response.get("document_chunks_used", 0)
+                }
             }
             
         except Exception as e:
@@ -123,38 +128,30 @@ class ImprovedChatOrchestrator:
                 "error": str(e)
             }
     
-    def _needs_clarification(self, session: ConversationContext, user_input: str) -> bool:
+    def _needs_clarification(self, session, user_input: str) -> bool:
         """
-        Determine if user input needs clarification
+        Determine if we need clarification - SIMPLIFIED VERSION
         """
-        # Don't ask for clarification if this is a follow-up message
-        user_messages = session.get_messages_by_role("user")
-        if len(user_messages) > 1:
-            return False
-        
-        # Check if input is vague
-        vague_patterns = [
-            r"i'm (not sure|unsure|confused|lost)",
-            r"i (don't know|dunno) (what|how|where)",
-            r"help me with (my|the|a) (thesis|research|phd)",
-            r"(what should i|how do i|where do i start)",
-            r"i need (help|advice|guidance)$",
-            r"(stuck|struggling) with",
-            r"(any|some) (advice|suggestions|ideas)$",
-            r"^(help|advice|guidance)$",
-        ]
-        
-        user_lower = user_input.lower().strip()
-        
-        for pattern in vague_patterns:
-            if re.search(pattern, user_lower):
+        try:
+            # Simple heuristics for clarification
+            if len(user_input.strip()) < 10:
                 return True
-        
-        # Check if input is too short (likely vague)
-        if len(user_input.split()) < 8:
-            return True
-        
-        return False
+            
+            # Check if it's a greeting or very general question
+            greeting_words = ['hi', 'hello', 'hey', 'good morning', 'good afternoon']
+            if any(word in user_input.lower() for word in greeting_words):
+                return True
+            
+            # Very vague questions
+            vague_words = ['help', 'advice', 'what should i do', 'i need help']
+            if len([word for word in vague_words if word in user_input.lower()]) > 0 and len(user_input.split()) < 8:
+                return True
+            
+            return False
+            
+        except Exception as e:
+            logger.error(f"Error in clarification check: {str(e)}")
+            return False  # Default to no clarification needed
     
     async def _generate_clarification_question(self, session: ConversationContext) -> str:
         """
@@ -196,62 +193,84 @@ class ImprovedChatOrchestrator:
         else:
             return "Could you provide more details about your specific situation or question?"
     
-    async def _generate_persona_responses(self, 
-                                        session: ConversationContext, 
-                                        response_length: str) -> List[Dict[str, Any]]:
+    async def _generate_persona_responses(self, session, response_length: str = "medium"):
         """
-        Generate responses from all personas
+        Generate responses from all personas with RAG integration
+        
+        Each persona gets persona-specific document retrieval for more targeted responses
         """
         responses = []
         
-        # Get the conversation context for personas
-        context_messages = session.get_recent_messages(limit=20)
-        
         for persona_id, persona in self.personas.items():
-            try:
-                response_data = await self._generate_single_persona_response(
-                    session, persona, response_length
-                )
-                responses.append(response_data)
-                
-                # Add persona response to session
-                session.append_message(persona_id, response_data['response'])
-                
-            except Exception as e:
-                logger.error(f"Error generating response for persona {persona_id}: {str(e)}")
-                # Add fallback response
-                responses.append({
-                    "persona_id": persona_id,
-                    "persona_name": persona.name,
-                    "response": self._get_fallback_response(persona_id),
-                    "error": True
-                })
+            logger.info(f"Generating response for {persona_id} with RAG")
+            
+            # Generate persona response with RAG
+            response_data = await self._generate_single_persona_response(session, persona, response_length)
+            
+            # Add persona response to session context
+            session.append_message(persona_id, response_data["response"])
+            
+            responses.append(response_data)
         
         return responses
     
-    async def _generate_single_persona_response(self, 
-                                              session: ConversationContext,
-                                              persona: Persona, 
-                                              response_length: str) -> Dict[str, Any]:
+    async def _generate_single_persona_response(self, session, persona, response_length: str = "medium"):
         """
-        Generate response from a single persona
+        Generate response from a single persona with RAG-enhanced context - FIXED VERSION
         """
         try:
-            # Get conversation context
-            context_messages = session.get_recent_messages(limit=20)
+            # Get conversation context (recent messages only for efficiency)
+            recent_messages = session.messages[-5:] if len(session.messages) > 5 else session.messages
+            
+            # Get the user's latest message for document retrieval
+            user_message = ""
+            try:
+                # Use the new method to get latest user message
+                user_message = session.get_latest_user_message() or ""
+            except AttributeError:
+                # Fallback: manually find latest user message
+                for msg in reversed(recent_messages):
+                    if msg.get('role') == 'user':
+                        user_message = msg.get('content', '')
+                        break
+            
+            # Retrieve relevant document context using RAG
+            document_context = ""
+            if user_message:
+                document_context = await self._retrieve_relevant_documents(
+                    user_input=user_message,
+                    session_id=session.session_id,
+                    persona_id=persona.id
+                )
+            
+            # Build enhanced context for the LLM
+            enhanced_context = []
+            
+            # Add document context at the beginning if available
+            if document_context:
+                enhanced_context.append({
+                    "role": "system",
+                    "content": f"{document_context}Based on the above document context and conversation history, please respond as {persona.name}."
+                })
             
-            # Generate response using persona
-            response = await persona.respond(context_messages, response_length)
+            # Add recent conversation messages
+            enhanced_context.extend(recent_messages)
+            
+            # Generate response with enhanced context
+            response = await persona.respond(enhanced_context, response_length)
             
             # Validate response
             if not self._is_valid_response(response, persona.id):
+                logger.warning(f"Invalid response from {persona.id}, using fallback")
                 response = self._get_fallback_response(persona.id)
             
             return {
                 "persona_id": persona.id,
                 "persona_name": persona.name,
                 "response": response,
-                "error": False
+                "error": False,
+                "used_documents": bool(document_context),
+                "document_chunks_used": document_context.count("[Document") if document_context else 0
             }
             
         except Exception as e:
@@ -260,7 +279,9 @@ class ImprovedChatOrchestrator:
                 "persona_id": persona.id,
                 "persona_name": persona.name,
                 "response": self._get_fallback_response(persona.id),
-                "error": True
+                "error": True,
+                "used_documents": False,
+                "document_chunks_used": 0
             }
     
     def _is_valid_response(self, response: str, persona_id: str) -> bool:
@@ -314,4 +335,73 @@ class ImprovedChatOrchestrator:
     
     def delete_session(self, session_id: str) -> bool:
         """Delete a session completely"""
-        return self.session_manager.delete_session(session_id)
+        return self.session_manager.delete_session(session_id)
+    
+    def _get_persona_context_keywords(self, persona_id: str) -> str:
+        """
+        Get persona-specific keywords for enhanced document retrieval
+        
+        This ensures each advisor gets the most relevant document chunks
+        based on their specialization
+        """
+        persona_keywords = {
+            "methodist": "methodology research design analysis methods data collection sampling validity reliability statistical approach quantitative qualitative",
+            "theorist": "theory theoretical framework conceptual model literature review philosophy epistemology ontology paradigm abstract concepts",
+            "pragmatist": "practical application implementation action steps next steps recommendation solution strategy timeline concrete advice"
+        }
+        return persona_keywords.get(persona_id, "")
+    
+    async def _retrieve_relevant_documents(self, user_input: str, session_id: str, persona_id: str = "") -> str:
+        """
+        Retrieve relevant document chunks using RAG - FIXED VERSION
+        """
+        try:
+            rag_manager = get_rag_manager()
+            
+            # Get persona-specific context for better retrieval
+            persona_context = self._get_persona_context_keywords(persona_id)
+            
+            # Search for relevant chunks
+            relevant_chunks = rag_manager.search_documents(
+                query=user_input,
+                session_id=session_id,
+                persona_context=persona_context,
+                n_results=3  # Limit to top 3 most relevant chunks
+            )
+            
+            logger.info(f"Retrieved {len(relevant_chunks)} total chunks for {persona_id}")
+            
+            if not relevant_chunks:
+                logger.info(f"No relevant documents found for query: {user_input[:50]}...")
+                return ""
+            
+            # Format retrieved content - LOWERED THRESHOLD
+            context_parts = []
+            for i, chunk in enumerate(relevant_chunks, 1):
+                filename = chunk["metadata"].get("filename", "Unknown")
+                relevance = chunk["relevance_score"]
+                distance = chunk.get("distance", "unknown")
+                text = chunk["text"]
+                
+                logger.info(f"Chunk {i} for {persona_id}: relevance={relevance:.3f}, distance={distance}")
+                
+                # LOWERED threshold from 0.3 to 0.1 to allow more chunks through
+                if relevance > 0.1:
+                    context_parts.append(
+                        f"[Document {i}: {filename} (relevance: {relevance:.3f})]\n{text[:500]}..."  # Limit text length
+                    )
+                    logger.info(f"Including chunk {i} for {persona_id} (relevance: {relevance:.3f})")
+                else:
+                    logger.info(f"Excluding chunk {i} for {persona_id} (relevance too low: {relevance:.3f})")
+            
+            if context_parts:
+                retrieved_context = "\n\n".join(context_parts)
+                logger.info(f"Using {len(context_parts)} relevant chunks for {persona_id}")
+                return f"RELEVANT DOCUMENT CONTEXT:\n{retrieved_context}\n\n"
+            
+            logger.info(f"No chunks met relevance threshold for {persona_id}")
+            return ""
+            
+        except Exception as e:
+            logger.error(f"Error retrieving documents for {persona_id}: {str(e)}")
+            return ""

multi_llm_chatbot_backend/app/core/rag_manager.py

@@ -0,0 +1,469 @@
+import chromadb
+from chromadb.config import Settings
+from sentence_transformers import SentenceTransformer
+import nltk
+import tiktoken
+from typing import List, Dict, Any, Optional
+import uuid
+import logging
+import os
+import re
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Download required NLTK data
+try:
+    nltk.data.find('tokenizers/punkt')
+except LookupError:
+    try:
+        nltk.download('punkt')
+    except Exception as e:
+        logger.warning(f"Could not download NLTK punkt: {e}")
+
+class DocumentChunker:
+    """Intelligent document chunking with overlaps and semantic boundaries"""
+    
+    def __init__(self, chunk_size: int = 500, overlap: int = 50):
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+        try:
+            self.encoding = tiktoken.get_encoding("cl100k_base")  # GPT-4 encoding
+        except Exception as e:
+            logger.warning(f"Could not load tiktoken encoding: {e}")
+            self.encoding = None
+    
+    def _count_tokens(self, text: str) -> int:
+        """Count tokens in text"""
+        if self.encoding:
+            return len(self.encoding.encode(text))
+        else:
+            # Fallback: approximate 4 chars per token
+            return len(text) // 4
+    
+    def _encode_text(self, text: str) -> List[int]:
+        """Encode text to tokens"""
+        if self.encoding:
+            return self.encoding.encode(text)
+        else:
+            # Fallback: return character indices
+            return list(range(len(text)))
+    
+    def _decode_tokens(self, tokens: List[int]) -> str:
+        """Decode tokens back to text"""
+        if self.encoding:
+            return self.encoding.decode(tokens)
+        else:
+            # Fallback: can't properly decode without tiktoken
+            return ""
+    
+    def chunk_text(self, text: str, metadata: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Chunk text intelligently with semantic boundaries
+        """
+        # Clean and normalize text
+        text = self._clean_text(text)
+        
+        # Simple sentence splitting fallback if NLTK isn't available
+        try:
+            sentences = nltk.sent_tokenize(text)
+        except Exception:
+            # Fallback sentence splitting
+            sentences = re.split(r'[.!?]+', text)
+            sentences = [s.strip() for s in sentences if s.strip()]
+        
+        chunks = []
+        current_chunk = ""
+        current_tokens = 0
+        chunk_index = 0
+        
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if not sentence:
+                continue
+                
+            sentence_tokens = self._count_tokens(sentence)
+            
+            # If adding this sentence would exceed chunk size, save current chunk
+            if current_tokens + sentence_tokens > self.chunk_size and current_chunk:
+                chunk_data = self._create_chunk_metadata(
+                    current_chunk.strip(), 
+                    metadata, 
+                    chunk_index
+                )
+                chunks.append(chunk_data)
+                
+                # Start new chunk with overlap
+                overlap_text = self._get_overlap_text(current_chunk)
+                current_chunk = overlap_text + " " + sentence
+                current_tokens = self._count_tokens(current_chunk)
+                chunk_index += 1
+            else:
+                current_chunk += " " + sentence
+                current_tokens += sentence_tokens
+        
+        # Add final chunk if it has content
+        if current_chunk.strip():
+            chunk_data = self._create_chunk_metadata(
+                current_chunk.strip(), 
+                metadata, 
+                chunk_index
+            )
+            chunks.append(chunk_data)
+        
+        logger.info(f"Created {len(chunks)} chunks from document: {metadata.get('filename', 'unknown')}")
+        return chunks
+    
+    def _clean_text(self, text: str) -> str:
+        """Clean and normalize text"""
+        # Remove excessive whitespace and newlines
+        text = re.sub(r'\n+', '\n', text)
+        text = re.sub(r'\s+', ' ', text)
+        return text.strip()
+    
+    def _get_overlap_text(self, text: str) -> str:
+        """Get the last N tokens for overlap"""
+        if self.encoding:
+            tokens = self._encode_text(text)
+            if len(tokens) <= self.overlap:
+                return text
+            
+            overlap_tokens = tokens[-self.overlap:]
+            return self._decode_tokens(overlap_tokens)
+        else:
+            # Fallback: use character-based overlap
+            words = text.split()
+            overlap_words = words[-self.overlap:] if len(words) > self.overlap else words
+            return " ".join(overlap_words)
+    
+    def _create_chunk_metadata(self, chunk_text: str, base_metadata: Dict[str, Any], chunk_index: int) -> Dict[str, Any]:
+        """Create comprehensive metadata for a chunk"""
+        return {
+            "text": chunk_text,
+            "chunk_id": str(uuid.uuid4()),
+            "chunk_index": chunk_index,
+            "token_count": self._count_tokens(chunk_text),
+            "filename": base_metadata.get("filename", "unknown"),
+            "file_type": base_metadata.get("file_type", "unknown"),
+            "session_id": base_metadata.get("session_id"),
+            "upload_timestamp": base_metadata.get("upload_timestamp"),
+            "file_size": base_metadata.get("file_size", 0)
+        }
+
+class SimpleEmbeddingFunction:
+    """Simple embedding function wrapper for ChromaDB - FIXED for new interface"""
+    
+    def __init__(self, model):
+        self.model = model
+    
+    def __call__(self, input):
+        """Generate embeddings for input texts - Updated signature for ChromaDB 0.4.16+"""
+        try:
+            # Handle both single string and list of strings
+            if isinstance(input, str):
+                input = [input]
+            
+            embeddings = self.model.encode(input, convert_to_tensor=False)
+            return embeddings.tolist() if hasattr(embeddings, 'tolist') else embeddings
+        except Exception as e:
+            logger.error(f"Error generating embeddings: {e}")
+            # Return dummy embeddings as fallback
+            return [[0.0] * 384 for _ in input]  # 384 is dimension for all-MiniLM-L6-v2
+
+class RAGManager:
+    """
+    Retrieval-Augmented Generation Manager - FIXED VERSION
+    
+    Handles document storage, embedding, and retrieval using ChromaDB
+    """
+    
+    def __init__(self, embedding_model: str = "all-MiniLM-L6-v2", persist_directory: str = "./chroma_db"):
+        self.embedding_model_name = embedding_model
+        self.persist_directory = Path(persist_directory)
+        self.persist_directory.mkdir(exist_ok=True)
+        
+        # Initialize embedding model
+        logger.info(f"Loading embedding model: {embedding_model}")
+        try:
+            self.embedding_model = SentenceTransformer(embedding_model)
+            logger.info("Embedding model loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load embedding model: {e}")
+            raise
+        
+        # Initialize ChromaDB client
+        try:
+            self.client = chromadb.PersistentClient(
+                path=str(self.persist_directory),
+                settings=Settings(
+                    anonymized_telemetry=False,
+                    allow_reset=True
+                )
+            )
+            logger.info("ChromaDB client initialized")
+        except Exception as e:
+            logger.error(f"Failed to initialize ChromaDB client: {e}")
+            raise
+        
+        # Initialize collection
+        self.collection_name = "phd_advisor_documents"
+        self.collection = self._get_or_create_collection()
+        
+        # Initialize chunker
+        self.chunker = DocumentChunker()
+    
+    def _get_or_create_collection(self):
+        """Get or create the ChromaDB collection"""
+        try:
+            # First, try to get existing collection
+            try:
+                collection = self.client.get_collection(
+                    name=self.collection_name
+                )
+                logger.info(f"Found existing collection: {self.collection_name}")
+                return collection
+            except ValueError:
+                # Collection doesn't exist, create it
+                logger.info(f"Creating new collection: {self.collection_name}")
+                collection = self.client.create_collection(
+                    name=self.collection_name,
+                    embedding_function=SimpleEmbeddingFunction(self.embedding_model),
+                    metadata={"description": "PhD Advisor document storage"}
+                )
+                logger.info(f"Created collection: {self.collection_name}")
+                return collection
+                
+        except Exception as e:
+            logger.error(f"Error with collection management: {e}")
+            # Try to reset and recreate
+            try:
+                logger.info("Attempting to reset and recreate collection...")
+                self.client.reset()
+                collection = self.client.create_collection(
+                    name=self.collection_name,
+                    embedding_function=SimpleEmbeddingFunction(self.embedding_model),
+                    metadata={"description": "PhD Advisor document storage"}
+                )
+                logger.info("Successfully recreated collection")
+                return collection
+            except Exception as e2:
+                logger.error(f"Failed to recreate collection: {e2}")
+                raise
+    
+    def add_document(self, content: str, filename: str, session_id: str, file_type: str = "unknown") -> Dict[str, Any]:
+        """
+        Add a document to the vector database
+        """
+        try:
+            # Create base metadata
+            base_metadata = {
+                "filename": filename,
+                "file_type": file_type,
+                "session_id": session_id,
+                "upload_timestamp": str(uuid.uuid4()),  # Using UUID as timestamp for simplicity
+                "file_size": len(content)
+            }
+            
+            # Chunk the document
+            chunks = self.chunker.chunk_text(content, base_metadata)
+            
+            if not chunks:
+                raise ValueError("No chunks created from document")
+            
+            # Prepare data for ChromaDB
+            chunk_ids = [chunk["chunk_id"] for chunk in chunks]
+            chunk_texts = [chunk["text"] for chunk in chunks]
+            chunk_metadatas = [
+                {k: v for k, v in chunk.items() if k != "text" and k != "chunk_id"}
+                for chunk in chunks
+            ]
+            
+            # Add to ChromaDB with error handling
+            try:
+                self.collection.add(
+                    ids=chunk_ids,
+                    documents=chunk_texts,
+                    metadatas=chunk_metadatas
+                )
+                logger.info(f"Successfully added {len(chunks)} chunks for document: {filename}")
+            except Exception as e:
+                logger.error(f"Error adding chunks to ChromaDB: {e}")
+                # Try to recreate collection and try again
+                self.collection = self._get_or_create_collection()
+                self.collection.add(
+                    ids=chunk_ids,
+                    documents=chunk_texts,
+                    metadatas=chunk_metadatas
+                )
+                logger.info(f"Successfully added {len(chunks)} chunks after collection reset")
+            
+            return {
+                "success": True,
+                "filename": filename,
+                "chunks_created": len(chunks),
+                "total_tokens": sum(chunk["token_count"] for chunk in chunks),
+                "chunk_ids": chunk_ids
+            }
+            
+        except Exception as e:
+            logger.error(f"Error adding document {filename}: {str(e)}")
+            return {
+                "success": False,
+                "filename": filename,
+                "error": str(e)
+            }
+    
+    def search_documents(self, query: str, session_id: str, persona_context: str = "", n_results: int = 5) -> List[Dict[str, Any]]:
+        """
+        Search for relevant document chunks
+        """
+        try:
+            # Enhance query with persona context for better retrieval
+            enhanced_query = f"{query} {persona_context}".strip()
+            
+            # Search the collection
+            results = self.collection.query(
+                query_texts=[enhanced_query],
+                n_results=n_results,
+                where={"session_id": session_id}  # Filter by session
+            )
+            
+            # Format results
+            retrieved_chunks = []
+            if results['documents'] and results['documents'][0]:
+                for i, (doc, metadata, distance) in enumerate(zip(
+                    results['documents'][0],
+                    results['metadatas'][0],
+                    results['distances'][0]
+                )):
+                    # ChromaDB returns squared euclidean distance, convert to similarity
+                    # For better similarity scores, we'll use: similarity = 1 / (1 + distance)
+                    similarity_score = 1 / (1 + abs(distance)) if distance is not None else 0.5
+                    
+                    chunk_data = {
+                        "text": doc,
+                        "metadata": metadata,
+                        "relevance_score": similarity_score,
+                        "distance": distance,  # Keep original for debugging
+                        "rank": i + 1
+                    }
+                    retrieved_chunks.append(chunk_data)
+            
+            logger.info(f"Retrieved {len(retrieved_chunks)} chunks for query: {query[:50]}...")
+            return retrieved_chunks
+            
+        except Exception as e:
+            logger.error(f"Error searching documents: {str(e)}")
+            return []
+    
+    def get_document_stats(self, session_id: str) -> Dict[str, Any]:
+        """Get statistics about documents in a session"""
+        try:
+            # Get all chunks for this session
+            results = self.collection.get(
+                where={"session_id": session_id}
+            )
+            
+            if not results or not results.get('metadatas'):
+                return {
+                    "total_chunks": 0,
+                    "total_documents": 0,
+                    "documents": []
+                }
+            
+            # Analyze metadata
+            metadatas = results['metadatas']
+            documents = {}
+            
+            for metadata in metadatas:
+                filename = metadata.get('filename', 'unknown')
+                if filename not in documents:
+                    documents[filename] = {
+                        "filename": filename,
+                        "file_type": metadata.get('file_type', 'unknown'),
+                        "chunk_count": 0,
+                        "total_tokens": 0
+                    }
+                
+                documents[filename]["chunk_count"] += 1
+                documents[filename]["total_tokens"] += metadata.get('token_count', 0)
+            
+            return {
+                "total_chunks": len(metadatas),
+                "total_documents": len(documents),
+                "documents": list(documents.values())
+            }
+            
+        except Exception as e:
+            logger.error(f"Error getting document stats: {str(e)}")
+            return {"total_chunks": 0, "total_documents": 0, "documents": []}
+    
+    def delete_session_documents(self, session_id: str) -> bool:
+        """Delete all documents for a session"""
+        try:
+            # Get all document IDs for this session
+            results = self.collection.get(
+                where={"session_id": session_id}
+            )
+            
+            if results and results.get('ids'):
+                chunk_ids = results['ids']
+                self.collection.delete(ids=chunk_ids)
+                logger.info(f"Deleted {len(chunk_ids)} chunks for session: {session_id}")
+                return True
+            
+            return True  # No documents to delete is also success
+            
+        except Exception as e:
+            logger.error(f"Error deleting session documents: {str(e)}")
+            return False
+    
+    def health_check(self) -> Dict[str, Any]:
+        """Check if RAG system is working properly"""
+        try:
+            # Test basic operations
+            test_doc = "This is a test document for health checking."
+            test_session = "health_check_session"
+            
+            # Try adding a test document
+            result = self.add_document(test_doc, "test.txt", test_session, "txt")
+            if not result["success"]:
+                return {"status": "error", "message": "Failed to add test document"}
+            
+            # Try searching
+            search_results = self.search_documents("test", test_session)
+            
+            # Try getting stats
+            stats = self.get_document_stats(test_session)
+            
+            # Cleanup
+            self.delete_session_documents(test_session)
+            
+            return {
+                "status": "healthy",
+                "embedding_model": self.embedding_model_name,
+                "collection_name": self.collection_name,
+                "test_results": {
+                    "add_document": result["success"],
+                    "search_documents": len(search_results) > 0,
+                    "get_stats": stats["total_chunks"] > 0
+                }
+            }
+            
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+# Global RAG manager instance
+_rag_manager = None
+
+def get_rag_manager() -> RAGManager:
+    """Get the global RAG manager instance"""
+    global _rag_manager
+    if _rag_manager is None:
+        try:
+            _rag_manager = RAGManager()
+            logger.info("RAG Manager initialized successfully")
+        except Exception as e:
+            logger.error(f"Failed to initialize RAG Manager: {e}")
+            raise
+    return _rag_manager

multi_llm_chatbot_backend/app/core/session_manager.py

@@ -4,49 +4,106 @@ import uuid
 from dataclasses import dataclass, field
 import asyncio
 from threading import Lock
+from app.core.rag_manager import get_rag_manager
 
 @dataclass
 class ConversationContext:
-    """Individual conversation context for a session"""
-    session_id: str
-    messages: List[dict] = field(default_factory=list)
-    uploaded_files: List[str] = field(default_factory=list)
-    total_upload_size: int = 0
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    created_at: datetime = field(default_factory=datetime.now)
-    last_accessed: datetime = field(default_factory=datetime.now)
+    """Enhanced conversation context for RAG integration"""
     
+    def __init__(self, session_id: str = None):
+        self.session_id = session_id or str(uuid.uuid4())
+        self.messages: List[Dict[str, str]] = []
+        self.uploaded_files: List[str] = []  # Now just stores filenames, not content
+        self.total_upload_size: int = 0  # For tracking purposes only
+        self.created_at = datetime.now()
+        self.last_accessed = datetime.now()
+        
+        # New RAG-related attributes
+        self.document_chunks_count: int = 0  # Track total chunks in vector DB
+        self.last_retrieval_stats: Dict[str, Any] = {}  # Last RAG retrieval info
+
     def append_message(self, role: str, content: str):
-        """Add a message to the conversation"""
+        """Add a message to the conversation history"""
         self.messages.append({
             "role": role,
             "content": content,
             "timestamp": datetime.now().isoformat()
         })
         self.last_accessed = datetime.now()
-    
-    def get_recent_messages(self, limit: int = 20) -> List[dict]:
-        """Get recent messages with limit"""
-        return self.messages[-limit:] if len(self.messages) > limit else self.messages
-    
-    def get_messages_by_role(self, role: str) -> List[dict]:
-        """Get all messages from a specific role"""
-        return [msg for msg in self.messages if msg['role'] == role]
-    
+
     def clear_messages(self):
-        """Clear all messages but preserve metadata"""
-        self.messages = []
+        """Clear conversation messages but keep document references"""
+        self.messages.clear()
         self.last_accessed = datetime.now()
+
+    def get_messages_by_role(self, role: str) -> List[Dict[str, str]]:
+        """Get all messages by a specific role - ADDED METHOD"""
+        return [msg for msg in self.messages if msg.get('role') == role]
     
+    def get_recent_messages(self, count: int = 10) -> List[Dict[str, str]]:
+        """Get the most recent N messages"""
+        return self.messages[-count:] if len(self.messages) > count else self.messages
+    
+    def get_user_messages(self) -> List[Dict[str, str]]:
+        """Get all user messages"""
+        return self.get_messages_by_role('user')
+    
+    def get_latest_user_message(self) -> Optional[str]:
+        """Get the content of the most recent user message"""
+        user_messages = self.get_user_messages()
+        return user_messages[-1]['content'] if user_messages else None
+
     def add_uploaded_file(self, filename: str, content: str, file_size: int):
-        """Add uploaded file content to context"""
+        """
+        Add uploaded file - MODIFIED FOR RAG
+        
+        Now we only track the filename and size in session context.
+        The actual content goes to the vector database via RAG manager.
+        """
         self.uploaded_files.append(filename)
         self.total_upload_size += file_size
-        self.append_message("document", f"[Uploaded: {filename}]\n{content}")
-    
+        
+        # Add a system message noting the upload (not the full content)
+        self.append_message("system", f"Document '{filename}' uploaded and processed into vector database")
+        
+        # Update document chunk count from RAG manager
+        try:
+            rag_manager = get_rag_manager()
+            stats = rag_manager.get_document_stats(self.session_id)
+            self.document_chunks_count = stats.get("total_chunks", 0)
+        except Exception as e:
+            print(f"Warning: Could not update chunk count: {e}")
+
     def get_context_size(self) -> int:
-        """Calculate total context size in characters"""
+        """Calculate conversation context size in characters (excluding vector DB documents)"""
         return sum(len(msg['content']) for msg in self.messages)
+    
+    def get_rag_stats(self) -> Dict[str, Any]:
+        """Get statistics about documents in vector database for this session"""
+        try:
+            rag_manager = get_rag_manager()
+            return rag_manager.get_document_stats(self.session_id)
+        except Exception as e:
+            return {"error": str(e), "total_chunks": 0, "total_documents": 0}
+
+    def clear_all_data(self):
+        """Clear both conversation and vector database documents"""
+        # Clear conversation messages
+        self.clear_messages()
+        
+        # Clear vector database documents
+        try:
+            rag_manager = get_rag_manager()
+            success = rag_manager.delete_session_documents(self.session_id)
+            if success:
+                self.uploaded_files.clear()
+                self.total_upload_size = 0
+                self.document_chunks_count = 0
+                self.append_message("system", "All conversation history and documents cleared")
+            else:
+                self.append_message("system", "Conversation cleared, but some documents may remain")
+        except Exception as e:
+            self.append_message("system", f"Conversation cleared, document cleanup failed: {str(e)}")
 
 class SessionManager:
     """Thread-safe session manager for handling multiple user conversations"""
@@ -115,6 +172,42 @@ class SessionManager:
         
         if expired_sessions:
             print(f"Cleaned up {len(expired_sessions)} expired sessions")
+    
+    def reset_session_completely(self, session_id: str) -> bool:
+        """
+        Completely reset a session - both conversation and vector database documents
+        """
+        with self.lock:
+            if session_id in self.sessions:
+                session = self.sessions[session_id]
+                session.clear_all_data()
+                return True
+            return False
+    
+    def get_session_stats(self, session_id: str) -> Dict[str, Any]:
+        """Get comprehensive session statistics including RAG data"""
+        with self.lock:
+            if session_id not in self.sessions:
+                return {"error": "Session not found"}
+            
+            session = self.sessions[session_id]
+            
+            # Get basic session stats
+            basic_stats = {
+                "session_id": session_id,
+                "message_count": len(session.messages),
+                "uploaded_files": session.uploaded_files,
+                "total_upload_size": session.total_upload_size,
+                "context_size_chars": session.get_context_size(),
+                "created_at": session.created_at.isoformat(),
+                "last_accessed": session.last_accessed.isoformat()
+            }
+            
+            # Get RAG stats
+            rag_stats = session.get_rag_stats()
+            
+            # Combine stats
+            return {**basic_stats, "rag_stats": rag_stats}
 
 # Global session manager instance
 session_manager = SessionManager()

multi_llm_chatbot_backend/app/tests/debug_rag.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Debug RAG Chat Integration
+
+This script helps debug why RAG isn't working in chat responses.
+"""
+
+import requests
+import json
+import time
+
+BASE_URL = "http://localhost:8000"
+
+def test_direct_search():
+    """Test direct document search with detailed output"""
+    print("🔍 Testing direct document search...")
+    
+    test_queries = [
+        {"query": "research methodology approach", "persona": "methodist"},
+        {"query": "theoretical framework theory", "persona": "theorist"},
+        {"query": "practical steps implementation", "persona": "pragmatist"}
+    ]
+    
+    for test in test_queries:
+        print(f"\nTesting: '{test['query']}' for {test['persona']}")
+        
+        try:
+            response = requests.post(
+                f"{BASE_URL}/search-documents",
+                json=test,
+                timeout=10
+            )
+            
+            if response.status_code == 200:
+                result = response.json()
+                print(f"✅ Found {result['results_count']} results")
+                
+                for i, chunk in enumerate(result['results'], 1):
+                    score = chunk['relevance_score']
+                    distance = chunk.get('distance', 'unknown')
+                    text_preview = chunk['text'][:100]
+                    
+                    print(f"  Result {i}:")
+                    print(f"    Relevance: {score:.4f}")
+                    print(f"    Distance: {distance}")
+                    print(f"    Text: {text_preview}...")
+                    print(f"    Meets 0.1 threshold: {'✅' if score > 0.1 else '❌'}")
+            else:
+                print(f"❌ Search failed: {response.status_code}")
+                
+        except Exception as e:
+            print(f"❌ Search error: {e}")
+
+def test_single_persona_chat():
+    """Test individual persona chat to isolate issues"""
+    print("\n💬 Testing individual persona chat...")
+    
+    personas = ["methodist", "theorist", "pragmatist"]
+    query = "What research methodology should I use based on the uploaded document?"
+    
+    for persona in personas:
+        print(f"\nTesting {persona}...")
+        
+        try:
+            # Note: This endpoint might not exist yet, but we can try
+            response = requests.post(
+                f"{BASE_URL}/chat/{persona}",
+                json={"user_input": query, "response_length": "short"},
+                timeout=15
+            )
+            
+            if response.status_code == 200:
+                result = response.json()
+                print(f"✅ {persona} response received")
+                
+                if 'rag_info' in result:
+                    rag_info = result['rag_info']
+                    print(f"   Used documents: {rag_info.get('used_documents', False)}")
+                    print(f"   Chunks used: {rag_info.get('document_chunks_used', 0)}")
+                
+                if 'persona' in result:
+                    response_text = result['persona'].get('response', '')[:150]
+                    print(f"   Response: {response_text}...")
+            else:
+                print(f"❌ {persona} chat failed: {response.status_code}")
+                
+        except Exception as e:
+            print(f"❌ {persona} chat error: {e}")
+
+def test_sequential_chat_debug():
+    """Test sequential chat with more detailed debugging"""
+    print("\n🔄 Testing sequential chat with debug info...")
+    
+    query = "Based on the methodology document I uploaded, what approach should I take?"
+    
+    try:
+        response = requests.post(
+            f"{BASE_URL}/chat-sequential",
+            json={"user_input": query, "response_length": "short"},
+            timeout=30
+        )
+        
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ Sequential chat response received")
+            print(f"   Response type: {result.get('type')}")
+            
+            if 'rag_info' in result:
+                rag_info = result['rag_info']
+                print(f"   RAG enabled: {rag_info.get('rag_enabled', False)}")
+                print(f"   Personas using documents: {rag_info.get('personas_using_documents', 0)}")
+                print(f"   Total chunks used: {rag_info.get('total_document_chunks_used', 0)}")
+            
+            if 'responses' in result:
+                print(f"\n   Individual persona results:")
+                for resp in result['responses']:
+                    persona_name = resp.get('persona_name', 'Unknown')
+                    used_docs = resp.get('used_documents', False)
+                    chunks_used = resp.get('document_chunks_used', 0)
+                    error = resp.get('error', False)
+                    
+                    print(f"   {persona_name}:")
+                    print(f"     Used documents: {used_docs}")
+                    print(f"     Chunks used: {chunks_used}")
+                    print(f"     Error: {error}")
+                    
+                    if resp.get('response'):
+                        response_preview = resp['response'][:100]
+                        print(f"     Response: {response_preview}...")
+        else:
+            print(f"❌ Sequential chat failed: {response.status_code}")
+            print(f"   Response: {response.text}")
+            
+    except Exception as e:
+        print(f"❌ Sequential chat error: {e}")
+
+def upload_better_test_document():
+    """Upload a more targeted test document"""
+    print("\n📄 Uploading better test document...")
+    
+    better_doc = """
+    PhD Research Methodology Guide for Machine Learning
+    
+    METHODOLOGY SECTION:
+    For machine learning research, you should use a mixed-methods approach combining quantitative experiments with qualitative analysis. Start with baseline models and incrementally add complexity. Use proper train/validation/test splits with stratified sampling. Statistical significance testing is crucial for methodology validation.
+    
+    THEORETICAL FRAMEWORK SECTION:
+    Ground your work in established learning theory and computational complexity theory. Consider information theory principles and statistical learning theory. The epistemological assumptions should assume that knowledge can be extracted from data through systematic analysis. Review relevant literature and position your work within existing theoretical frameworks.
+    
+    PRACTICAL IMPLEMENTATION SECTION:
+    Your next steps should include: 1) Define clear research questions, 2) Design data collection methods, 3) Implement baseline models, 4) Conduct hyperparameter tuning, 5) Perform comprehensive evaluation, 6) Document all results. Create a timeline with specific milestones and success metrics for each phase.
+    """
+    
+    try:
+        with open("better_test_doc.txt", "w") as f:
+            f.write(better_doc)
+        
+        with open("better_test_doc.txt", "rb") as f:
+            files = {"file": ("ml_methodology_guide.txt", f, "text/plain")}
+            response = requests.post(f"{BASE_URL}/upload-document", files=files)
+        
+        import os
+        os.remove("better_test_doc.txt")
+        
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ Better document uploaded: {result['chunks_created']} chunks")
+            return True
+        else:
+            print(f"❌ Upload failed: {response.status_code}")
+            return False
+            
+    except Exception as e:
+        print(f"❌ Upload error: {e}")
+        return False
+
+def main():
+    """Run debug sequence"""
+    print("🔧 RAG Chat Debug Tool")
+    print("=" * 40)
+    
+    # Test 1: Upload better document
+    if not upload_better_test_document():
+        print("❌ Document upload failed, continuing with existing document...")
+    
+    # Test 2: Direct search
+    test_direct_search()
+    
+    # Test 3: Individual persona chat (if endpoint exists)
+    test_single_persona_chat()
+    
+    # Test 4: Sequential chat with debug
+    test_sequential_chat_debug()
+    
+    print("\n" + "=" * 40)
+    print("🔍 Debug Analysis:")
+    print("1. Check if direct search shows relevance scores > 0.1")
+    print("2. Check if individual personas work (may not be implemented)")
+    print("3. Check if sequential chat shows 'used_documents: true' for any persona")
+    print("4. Look for server logs showing document retrieval attempts")
+    
+    print("\nIf RAG still isn't working:")
+    print("- Check server logs for 'Retrieved X chunks for persona_name'")
+    print("- Verify relevance scores are above 0.1 threshold")
+    print("- Ensure document context is being passed to personas")
+
+if __name__ == "__main__":
+    main()

multi_llm_chatbot_backend/app/tests/test_rag_system.py

@@ -0,0 +1,155 @@
+import requests
+import json
+import time
+
+BASE_URL = "http://localhost:8000"
+
+def test_rag_system():
+    """
+    Comprehensive test of the RAG system
+    """
+    print("🚀 Testing RAG System Integration\n")
+    
+    # Test 1: Upload a document
+    print("📄 Test 1: Uploading a document...")
+    
+    # Create a sample research document
+    sample_document = """
+    Research Methodology for Machine Learning PhD
+    
+    Abstract: This document outlines the methodology for conducting research in machine learning, 
+    focusing on experimental design, data collection, and validation techniques.
+    
+    1. Introduction
+    Machine learning research requires rigorous methodology to ensure reproducible results.
+    The theoretical framework must be grounded in statistical learning theory.
+    
+    2. Methodology
+    Our research design follows a mixed-methods approach combining quantitative experiments
+    with qualitative analysis. Data collection involves sampling from multiple datasets
+    to ensure statistical validity and external validity.
+    
+    3. Practical Implementation
+    The implementation strategy focuses on actionable steps: first, establish baseline models,
+    then implement incremental improvements, and finally conduct comprehensive evaluation.
+    Each step should have clear success metrics and timelines.
+    
+    4. Theoretical Framework
+    The conceptual foundation draws from information theory, statistical learning theory,
+    and computational complexity theory. The epistemological assumptions underlying
+    our approach assume that knowledge can be extracted from data through systematic analysis.
+    
+    5. Next Steps
+    Immediate action items include: data preprocessing, model selection, hyperparameter tuning,
+    and result validation. The timeline should prioritize high-impact activities first.
+    """
+    
+    # Save as temporary file
+    with open("temp_research_doc.txt", "w") as f:
+        f.write(sample_document)
+    
+    # Upload the document
+    with open("temp_research_doc.txt", "rb") as f:
+        files = {"file": ("research_methodology.txt", f, "text/plain")}
+        upload_response = requests.post(f"{BASE_URL}/upload-document", files=files)
+    
+    print(f"Upload Status: {upload_response.status_code}")
+    print(f"Response: {json.dumps(upload_response.json(), indent=2)}\n")
+    
+    # Test 2: Get document statistics
+    print("📊 Test 2: Getting document statistics...")
+    stats_response = requests.get(f"{BASE_URL}/document-stats")
+    print(f"Stats: {json.dumps(stats_response.json(), indent=2)}\n")
+    
+    # Test 3: Test direct document search
+    print("🔍 Test 3: Testing direct document search...")
+    
+    search_queries = [
+        {"query": "What methodology should I use?", "persona": "methodist"},
+        {"query": "What is the theoretical framework?", "persona": "theorist"},
+        {"query": "What are the next steps?", "persona": "pragmatist"}
+    ]
+    
+    for search in search_queries:
+        print(f"\nSearching: '{search['query']}' for {search['persona']}")
+        search_response = requests.post(
+            f"{BASE_URL}/search-documents",
+            json=search
+        )
+        
+        if search_response.status_code == 200:
+            results = search_response.json()
+            print(f"Found {results['results_count']} results")
+            for i, result in enumerate(results['results'][:2], 1):  # Show top 2
+                print(f"  Result {i}: {result['text'][:100]}... (score: {result['relevance_score']:.3f})")
+        else:
+            print(f"Search failed: {search_response.status_code}")
+    
+    print("\n" + "="*60)
+    
+    # Test 4: Test RAG-enhanced chat
+    print("💬 Test 4: Testing RAG-enhanced chat responses...\n")
+    
+    chat_queries = [
+        "I need help with my research methodology. What approach should I take?",
+        "Can you explain the theoretical framework I should consider?", 
+        "What are the practical next steps I should focus on?"
+    ]
+    
+    for query in chat_queries:
+        print(f"Query: {query}")
+        chat_response = requests.post(
+            f"{BASE_URL}/chat-sequential",
+            json={"user_input": query, "response_length": "short"}
+        )
+        
+        if chat_response.status_code == 200:
+            result = chat_response.json()
+            print(f"Response type: {result.get('type')}")
+            
+            if 'rag_info' in result:
+                rag_info = result['rag_info']
+                print(f"RAG Info: {rag_info['personas_using_documents']}/{len(result.get('responses', []))} personas used documents")
+                print(f"Total chunks used: {rag_info['total_document_chunks_used']}")
+            
+            # Show one advisor response
+            if 'responses' in result and result['responses']:
+                advisor = result['responses'][0]
+                print(f"\n{advisor['persona_name']}: {advisor['response'][:200]}...")
+                if advisor.get('used_documents'):
+                    print(f"✅ Used {advisor.get('document_chunks_used', 0)} document chunks")
+                else:
+                    print("❌ No documents used")
+        else:
+            print(f"Chat failed: {chat_response.status_code}")
+        
+        print("\n" + "-"*40 + "\n")
+    
+    # Test 5: Session statistics
+    print("📈 Test 5: Getting comprehensive session statistics...")
+    session_stats = requests.get(f"{BASE_URL}/session-stats")
+    if session_stats.status_code == 200:
+        stats = session_stats.json()
+        print(f"Session Stats:")
+        print(f"  Messages: {stats.get('message_count', 0)}")
+        print(f"  Uploaded files: {len(stats.get('uploaded_files', []))}")
+        print(f"  RAG documents: {stats.get('rag_stats', {}).get('total_documents', 0)}")
+        print(f"  RAG chunks: {stats.get('rag_stats', {}).get('total_chunks', 0)}")
+    
+    # Cleanup
+    import os
+    try:
+        os.remove("temp_research_doc.txt")
+    except:
+        pass
+    
+    print("\n🎉 RAG System Testing Complete!")
+    print("\nKey Improvements:")
+    print("✅ Documents are now chunked and stored in vector database")
+    print("✅ Each advisor gets persona-specific document chunks") 
+    print("✅ Only relevant content is retrieved for each query")
+    print("✅ Session context is much more efficient")
+    print("✅ Supports semantic search across all uploaded documents")
+
+if __name__ == "__main__":
+    test_rag_system()

multi_llm_chatbot_backend/requirements.txt

@@ -1,3 +1,10 @@
 fastapi
 uvicorn
-httpx
+httpx
+PyPDF2
+docx2txt
+python-dotenv
+chromadb
+sentence-transformers
+nltk
+tiktoken

phd-advisor-frontend/src/components/MessageBubble.js

@@ -1,5 +1,5 @@
-import React, { useState } from 'react';
-import { Reply, Copy, Check, Maximize2 } from 'lucide-react';
+import React, { useState, useRef, useEffect } from 'react';
+import { Reply, Copy, Check, Maximize2, Info, FileText, Hash, Target } from 'lucide-react';
 import { advisors, getAdvisorColors } from '../data/advisors';
 import { useTheme } from '../contexts/ThemeContext';
 
@@ -13,6 +13,8 @@ const MessageBubble = ({
   const { isDark } = useTheme();
   const [showTooltip, setShowTooltip] = useState(null);
   const [copiedStates, setCopiedStates] = useState({});
+  const [showInfoOverlay, setShowInfoOverlay] = useState(false);
+  const overlayRef = useRef(null);
 
   const handleCopy = async (messageId, content) => {
     try {
@@ -33,6 +35,10 @@ const MessageBubble = ({
     if (onExpand) onExpand(messageId, advisorId);
   };
 
+  const handleInfoToggle = () => {
+    setShowInfoOverlay(!showInfoOverlay);
+  };
+
   const showTooltipWithDelay = (tooltipType) => {
     setTimeout(() => setShowTooltip(tooltipType), 500);
   };
@@ -41,6 +47,99 @@ const MessageBubble = ({
     setShowTooltip(null);
   };
 
+  // Close overlay when clicking outside
+  useEffect(() => {
+    const handleClickOutside = (event) => {
+      if (overlayRef.current && !overlayRef.current.contains(event.target)) {
+        setShowInfoOverlay(false);
+      }
+    };
+
+    if (showInfoOverlay) {
+      document.addEventListener('mousedown', handleClickOutside);
+      return () => {
+        document.removeEventListener('mousedown', handleClickOutside);
+      };
+    }
+  }, [showInfoOverlay]);
+
+  // RAG Metadata Component
+  const RagInfoOverlay = ({ ragMetadata, colors }) => {
+    const hasDocuments = ragMetadata?.usedDocuments || false;
+    const chunksUsed = ragMetadata?.chunksUsed || 0;
+    const documentChunks = ragMetadata?.documentChunks || [];
+
+    return (
+      <div 
+        ref={overlayRef}
+        className="rag-info-overlay"
+        style={{ 
+          borderColor: colors.color + '40',
+          backgroundColor: isDark ? '#1f2937' : '#ffffff'
+        }}
+      >
+        <div className="rag-overlay-header" style={{ color: colors.color }}>
+          <Info size={14} />
+          <span>RAG Information</span>
+        </div>
+        
+        <div className="rag-overlay-content">
+          {/* Basic Stats */}
+          <div className="rag-stat-row">
+            <div className="rag-stat-label">Used Documents:</div>
+            <div className={`rag-stat-value ${hasDocuments ? 'positive' : 'negative'}`}>
+              {hasDocuments ? 'Yes' : 'No'}
+            </div>
+          </div>
+
+          <div className="rag-stat-row">
+            <div className="rag-stat-label">Document Chunks:</div>
+            <div className="rag-stat-value">{chunksUsed}</div>
+          </div>
+
+          {/* Document Details */}
+          {hasDocuments && documentChunks.length > 0 && (
+            <div className="rag-documents-section">
+              <div className="rag-section-title">
+                <FileText size={12} />
+                Referenced Sources
+              </div>
+              
+              {documentChunks.map((chunk, index) => (
+                <div key={index} className="rag-document-item">
+                  <div className="rag-document-header">
+                    <span className="rag-filename">
+                      {chunk.metadata?.filename || 'Unknown file'}
+                    </span>
+                    <span className="rag-relevance">
+                      <Target size={10} />
+                      {Math.round((chunk.relevance_score || 0) * 100)}%
+                    </span>
+                  </div>
+                  
+                  {chunk.text && (
+                    <div className="rag-chunk-preview">
+                      {chunk.text.substring(0, 120)}
+                      {chunk.text.length > 120 && '...'}
+                    </div>
+                  )}
+                </div>
+              ))}
+            </div>
+          )}
+
+          {/* No Documents Message */}
+          {!hasDocuments && (
+            <div className="rag-no-documents">
+              <Hash size={12} />
+              <span>This response was generated without referencing uploaded documents.</span>
+            </div>
+          )}
+        </div>
+      </div>
+    );
+  };
+
   if (message.type === 'user') {
     return (
       <div className="user-message-container">
@@ -75,7 +174,8 @@ const MessageBubble = ({
           className="advisor-message-bubble"
           style={{ 
             backgroundColor: colors.bgColor,
-            borderColor: colors.color + '40'
+            borderColor: colors.color + '40',
+            position: 'relative' // For overlay positioning
           }}
         >
           <div className="advisor-message-header">
@@ -169,9 +269,37 @@ const MessageBubble = ({
                     <div className="tooltip">Expand on this response</div>
                   )}
                 </div>
+
+                {/* NEW: Info Button */}
+                <div className="tooltip-container">
+                  <button 
+                    className="action-button"
+                    onClick={handleInfoToggle}
+                    onMouseEnter={() => showTooltipWithDelay('info')}
+                    onMouseLeave={hideTooltip}
+                    style={{ 
+                      color: showInfoOverlay ? colors.color : colors.color,
+                      borderColor: showInfoOverlay ? colors.color : colors.color + '40',
+                      backgroundColor: showInfoOverlay ? colors.color + '20' : 'transparent'
+                    }}
+                  >
+                    <Info size={14} />
+                  </button>
+                  {showTooltip === 'info' && (
+                    <div className="tooltip">RAG Information</div>
+                  )}
+                </div>
               </div>
             </div>
           )}
+
+          {/* RAG Info Overlay */}
+          {showInfoOverlay && (
+            <RagInfoOverlay 
+              ragMetadata={message.ragMetadata} 
+              colors={colors}
+            />
+          )}
         </div>
       </div>
     );

phd-advisor-frontend/src/pages/ChatPage.js

@@ -138,270 +138,245 @@ const ChatPage = ({ onNavigateToHome }) => {
   };
 
   const handleSendMessage = async (inputMessage) => {
-    if (replyingTo) {
-      await handleReplyToAdvisor(inputMessage, replyingTo);
-      return;
-    }
+  if (replyingTo) {
+    await handleReplyToAdvisor(inputMessage, replyingTo);
+    return;
+  }
+
+  const userMessage = {
+    id: generateMessageId(),
+    type: 'user',
+    content: inputMessage,
+    timestamp: new Date()
+  };
+  setMessages(prev => [...prev, userMessage]);
+  
+  setIsLoading(true);
+  setThinkingAdvisors(['system']);
+
+  try {
+    const response = await fetch('http://localhost:8000/chat-sequential', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        user_input: inputMessage
+      }),
+    });
 
-    const userMessage = {
-      id: generateMessageId(),
-      type: 'user',
-      content: inputMessage,
-      timestamp: new Date()
-    };
-    setMessages(prev => [...prev, userMessage]);
-    
-    setIsLoading(true);
-    setThinkingAdvisors(['system']);
+    if (!response.ok) {
+      throw new Error(`HTTP error! Status: ${response.status}`);
+    }
 
-    try {
-      const response = await fetch('http://localhost:8000/chat-sequential', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          user_input: inputMessage
-        }),
-      });
+    const data = await response.json();
+    console.log('Backend response:', data); // Debug log
 
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
+    if (data.type === 'persona_responses' && data.responses) {
+      // Map each advisor response with RAG metadata
+      const advisorMessages = data.responses.map((advisor, index) => ({
+        id: generateMessageId(),
+        type: 'advisor',
+        advisorId: advisor.persona_id,
+        advisorName: advisor.persona_name,
+        content: advisor.response,
+        timestamp: new Date(),
+        // NEW: Map RAG metadata to the structure MessageBubble expects
+        ragMetadata: {
+          usedDocuments: advisor.used_documents || false,
+          chunksUsed: advisor.document_chunks_used || 0,
+          documentChunks: advisor.retrieved_chunks || []
+        }
+      }));
 
-      const data = await response.json();
-      setCollectedInfo(data.collected_info || {});
-      setThinkingAdvisors([]);
+      setMessages(prev => [...prev, ...advisorMessages]);
 
-      if (data.type === 'orchestrator_question') {
-        const orchestratorMessage = {
-          id: generateMessageId(),
-          type: 'orchestrator',
-          content: data.responses[0].response,
-          timestamp: new Date()
-        };
-        setMessages(prev => [...prev, orchestratorMessage]);
-      } else if (data.type === 'sequential_responses') {
-        const advisorIds = ['methodist', 'theorist', 'pragmatist'];
-        
-        for (let i = 0; i < advisorIds.length; i++) {
-          const advisorId = advisorIds[i];
-          const response = data.responses.find(r => r.persona_id === advisorId);
-          
-          if (response) {
-            setThinkingAdvisors([advisorId]);
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            
-            const advisorMessage = {
-              id: generateMessageId(),
-              type: 'advisor',
-              advisorId: advisorId,
-              content: response.response,
-              timestamp: new Date()
-            };
-            
-            setMessages(prev => [...prev, advisorMessage]);
-            setThinkingAdvisors([]);
-            
-            if (i < advisorIds.length - 1) {
-              await new Promise(resolve => setTimeout(resolve, 500));
-            }
-          }
-        }
-      } else if (data.type === 'error') {
-        const errorMessage = {
+      // Optional: Add RAG summary message if documents were used
+      const ragInfo = data.rag_info || {};
+      if (ragInfo.personas_using_documents > 0) {
+        const ragSummaryMessage = {
           id: generateMessageId(),
-          type: 'error',
-          content: data.responses[0].response,
+          type: 'system',
+          content: `📚 ${ragInfo.personas_using_documents}/${data.responses.length} advisors referenced your uploaded documents (${ragInfo.total_document_chunks_used} chunks used)`,
           timestamp: new Date()
         };
-        setMessages(prev => [...prev, errorMessage]);
+        setMessages(prev => [...prev, ragSummaryMessage]);
       }
 
-    } catch (error) {
-      console.error('Error sending message:', error);
+    } else if (data.type === 'error') {
       const errorMessage = {
         id: generateMessageId(),
         type: 'error',
-        content: 'Sorry, I encountered an error. Please try again.',
+        content: data.message || 'An error occurred. Please try again.',
         timestamp: new Date()
       };
       setMessages(prev => [...prev, errorMessage]);
-    } finally {
-      setIsLoading(false);
-      setThinkingAdvisors([]);
     }
-  };
-
-  const handleReplyToAdvisor = async (inputMessage, replyInfo) => {
-    const userMessage = {
-      id: generateMessageId(),
-      type: 'user',
-      content: inputMessage,
-      timestamp: new Date(),
-      replyingTo: replyInfo
-    };
-    setMessages(prev => [...prev, userMessage]);
-    
-    setIsLoading(true);
-    setThinkingAdvisors([replyInfo.advisorId]);
-
-    try {
-      const response = await fetch('http://localhost:8000/reply-to-advisor', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          user_input: inputMessage,
-          advisor_id: replyInfo.advisorId,
-          original_message_id: replyInfo.messageId
-        }),
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-
-      const data = await response.json();
-      setThinkingAdvisors([]);
 
-      if (data.type === 'advisor_reply') {
-        const replyMessage = {
-          id: generateMessageId(),
-          type: 'advisor',
-          advisorId: replyInfo.advisorId,
-          content: data.response,
-          timestamp: new Date(),
-          isReply: true
-        };
-        setMessages(prev => [...prev, replyMessage]);
-      } else if (data.type === 'error') {
+    } catch (error) {
+        console.error('Error sending message:', error);
         const errorMessage = {
           id: generateMessageId(),
           type: 'error',
-          content: data.response,
+          content: 'Sorry, I encountered an error. Please try again.',
           timestamp: new Date()
-        };
-        setMessages(prev => [...prev, errorMessage]);
-      }
+      };
+      setMessages(prev => [...prev, errorMessage]);
+  }
 
-    } catch (error) {
-      console.error('Error sending reply:', error);
-      const errorMessage = {
+    setIsLoading(false);
+    setThinkingAdvisors([]);
+  };
+
+  const handleReplyToAdvisor = async (inputMessage, replyContext) => {
+  const replyMessage = {
+    id: generateMessageId(),
+    type: 'user',
+    content: inputMessage,
+    replyTo: {
+      advisorId: replyContext.advisorId,
+      advisorName: replyContext.advisorName,
+      messageId: replyContext.messageId
+    },
+    timestamp: new Date()
+  };
+  setMessages(prev => [...prev, replyMessage]);
+  
+  setIsLoading(true);
+  setThinkingAdvisors([replyContext.advisorId]);
+
+  try {
+    const response = await fetch('http://localhost:8000/reply-to-advisor', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        user_input: inputMessage,
+        advisor_id: replyContext.advisorId,
+        original_message_id: replyContext.messageId
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! Status: ${response.status}`);
+    }
+
+    const data = await response.json();
+
+    if (data.type === 'advisor_reply') {
+      const replyResponseMessage = {
         id: generateMessageId(),
-        type: 'error',
-        content: 'Sorry, I encountered an error. Please try again.',
-        timestamp: new Date()
+        type: 'advisor',
+        advisorId: data.persona_id,
+        advisorName: data.persona,
+        content: data.response,
+        isReply: true,
+        timestamp: new Date(),
+        // NEW: Map RAG metadata for reply responses too
+        ragMetadata: {
+          usedDocuments: data.used_documents || false,
+          chunksUsed: data.document_chunks_used || 0,
+          documentChunks: data.retrieved_chunks || []
+        }
       };
-      setMessages(prev => [...prev, errorMessage]);
-    } finally {
-      setIsLoading(false);
-      setThinkingAdvisors([]);
-      setReplyingTo(null);
+      setMessages(prev => [...prev, replyResponseMessage]);
     }
+
+  } catch (error) {
+    console.error('Error replying to advisor:', error);
+    const errorMessage = {
+      id: generateMessageId(),
+      type: 'error',
+      content: 'Sorry, I encountered an error with your reply. Please try again.',
+      timestamp: new Date()
+    };
+    setMessages(prev => [...prev, errorMessage]);
+  }
+
+    setIsLoading(false);
+    setThinkingAdvisors([]);
+    setReplyingTo(null);
   };
 
   const handleCopyMessage = (messageId, content) => {
   // Optional: Show a toast notification or add to message history
   console.log(`Copied message ${messageId}: ${content.substring(0, 50)}...`);
-  
-  // could add a temporary notification here if desired
-  // const notificationMessage = {
-  //   id: generateMessageId(),
-  //   type: 'system',
-  //   content: '✅ Response copied to clipboard',
-  //   timestamp: new Date()
-  // };
-  
-  // setMessages(prev => [...prev, notificationMessage]);
-  
-  // // Remove the notification after 3 seconds
-  // setTimeout(() => {
-  //   setMessages(prev => prev.filter(msg => msg.id !== notificationMessage.id));
-  // }, 3000);
   };
 
-  const handleExpandMessage = async (messageId, advisorId) => {
-    const advisor = advisors[advisorId];
-    
-    try {
-      setIsLoading(true);
-      setThinkingAdvisors([advisorId]);
-
-      // Find the original message to expand on
-      const originalMessage = messages.find(msg => msg.id === messageId);
-      
-      if (!originalMessage) {
-        console.error('Original message not found');
-        return;
-      }
-
-      // Create advisor-specific expansion prompts
-      const advisorSpecificPrompts = {
-        'methodologist': `Please expand on your previous response with more methodological detail. Include specific research methods, data collection techniques, analytical approaches, and methodological considerations that would be relevant. Provide practical examples and step-by-step guidance where applicable.`,
-        'theorist': `Please elaborate on your previous response by exploring the theoretical frameworks and conceptual foundations in greater depth. Include additional theoretical perspectives, scholarly context, and conceptual analysis that would enrich understanding of the topic.`,
-        'pragmatist': `Please expand on your previous response with more practical, actionable details. Include specific examples, concrete next steps, real-world applications, and practical strategies that can be immediately implemented.`
-      };
+const handleExpandMessage = async (messageId, advisorId) => {
+  const advisor = advisors[advisorId];
+  if (!advisor) return;
 
-      // Use advisor-specific prompt or general expansion prompt
-      const expandPrompt = advisorSpecificPrompts[advisorId] || 
-        `Please expand and elaborate on your previous response in more detail. Provide additional insights, practical examples, and deeper analysis that would be helpful for understanding and implementing your advice.`;
+  const originalMessage = messages.find(msg => msg.id === messageId);
+  if (!originalMessage) return;
 
-      // Use the existing /reply-to-advisor endpoint
-      const response = await fetch('http://localhost:8000/reply-to-advisor', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          advisor_id: advisorId,
-          user_input: expandPrompt,
-          original_message_id: messageId
-        }),
-      });
+  const expandPrompt = `Please expand on your previous response: "${originalMessage.content.substring(0, 100)}..." Provide more detail and depth.`;
+  
+  const expandMessage = {
+    id: generateMessageId(),
+    type: 'user',
+    content: expandPrompt,
+    timestamp: new Date(),
+    isExpandRequest: true,
+    expandsMessageId: messageId
+  };
+  setMessages(prev => [...prev, expandMessage]);
+  
+  setIsLoading(true);
+  setThinkingAdvisors([advisorId]);
+
+  try {
+    const response = await fetch(`http://localhost:8000/chat/${advisorId}`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        user_input: expandPrompt,
+        response_length: 'long'
+      }),
+    });
 
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
+    if (!response.ok) {
+      throw new Error(`HTTP error! Status: ${response.status}`);
+    }
 
-      const data = await response.json();
-      setThinkingAdvisors([]);
+    const data = await response.json();
 
-      if (data.type === 'advisor_reply') {
-        const expandedMessage = {
-          id: generateMessageId(),
-          type: 'advisor',
-          advisorId: advisorId,
-          content: data.response,
-          timestamp: new Date(),
-          isExpansion: true, // Mark this as an expansion
-          expandedFrom: messageId // Reference to original message
-        };
-        setMessages(prev => [...prev, expandedMessage]);
-      } else if (data.type === 'error') {
-        const errorMessage = {
-          id: generateMessageId(),
-          type: 'error',
-          content: data.response,
-          timestamp: new Date()
-        };
-        setMessages(prev => [...prev, errorMessage]);
-      }
-
-    } catch (error) {
-      console.error('Error expanding message:', error);
-      const errorMessage = {
+    if (data.type === 'single_persona_response' && data.persona) {
+      const expandedMessage = {
         id: generateMessageId(),
-        type: 'error',
-        content: 'Sorry, I encountered an error while expanding the response. Please try again.',
-        timestamp: new Date()
+        type: 'advisor',
+        advisorId: advisorId,
+        advisorName: advisor.name,
+        content: data.persona.response,
+        isExpansion: true,
+        expandsMessageId: messageId,
+        timestamp: new Date(),
+        // NEW: Map RAG metadata for expanded responses
+        ragMetadata: {
+          usedDocuments: data.persona.used_documents || false,
+          chunksUsed: data.persona.document_chunks_used || 0,
+          documentChunks: data.persona.retrieved_chunks || []
+        }
       };
-      setMessages(prev => [...prev, errorMessage]);
-    } finally {
-      setIsLoading(false);
-      setThinkingAdvisors([]);
+      setMessages(prev => [...prev, expandedMessage]);
     }
+
+  } catch (error) {
+    console.error('Error expanding message:', error);
+    const errorMessage = {
+      id: generateMessageId(),
+      type: 'error',
+      content: 'Sorry, I encountered an error expanding the response. Please try again.',
+      timestamp: new Date()
+    };
+    setMessages(prev => [...prev, errorMessage]);
+  }
+
+    setIsLoading(false);
+    setThinkingAdvisors([]);
   };
 
   const handleReplyToMessage = (message) => {

phd-advisor-frontend/src/styles/components.css

@@ -830,6 +830,230 @@
   display: none;
 }
 
+.rag-info-overlay {
+  position: absolute;
+  top: 100%;
+  right: 0;
+  z-index: 1000;
+  width: 320px;
+  max-width: 90vw;
+  border: 1px solid;
+  border-radius: 8px;
+  box-shadow: 0 8px 25px rgba(0, 0, 0, 0.15);
+  margin-top: 8px;
+  backdrop-filter: blur(10px);
+  animation: ragOverlaySlideIn 0.2s ease-out;
+}
+
+@keyframes ragOverlaySlideIn {
+  from {
+    opacity: 0;
+    transform: translateY(-10px) scale(0.95);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0) scale(1);
+  }
+}
+
+.rag-overlay-header {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 12px 16px;
+  font-weight: 600;
+  font-size: 14px;
+  border-bottom: 1px solid var(--border-color);
+  background: var(--bg-secondary);
+  border-radius: 8px 8px 0 0;
+}
+
+.rag-overlay-content {
+  padding: 16px;
+  max-height: 400px;
+  overflow-y: auto;
+}
+
+.rag-stat-row {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 12px;
+  padding: 8px 0;
+  border-bottom: 1px solid var(--border-secondary);
+}
+
+.rag-stat-row:last-child {
+  border-bottom: none;
+  margin-bottom: 0;
+}
+
+.rag-stat-label {
+  font-size: 13px;
+  color: var(--text-secondary);
+  font-weight: 500;
+}
+
+.rag-stat-value {
+  font-size: 13px;
+  font-weight: 600;
+  color: var(--text-primary);
+}
+
+.rag-stat-value.positive {
+  color: #10B981;
+}
+
+.rag-stat-value.negative {
+  color: #6B7280;
+}
+
+.rag-documents-section {
+  margin-top: 16px;
+  padding-top: 16px;
+  border-top: 1px solid var(--border-secondary);
+}
+
+.rag-section-title {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  font-size: 12px;
+  font-weight: 600;
+  color: var(--text-secondary);
+  margin-bottom: 12px;
+  text-transform: uppercase;
+  letter-spacing: 0.5px;
+}
+
+.rag-document-item {
+  background: var(--bg-tertiary);
+  border: 1px solid var(--border-secondary);
+  border-radius: 6px;
+  padding: 10px;
+  margin-bottom: 8px;
+}
+
+.rag-document-item:last-child {
+  margin-bottom: 0;
+}
+
+.rag-document-header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 6px;
+}
+
+.rag-filename {
+  font-size: 12px;
+  font-weight: 600;
+  color: var(--text-primary);
+  flex: 1;
+  margin-right: 8px;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+
+.rag-relevance {
+  display: flex;
+  align-items: center;
+  gap: 3px;
+  font-size: 11px;
+  font-weight: 600;
+  color: var(--accent-color);
+  background: var(--accent-color)15;
+  padding: 2px 6px;
+  border-radius: 10px;
+  white-space: nowrap;
+}
+
+.rag-chunk-preview {
+  font-size: 11px;
+  line-height: 1.4;
+  color: var(--text-tertiary);
+  background: var(--bg-quaternary);
+  padding: 6px 8px;
+  border-radius: 4px;
+  border-left: 2px solid var(--accent-color);
+  margin-top: 6px;
+}
+
+.rag-no-documents {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 12px;
+  background: var(--bg-secondary);
+  border: 1px dashed var(--border-secondary);
+  border-radius: 6px;
+  color: var(--text-secondary);
+  font-size: 12px;
+  font-style: italic;
+  margin-top: 8px;
+}
+
+/* Accessibility improvements */
+.rag-info-overlay:focus-within {
+  outline: 2px solid var(--accent-color);
+  outline-offset: 2px;
+}
+
+/* Smooth transitions */
+.action-button {
+  transition: all 0.2s ease;
+}
+
+.action-button:hover {
+  transform: translateY(-1px);
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+/* Custom scrollbar for overlay content */
+.rag-overlay-content::-webkit-scrollbar {
+  width: 4px;
+}
+
+.rag-overlay-content::-webkit-scrollbar-track {
+  background: var(--bg-secondary);
+  border-radius: 2px;
+}
+
+.rag-overlay-content::-webkit-scrollbar-thumb {
+  background: var(--border-color);
+  border-radius: 2px;
+}
+
+.rag-overlay-content::-webkit-scrollbar-thumb:hover {
+  background: var(--text-secondary);
+}
+
+/* Dark theme adjustments */
+[data-theme="dark"] .rag-info-overlay {
+  background: #1f2937;
+  box-shadow: 0 8px 25px rgba(0, 0, 0, 0.4);
+}
+
+[data-theme="dark"] .rag-overlay-header {
+  background: #374151;
+}
+
+[data-theme="dark"] .rag-document-item {
+  background: #374151;
+  border-color: #4B5563;
+}
+
+[data-theme="dark"] .rag-chunk-preview {
+  background: #4B5563;
+  color: #D1D5DB;
+}
+
+[data-theme="dark"] .rag-no-documents {
+  background: #374151;
+  border-color: #4B5563;
+}
+
 /* Animations */
 @keyframes bounce {
   0%, 80%, 100% {
@@ -911,4 +1135,23 @@
     padding: 4px 8px;
   }
 
+  .rag-info-overlay {
+    width: 280px;
+    left: 0;
+    right: auto;
+  }
+  
+  .rag-document-header {
+    flex-direction: column;
+    align-items: flex-start;
+    gap: 4px;
+  }
+  
+  .rag-filename {
+    margin-right: 0;
+    white-space: normal;
+    overflow: visible;
+    text-overflow: initial;
+  }
+
 }