add

Dockerfile

@@ -1,44 +1,12 @@
-# Use Python 3.11 slim image as base
 FROM python:3.11-slim
 
-# Set environment variables
-ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    PORT=7860
-
-# Set work directory
 WORKDIR /app
 
-# Install system dependencies
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-        build-essential \
-        gcc \
-        curl \
-    && rm -rf /var/lib/apt/lists/*
-
-# Copy requirements first to leverage Docker cache
 COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 
-# Install Python dependencies
-RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir -r requirements.txt
-
-# Copy the rest of the application
 COPY . .
 
-# Create a non-root user and set permissions
-RUN adduser --disabled-password --gecos '' appuser \
-    && chown -R appuser:appuser /app
-USER appuser
-
-# Expose port (Hugging Face typically uses port 7860 or 8080)
-EXPOSE $PORT
-
-# Health check endpoint
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-    CMD curl -f http://localhost:$PORT/health || exit 1
+EXPOSE 7860
 
-# Run the application with uvicorn directly for better production performance
-CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port $PORT"]
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "7860"]

agent.py

@@ -0,0 +1,249 @@
+import os
+import json
+import logging
+from typing import Dict, List, Any
+from dotenv import load_dotenv
+from agents import Agent, Runner
+from agents import function_tool
+import asyncio
+import time
+
+# Load environment variables
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@function_tool
+def retrieve_information(query: str) -> Dict:
+    """
+    Retrieve information from the knowledge base based on a query
+    """
+    from retrieving import RAGRetriever
+    retriever = RAGRetriever()
+
+    try:
+        # Call the existing retrieve method from the RAGRetriever instance
+        json_response = retriever.retrieve(query_text=query, top_k=5, threshold=0.3)
+        results = json.loads(json_response)
+
+        # Format the results for the assistant
+        formatted_results = []
+        for result in results.get('results', []):
+            formatted_results.append({
+                'content': result['content'],
+                'url': result['url'],
+                'position': result['position'],
+                'similarity_score': result['similarity_score']
+            })
+
+        return {
+            'query': query,
+            'retrieved_chunks': formatted_results,
+            'total_results': len(formatted_results)
+        }
+    except Exception as e:
+        logger.error(f"Error in retrieve_information: {e}")
+        return {
+            'query': query,
+            'retrieved_chunks': [],
+            'total_results': 0,
+            'error': str(e)
+        }
+
+class RAGAgent:
+    def __init__(self):
+        # Create the agent with retrieval tool using the new OpenAI Agents SDK
+        self.agent = Agent(
+            name="RAG Assistant",
+            instructions="You are a helpful assistant that answers questions based on retrieved documents. When asked a question, retrieve relevant documents first using the retrieve_information tool, then answer based on them. Always cite your sources and provide the information that was used to generate the answer.",
+            tools=[retrieve_information]
+        )
+
+        logger.info("RAG Agent initialized with OpenAI Agents SDK")
+
+    def query_agent(self, query_text: str) -> Dict:
+        """
+        Process a query through the RAG agent and return structured response
+        """
+        start_time = time.time()
+
+        logger.info(f"Processing query through RAG agent: '{query_text[:50]}...'")
+
+        try:
+            # Run the agent with the query using the new OpenAI Agents SDK
+            # Since Runner.run is async, we need to run it in an event loop
+            import asyncio
+            if asyncio.get_event_loop().is_running():
+                # If we're already in an event loop, we need to use a different approach
+                import concurrent.futures
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future = executor.submit(asyncio.run, self._async_query_agent(query_text))
+                    result = future.result()
+            else:
+                result = asyncio.run(self._async_query_agent(query_text))
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error processing query: {e}")
+            return {
+                "answer": "Sorry, I encountered an error processing your request.",
+                "sources": [],
+                "matched_chunks": [],
+                "error": str(e),
+                "query_time_ms": (time.time() - start_time) * 1000
+            }
+
+    async def _async_query_agent(self, query_text: str) -> Dict:
+        """
+        Internal async method to run the agent query
+        """
+        start_time = time.time()
+
+        try:
+            result = await Runner.run(self.agent, query_text)
+
+            # Extract the assistant's response
+            assistant_response = result.final_output
+
+            if not assistant_response:
+                return {
+                    "answer": "Sorry, I couldn't generate a response.",
+                    "sources": [],
+                    "matched_chunks": [],
+                    "error": "No response from assistant",
+                    "query_time_ms": (time.time() - start_time) * 1000
+                }
+
+            # Extract sources and matched chunks from the tool calls
+            sources = set()
+            matched_chunks = []
+
+            # The new SDK might store tool call results differently
+            # Let's try to access them in the most likely way based on the documentation
+            if hasattr(result, 'final_output') and result.final_output:
+                # If the result contains tool call results in final_output
+                # For now, we'll rely on the agent's processing of the tool results
+                # The agent itself will incorporate the tool results into the final response
+                pass
+
+            # Calculate query time
+            query_time_ms = (time.time() - start_time) * 1000
+
+            # Format the response
+            # For the new SDK, we may need to extract the sources and chunks differently
+            # based on how the agent processes the tool results
+            response = {
+                "answer": str(assistant_response),
+                "sources": list(sources),
+                "matched_chunks": matched_chunks,
+                "query_time_ms": query_time_ms,
+                "confidence": self._calculate_confidence(matched_chunks)
+            }
+
+            logger.info(f"Query processed in {query_time_ms:.2f}ms")
+            return response
+
+        except Exception as e:
+            logger.error(f"Error in async query: {e}")
+            raise
+
+    def _calculate_confidence(self, matched_chunks: List[Dict]) -> str:
+        """
+        Calculate confidence level based on similarity scores and number of matches
+        """
+        if not matched_chunks:
+            return "low"
+
+        avg_score = sum(chunk.get('similarity_score', 0.0) for chunk in matched_chunks) / len(matched_chunks)
+
+        if avg_score >= 0.7:
+            return "high"
+        elif avg_score >= 0.4:
+            return "medium"
+        else:
+            return "low"
+
+def query_agent(query_text: str) -> Dict:
+    """
+    Convenience function to query the RAG agent
+    """
+    agent = RAGAgent()
+    return agent.query_agent(query_text)
+
+def run_agent_sync(query_text: str) -> Dict:
+    """
+    Synchronous function to run the agent for direct usage
+    """
+    import asyncio
+
+    async def run_async():
+        agent = RAGAgent()
+        return await agent._async_query_agent(query_text)
+
+    # Check if there's already a running event loop
+    try:
+        loop = asyncio.get_running_loop()
+        # If there's already a loop, run in a separate thread
+        import concurrent.futures
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future = executor.submit(asyncio.run, run_async())
+            return future.result()
+    except RuntimeError:
+        # No running loop, safe to use asyncio.run
+        return asyncio.run(run_async())
+
+def main():
+    """
+    Main function to demonstrate the RAG agent functionality
+    """
+    logger.info("Initializing RAG Agent...")
+
+    # Initialize the agent
+    agent = RAGAgent()
+
+    # Example queries to test the system
+    test_queries = [
+        "What is ROS2?",
+        "Explain humanoid design principles",
+        "How does VLA work?",
+        "What are simulation techniques?",
+        "Explain AI control systems"
+    ]
+
+    print("RAG Agent - Testing Queries")
+    print("=" * 50)
+
+    for i, query in enumerate(test_queries, 1):
+        print(f"\nQuery {i}: {query}")
+        print("-" * 30)
+
+        # Process query through agent
+        response = agent.query_agent(query)
+
+        # Print formatted results
+        print(f"Answer: {response['answer']}")
+
+        if response.get('sources'):
+            print(f"Sources: {len(response['sources'])} documents")
+            for source in response['sources'][:3]:  # Show first 3 sources
+                print(f"  - {source}")
+
+        if response.get('matched_chunks'):
+            print(f"Matched chunks: {len(response['matched_chunks'])}")
+            for j, chunk in enumerate(response['matched_chunks'][:2], 1):  # Show first 2 chunks
+                content_preview = chunk['content'][:100] + "..." if len(chunk['content']) > 100 else chunk['content']
+                print(f"  Chunk {j}: {content_preview}")
+                print(f"    Source: {chunk['url']}")
+                print(f"    Score: {chunk['similarity_score']:.3f}")
+
+        print(f"Query time: {response['query_time_ms']:.2f}ms")
+        print(f"Confidence: {response.get('confidence', 'unknown')}")
+
+        if i < len(test_queries):  # Don't sleep after the last query
+            time.sleep(1)  # Small delay between queries
+
+if __name__ == "__main__":
+    main()

api.py

@@ -0,0 +1,215 @@
+import os
+import asyncio
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Optional, Dict
+from dotenv import load_dotenv
+import logging
+
+# Load environment variables
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Import the existing RAG agent functionality
+from agent import RAGAgent
+
+# Create FastAPI app
+app = FastAPI(
+    title="RAG Agent API",
+    description="API for RAG Agent with document retrieval and question answering",
+    version="1.0.0"
+)
+
+# Add CORS middleware for development
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # In production, replace with specific origins
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Pydantic models
+class QueryRequest(BaseModel):
+    query: str
+
+class ChatRequest(BaseModel):
+    query: str
+    message: str
+    session_id: str
+    selected_text: Optional[str] = None
+    query_type: str = "global"
+    top_k: int = 5
+
+class MatchedChunk(BaseModel):
+    content: str
+    url: str
+    position: int
+    similarity_score: float
+
+class QueryResponse(BaseModel):
+    answer: str
+    sources: List[str]
+    matched_chunks: List[MatchedChunk]
+    error: Optional[str] = None
+    status: str  # "success", "error", "empty"
+    query_time_ms: Optional[float] = None
+    confidence: Optional[str] = None
+
+class ChatResponse(BaseModel):
+    response: str
+    citations: List[Dict[str, str]]
+    session_id: str
+    query_type: str
+    timestamp: str
+
+class HealthResponse(BaseModel):
+    status: str
+    message: str
+
+# Global RAG agent instance
+rag_agent = None
+
+@app.on_event("startup")
+async def startup_event():
+    """Initialize the RAG agent on startup"""
+    global rag_agent
+    logger.info("Initializing RAG Agent...")
+    try:
+        rag_agent = RAGAgent()
+        logger.info("RAG Agent initialized successfully")
+    except Exception as e:
+        logger.error(f"Failed to initialize RAG Agent: {e}")
+        raise
+
+@app.post("/ask", response_model=QueryResponse)
+async def ask_rag(request: QueryRequest):
+    """
+    Process a user query through the RAG agent and return the response
+    """
+    logger.info(f"Processing query: {request.query[:50]}...")
+
+    try:
+        # Validate input
+        if not request.query or len(request.query.strip()) == 0:
+            raise HTTPException(status_code=400, detail="Query cannot be empty")
+
+        if len(request.query) > 2000:
+            raise HTTPException(status_code=400, detail="Query too long, maximum 2000 characters")
+
+        # Process query through RAG agent
+        response = rag_agent.query_agent(request.query)
+
+        # Format response
+        formatted_response = QueryResponse(
+            answer=response.get("answer", ""),
+            sources=response.get("sources", []),
+            matched_chunks=[
+                MatchedChunk(
+                    content=chunk.get("content", ""),
+                    url=chunk.get("url", ""),
+                    position=chunk.get("position", 0),
+                    similarity_score=chunk.get("similarity_score", 0.0)
+                )
+                for chunk in response.get("matched_chunks", [])
+            ],
+            error=response.get("error"),
+            status="error" if response.get("error") else "success",
+            query_time_ms=response.get("query_time_ms"),
+            confidence=response.get("confidence")
+        )
+
+        logger.info(f"Query processed successfully in {response.get('query_time_ms', 0):.2f}ms")
+        return formatted_response
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error processing query: {e}")
+        return QueryResponse(
+            answer="",
+            sources=[],
+            matched_chunks=[],
+            error=str(e),
+            status="error"
+        )
+
+@app.post("/api", response_model=ChatResponse)
+async def chat_endpoint(request: ChatRequest):
+    """
+    Main chat endpoint that handles conversation with RAG capabilities
+    """
+    logger.info(f"Processing chat query: {request.query[:50]}...")
+
+    try:
+        # Validate input
+        if not request.query or len(request.query.strip()) == 0:
+            raise HTTPException(status_code=400, detail="Query cannot be empty")
+
+        if not request.session_id or len(request.session_id.strip()) == 0:
+            raise HTTPException(status_code=400, detail="Session ID cannot be empty")
+
+        if len(request.query) > 2000:
+            raise HTTPException(status_code=400, detail="Query too long, maximum 2000 characters")
+
+        # Process query through RAG agent
+        response = rag_agent.query_agent(request.query)
+
+        # Format response to match expected structure
+        from datetime import datetime
+        timestamp = datetime.utcnow().isoformat()
+
+        # Convert matched chunks to citations format
+        citations = []
+        for chunk in response.get("matched_chunks", []):
+            citation = {
+                "document_id": "",
+                "title": chunk.get("url", ""),
+                "chapter": "",
+                "section": "",
+                "page_reference": ""
+            }
+            citations.append(citation)
+
+        formatted_response = ChatResponse(
+            response=response.get("answer", ""),
+            citations=citations,
+            session_id=request.session_id,
+            query_type=request.query_type,
+            timestamp=timestamp
+        )
+
+        logger.info(f"Chat query processed successfully")
+        return formatted_response
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error processing chat query: {e}")
+        from datetime import datetime
+        return ChatResponse(
+            response="",
+            citations=[],
+            session_id=request.session_id,
+            query_type=request.query_type,
+            timestamp=datetime.utcnow().isoformat()
+        )
+
+@app.get("/health", response_model=HealthResponse)
+async def health_check():
+    """
+    Health check endpoint
+    """
+    return HealthResponse(
+        status="healthy",
+        message="RAG Agent API is running"
+    )
+
+# For running with uvicorn
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app/__init__.py

@@ -1,3 +0,0 @@
-"""
-RAG Chatbot Backend Application
-"""

app/api/chat.py

@@ -1,249 +0,0 @@
-from fastapi import APIRouter, HTTPException, Depends, Header
-from typing import Dict, Any, List, Optional
-from pydantic import BaseModel
-from datetime import datetime
-import uuid
-import asyncio
-from functools import lru_cache
-
-from app.retrieval.retriever import retriever
-from app.prompting.context_filter import context_filter
-from app.generation.response_generator import response_generator
-from app.database.repositories import ChatSessionRepository, ChatMessageRepository, QueryContextRepository
-from app.database.database import get_db
-from app.config import settings
-
-
-router = APIRouter()
-
-
-class ChatRequest(BaseModel):
-    session_id: str
-    message: str
-    selected_text: Optional[str] = None
-    query_type: str = "global"  # "global" or "selection"
-    top_k: int = 5
-
-
-class ChatResponse(BaseModel):
-    response: str
-    citations: List[Dict[str, str]]
-    session_id: str
-    query_type: str
-    timestamp: str
-
-
-class ChatMessage(BaseModel):
-    message_id: str
-    session_id: str
-    role: str  # "user" or "assistant"
-    content: str
-    citations: Optional[List[Dict[str, str]]] = None
-    timestamp: str
-
-
-@router.post("", response_model=ChatResponse)
-async def chat_endpoint(
-    request: ChatRequest,
-    x_api_key: str = Header(None)
-):
-    """
-    Main chat endpoint that handles conversation with RAG capabilities
-    """
-    # Validate API key if configured
-    if settings.BACKEND_API_KEY and x_api_key != settings.BACKEND_API_KEY:
-        raise HTTPException(status_code=401, detail="Invalid API key")
-
-    try:
-        # Validate query type
-        if request.query_type not in ["global", "selection"]:
-            raise HTTPException(
-                status_code=400,
-                detail="query_type must be either 'global' or 'selection'"
-            )
-
-        # Retrieve relevant documents based on query and query type
-        retrieved_docs = await retriever.retrieve_with_context_filtering(
-            query=request.message,
-            top_k=request.top_k,
-            query_type=request.query_type,
-            selected_text=request.selected_text
-        )
-
-        # Apply context filtering to ensure proper isolation
-        filtered_docs = context_filter.enforce_context_boundaries(
-            contexts=retrieved_docs,
-            query_type=request.query_type,
-            selected_text=request.selected_text
-        )
-
-        # Generate response using Claude
-        response_data = await response_generator.generate_response_with_validation(
-            query=request.message,
-            retrieved_contexts=filtered_docs,
-            query_type=request.query_type,
-            selected_text=request.selected_text,
-            session_id=request.session_id
-        )
-
-        # Store the conversation in the database
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            # Create or update session
-            session_repo = ChatSessionRepository(db)
-            existing_session = session_repo.get_session_by_id(request.session_id)
-            if not existing_session:
-                session_repo.create_session(session_id=request.session_id)
-
-            # Store user message
-            user_message_id = f"msg_{uuid.uuid4().hex[:8]}"
-            message_repo = ChatMessageRepository(db)
-            message_repo.create_message(
-                message_id=user_message_id,
-                session_id=request.session_id,
-                role="user",
-                content=request.message
-            )
-
-            # Store assistant response
-            assistant_message_id = f"msg_{uuid.uuid4().hex[:8]}"
-            citations_for_storage = response_data.get("citations", [])
-            message_repo.create_message(
-                message_id=assistant_message_id,
-                session_id=request.session_id,
-                role="assistant",
-                content=response_data.get("response", ""),
-                citations=citations_for_storage
-            )
-        finally:
-            next(db_gen, None)  # Close the db session
-
-        return ChatResponse(
-            response=response_data.get("response", ""),
-            citations=response_data.get("citations", []),
-            session_id=request.session_id,
-            query_type=request.query_type,
-            timestamp=datetime.utcnow().isoformat()
-        )
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error processing chat request: {str(e)}"
-        )
-
-
-@router.post("/stream")
-async def chat_stream_endpoint(request: ChatRequest):
-    """
-    Streaming chat endpoint (placeholder - actual streaming implementation would be more complex)
-    """
-    # For now, this just calls the regular endpoint
-    # In a production implementation, you would use FastAPI's StreamingResponse
-    result = await chat_endpoint(request, None)
-    return result
-
-
-@router.get("/session/{session_id}")
-async def get_session_history(session_id: str):
-    """
-    Retrieve chat history for a specific session
-    """
-    try:
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            message_repo = ChatMessageRepository(db)
-            messages = message_repo.get_messages_by_session(session_id)
-
-            return {
-                "session_id": session_id,
-                "messages": [
-                    {
-                        "message_id": msg.message_id,
-                        "role": msg.role,
-                        "content": msg.content,
-                        "timestamp": msg.timestamp.isoformat() if msg.timestamp else None,
-                        "citations": msg.citations
-                    }
-                    for msg in messages
-                ],
-                "timestamp": datetime.utcnow().isoformat()
-            }
-        finally:
-            next(db_gen, None)
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error retrieving session history: {str(e)}"
-        )
-
-
-@router.delete("/session/{session_id}")
-async def delete_session(session_id: str):
-    """
-    Delete a chat session and all associated messages
-    """
-    try:
-        # In a real implementation, you would have a method to delete all messages
-        # associated with a session. For now, we'll just return a success message.
-        return {
-            "status": "deleted",
-            "session_id": session_id,
-            "timestamp": datetime.utcnow().isoformat()
-        }
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error deleting session: {str(e)}"
-        )
-
-
-@router.post("/validate")
-async def validate_query(request: ChatRequest):
-    """
-    Validate a query without generating a response (for testing purposes)
-    """
-    try:
-        # Validate query type
-        if request.query_type not in ["global", "selection"]:
-            return {"valid": False, "error": "query_type must be either 'global' or 'selection'"}
-
-        # Validate that if query_type is 'selection', selected_text is provided
-        if request.query_type == "selection" and not request.selected_text:
-            return {"valid": False, "error": "selected_text is required for selection-based queries"}
-
-        # Check if we can retrieve relevant documents
-        retrieved_docs = await retriever.retrieve_relevant_documents(
-            query=request.message,
-            top_k=request.top_k,
-            query_type=request.query_type,
-            selected_text=request.selected_text
-        )
-
-        return {
-            "valid": True,
-            "query_type": request.query_type,
-            "documents_found": len(retrieved_docs),
-            "has_context": len(retrieved_docs) > 0
-        }
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error validating query: {str(e)}"
-        )
-
-
-# Rate limiting middleware would be implemented here in a production system
-# For now, we'll add a simple rate limiting check based on settings
-async def check_rate_limit(session_id: str) -> bool:
-    """
-    Check if the session has exceeded rate limits
-    This is a simplified implementation - a production system would use Redis or similar
-    """
-    # In a real implementation, you would check against a rate limit store
-    # For now, we'll just return True to allow all requests
-    return True

app/api/health.py

@@ -1,160 +0,0 @@
-from fastapi import APIRouter, HTTPException, Depends
-from typing import Dict, Any
-from datetime import datetime
-import time
-import asyncio
-
-from app.config import settings
-from app.vector_store.qdrant_client import qdrant_client
-from app.database.models import create_tables, get_db
-from app.database.database import get_db
-from sqlalchemy.orm import Session
-
-
-router = APIRouter()
-
-
-@router.get("/health", response_model=Dict[str, Any])
-async def health_check():
-    """
-    Health check endpoint that verifies the system is operational
-    """
-    # Basic system status
-    health_status = {
-        "status": "healthy",
-        "timestamp": datetime.utcnow().isoformat(),
-        "service": "RAG Chatbot API",
-        "version": "1.0.0",
-        "checks": {
-            "database": {"status": "unknown", "message": ""},
-            "vector_store": {"status": "unknown", "message": ""},
-            "api_config": {"status": "ok", "message": "API configuration loaded"},
-            "external_services": {"status": "unknown", "message": ""}
-        }
-    }
-
-    # Check database connection
-    try:
-        # Try to access the database
-        from app.database.database import engine
-        with engine.connect() as conn:
-            # Simple query to test connection using SQLAlchemy 2.0 syntax
-            from sqlalchemy import text
-            result = conn.execute(text("SELECT 1"))
-            health_status["checks"]["database"]["status"] = "ok"
-            health_status["checks"]["database"]["message"] = "Database connection successful"
-    except Exception as e:
-        health_status["checks"]["database"]["status"] = "error"
-        health_status["checks"]["database"]["message"] = f"Database connection failed: {str(e)}"
-
-    # Check vector store connection
-    try:
-        # Try to get collection info as a simple connectivity test
-        collection_info = qdrant_client.get_collection_info()
-        health_status["checks"]["vector_store"]["status"] = "ok"
-        health_status["checks"]["vector_store"]["message"] = f"Vector store connected, {collection_info.get('point_count', 0)} points"
-    except Exception as e:
-        health_status["checks"]["vector_store"]["status"] = "error"
-        health_status["checks"]["vector_store"]["message"] = f"Vector store connection failed: {str(e)}"
-
-    # Check external services (API keys)
-    try:
-        if not settings.OPENROUTER_API_KEY or settings.OPENROUTER_API_KEY == "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":
-            health_status["checks"]["external_services"]["status"] = "warning"
-            health_status["checks"]["external_services"]["message"] = "API key not configured or using default value"
-        else:
-            health_status["checks"]["external_services"]["status"] = "ok"
-            health_status["checks"]["external_services"]["message"] = "External services configured"
-    except Exception as e:
-        health_status["checks"]["external_services"]["status"] = "error"
-        health_status["checks"]["external_services"]["message"] = f"External service config error: {str(e)}"
-
-    # Overall status based on individual checks
-    overall_status = "healthy"
-    for check_name, check in health_status["checks"].items():
-        if check["status"] == "error":
-            overall_status = "error"
-            break
-        elif check["status"] == "warning" and overall_status != "error":
-            overall_status = "warning"
-
-    health_status["status"] = overall_status
-
-    return health_status
-
-
-@router.get("/ready", response_model=Dict[str, str])
-async def readiness_check():
-    """
-    Readiness check - verifies the service is ready to accept traffic
-    """
-    # For readiness, we mainly check if critical services are available
-    try:
-        # Test database connection
-        from app.database.database import engine
-        from sqlalchemy import text
-        with engine.connect() as conn:
-            conn.execute(text("SELECT 1"))
-
-        # Test vector store connection
-        qdrant_client.get_collection_info()
-
-        return {"status": "ready"}
-    except Exception:
-        raise HTTPException(status_code=503, detail="Service not ready")
-
-
-@router.get("/live", response_model=Dict[str, str])
-async def liveness_check():
-    """
-    Liveness check - verifies the service is alive and responding
-    """
-    return {"status": "alive", "timestamp": datetime.utcnow().isoformat()}
-
-
-@router.get("/stats", response_model=Dict[str, Any])
-async def get_service_stats():
-    """
-    Get detailed service statistics
-    """
-    stats = {
-        "timestamp": datetime.utcnow().isoformat(),
-        "uptime": "tracking needed",  # This would need to be implemented with a global start time
-        "database": {},
-        "vector_store": {},
-        "api_usage": {}
-    }
-
-    # Database stats
-    try:
-        from app.database.database import engine
-        with engine.connect() as conn:
-            # Get basic database info using SQLAlchemy 2.0 syntax
-            from sqlalchemy import text
-            result = conn.execute(text("SELECT COUNT(*) FROM book_content_documents"))
-            doc_count = result.scalar()
-            stats["database"] = {
-                "documents_count": doc_count,
-                "status": "connected"
-            }
-    except Exception as e:
-        stats["database"] = {"status": "error", "error": str(e)}
-
-    # Vector store stats
-    try:
-        collection_info = qdrant_client.get_collection_info()
-        stats["vector_store"] = {
-            "point_count": collection_info.get('point_count', 0),
-            "vector_size": collection_info.get('vector_size', 0),
-            "status": "connected"
-        }
-    except Exception as e:
-        stats["vector_store"] = {"status": "error", "error": str(e)}
-
-    # API usage would be tracked separately in a production system
-    stats["api_usage"] = {
-        "requests_served": 0,  # This would be implemented with actual tracking
-        "avg_response_time": 0.0  # This would be calculated from actual metrics
-    }
-
-    return stats

app/api/ingest.py

@@ -1,316 +0,0 @@
-from fastapi import APIRouter, HTTPException, BackgroundTasks, UploadFile, File, Form
-from typing import Dict, Any, List, Optional
-from pydantic import BaseModel
-from datetime import datetime
-import asyncio
-import os
-from pathlib import Path
-
-from app.ingestion.file_scanner import FileScanner
-from app.ingestion.chunker import TextChunker, chunk_documents
-from app.embeddings.minimal_embedding_generator import minimal_embedding_generator
-from app.vector_store.vector_repository import vector_repository
-from app.database.repositories import BookContentRepository
-from app.database.database import get_db
-from app.config import settings
-
-
-router = APIRouter()
-
-# Initialize components
-chunker = TextChunker()
-file_scanner = FileScanner()
-
-
-class IngestRequest(BaseModel):
-    book_path: str = "docusaurus/docs"  # Default path for Docusaurus docs
-    chunk_size: int = 800
-    force_reprocess: bool = False
-
-
-class IngestResponse(BaseModel):
-    status: str
-    message: str
-    documents_processed: int
-    chunks_created: int
-    embeddings_generated: int
-    timestamp: str
-
-
-class IngestStatusResponse(BaseModel):
-    status: str
-    progress: float
-    message: str
-    details: Dict[str, Any]
-
-
-# In-memory storage for tracking ingestion jobs (in production, use a proper task queue)
-ingestion_jobs: Dict[str, Dict[str, Any]] = {}
-
-
-@router.post("/ingest", response_model=IngestResponse)
-async def ingest_documents(request: IngestRequest, background_tasks: BackgroundTasks):
-    """
-    Ingest book content from markdown files, process, and store in vector database
-    """
-    try:
-        # Validate the book path exists
-        if not os.path.exists(request.book_path):
-            raise HTTPException(
-                status_code=400,
-                detail=f"Book path does not exist: {request.book_path}"
-            )
-
-        # Update chunker settings if provided
-        if request.chunk_size:
-            chunker.max_tokens = request.chunk_size
-
-        # Scan and parse documents
-        documents = file_scanner.scan_and_parse_documents()
-
-        if not documents:
-            raise HTTPException(
-                status_code=400,
-                detail=f"No markdown documents found in path: {request.book_path}"
-            )
-
-        # Validate documents
-        valid_documents = [doc for doc in documents if file_scanner.validate_document(doc)]
-        if not valid_documents:
-            raise HTTPException(
-                status_code=400,
-                detail="No valid documents found after validation"
-            )
-
-        # Chunk documents
-        all_chunks = chunk_documents(valid_documents)
-
-        if not all_chunks:
-            raise HTTPException(
-                status_code=500,
-                detail="No chunks were created from documents"
-            )
-
-        # Generate embeddings for chunks
-        chunks_with_metadata = []
-        for chunk in all_chunks:
-            # Generate embedding using minimal embedding generator
-            embedding = minimal_embedding_generator.encode_query(chunk.content)
-            if embedding:
-                chunk_data = {
-                    'id': chunk.id,
-                    'content': chunk.content,
-                    'title': chunk.title,
-                    'chapter': chunk.chapter,
-                    'section': chunk.section,
-                    'page_reference': chunk.page_reference,
-                    'token_count': chunk.token_count,
-                    'embedding': embedding
-                }
-                chunks_with_metadata.append(chunk_data)
-            else:
-                # Skip chunks that couldn't generate embeddings
-                continue
-
-        # Store embeddings in vector database
-        if chunks_with_metadata:
-            vector_repository.store_document_chunks(chunks_with_metadata)
-
-        # Store document metadata in SQL database
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            content_repo = BookContentRepository(db)
-
-            for chunk_data in chunks_with_metadata:
-                # Create or update document in SQL database
-                existing_doc = content_repo.get_document_by_id(chunk_data['id'])
-                if not existing_doc:
-                    content_repo.create_document(chunk_data)
-        finally:
-            next(db_gen, None)  # Close the db session
-
-        return IngestResponse(
-            status="success",
-            message=f"Successfully ingested {len(valid_documents)} documents, "
-                   f"created {len(all_chunks)} chunks, "
-                   f"generated {len(chunks_with_metadata)} embeddings",
-            documents_processed=len(valid_documents),
-            chunks_created=len(all_chunks),
-            embeddings_generated=len(chunks_with_metadata),
-            timestamp=datetime.utcnow().isoformat()
-        )
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error during ingestion: {str(e)}"
-        )
-
-
-@router.post("/ingest-from-content")
-async def ingest_from_content(content: str = Form(...), title: str = Form(...), chapter: str = Form("Unknown"), section: str = Form("Unknown")):
-    """
-    Ingest content directly from provided text
-    """
-    try:
-        # Create a mock document from the provided content
-        document = {
-            'title': title,
-            'content': content,
-            'chapter': chapter,
-            'section': section,
-            'file_path': f"api_upload_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}",
-            'metadata': {},
-            'structure': []
-        }
-
-        # Validate document
-        if not document['content'].strip():
-            raise HTTPException(status_code=400, detail="Content cannot be empty")
-
-        # Chunk the document
-        all_chunks = chunk_documents([document])
-
-        if not all_chunks:
-            raise HTTPException(status_code=500, detail="No chunks were created from content")
-
-        # Generate embeddings for chunks
-        chunks_with_metadata = []
-        for chunk in all_chunks:
-            # Generate embedding using minimal embedding generator
-            embedding = minimal_embedding_generator.encode_query(chunk.content)
-            if embedding:
-                chunk_data = {
-                    'id': chunk.id,
-                    'content': chunk.content,
-                    'title': chunk.title,
-                    'chapter': chunk.chapter,
-                    'section': chunk.section,
-                    'page_reference': chunk.page_reference,
-                    'token_count': chunk.token_count,
-                    'embedding': embedding
-                }
-                chunks_with_metadata.append(chunk_data)
-            else:
-                # Skip chunks that couldn't generate embeddings
-                continue
-
-        # Store embeddings in vector database
-        if chunks_with_metadata:
-            vector_repository.store_document_chunks(chunks_with_metadata)
-
-        # Store document metadata in SQL database
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            content_repo = BookContentRepository(db)
-
-            for chunk_data in chunks_with_metadata:
-                # Create or update document in SQL database
-                existing_doc = content_repo.get_document_by_id(chunk_data['id'])
-                if not existing_doc:
-                    content_repo.create_document(chunk_data)
-        finally:
-            next(db_gen, None)  # Close the db session
-
-        return {
-            "status": "success",
-            "message": f"Successfully ingested content, created {len(all_chunks)} chunks",
-            "chunks_created": len(all_chunks),
-            "timestamp": datetime.utcnow().isoformat()
-        }
-
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error during content ingestion: {str(e)}"
-        )
-
-
-@router.post("/ingest-file")
-async def ingest_from_file(file: UploadFile = File(...), title: str = Form(None), chapter: str = Form("Unknown")):
-    """
-    Ingest content from an uploaded file
-    """
-    try:
-        # Read the uploaded file
-        content = await file.read()
-        content_str = content.decode('utf-8')
-
-        # Use filename as title if not provided
-        if not title:
-            title = Path(file.filename).stem
-
-        # Ingest the content
-        return await ingest_from_content(
-            content=content_str,
-            title=title,
-            chapter=chapter,
-            section=Path(file.filename).stem
-        )
-
-    except UnicodeDecodeError:
-        raise HTTPException(
-            status_code=400,
-            detail="File must be a UTF-8 encoded text file"
-        )
-    except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error processing uploaded file: {str(e)}"
-        )
-
-
-@router.get("/ingest-status/{job_id}")
-async def get_ingest_status(job_id: str):
-    """
-    Get the status of an ingestion job
-    """
-    if job_id not in ingestion_jobs:
-        raise HTTPException(status_code=404, detail="Job not found")
-
-    return IngestStatusResponse(
-        status=ingestion_jobs[job_id]["status"],
-        progress=ingestion_jobs[job_id]["progress"],
-        message=ingestion_jobs[job_id]["message"],
-        details=ingestion_jobs[job_id]["details"]
-    )
-
-
-@router.get("/ingest-stats")
-async def get_ingest_stats():
-    """
-    Get ingestion statistics
-    """
-    # Get vector store stats
-    vector_stats = vector_repository.get_collection_stats()
-
-    # Get database stats
-    db_gen = get_db()
-    db = next(db_gen)
-    try:
-        content_repo = BookContentRepository(db)
-        all_docs = content_repo.get_all_documents()
-        total_docs = len(all_docs)
-
-        # Group by chapter
-        chapters = {}
-        for doc in all_docs:
-            chapter = doc.chapter
-            if chapter not in chapters:
-                chapters[chapter] = 0
-            chapters[chapter] += 1
-    finally:
-        next(db_gen, None)
-
-    return {
-        "vector_store": vector_stats,
-        "database": {
-            "total_documents": total_docs,
-            "documents_by_chapter": chapters
-        },
-        "timestamp": datetime.utcnow().isoformat()
-    }

app/config.py

@@ -1,45 +0,0 @@
-from pydantic_settings import BaseSettings
-from typing import Optional
-from functools import lru_cache
-
-
-class Settings(BaseSettings):
-    # OpenRouter API
-    OPENROUTER_API_KEY: str
-    OPENROUTER_BASE_URL: str = "https://openrouter.ai/api/v1"
-
-    # Qdrant Vector Database
-    QDRANT_URL: str
-    QDRANT_API_KEY: str
-    QDRANT_CLUSTER_ID: str
-
-    # Neon PostgreSQL Database
-    NEON_DATABASE_URL: str
-
-    # Cohere API (if needed)
-    COHERE_API_KEY: Optional[str] = None
-
-    # Google Gemini API
-    GEMINI_API_KEY: Optional[str] = None
-
-    # Backend API
-    BACKEND_API_KEY: str
-
-    # Application settings
-    DEBUG: bool = False
-    LOG_LEVEL: str = "INFO"
-    MAX_CONTENT_LENGTH: int = 5000
-    RATE_LIMIT_REQUESTS: int = 100
-    RATE_LIMIT_WINDOW: int = 60  # in seconds
-
-    class Config:
-        env_file = ".env"
-        case_sensitive = False
-
-
-@lru_cache()
-def get_settings():
-    return Settings()
-
-
-settings = get_settings()

app/database/database.py

@@ -1,56 +0,0 @@
-from sqlalchemy import create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
-from app.config import settings
-
-
-# Create database engine with connection pooling
-engine = create_engine(
-    settings.NEON_DATABASE_URL,
-    pool_pre_ping=True,  # Verify connections before use
-    pool_recycle=300,    # Recycle connections after 5 minutes
-    pool_size=10,        # Number of connection to keep open
-    max_overflow=20,     # Additional connections beyond pool_size
-    echo=False           # Set to True for SQL query logging
-)
-
-# Create session factory
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
-# Base class for models
-Base = declarative_base()
-
-
-def get_db():
-    """
-    Dependency for getting database session
-    """
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()
-
-
-def create_tables():
-    """
-    Create all database tables based on models
-    This should be called during application startup
-    """
-    from .models import Base
-    Base.metadata.create_all(bind=engine)
-
-
-def get_engine():
-    """
-    Return the database engine (useful for direct access)
-    """
-    return engine
-
-
-# Global database instance for easy access
-db_instance = {
-    'engine': engine,
-    'SessionLocal': SessionLocal,
-    'Base': Base
-}

app/database/models.py

@@ -1,138 +0,0 @@
-from sqlalchemy import create_engine, Column, Integer, String, Text, DateTime, JSON, Index
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
-from datetime import datetime
-import uuid
-from app.config import settings
-
-
-Base = declarative_base()
-
-
-class BookContentDocument(Base):
-    """
-    Model for storing book content document metadata
-    """
-    __tablename__ = "book_content_documents"
-
-    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
-    document_id = Column(String, unique=True, nullable=False, index=True)  # The ID from our chunking process
-    title = Column(String, nullable=False)
-    content = Column(Text, nullable=False)  # Content summary or the actual content
-    chapter = Column(String, nullable=False)
-    section = Column(String, nullable=False)
-    page_reference = Column(String, nullable=True)
-    embedding_vector = Column(JSON, nullable=True)  # Store as JSON for flexibility
-    token_count = Column(Integer, nullable=True)
-    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
-    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
-
-    # Indexes for better query performance
-    # Note: Indexes are handled separately to avoid creation conflicts
-    # __table_args__ = (
-    #     Index('idx_chapter_section', 'chapter', 'section'),
-    #     Index('idx_document_id', 'document_id'),
-    # )
-
-
-class ChatSession(Base):
-    """
-    Model for storing chat session information
-    """
-    __tablename__ = "chat_sessions"
-
-    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
-    session_id = Column(String, unique=True, nullable=False, index=True)
-    user_id = Column(String, nullable=True)  # Optional user identifier
-    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
-    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
-    session_metadata = Column(JSON, nullable=True)  # Additional session metadata
-
-    # Note: Indexes are handled separately to avoid creation conflicts
-    # __table_args__ = (
-    #     Index('idx_session_id', 'session_id'),
-    #     Index('idx_user_id', 'user_id'),
-    # )
-
-
-class ChatMessage(Base):
-    """
-    Model for storing individual chat messages
-    """
-    __tablename__ = "chat_messages"
-
-    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
-    message_id = Column(String, unique=True, nullable=False, index=True)
-    session_id = Column(String, nullable=False, index=True)  # References chat_sessions.session_id
-    role = Column(String, nullable=False)  # 'user' or 'assistant'
-    content = Column(Text, nullable=False)
-    citations = Column(JSON, nullable=True)  # List of document IDs used in response
-    query_context_id = Column(String, nullable=True, index=True)  # References query_context.context_id
-    timestamp = Column(DateTime, default=datetime.utcnow, nullable=False)
-
-    # Note: Indexes are handled separately to avoid creation conflicts
-    # __table_args__ = (
-    #     Index('idx_session_id', 'session_id'),
-    #     Index('idx_message_id', 'message_id'),
-    #     Index('idx_query_context_id', 'query_context_id'),
-    # )
-
-
-class QueryContext(Base):
-    """
-    Model for storing query context information
-    """
-    __tablename__ = "query_contexts"
-
-    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
-    context_id = Column(String, unique=True, nullable=False, index=True)
-    session_id = Column(String, nullable=False, index=True)  # References chat_sessions.session_id
-    selected_text = Column(Text, nullable=True)  # Text selected by user (for selection-based queries)
-    query_type = Column(String, nullable=False)  # 'global' or 'selection'
-    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
-
-    # Note: Indexes are handled separately to avoid creation conflicts
-    # __table_args__ = (
-    #     Index('idx_context_id', 'context_id'),
-    #     Index('idx_session_id', 'session_id'),
-    # )
-
-
-# Create engine and session
-engine = create_engine(settings.NEON_DATABASE_URL, pool_pre_ping=True)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
-
-def drop_tables():
-    """
-    Drop all database tables
-    """
-    # Drop all tables with CASCADE to handle foreign key dependencies
-    # First drop any dependent objects, then drop tables
-    from sqlalchemy import text
-    with engine.connect() as conn:
-        # Drop all tables in the schema with CASCADE
-        conn.execute(text("DROP SCHEMA public CASCADE"))
-        conn.execute(text("CREATE SCHEMA public"))
-        conn.execute(text("GRANT ALL ON SCHEMA public TO public"))
-        conn.execute(text("GRANT ALL ON SCHEMA public TO neondb_owner"))
-        conn.commit()
-
-
-def create_tables():
-    """
-    Create all database tables
-    """
-    # Create tables with checkfirst to avoid errors if they already exist
-    Base.metadata.create_all(bind=engine, checkfirst=True)
-
-
-def get_db():
-    """
-    Dependency for getting database session
-    """
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()

app/database/repositories.py

@@ -1,217 +0,0 @@
-from typing import List, Optional, Dict, Any
-from sqlalchemy.orm import Session
-from sqlalchemy import and_, or_
-from app.database.models import BookContentDocument, ChatSession, ChatMessage, QueryContext
-from app.ingestion.chunker import TextChunk
-
-
-class BookContentRepository:
-    """
-    Repository for managing book content documents
-    """
-
-    def __init__(self, db: Session):
-        self.db = db
-
-    def create_document(self, chunk: Dict[str, Any]) -> BookContentDocument:
-        """
-        Create a new book content document
-        """
-        db_document = BookContentDocument(
-            document_id=chunk.get('id'),
-            title=chunk.get('title', ''),
-            content=chunk.get('content', ''),
-            chapter=chunk.get('chapter', ''),
-            section=chunk.get('section', ''),
-            page_reference=chunk.get('page_reference', ''),
-            embedding_vector=chunk.get('embedding'),
-            token_count=chunk.get('token_count', 0)
-        )
-        self.db.add(db_document)
-        self.db.commit()
-        self.db.refresh(db_document)
-        return db_document
-
-    def get_document_by_id(self, document_id: str) -> Optional[BookContentDocument]:
-        """
-        Get a document by its ID
-        """
-        return self.db.query(BookContentDocument).filter(
-            BookContentDocument.document_id == document_id
-        ).first()
-
-    def get_documents_by_chapter(self, chapter: str) -> List[BookContentDocument]:
-        """
-        Get all documents for a specific chapter
-        """
-        return self.db.query(BookContentDocument).filter(
-            BookContentDocument.chapter == chapter
-        ).all()
-
-    def get_all_documents(self) -> List[BookContentDocument]:
-        """
-        Get all documents
-        """
-        return self.db.query(BookContentDocument).all()
-
-    def update_document(self, document_id: str, **kwargs) -> Optional[BookContentDocument]:
-        """
-        Update a document
-        """
-        document = self.get_document_by_id(document_id)
-        if document:
-            for key, value in kwargs.items():
-                setattr(document, key, value)
-            self.db.commit()
-            self.db.refresh(document)
-        return document
-
-    def delete_document(self, document_id: str) -> bool:
-        """
-        Delete a document
-        """
-        document = self.get_document_by_id(document_id)
-        if document:
-            self.db.delete(document)
-            self.db.commit()
-            return True
-        return False
-
-
-class ChatSessionRepository:
-    """
-    Repository for managing chat sessions
-    """
-
-    def __init__(self, db: Session):
-        self.db = db
-
-    def create_session(self, session_id: str, user_id: Optional[str] = None, metadata: Optional[Dict] = None) -> ChatSession:
-        """
-        Create a new chat session
-        """
-        db_session = ChatSession(
-            session_id=session_id,
-            user_id=user_id,
-            session_metadata=metadata
-        )
-        self.db.add(db_session)
-        self.db.commit()
-        self.db.refresh(db_session)
-        return db_session
-
-    def get_session_by_id(self, session_id: str) -> Optional[ChatSession]:
-        """
-        Get a session by its ID
-        """
-        return self.db.query(ChatSession).filter(
-            ChatSession.session_id == session_id
-        ).first()
-
-    def update_session(self, session_id: str, **kwargs) -> Optional[ChatSession]:
-        """
-        Update a session
-        """
-        session = self.get_session_by_id(session_id)
-        if session:
-            for key, value in kwargs.items():
-                setattr(session, key, value)
-            self.db.commit()
-            self.db.refresh(session)
-        return session
-
-
-class ChatMessageRepository:
-    """
-    Repository for managing chat messages
-    """
-
-    def __init__(self, db: Session):
-        self.db = db
-
-    def create_message(
-        self,
-        message_id: str,
-        session_id: str,
-        role: str,
-        content: str,
-        citations: Optional[List[Dict]] = None,
-        query_context_id: Optional[str] = None
-    ) -> ChatMessage:
-        """
-        Create a new chat message
-        """
-        db_message = ChatMessage(
-            message_id=message_id,
-            session_id=session_id,
-            role=role,
-            content=content,
-            citations=citations,
-            query_context_id=query_context_id
-        )
-        self.db.add(db_message)
-        self.db.commit()
-        self.db.refresh(db_message)
-        return db_message
-
-    def get_messages_by_session(self, session_id: str) -> List[ChatMessage]:
-        """
-        Get all messages for a session
-        """
-        return self.db.query(ChatMessage).filter(
-            ChatMessage.session_id == session_id
-        ).order_by(ChatMessage.timestamp).all()
-
-    def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]:
-        """
-        Get a message by its ID
-        """
-        return self.db.query(ChatMessage).filter(
-            ChatMessage.message_id == message_id
-        ).first()
-
-
-class QueryContextRepository:
-    """
-    Repository for managing query contexts
-    """
-
-    def __init__(self, db: Session):
-        self.db = db
-
-    def create_query_context(
-        self,
-        context_id: str,
-        session_id: str,
-        selected_text: Optional[str] = None,
-        query_type: str = "global"
-    ) -> QueryContext:
-        """
-        Create a new query context
-        """
-        db_context = QueryContext(
-            context_id=context_id,
-            session_id=session_id,
-            selected_text=selected_text,
-            query_type=query_type
-        )
-        self.db.add(db_context)
-        self.db.commit()
-        self.db.refresh(db_context)
-        return db_context
-
-    def get_context_by_id(self, context_id: str) -> Optional[QueryContext]:
-        """
-        Get a query context by its ID
-        """
-        return self.db.query(QueryContext).filter(
-            QueryContext.context_id == context_id
-        ).first()
-
-    def get_contexts_by_session(self, session_id: str) -> List[QueryContext]:
-        """
-        Get all query contexts for a session
-        """
-        return self.db.query(QueryContext).filter(
-            QueryContext.session_id == session_id
-        ).all()

app/embeddings/minimal_embedding_generator.py

@@ -1,79 +0,0 @@
-import numpy as np
-from typing import List
-import hashlib
-import re
-from collections import Counter
-
-
-class MinimalEmbeddingGenerator:
-    """
-    Minimal embedding generator using simple hashing and basic statistics
-    This avoids heavy dependencies and memory issues
-    """
-
-    def __init__(self):
-        self.vocab = set()
-        self.fitted = False
-
-    def _pad_embedding(self, embedding: List[float], target_size: int = 1536) -> List[float]:
-        """
-        Pad embedding to target size with zeros
-        """
-        current_size = len(embedding)
-        if current_size >= target_size:
-            return embedding[:target_size]
-        else:
-            padded = [0.0] * target_size
-            padded[:current_size] = embedding
-            return padded
-
-    def _text_to_vector(self, text: str) -> List[float]:
-        """
-        Convert text to a vector using simple hashing approach
-        """
-        # Clean text
-        text = re.sub(r'[^\w\s]', ' ', text.lower())
-        words = text.split()
-
-        if not words:
-            return [0.0] * 1536
-
-        # Create a simple vector based on word hashes
-        vector = [0.0] * 1536
-
-        for word in words:
-            # Use hash to determine position in vector
-            hash_val = int(hashlib.md5(word.encode()).hexdigest(), 16)
-            pos = hash_val % 1536
-            # Add to vector with some normalization
-            vector[pos] += 1.0 / len(words)  # Normalize by document length
-
-        # Normalize the vector
-        norm = sum(x**2 for x in vector) ** 0.5
-        if norm > 0:
-            vector = [x / norm for x in vector]
-
-        return vector
-
-    def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
-        """
-        Generate embeddings for a list of texts using simple hashing
-        """
-        embeddings = []
-        for text in texts:
-            embedding = self._text_to_vector(text)
-            padded_embedding = self._pad_embedding(embedding)
-            embeddings.append(padded_embedding)
-
-        return embeddings
-
-    def encode_query(self, query: str) -> List[float]:
-        """
-        Encode a single query for similarity search
-        """
-        embedding = self._text_to_vector(query)
-        return self._pad_embedding(embedding)
-
-
-# Global instance
-minimal_embedding_generator = MinimalEmbeddingGenerator()

app/generation/response_generator.py

@@ -1,387 +0,0 @@
-import asyncio
-import json
-from typing import List, Dict, Any, Optional
-from app.services.openrouter_client import openrouter_client
-from app.prompting.prompt_builder import PromptBuilder
-from app.retrieval.retriever import Retriever
-from app.config import settings
-
-
-class ResponseGenerator:
-    """
-    Generates responses using Google Gemini with proper context and citation tracking
-    """
-
-    def __init__(self):
-        self.openrouter_client = openrouter_client
-        self.prompt_builder = PromptBuilder()
-        self.retriever = Retriever()
-
-    async def generate_response(
-        self,
-        query: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        query_type: str = "global",
-        selected_text: Optional[str] = None,
-        session_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Generate a response to a query using Google Gemini
-        """
-        try:
-            # Build context string from retrieved contexts
-            context_parts = []
-            for ctx in retrieved_contexts:
-                context_text = ctx.get('content', '')
-                if context_text:
-                    context_parts.append(context_text)
-
-            context_string = "\n\n".join(context_parts)
-
-            # Prepare messages for OpenRouter
-            messages = [
-                {
-                    "role": "system",
-                    "content": "You are an expert assistant for the Physical AI & Humanoid Robotics curriculum. Provide helpful, conversational responses based on the provided context. Always use information only from the provided context and be factual."
-                },
-                {
-                    "role": "user",
-                    "content": f"Context: {context_string}\n\nQuestion: {query}\n\nProvide a helpful response based on the context:"
-                }
-            ]
-
-            # Generate response using OpenRouter
-            openrouter_response = await self.openrouter_client.generate_completion(
-                messages=messages,
-                model="mistralai/devstral-2512:free",
-                temperature=0.3,
-                max_tokens=1000
-            )
-
-            if not openrouter_response:
-                return {
-                    "response": "I encountered an issue generating a response. Please try again.",
-                    "citations": [],
-                    "query_type": query_type,
-                    "session_id": session_id
-                }
-
-            # Extract citations from contexts used
-            citations = self._extract_citations(retrieved_contexts)
-
-            return {
-                "response": openrouter_response,
-                "citations": citations,
-                "query_type": query_type,
-                "session_id": session_id
-            }
-
-        except Exception as e:
-            print(f"[ERROR] Error generating response: {str(e)}")
-            # If AI service fails, try to provide a helpful response based on the context
-            if retrieved_contexts:
-                # Extract key information from contexts to provide a basic response
-                context_titles = [ctx.get('title', '') for ctx in retrieved_contexts if ctx.get('title')]
-                unique_titles = list(set(context_titles))
-
-                if unique_titles:
-                    response = f"Based on the Physical AI & Humanoid Robotics curriculum, I found information related to: {', '.join(unique_titles[:3])}. "
-                    response += "Here's what I can share from the book content: "
-                    # Include some of the actual content from the contexts
-                    first_context = retrieved_contexts[0]
-                    content_preview = first_context.get('content', '')[:200]
-                    response += content_preview + ("..." if len(first_context.get('content', '')) > 200 else "")
-                else:
-                    response = "Based on the Physical AI & Humanoid Robotics curriculum, I found relevant content. "
-                    response += "Could you ask a more specific question about the topic you're interested in?"
-            else:
-                response = "I couldn't find relevant information in the Physical AI & Humanoid Robotics curriculum to answer your question. "
-                response += "Please try asking about specific topics from the curriculum like ROS 2, Digital Twins, AI-Brain, or VLA."
-
-            return {
-                "response": response,
-                "citations": self._extract_citations(retrieved_contexts),
-                "query_type": query_type,
-                "session_id": session_id,
-                "error": str(e)
-            }
-
-    async def generate_response_with_validation(
-        self,
-        query: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        query_type: str = "global",
-        selected_text: Optional[str] = None,
-        session_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Generate response with additional validation to ensure it's based only on provided context
-        """
-        try:
-            # First, generate the initial response
-            result = await self.generate_response(
-                query=query,
-                retrieved_contexts=retrieved_contexts,
-                query_type=query_type,
-                selected_text=selected_text,
-                session_id=session_id
-            )
-
-            # Validate that the response is grounded in the provided context
-            # But be more lenient for high-level book/module questions
-            if retrieved_contexts and "The provided context does not contain" not in result.get("response", ""):
-                is_valid = self._validate_response_uses_context(
-                    response=result["response"],
-                    contexts=retrieved_contexts
-                )
-
-                # For high-level book/module questions, be more lenient with validation
-                query_lower = query.lower()
-                is_book_overview_query = any(phrase in query_lower for phrase in [
-                    'what is', 'tell me about', 'describe', 'overview', 'introduction',
-                    'physical ai', 'humanoid robotics', 'book', 'curriculum', 'module',
-                    'quick start', 'setup', 'getting started', 'chapter', 'section'
-                ])
-
-                # Only retry with stronger guidance if it's not a book overview query and validation fails
-                if not is_valid and not is_book_overview_query and query_type != "selection":
-                    # If response doesn't seem to use context, try again with stronger instructions
-                    result = await self._generate_with_stronger_context_guidance(
-                        query=query,
-                        retrieved_contexts=retrieved_contexts,
-                        query_type=query_type,
-                        selected_text=selected_text,
-                        session_id=session_id
-                    )
-
-            return result
-        except Exception as e:
-            print(f"[ERROR] Error in generate_response_with_validation: {str(e)}")
-            # If AI service fails, try to provide a helpful response based on the context
-            if retrieved_contexts:
-                # Extract key information from contexts to provide a basic response
-                context_titles = [ctx.get('title', '') for ctx in retrieved_contexts if ctx.get('title')]
-                unique_titles = list(set(context_titles))
-
-                if unique_titles:
-                    response = f"Based on the Physical AI & Humanoid Robotics curriculum, I found information related to: {', '.join(unique_titles[:3])}. "
-                    response += "Here's what I can share from the book content: "
-                    # Include some of the actual content from the contexts
-                    first_context = retrieved_contexts[0]
-                    content_preview = first_context.get('content', '')[:200]
-                    response += content_preview + ("..." if len(first_context.get('content', '')) > 200 else "")
-                else:
-                    response = "Based on the Physical AI & Humanoid Robotics curriculum, I found relevant content. "
-                    response += "Could you ask a more specific question about the topic you're interested in?"
-            else:
-                response = "I couldn't find relevant information in the Physical AI & Humanoid Robotics curriculum to answer your question. "
-                response += "Please try asking about specific topics from the curriculum like ROS 2, Digital Twins, AI-Brain, or VLA."
-
-            return {
-                "response": response,
-                "citations": self._extract_citations(retrieved_contexts),
-                "query_type": query_type,
-                "session_id": session_id,
-                "error": str(e)
-            }
-
-    async def _generate_with_stronger_context_guidance(
-        self,
-        query: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        query_type: str,
-        selected_text: Optional[str] = None,
-        session_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Generate response with stronger instructions to use only provided context
-        """
-        # Build context string from retrieved contexts
-        context_parts = []
-        for ctx in retrieved_contexts:
-            context_text = ctx.get('content', '')
-            if context_text:
-                context_parts.append(context_text)
-
-        context_string = "\n\n".join(context_parts)
-
-        # Add stronger instructions to the context
-        stronger_context = (
-            context_string +
-            "\n\nIMPORTANT: The response MUST be based ONLY on the provided context above. "
-            "Do not use any external knowledge or make up information. "
-            "If the context does not contain the answer, explicitly state this fact."
-        )
-
-        # Prepare messages for OpenRouter
-        messages = [
-            {
-                "role": "system",
-                "content": "You are an expert assistant for the Physical AI & Humanoid Robotics curriculum. Provide helpful, conversational responses based on the provided context. Always use information only from the provided context and be factual."
-            },
-            {
-                "role": "user",
-                "content": f"Context: {stronger_context}\n\nQuestion: {query}\n\nProvide a helpful response based on the context:"
-            }
-        ]
-
-        # Generate response using OpenRouter
-        openrouter_response = await self.openrouter_client.generate_completion(
-            messages=messages,
-            model="mistralai/devstral-2512:free",
-            temperature=0.3,
-            max_tokens=1000
-        )
-
-        if not openrouter_response:
-            return {
-                "response": "I encountered an issue generating a response. Please try again.",
-                "citations": [],
-                "query_type": query_type,
-                "session_id": session_id
-            }
-
-        citations = self._extract_citations(retrieved_contexts)
-
-        return {
-            "response": openrouter_response,
-            "citations": citations,
-            "query_type": query_type,
-            "session_id": session_id
-        }
-
-    def _extract_citations(self, contexts: List[Dict[str, Any]]) -> List[Dict[str, str]]:
-        """
-        Extract citation information from retrieved contexts
-        """
-        citations = []
-        for ctx in contexts:
-            citation = {
-                "document_id": ctx.get('id', ''),
-                "title": ctx.get('title', ''),
-                "chapter": ctx.get('chapter', ''),
-                "section": ctx.get('section', ''),
-                "page_reference": ctx.get('page_reference', '')
-            }
-            citations.append(citation)
-        return citations
-
-    def _validate_response_uses_context(
-        self,
-        response: str,
-        contexts: List[Dict[str, Any]]
-    ) -> bool:
-        """
-        Validate if response uses the provided context, allowing for summarization and synthesis
-        """
-        if not contexts:
-            return False
-
-        response_lower = response.lower()
-
-        # Check for semantic relevance rather than exact keyword matching
-        # Look for broader topic matches and concepts
-        context_text = " ".join([ctx.get('content', '') for ctx in contexts if ctx.get('content')])
-        context_lower = context_text.lower()
-
-        # Check for relevant topics and concepts that indicate proper grounding
-        book_related_terms = [
-            'physical ai', 'humanoid', 'robotics', 'module', 'ros', 'digital twin',
-            'ai-brain', 'vla', 'curriculum', 'course', 'book', 'chapter', 'section',
-            'setup', 'quickstart', 'introduction', 'learning', 'education'
-        ]
-
-        # Check if response mentions book-related concepts that are in context
-        response_has_relevant_terms = any(term in response_lower for term in book_related_terms)
-        context_has_relevant_terms = any(term in context_lower for term in book_related_terms)
-
-        if response_has_relevant_terms and context_has_relevant_terms:
-            return True
-
-        # Also check for title and chapter references
-        context_titles = " ".join([ctx.get('title', '') + ' ' + ctx.get('chapter', '') + ' ' + ctx.get('section', '')
-                                  for ctx in contexts if ctx.get('title') or ctx.get('chapter') or ctx.get('section')])
-        context_titles_lower = context_titles.lower()
-
-        # If response mentions specific titles/chapters that exist in context, it's likely valid
-        if any(title_term in response_lower for title_term in context_titles_lower.split() if len(title_term) > 3):
-            return True
-
-        # Fallback: check if there's general overlap in concepts
-        response_words = set(response_lower.split())
-        context_words = set(context_lower.split())
-
-        # Find intersection of meaningful words (longer than 3 chars, not common words)
-        common_words = response_words.intersection(context_words)
-        meaningful_common_words = [w for w in common_words if len(w) > 3 and w not in ['the', 'and', 'for', 'are', 'but', 'not', 'you', 'have', 'with', 'this', 'that', 'from']]
-
-        # If there are meaningful overlapping words, consider it valid
-        return len(meaningful_common_words) >= 2
-
-    async def generate_citation_aware_response(
-        self,
-        query: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        query_type: str = "global",
-        selected_text: Optional[str] = None,
-        session_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Generate a response that explicitly mentions which sources were used
-        """
-        # Build context string from retrieved contexts
-        context_parts = []
-        for ctx in retrieved_contexts:
-            context_text = ctx.get('content', '')
-            if context_text:
-                context_parts.append(context_text)
-
-        context_string = "\n\n".join(context_parts)
-
-        # Add specific instruction about citations to the context
-        citation_context = (
-            context_string +
-            "\n\nWhen answering, please indicate which sources you used by referencing them as "
-            "[Source 1], [Source 2], etc., corresponding to the order they appear in the context section."
-        )
-
-        # Prepare messages for OpenRouter
-        messages = [
-            {
-                "role": "system",
-                "content": "You are an expert assistant for the Physical AI & Humanoid Robotics curriculum. Provide helpful, conversational responses based on the provided context. Always use information only from the provided context and be factual. When answering, please indicate which sources you used by referencing them as [Source 1], [Source 2], etc., corresponding to the order they appear in the context section."
-            },
-            {
-                "role": "user",
-                "content": f"Context: {citation_context}\n\nQuestion: {query}\n\nProvide a helpful response based on the context:"
-            }
-        ]
-
-        # Generate response using OpenRouter
-        openrouter_response = await self.openrouter_client.generate_completion(
-            messages=messages,
-            model="mistralai/devstral-2512:free",
-            temperature=0.3,
-            max_tokens=1000
-        )
-
-        if not openrouter_response:
-            return {
-                "response": "I encountered an issue generating a response. Please try again.",
-                "citations": [],
-                "query_type": query_type,
-                "session_id": session_id
-            }
-
-        citations = self._extract_citations(retrieved_contexts)
-
-        return {
-            "response": openrouter_response,
-            "citations": citations,
-            "query_type": query_type,
-            "session_id": session_id
-        }
-
-
-# Global instance
-response_generator = ResponseGenerator()

app/ingestion/chunker.py

@@ -1,291 +0,0 @@
-import re
-import uuid
-from typing import List, Dict, Tuple
-from dataclasses import dataclass
-from .document_parser import DocumentParser
-
-
-@dataclass
-class TextChunk:
-    """Represents a chunk of text with metadata"""
-    id: str
-    content: str
-    title: str
-    chapter: str
-    section: str
-    page_reference: str
-    token_count: int
-    original_start_pos: int
-    original_end_pos: int
-
-
-class TextChunker:
-    """
-    Implements heading-aware text chunking to maintain semantic boundaries
-    """
-
-    def __init__(self, max_tokens: int = 800, min_tokens: int = 100, overlap_ratio: float = 0.1):
-        self.max_tokens = max_tokens
-        self.min_tokens = min_tokens
-        self.overlap_ratio = overlap_ratio
-        self.parser = DocumentParser()
-
-    def chunk_document(self, document: Dict) -> List[TextChunk]:
-        """
-        Chunk a document while preserving semantic boundaries
-        """
-        content = document['content']
-        title = document['title']
-        chapter = document['chapter']
-        section = document['section']
-        file_path = document['file_path']
-
-        # First, identify structural boundaries (headings)
-        structure = self._identify_structure(content)
-
-        if not structure:
-            # If no headings found, chunk by max token size
-            return self._chunk_by_size(content, title, chapter, section, file_path)
-
-        # Split content by structural boundaries
-        structured_chunks = self._split_by_structure(content, structure)
-
-        # Further chunk large structural sections if needed
-        final_chunks = []
-        for i, (start_pos, end_pos, heading_context) in enumerate(structured_chunks):
-            chunk_content = content[start_pos:end_pos]
-
-            if self._count_tokens(chunk_content) > self.max_tokens:
-                # Split large sections while preserving heading context
-                sub_chunks = self._chunk_large_section(
-                    chunk_content,
-                    title,
-                    chapter,
-                    section,
-                    file_path,
-                    heading_context,
-                    start_pos
-                )
-                final_chunks.extend(sub_chunks)
-            else:
-                # Create a single chunk for this section
-                chunk_id = str(uuid.uuid4())
-                token_count = self._count_tokens(chunk_content)
-
-                chunk = TextChunk(
-                    id=chunk_id,
-                    content=chunk_content,
-                    title=title,
-                    chapter=chapter,
-                    section=section,
-                    page_reference=file_path,
-                    token_count=token_count,
-                    original_start_pos=start_pos,
-                    original_end_pos=end_pos
-                )
-                final_chunks.append(chunk)
-
-        return final_chunks
-
-    def _identify_structure(self, content: str) -> List[Tuple[int, str]]:
-        """
-        Identify structural boundaries in the content (headings)
-        Returns list of (position, heading_text) tuples
-        """
-        lines = content.split('\n')
-        structure = []
-        pos = 0
-
-        for line in lines:
-            # Check for markdown headings
-            heading_match = re.match(r'^(\#{1,6})\s+(.+)', line)
-            if heading_match:
-                level = len(heading_match.group(1))
-                heading_text = heading_match.group(2).strip()
-                structure.append((pos, f"{'#' * level} {heading_text}"))
-
-            pos += len(line) + 1  # +1 for newline
-
-        return structure
-
-    def _split_by_structure(self, content: str, structure: List[Tuple[int, str]]) -> List[Tuple[int, int, str]]:
-        """
-        Split content by structural boundaries
-        Returns list of (start_pos, end_pos, heading_context) tuples
-        """
-        if not structure:
-            return [(0, len(content), "")]
-
-        splits = []
-        start_pos = 0
-
-        for pos, heading in structure:
-            if pos > start_pos:
-                # Add the chunk before this heading
-                splits.append((start_pos, pos, heading))
-            start_pos = pos
-
-        # Add the final chunk if there's remaining content
-        if start_pos < len(content):
-            splits.append((start_pos, len(content), structure[-1][1]))
-
-        return splits
-
-    def _chunk_by_size(self, content: str, title: str, chapter: str, section: str, file_path: str) -> List[TextChunk]:
-        """
-        Fallback method to chunk content by size when no structure is available
-        """
-        chunks = []
-        tokens_per_chunk = self._estimate_tokens_per_chunk()
-        chunk_size = tokens_per_chunk * 4  # Rough estimate: 4 chars per token
-
-        for i in range(0, len(content), chunk_size):
-            chunk_content = content[i:i + chunk_size]
-            chunk_id = str(uuid.uuid4())
-
-            chunk = TextChunk(
-                id=chunk_id,
-                content=chunk_content,
-                title=title,
-                chapter=chapter,
-                section=section,
-                page_reference=file_path,
-                token_count=self._count_tokens(chunk_content),
-                original_start_pos=i,
-                original_end_pos=min(i + chunk_size, len(content))
-            )
-            chunks.append(chunk)
-
-        return chunks
-
-    def _chunk_large_section(
-        self,
-        content: str,
-        title: str,
-        chapter: str,
-        section: str,
-        file_path: str,
-        heading_context: str,
-        offset: int
-    ) -> List[TextChunk]:
-        """
-        Further chunk a large section while preserving heading context
-        """
-        chunks = []
-
-        # If we have heading context, prepend it to each chunk
-        context_prefix = f"{heading_context}\n\n" if heading_context else ""
-
-        # Split content into sentences to find good break points
-        sentences = self._split_into_sentences(content)
-
-        current_chunk = ""
-        current_tokens = 0
-        chunk_idx = 0
-
-        for sentence in sentences:
-            sentence_tokens = self._count_tokens(sentence)
-
-            # If adding this sentence would exceed the token limit
-            if current_tokens + sentence_tokens > self.max_tokens and current_chunk:
-                # Save the current chunk
-                chunk_id = str(uuid.uuid4())
-                chunk = TextChunk(
-                    id=chunk_id,
-                    content=context_prefix + current_chunk,
-                    title=title,
-                    chapter=chapter,
-                    section=section,
-                    page_reference=file_path,
-                    token_count=current_tokens,
-                    original_start_pos=offset + content.find(current_chunk),
-                    original_end_pos=offset + content.find(current_chunk) + len(current_chunk)
-                )
-                chunks.append(chunk)
-
-                # Start a new chunk with potential overlap
-                overlap_size = int(len(current_chunk) * self.overlap_ratio)
-                overlap_content = current_chunk[-overlap_size:] if overlap_size > 0 else ""
-
-                current_chunk = overlap_content + sentence
-                current_tokens = self._count_tokens(context_prefix + current_chunk)
-                chunk_idx += 1
-            else:
-                # Add sentence to current chunk
-                current_chunk += sentence
-                current_tokens += sentence_tokens
-
-        # Add the final chunk if it has content
-        if current_chunk.strip():
-            chunk_id = str(uuid.uuid4())
-            chunk = TextChunk(
-                id=chunk_id,
-                content=context_prefix + current_chunk,
-                title=title,
-                chapter=chapter,
-                section=section,
-                page_reference=file_path,
-                token_count=self._count_tokens(context_prefix + current_chunk),
-                original_start_pos=offset + content.find(current_chunk),
-                original_end_pos=offset + content.find(current_chunk) + len(current_chunk)
-            )
-            chunks.append(chunk)
-
-        return chunks
-
-    def _split_into_sentences(self, text: str) -> List[str]:
-        """
-        Split text into sentences using common sentence boundaries
-        """
-        # Use regex to split on sentence boundaries while preserving the delimiters
-        sentence_pattern = r'(?<=[.!?])\s+'
-        sentences = re.split(sentence_pattern, text)
-
-        # Re-add the punctuation to each sentence
-        result = []
-        for i, sentence in enumerate(sentences):
-            if i < len(sentences) - 1:
-                # Check if the original text had punctuation at the end of this sentence
-                # by looking at the character that followed this sentence in the original text
-                next_char_idx = len(''.join(sentences[:i+1])) + i  # +i for spaces
-                if next_char_idx < len(text):
-                    next_char = text[next_char_idx] if next_char_idx < len(text) else ''
-                    if next_char in '.!?':
-                        sentence += next_char
-            result.append(sentence + ' ')
-
-        # Clean up and ensure each sentence is properly formatted
-        result = [s.strip() for s in result if s.strip()]
-        return result
-
-    def _count_tokens(self, text: str) -> int:
-        """
-        Count approximate number of tokens in text
-        This is a simple estimation; for more accurate counting, use tiktoken
-        """
-        import tiktoken
-        # Use cl100k_base encoding which is used by many OpenAI models
-        encoding = tiktoken.get_encoding("cl100k_base")
-        return len(encoding.encode(text))
-
-    def _estimate_tokens_per_chunk(self) -> int:
-        """
-        Estimate number of tokens that would fit in a chunk based on max_tokens
-        """
-        # This is a rough estimation - in practice, you might want to use
-        # a more sophisticated approach based on your specific content
-        return min(self.max_tokens, 800)  # Conservative estimate
-
-
-def chunk_documents(documents: List[Dict]) -> List[TextChunk]:
-    """
-    Convenience function to chunk a list of documents
-    """
-    chunker = TextChunker()
-    all_chunks = []
-
-    for document in documents:
-        chunks = chunker.chunk_document(document)
-        all_chunks.extend(chunks)
-
-    return all_chunks

app/ingestion/document_parser.py

@@ -1,146 +0,0 @@
-import os
-import re
-from typing import List, Dict, Optional
-from pathlib import Path
-import markdown
-from bs4 import BeautifulSoup
-
-
-class DocumentParser:
-    """
-    Parses markdown files and extracts content with structural information
-    """
-
-    def __init__(self):
-        pass
-
-    def parse_markdown_file(self, file_path: str) -> Dict:
-        """
-        Parse a markdown file and extract content with structural information
-        """
-        with open(file_path, 'r', encoding='utf-8') as file:
-            content = file.read()
-
-        # Extract metadata from frontmatter if present
-        metadata = self._extract_frontmatter(content)
-
-        # Extract structural information (headings)
-        structure = self._extract_structure(content)
-
-        # Get clean content without frontmatter
-        clean_content = self._remove_frontmatter(content)
-
-        # Extract title from the first heading or filename
-        title = metadata.get('title') or self._extract_title(clean_content) or Path(file_path).stem
-
-        # Determine chapter/section from file path
-        path_parts = Path(file_path).parts
-        chapter = self._extract_chapter_info(path_parts)
-        section = self._extract_section_info(path_parts)
-
-        return {
-            'title': title,
-            'content': clean_content,
-            'chapter': chapter,
-            'section': section,
-            'file_path': file_path,
-            'metadata': metadata,
-            'structure': structure
-        }
-
-    def _extract_frontmatter(self, content: str) -> Dict:
-        """
-        Extract YAML frontmatter from markdown content
-        """
-        import yaml
-
-        # Look for YAML frontmatter between --- delimiters
-        frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
-
-        if frontmatter_match:
-            try:
-                frontmatter = yaml.safe_load(frontmatter_match.group(1))
-                return frontmatter or {}
-            except yaml.YAMLError:
-                return {}
-
-        return {}
-
-    def _remove_frontmatter(self, content: str) -> str:
-        """
-        Remove YAML frontmatter from content
-        """
-        frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
-
-        if frontmatter_match:
-            return content[frontmatter_match.end():]
-
-        return content
-
-    def _extract_structure(self, content: str) -> List[Dict]:
-        """
-        Extract structural information (headings) from markdown content
-        """
-        # Convert markdown to HTML to easily extract headings
-        html = markdown.markdown(content)
-        soup = BeautifulSoup(html, 'html.parser')
-
-        structure = []
-        for i, heading in enumerate(soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])):
-            structure.append({
-                'level': int(heading.name[1]),
-                'text': heading.get_text().strip(),
-                'position': i
-            })
-
-        return structure
-
-    def _extract_title(self, content: str) -> Optional[str]:
-        """
-        Extract title from the first heading in the content
-        """
-        lines = content.split('\n')
-        for line in lines:
-            # Check for markdown heading pattern
-            heading_match = re.match(r'^#+\s+(.+)', line)
-            if heading_match:
-                return heading_match.group(1).strip()
-
-        return None
-
-    def _extract_chapter_info(self, path_parts: tuple) -> str:
-        """
-        Extract chapter information from file path
-        """
-        # Look for common chapter-related directory names
-        for part in path_parts:
-            if 'chapter' in part.lower() or 'module' in part.lower():
-                return part
-
-        # If no chapter directory found, use the directory name
-        if len(path_parts) > 1:
-            return path_parts[-2]  # Parent directory of the file
-
-        return 'unknown'
-
-    def _extract_section_info(self, path_parts: tuple) -> str:
-        """
-        Extract section information from file path
-        """
-        file_name = path_parts[-1]
-        # Remove file extension
-        section = Path(file_name).stem
-        return section
-
-
-def scan_markdown_files(directory: str) -> List[str]:
-    """
-    Scan a directory for markdown files
-    """
-    markdown_files = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.lower().endswith(('.md', '.markdown')):
-                markdown_files.append(os.path.join(root, file))
-
-    return markdown_files

app/ingestion/file_scanner.py

@@ -1,92 +0,0 @@
-import os
-from typing import List, Dict, Optional
-from pathlib import Path
-from .document_parser import DocumentParser, scan_markdown_files
-
-
-class FileScanner:
-    """
-    Scans and processes markdown files from the Docusaurus documentation directory
-    """
-
-    def __init__(self, base_path: str = "docusaurus/docs"):
-        self.base_path = base_path
-        self.parser = DocumentParser()
-
-    def scan_and_parse_documents(self) -> List[Dict]:
-        """
-        Scan the documentation directory and parse all markdown files
-        """
-        if not os.path.exists(self.base_path):
-            raise FileNotFoundError(f"Documentation directory not found: {self.base_path}")
-
-        markdown_files = scan_markdown_files(self.base_path)
-        documents = []
-
-        for file_path in markdown_files:
-            try:
-                document = self.parser.parse_markdown_file(file_path)
-                documents.append(document)
-            except Exception as e:
-                print(f"Error parsing file {file_path}: {str(e)}")
-                continue
-
-        return documents
-
-    def validate_document(self, document: Dict) -> bool:
-        """
-        Validate document structure and content
-        """
-        required_fields = ['title', 'content', 'chapter', 'section', 'file_path']
-        for field in required_fields:
-            if field not in document or not document[field]:
-                return False
-
-        # Check content length
-        if len(document['content'].strip()) < 10:
-            return False
-
-        return True
-
-    def get_document_stats(self, documents: List[Dict]) -> Dict:
-        """
-        Get statistics about the parsed documents
-        """
-        total_docs = len(documents)
-        valid_docs = sum(1 for doc in documents if self.validate_document(doc))
-        total_chars = sum(len(doc['content']) for doc in documents)
-        unique_chapters = len(set(doc['chapter'] for doc in documents))
-
-        return {
-            'total_documents': total_docs,
-            'valid_documents': valid_docs,
-            'invalid_documents': total_docs - valid_docs,
-            'total_characters': total_chars,
-            'unique_chapters': unique_chapters,
-            'average_length': total_chars // total_docs if total_docs > 0 else 0
-        }
-
-
-def main():
-    """
-    Main function to demonstrate file scanning
-    """
-    # Use the docusaurus docs directory by default, or allow override
-    scanner = FileScanner()
-    documents = scanner.scan_and_parse_documents()
-
-    print(f"Found {len(documents)} documents")
-    stats = scanner.get_document_stats(documents)
-    print(f"Statistics: {stats}")
-
-    # Print first document as example
-    if documents:
-        print(f"\nFirst document example:")
-        print(f"Title: {documents[0]['title']}")
-        print(f"Chapter: {documents[0]['chapter']}")
-        print(f"Section: {documents[0]['section']}")
-        print(f"Content preview: {documents[0]['content'][:200]}...")
-
-
-if __name__ == "__main__":
-    main()

app/main.py

@@ -1,44 +0,0 @@
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-
-app = FastAPI(
-    title="RAG Chatbot API",
-    description="API for RAG-based question answering for Physical AI & Humanoid Robotics book",
-    version="1.0.0"
-)
-
-# Add CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],  # In production, replace with specific origins
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-@app.get("/")
-async def root():
-    return {"message": "RAG Chatbot API is running!"}
-
-@app.get("/health")
-async def health_check():
-    return {"status": "healthy", "service": "RAG Chatbot API"}
-
-
-@app.on_event("startup")
-async def startup_event():
-    """
-    Create database tables on startup
-    """
-    from app.database.models import create_tables
-    create_tables()  # Create tables with correct schema (using checkfirst to avoid errors)
-
-# Include API routes
-from app.api import health, ingest, chat
-app.include_router(health.router, prefix="/api")
-app.include_router(ingest.router, prefix="/api")
-app.include_router(chat.router, prefix="/api")
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)

app/models/chat.py

@@ -1,69 +0,0 @@
-from pydantic import BaseModel
-from typing import List, Dict, Optional
-from datetime import datetime
-
-
-class ChatSession(BaseModel):
-    """
-    Model for chat session data
-    """
-    session_id: str
-    user_id: Optional[str] = None
-    created_at: datetime
-    updated_at: datetime
-    metadata: Optional[Dict] = None
-
-
-class ChatMessage(BaseModel):
-    """
-    Model for chat message data
-    """
-    message_id: str
-    session_id: str
-    role: str  # "user" or "assistant"
-    content: str
-    citations: Optional[List[Dict[str, str]]] = None
-    query_context_id: Optional[str] = None
-    timestamp: datetime
-
-
-class QueryContext(BaseModel):
-    """
-    Model for query context data
-    """
-    context_id: str
-    session_id: str
-    selected_text: Optional[str] = None
-    query_type: str  # "global" or "selection"
-    created_at: datetime
-
-
-class ChatRequest(BaseModel):
-    """
-    Model for chat API request
-    """
-    session_id: str
-    message: str
-    selected_text: Optional[str] = None
-    query_type: str = "global"  # "global" or "selection"
-    top_k: int = 5
-
-
-class ChatResponse(BaseModel):
-    """
-    Model for chat API response
-    """
-    response: str
-    citations: List[Dict[str, str]]
-    session_id: str
-    query_type: str
-    timestamp: str
-
-
-class ChatHistoryResponse(BaseModel):
-    """
-    Model for chat history API response
-    """
-    session_id: str
-    messages: List[ChatMessage]
-    timestamp: str

app/prompting/context_filter.py

@@ -1,205 +0,0 @@
-from typing import List, Dict, Any, Optional
-from app.retrieval.retriever import Retriever
-from app.prompting.prompt_builder import PromptBuilder
-
-
-class ContextFilter:
-    """
-    Filters and validates contexts to prevent information leakage between query types
-    """
-
-    def __init__(self):
-        self.retriever = Retriever()
-        self.prompt_builder = PromptBuilder()
-
-    def filter_context_for_query_type(
-        self,
-        contexts: List[Dict[str, Any]],
-        query_type: str,
-        selected_text: Optional[str] = None
-    ) -> List[Dict[str, Any]]:
-        """
-        Filter contexts based on query type to prevent information leakage
-        """
-        if query_type == "selection" and selected_text:
-            # For selection-based queries, we need to ensure contexts are relevant
-            # to the selected text and don't introduce unrelated global knowledge
-            return self._filter_selection_contexts(contexts, selected_text)
-        elif query_type == "global":
-            # For global queries, we can use all retrieved contexts
-            return contexts
-        else:
-            # Default to global behavior
-            return contexts
-
-    def _filter_selection_contexts(
-        self,
-        contexts: List[Dict[str, Any]],
-        selected_text: str
-    ) -> List[Dict[str, Any]]:
-        """
-        Filter contexts to ensure they're relevant to the selected text for selection-based queries
-        """
-        if not contexts or not selected_text:
-            return contexts
-
-        # Simple relevance check: ensure contexts have some connection to the selected text
-        # In a more sophisticated implementation, you might use semantic similarity
-        selected_keywords = set(selected_text.lower().split()[:10])  # Use first 10 words as keywords
-        filtered_contexts = []
-
-        for context in contexts:
-            content = context.get('content', '').lower()
-            content_words = set(content.split())
-
-            # Check if there's significant overlap in keywords
-            keyword_overlap = len(selected_keywords.intersection(content_words))
-            keyword_ratio = keyword_overlap / len(selected_keywords) if selected_keywords else 0
-
-            # Include context if it has some relevance to the selected text
-            # or if we don't have enough contexts yet
-            if keyword_ratio > 0.1 or len(filtered_contexts) < 2:  # At least 10% overlap or include first few
-                filtered_contexts.append(context)
-
-        return filtered_contexts
-
-    def validate_context_isolation(
-        self,
-        contexts: List[Dict[str, Any]],
-        query_type: str,
-        selected_text: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Validate that contexts are properly isolated based on query type
-        """
-        validation_result = {
-            'is_valid': True,
-            'query_type': query_type,
-            'context_count': len(contexts),
-            'issues': []
-        }
-
-        if query_type == "selection" and selected_text:
-            # Validate that contexts are related to selected text
-            validation_result.update(self._validate_selection_contexts(contexts, selected_text))
-        elif query_type == "global":
-            # For global queries, validate that we have diverse contexts
-            validation_result.update(self._validate_global_contexts(contexts))
-
-        return validation_result
-
-    def _validate_selection_contexts(
-        self,
-        contexts: List[Dict[str, Any]],
-        selected_text: str
-    ) -> Dict[str, Any]:
-        """
-        Validate selection-based contexts
-        """
-        result = {
-            'is_valid': True,
-            'issues': []
-        }
-
-        if not contexts:
-            result['is_valid'] = False
-            result['issues'].append("No contexts provided for selection-based query")
-            return result
-
-        # Check relevance to selected text
-        relevant_count = 0
-        selected_keywords = set(selected_text.lower().split()[:10])
-
-        for context in contexts:
-            content = context.get('content', '').lower()
-            content_words = set(content.split())
-            keyword_overlap = len(selected_keywords.intersection(content_words))
-
-            if keyword_overlap > 0:
-                relevant_count += 1
-
-        relevance_ratio = relevant_count / len(contexts)
-        if relevance_ratio < 0.3:  # Less than 30% of contexts are relevant
-            result['is_valid'] = False
-            result['issues'].append(
-                f"Only {relevant_count}/{len(contexts)} contexts ({relevance_ratio:.1%}) "
-                f"are relevant to the selected text"
-            )
-
-        return result
-
-    def _validate_global_contexts(self, contexts: List[Dict[str, Any]]) -> Dict[str, Any]:
-        """
-        Validate global contexts
-        """
-        result = {
-            'is_valid': True,
-            'issues': []
-        }
-
-        if not contexts:
-            result['is_valid'] = False
-            result['issues'].append("No contexts provided for global query")
-            return result
-
-        # Check for diversity in chapters/sections
-        unique_chapters = set(ctx.get('chapter', '') for ctx in contexts)
-        unique_sections = set(ctx.get('section', '') for ctx in contexts)
-
-        if len(unique_chapters) < 2 and len(contexts) > 1:
-            result['issues'].append("Contexts lack diversity - all from same chapter")
-
-        return result
-
-    def enforce_context_boundaries(
-        self,
-        contexts: List[Dict[str, Any]],
-        query_type: str,
-        selected_text: Optional[str] = None,
-        max_contexts: int = 5
-    ) -> List[Dict[str, Any]]:
-        """
-        Enforce strict boundaries on contexts to prevent information leakage
-        """
-        # First, filter based on query type
-        filtered_contexts = self.filter_context_for_query_type(contexts, query_type, selected_text)
-
-        # Then enforce maximum count
-        if len(filtered_contexts) > max_contexts:
-            filtered_contexts = filtered_contexts[:max_contexts]
-
-        # Validate the final contexts
-        validation = self.validate_context_isolation(filtered_contexts, query_type, selected_text)
-
-        if not validation['is_valid']:
-            print(f"Context validation warning: {validation['issues']}")
-
-        return filtered_contexts
-
-    def build_isolated_context_string(
-        self,
-        contexts: List[Dict[str, Any]],
-        query_type: str,
-        selected_text: Optional[str] = None
-    ) -> str:
-        """
-        Build a context string that enforces isolation between query types
-        """
-        if query_type == "selection" and selected_text:
-            # Build context string focused on selected text and related content
-            context_str = f"SELECTED TEXT:\n{selected_text}\n\nRELATED CONTENT:\n"
-            for i, ctx in enumerate(contexts):
-                context_str += f"[{i+1}] {ctx.get('content', '')}\n"
-                context_str += f"Source: {ctx.get('chapter', '')} - {ctx.get('section', '')}\n\n"
-        else:
-            # Build global context string
-            context_str = "RETRIEVED CONTENT:\n"
-            for i, ctx in enumerate(contexts):
-                context_str += f"[{i+1}] {ctx.get('content', '')}\n"
-                context_str += f"Source: {ctx.get('chapter', '')} - {ctx.get('section', '')}\n\n"
-
-        return context_str
-
-
-# Global instance
-context_filter = ContextFilter()

app/prompting/prompt_builder.py

@@ -1,187 +0,0 @@
-from typing import List, Dict, Any, Optional
-from app.retrieval.retriever import Retriever
-from app.config import settings
-
-
-class PromptBuilder:
-    """
-    Builds context-aware prompts for the LLM while preventing hallucinations
-    """
-
-    def __init__(self):
-        self.retriever = Retriever()
-
-    def build_global_query_prompt(
-        self,
-        query: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        max_context_length: int = 2000
-    ) -> str:
-        """
-        Build a prompt for global book queries using all retrieved contexts
-        """
-        # Start with system message to prevent hallucinations
-        system_prompt = (
-            "You are an AI assistant that answers questions based only on the provided context. "
-            "Do not use any prior knowledge or information not present in the context. "
-            "If the answer cannot be found in the provided context, respond with: "
-            "'The provided context does not contain information to answer this question.'\n\n"
-        )
-
-        # Add retrieved contexts
-        context_section = "### CONTEXT:\n\n"
-        total_context_length = 0
-
-        for i, ctx in enumerate(retrieved_contexts):
-            if total_context_length >= max_context_length:
-                break
-
-            context_text = ctx.get('content', '')
-            # Truncate if too long
-            if len(context_text) + total_context_length > max_context_length:
-                available_length = max_context_length - total_context_length
-                context_text = context_text[:available_length]
-
-            context_section += f"**Source {i+1} ({ctx.get('title', 'Unknown')} - {ctx.get('chapter', 'Unknown')}):**\n"
-            context_section += f"{context_text}\n\n"
-            total_context_length += len(context_text)
-
-        # Add user query
-        user_query_section = f"### QUESTION:\n{query}\n\n"
-
-        # Add instruction for response format
-        response_format = (
-            "### INSTRUCTIONS:\n"
-            "1. Answer the question based ONLY on the provided context\n"
-            "2. If the context doesn't contain the answer, say so explicitly\n"
-            "3. Include relevant citations to the sources in your response\n"
-            "4. Keep your response concise and to the point\n\n"
-        )
-
-        # Combine all parts
-        full_prompt = system_prompt + context_section + user_query_section + response_format
-        return full_prompt
-
-    def build_selection_based_prompt(
-        self,
-        query: str,
-        selected_text: str,
-        retrieved_contexts: List[Dict[str, Any]],
-        max_context_length: int = 2000
-    ) -> str:
-        """
-        Build a prompt for selection-based queries using only the selected text and relevant contexts
-        """
-        # Start with system message
-        system_prompt = (
-            "You are an AI assistant that answers questions based only on the provided selected text and related context. "
-            "Do not use any prior knowledge or information not present in the provided content. "
-            "Focus your answer on the relationship between the selected text and the question. "
-            "If the answer cannot be found in the provided content, respond with: "
-            "'The provided content does not contain information to answer this question.'\n\n"
-        )
-
-        # Add the selected text as primary context
-        primary_context = f"### SELECTED TEXT:\n{selected_text}\n\n"
-
-        # Add related contexts
-        related_context = "### RELATED CONTEXT:\n\n"
-        total_context_length = len(selected_text)
-
-        for i, ctx in enumerate(retrieved_contexts):
-            if total_context_length >= max_context_length:
-                break
-
-            context_text = ctx.get('content', '')
-            # Truncate if too long
-            if len(context_text) + total_context_length > max_context_length:
-                available_length = max_context_length - total_context_length
-                context_text = context_text[:available_length]
-
-            related_context += f"**Related Content {i+1} ({ctx.get('title', 'Unknown')} - {ctx.get('chapter', 'Unknown')}):**\n"
-            related_context += f"{context_text}\n\n"
-            total_context_length += len(context_text)
-
-        # Add user query
-        user_query_section = f"### QUESTION ABOUT SELECTED TEXT:\n{query}\n\n"
-
-        # Add instruction for response format
-        response_format = (
-            "### INSTRUCTIONS:\n"
-            "1. Answer the question based ONLY on the selected text and related context\n"
-            "2. Focus on how the question relates to the selected text\n"
-            "3. If the content doesn't contain the answer, say so explicitly\n"
-            "4. Include citations to the sources in your response\n"
-            "5. Keep your response concise and relevant to the selected text\n\n"
-        )
-
-        # Combine all parts
-        full_prompt = system_prompt + primary_context + related_context + user_query_section + response_format
-        return full_prompt
-
-    def build_context_filter_prompt(
-        self,
-        query: str,
-        available_contexts: List[Dict[str, Any]],
-        max_contexts_to_use: int = 3
-    ) -> str:
-        """
-        Build a prompt to filter relevant contexts from a larger set
-        """
-        system_prompt = (
-            "You are an AI assistant that helps filter relevant contexts for a given question. "
-            "Analyze the question and the provided contexts, then select only the most relevant ones.\n\n"
-        )
-
-        # Add contexts
-        contexts_section = "### AVAILABLE CONTEXTS:\n\n"
-        for i, ctx in enumerate(available_contexts):
-            contexts_section += f"**Context {i+1} ({ctx.get('title', 'Unknown')} - {ctx.get('chapter', 'Unknown')}):**\n"
-            contexts_section += f"{ctx.get('content', '')[:500]}...\n\n"  # Truncate for brevity
-
-        # Add query
-        query_section = f"### QUESTION:\n{query}\n\n"
-
-        # Add instructions
-        instruction_section = (
-            "### INSTRUCTIONS:\n"
-            "1. Identify which contexts are most relevant to answering the question\n"
-            "2. List the most relevant contexts by their number\n"
-            "3. Provide a brief reason for why each selected context is relevant\n"
-            f"4. Select at most {max_contexts_to_use} contexts\n\n"
-        )
-
-        # Add response format
-        response_format = (
-            "### RESPONSE FORMAT:\n"
-            "Respond with only the following JSON format:\n"
-            "{\n"
-            "  \"relevant_contexts\": [\n"
-            "    {\n"
-            "      \"index\": 0,\n"
-            "      \"reason\": \"Brief explanation of relevance\"\n"
-            "    }\n"
-            "  ]\n"
-            "}\n"
-        )
-
-        full_prompt = system_prompt + contexts_section + query_section + instruction_section + response_format
-        return full_prompt
-
-    def validate_prompt_context_isolation(self, prompt: str, query_type: str, original_context: Optional[str] = None) -> bool:
-        """
-        Validate that the prompt properly isolates contexts based on query type
-        """
-        if query_type == "selection" and original_context:
-            # For selection queries, ensure the prompt focuses on the selected text
-            # This is a basic validation - in practice, you'd want more sophisticated checks
-            return "SELECTED TEXT" in prompt or original_context[:50] in prompt
-        else:
-            # For global queries, ensure the prompt uses broader context
-            return "CONTEXT:" in prompt
-
-        return True
-
-
-# Global instance
-prompt_builder = PromptBuilder()

app/retrieval/retriever.py

@@ -1,149 +0,0 @@
-from typing import List, Dict, Any, Optional
-from app.retrieval.vector_search import VectorSearchEngine
-
-
-class Retriever:
-    """
-    High-level retriever that handles the complete retrieval process
-    """
-
-    def __init__(self):
-        self.vector_search = VectorSearchEngine()
-
-    async def retrieve_relevant_documents(
-        self,
-        query: str,
-        top_k: int = 5,
-        query_type: str = "global",  # "global" or "selection"
-        selected_text: Optional[str] = None,
-        filters: Optional[Dict] = None
-    ) -> List[Dict[str, Any]]:
-        """
-        Retrieve relevant documents based on the query and query type
-        """
-        if query_type == "selection" and selected_text:
-            # For selection-based queries, we use the selected text as context
-            # but still search for relevant content in the book
-            # This approach focuses on content related to the selected text
-            search_query = f"{selected_text} {query}".strip()
-        else:
-            # For global queries, we search with the original query
-            search_query = query
-
-        # Perform the search
-        results = await self.vector_search.search_with_query(
-            query=search_query,
-            top_k=top_k,
-            filters=filters
-        )
-
-        # Apply ranking and filtering
-        ranked_results = self.vector_search.rank_results_by_relevance(results, query)
-        filtered_results = self.vector_search.filter_results(ranked_results, filters)
-
-        return filtered_results
-
-    async def retrieve_with_context_filtering(
-        self,
-        query: str,
-        top_k: int = 5,
-        query_type: str = "global",
-        selected_text: Optional[str] = None
-    ) -> List[Dict[str, Any]]:
-        """
-        Retrieve documents with special handling for different query types
-        """
-        if query_type == "selection" and selected_text:
-            # For selection-based queries, we might want to prioritize results
-            # that are semantically similar to both the selected text and the query
-            combined_query = f"Context: {selected_text}\nQuestion: {query}"
-            results = await self.vector_search.search_with_query(
-                query=combined_query,
-                top_k=top_k
-            )
-        else:
-            # For global queries, search normally
-            results = await self.vector_search.search_with_query(
-                query=query,
-                top_k=top_k
-            )
-
-        # Apply additional filtering to ensure results are relevant
-        # For selection-based queries, we might want to ensure results are related to the selected text
-        if query_type == "selection" and selected_text:
-            # Filter results to ensure they're related to the selected text context
-            filtered_results = []
-            for result in results:
-                # This is a simple check - in practice, you might want more sophisticated filtering
-                content = result.get('content', '').lower()
-                selected_lower = selected_text.lower()
-
-                # Check if the result content has some relation to the selected text
-                # This could be improved with semantic similarity checks
-                if any(word in content for word in selected_lower.split()[:5]):  # Check first 5 words
-                    filtered_results.append(result)
-                elif len(filtered_results) < top_k:  # Add some results even if not perfectly matched
-                    filtered_results.append(result)
-
-            results = filtered_results
-
-        return results
-
-    async def retrieve_for_citation(self, query: str, top_k: int = 3) -> List[Dict[str, Any]]:
-        """
-        Retrieve documents specifically for citation purposes
-        This method focuses on getting clean, citable content
-        """
-        results = await self.retrieve_relevant_documents(query, top_k)
-
-        # Format results for citation
-        citations = []
-        for result in results:
-            citation = {
-                'document_id': result.get('id'),
-                'title': result.get('title'),
-                'chapter': result.get('chapter'),
-                'section': result.get('section'),
-                'page_reference': result.get('page_reference'),
-                'content': result.get('content'),
-                'relevance_score': result.get('score')
-            }
-            citations.append(citation)
-
-        return citations
-
-    def validate_retrieval_quality(self, results: List[Dict[str, Any]], query: str) -> Dict[str, Any]:
-        """
-        Validate the quality of the retrieval results
-        """
-        quality_metrics = {
-            'total_results': len(results),
-            'avg_relevance_score': sum(r.get('score', 0) for r in results) / len(results) if results else 0,
-            'has_high_quality_results': any(r.get('score', 0) > 0.7 for r in results) if results else False,
-            'query_coverage': self._assess_query_coverage(results, query)
-        }
-
-        return quality_metrics
-
-    def _assess_query_coverage(self, results: List[Dict[str, Any]], query: str) -> float:
-        """
-        Assess how well the results cover the query topics
-        This is a simplified implementation
-        """
-        if not results:
-            return 0.0
-
-        query_keywords = set(query.lower().split())
-        covered_keywords = set()
-
-        for result in results:
-            content = result.get('content', '').lower()
-            result_keywords = set(content.split())
-            covered_keywords.update(query_keywords.intersection(result_keywords))
-
-        coverage = len(covered_keywords) / len(query_keywords) if query_keywords else 0.0
-        return min(coverage, 1.0)  # Ensure value is between 0 and 1
-
-
-# Global instance
-retriever = Retriever()

app/retrieval/vector_search.py

@@ -1,103 +0,0 @@
-from typing import List, Dict, Any, Optional
-from app.vector_store.qdrant_client import QdrantVectorStore
-from app.embeddings.minimal_embedding_generator import minimal_embedding_generator
-from app.config import settings
-
-
-class VectorSearchEngine:
-    """
-    Core vector search engine that handles semantic search operations
-    """
-
-    def __init__(self):
-        self.qdrant_client = QdrantVectorStore()
-        self.top_k_default = 5
-
-    async def search_with_query(self, query: str, top_k: int = 5, filters: Optional[Dict] = None) -> List[Dict[str, Any]]:
-        """
-        Perform semantic search using a query string
-        """
-        # Generate embedding for the query using minimal generator
-        query_embedding = minimal_embedding_generator.encode_query(query)
-
-        if not query_embedding:
-            return []
-
-        # Perform vector search in Qdrant
-        chapter_filter = filters.get('chapter') if filters else None
-        search_results = self.qdrant_client.search_similar(
-            query_embedding=query_embedding,
-            top_k=top_k,
-            chapter_filter=chapter_filter
-        )
-
-        return search_results
-
-    async def search_with_embedding(self, query_embedding: List[float], top_k: int = 5, filters: Optional[Dict] = None) -> List[Dict[str, Any]]:
-        """
-        Perform semantic search using a pre-computed embedding
-        """
-        chapter_filter = filters.get('chapter') if filters else None
-        search_results = self.qdrant_client.search_similar(
-            query_embedding=query_embedding,
-            top_k=top_k,
-            chapter_filter=chapter_filter
-        )
-
-        return search_results
-
-    def rank_results_by_relevance(self, results: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
-        """
-        Apply additional ranking based on relevance to the query
-        This is a simple implementation; in production, you might want to use more sophisticated ranking
-        """
-        # For now, we'll just return the results as Qdrant already ranks by similarity score
-        # In the future, we could implement additional ranking based on:
-        # - keyword matching in title/content
-        # - recency of content
-        # - content length relative to query needs
-        return sorted(results, key=lambda x: x.get('score', 0), reverse=True)
-
-    def filter_results(self, results: List[Dict[str, Any]], filters: Optional[Dict] = None) -> List[Dict[str, Any]]:
-        """
-        Apply additional filtering to search results
-        """
-        if not filters:
-            return results
-
-        filtered_results = []
-        for result in results:
-            include = True
-
-            # Apply content-based filters
-            if 'min_score' in filters:
-                if result.get('score', 0) < filters['min_score']:
-                    include = False
-
-            if 'required_keywords' in filters:
-                content = result.get('content', '').lower()
-                for keyword in filters['required_keywords']:
-                    if keyword.lower() not in content:
-                        include = False
-                        break
-
-            if include:
-                filtered_results.append(result)
-
-        return filtered_results
-
-    async def get_document_content(self, doc_id: str) -> Optional[Dict[str, Any]]:
-        """
-        Retrieve content of a specific document by ID
-        """
-        return self.qdrant_client.get_document_by_id(doc_id)
-
-    def get_collection_stats(self) -> Dict[str, Any]:
-        """
-        Get statistics about the vector collection
-        """
-        return self.qdrant_client.get_collection_info()
-
-
-# Global instance
-vector_search_engine = VectorSearchEngine()

app/services/chat_service.py

@@ -1,144 +0,0 @@
-from typing import Dict, Any, List, Optional
-from datetime import datetime
-import uuid
-
-from app.retrieval.retriever import Retriever
-from app.prompting.context_filter import ContextFilter
-from app.generation.response_generator import ResponseGenerator
-from app.database.repositories import ChatSessionRepository, ChatMessageRepository, QueryContextRepository
-from app.database.database import get_db
-
-
-class ChatService:
-    """
-    Service class that orchestrates the chat functionality
-    """
-
-    def __init__(self):
-        self.retriever = Retriever()
-        self.context_filter = ContextFilter()
-        self.response_generator = ResponseGenerator()
-
-    async def process_chat_message(
-        self,
-        session_id: str,
-        message: str,
-        query_type: str = "global",
-        selected_text: Optional[str] = None,
-        top_k: int = 5
-    ) -> Dict[str, Any]:
-        """
-        Process a chat message through the full RAG pipeline
-        """
-        # Retrieve relevant documents
-        retrieved_docs = await self.retriever.retrieve_with_context_filtering(
-            query=message,
-            top_k=top_k,
-            query_type=query_type,
-            selected_text=selected_text
-        )
-
-        # Apply context filtering to ensure proper isolation
-        filtered_docs = self.context_filter.enforce_context_boundaries(
-            contexts=retrieved_docs,
-            query_type=query_type,
-            selected_text=selected_text
-        )
-
-        # Generate response
-        response_data = await self.response_generator.generate_response_with_validation(
-            query=message,
-            retrieved_contexts=filtered_docs,
-            query_type=query_type,
-            selected_text=selected_text,
-            session_id=session_id
-        )
-
-        # Store conversation in database
-        await self._store_conversation(session_id, message, response_data)
-
-        return response_data
-
-    async def _store_conversation(self, session_id: str, user_message: str, response_data: Dict[str, Any]):
-        """
-        Store the conversation in the database
-        """
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            # Create or update session
-            session_repo = ChatSessionRepository(db)
-            existing_session = session_repo.get_session_by_id(session_id)
-            if not existing_session:
-                session_repo.create_session(session_id=session_id)
-
-            # Store user message
-            user_message_id = f"msg_{uuid.uuid4().hex[:8]}"
-            message_repo = ChatMessageRepository(db)
-            message_repo.create_message(
-                message_id=user_message_id,
-                session_id=session_id,
-                role="user",
-                content=user_message
-            )
-
-            # Store assistant response
-            assistant_message_id = f"msg_{uuid.uuid4().hex[:8]}"
-            citations_for_storage = response_data.get("citations", [])
-            message_repo.create_message(
-                message_id=assistant_message_id,
-                session_id=session_id,
-                role="assistant",
-                content=response_data.get("response", ""),
-                citations=citations_for_storage
-            )
-        finally:
-            next(db_gen, None)  # Close the db session
-
-    async def get_chat_history(self, session_id: str) -> List[Dict[str, Any]]:
-        """
-        Retrieve chat history for a session
-        """
-        db_gen = get_db()
-        db = next(db_gen)
-        try:
-            message_repo = ChatMessageRepository(db)
-            messages = message_repo.get_messages_by_session(session_id)
-
-            return [
-                {
-                    "message_id": msg.message_id,
-                    "role": msg.role,
-                    "content": msg.content,
-                    "timestamp": msg.timestamp.isoformat() if msg.timestamp else None,
-                    "citations": msg.citations
-                }
-                for msg in messages
-            ]
-        finally:
-            next(db_gen, None)
-
-    def validate_query_params(
-        self,
-        query_type: str,
-        selected_text: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """
-        Validate query parameters
-        """
-        errors = []
-
-        if query_type not in ["global", "selection"]:
-            errors.append("query_type must be either 'global' or 'selection'")
-
-        if query_type == "selection" and not selected_text:
-            errors.append("selected_text is required for selection-based queries")
-
-        return {
-            "is_valid": len(errors) == 0,
-            "errors": errors
-        }
-
-
-# Global instance
-chat_service = ChatService()

app/services/openrouter_client.py

@@ -1,165 +0,0 @@
-import asyncio
-import httpx
-from typing import List, Dict, Any, Optional
-from app.config import settings
-
-
-class OpenRouterClient:
-    """
-    Client for interacting with OpenRouter API for both embeddings and chat completions
-    """
-
-    def __init__(self):
-        self.base_url = settings.OPENROUTER_BASE_URL
-        self.api_key = settings.OPENROUTER_API_KEY
-        self.max_retries = 3
-        self.retry_delay = 1
-
-    async def generate_embeddings(self, texts: List[str], model: str = "text-embedding-ada-002") -> List[List[float]]:
-        """
-        Generate embeddings for a list of texts using OpenRouter API
-        """
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
-        }
-
-        embeddings = []
-        for text in texts:
-            # Truncate text if it's too long
-            max_length = 8000  # Conservative limit
-            if len(text) > max_length:
-                text = text[:max_length]
-
-            data = {
-                "model": model,
-                "input": text
-            }
-
-            async with httpx.AsyncClient(timeout=30.0) as client:
-                for attempt in range(self.max_retries):
-                    try:
-                        response = await client.post(
-                            f"{self.base_url}/embeddings",
-                            headers=headers,
-                            json=data
-                        )
-
-                        if response.status_code == 200:
-                            result = response.json()
-                            embedding = result['data'][0]['embedding']
-                            embeddings.append(embedding)
-                            break
-                        elif response.status_code == 429:
-                            # Rate limited - wait and retry
-                            wait_time = self.retry_delay * (2 ** attempt)  # Exponential backoff
-                            print(f"Rate limited, waiting {wait_time}s before retry {attempt + 1}")
-                            await asyncio.sleep(wait_time)
-                            continue
-                        else:
-                            print(f"Error {response.status_code}: {response.text}")
-                            if attempt == self.max_retries - 1:
-                                # Last attempt, return zeros as fallback
-                                embeddings.append([0.0] * 1536)
-                            break
-
-                    except httpx.RequestError as e:
-                        print(f"Request error on attempt {attempt + 1}: {str(e)}")
-                        if attempt == self.max_retries - 1:
-                            embeddings.append([0.0] * 1536)
-                        await asyncio.sleep(self.retry_delay * (2 ** attempt))
-                    except Exception as e:
-                        print(f"Unexpected error on attempt {attempt + 1}: {str(e)}")
-                        if attempt == self.max_retries - 1:
-                            embeddings.append([0.0] * 1536)
-                        await asyncio.sleep(self.retry_delay * (2 ** attempt))
-
-        return embeddings
-
-    async def generate_completion(
-        self,
-        messages: List[Dict[str, str]],
-        model: str = "mistralai/devstral-2512:free",
-        temperature: float = 0.7,
-        max_tokens: int = 1000
-    ) -> Optional[str]:
-        """
-        Generate completion using OpenRouter API with specified model
-        """
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
-        }
-
-        data = {
-            "model": model,
-            "messages": messages,
-            "temperature": temperature,
-            "max_tokens": max_tokens
-        }
-
-        async with httpx.AsyncClient(timeout=60.0) as client:
-            for attempt in range(self.max_retries):
-                try:
-                    response = await client.post(
-                        f"{self.base_url}/chat/completions",
-                        headers=headers,
-                        json=data
-                    )
-
-                    if response.status_code == 200:
-                        result = response.json()
-                        return result['choices'][0]['message']['content']
-                    elif response.status_code == 429:
-                        # Rate limited - wait and retry
-                        wait_time = self.retry_delay * (2 ** attempt)  # Exponential backoff
-                        print(f"Rate limited, waiting {wait_time}s before retry {attempt + 1}")
-                        await asyncio.sleep(wait_time)
-                        continue
-                    else:
-                        print(f"Error {response.status_code}: {response.text}")
-                        if attempt == self.max_retries - 1:
-                            return None
-                        break
-
-                except httpx.RequestError as e:
-                    print(f"Request error on attempt {attempt + 1}: {str(e)}")
-                    if attempt == self.max_retries - 1:
-                        return None
-                    await asyncio.sleep(self.retry_delay * (2 ** attempt))
-                except Exception as e:
-                    print(f"Unexpected error on attempt {attempt + 1}: {str(e)}")
-                    if attempt == self.max_retries - 1:
-                        return None
-                    await asyncio.sleep(self.retry_delay * (2 ** attempt))
-
-        return None
-
-    async def get_model_info(self, model: str) -> Optional[Dict[str, Any]]:
-        """
-        Get information about a specific model
-        """
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json"
-        }
-
-        try:
-            async with httpx.AsyncClient(timeout=30.0) as client:
-                response = await client.get(
-                    f"{self.base_url}/models/{model}",
-                    headers=headers
-                )
-
-                if response.status_code == 200:
-                    return response.json()
-                else:
-                    print(f"Error getting model info {response.status_code}: {response.text}")
-                    return None
-        except Exception as e:
-            print(f"Error getting model info: {str(e)}")
-            return None
-
-
-# Global client instance
-openrouter_client = OpenRouterClient()

app/vector_store/qdrant_client.py

@@ -1,207 +0,0 @@
-from typing import List, Dict, Any, Optional
-from qdrant_client import QdrantClient
-from qdrant_client.http import models
-from app.config import settings
-from app.ingestion.chunker import TextChunk
-
-
-class QdrantVectorStore:
-    """
-    Qdrant vector database client for storing and retrieving embeddings
-    """
-
-    def __init__(self):
-        try:
-            self.client = QdrantClient(
-                url=settings.QDRANT_URL,
-                api_key=settings.QDRANT_API_KEY,
-                prefer_grpc=False,  # Using HTTP for better compatibility
-                timeout=60.0  # Increase timeout for large batch operations
-            )
-            self.collection_name = "book_content_chunks"
-            self.vector_size = 1536  # Standard embedding size for text-embedding-ada-002
-            self._initialize_collection()
-        except Exception as e:
-            print(f"[WARN] Could not connect to Qdrant: {e}")
-            print("[WARN] Qdrant functionality will be unavailable until connection is restored")
-            # Initialize with None values when connection fails
-            self.client = None
-            self.collection_name = "book_content_chunks"
-            self.vector_size = 1536
-
-    def _initialize_collection(self):
-        """
-        Initialize the collection if it doesn't exist
-        """
-        if self.client is None:
-            return  # Skip initialization if no client
-
-        try:
-            # Check if collection exists
-            self.client.get_collection(self.collection_name)
-            print(f"[INFO] Collection '{self.collection_name}' already exists")
-        except:
-            # Create collection if it doesn't exist
-            self.client.create_collection(
-                collection_name=self.collection_name,
-                vectors_config=models.VectorParams(
-                    size=self.vector_size,
-                    distance=models.Distance.COSINE
-                )
-            )
-            print(f"[INFO] Created collection '{self.collection_name}'")
-
-            # Create payload index for faster filtering
-            self.client.create_payload_index(
-                collection_name=self.collection_name,
-                field_name="chapter",
-                field_schema=models.PayloadSchemaType.KEYWORD
-            )
-
-            self.client.create_payload_index(
-                collection_name=self.collection_name,
-                field_name="section",
-                field_schema=models.PayloadSchemaType.KEYWORD
-            )
-
-    def store_embeddings(self, chunks_with_embeddings: List[Dict[str, Any]]):
-        """
-        Store chunks with their embeddings in Qdrant
-        """
-        if self.client is None:
-            print("[WARN] Cannot store embeddings - Qdrant not connected")
-            return
-
-        points = []
-        for item in chunks_with_embeddings:
-            point = models.PointStruct(
-                id=item['id'],
-                vector=item['embedding'],
-                payload={
-                    'content': item['content'],
-                    'title': item['title'],
-                    'chapter': item['chapter'],
-                    'section': item['section'],
-                    'page_reference': item['page_reference'],
-                    'token_count': item['token_count']
-                }
-            )
-            points.append(point)
-
-        # Upload points in smaller batches to avoid timeouts
-        batch_size = 16  # Smaller batch size to avoid timeouts
-        for i in range(0, len(points), batch_size):
-            batch = points[i:i + batch_size]
-            self.client.upsert(
-                collection_name=self.collection_name,
-                points=batch
-            )
-            # Add a small delay between batches to avoid overwhelming the server
-            import time
-            time.sleep(0.1)
-
-    def search_similar(self, query_embedding: List[float], top_k: int = 5, chapter_filter: Optional[str] = None) -> List[Dict[str, Any]]:
-        """
-        Search for similar content based on embedding similarity
-        """
-        if self.client is None:
-            print("[WARN] Cannot search - Qdrant not connected")
-            return []
-
-        # Build filters if needed
-        filters = None
-        if chapter_filter:
-            filters = models.Filter(
-                must=[
-                    models.FieldCondition(
-                        key="chapter",
-                        match=models.MatchValue(value=chapter_filter)
-                    )
-                ]
-            )
-
-        # Perform search
-        search_results = self.client.search(
-            collection_name=self.collection_name,
-            query_vector=query_embedding,
-            query_filter=filters,
-            limit=top_k,
-            with_payload=True
-        )
-
-        # Format results
-        results = []
-        for result in search_results:
-            results.append({
-                'id': result.id,
-                'content': result.payload['content'],
-                'title': result.payload['title'],
-                'chapter': result.payload['chapter'],
-                'section': result.payload['section'],
-                'page_reference': result.payload['page_reference'],
-                'score': result.score
-            })
-
-        return results
-
-    def get_document_by_id(self, doc_id: str) -> Optional[Dict[str, Any]]:
-        """
-        Retrieve a specific document by its ID
-        """
-        if self.client is None:
-            print("[WARN] Cannot retrieve document - Qdrant not connected")
-            return None
-
-        points = self.client.retrieve(
-            collection_name=self.collection_name,
-            ids=[doc_id],
-            with_payload=True
-        )
-
-        if points:
-            point = points[0]
-            return {
-                'id': point.id,
-                'content': point.payload['content'],
-                'title': point.payload['title'],
-                'chapter': point.payload['chapter'],
-                'section': point.payload['section'],
-                'page_reference': point.payload['page_reference']
-            }
-
-        return None
-
-    def delete_collection(self):
-        """
-        Delete the entire collection (use with caution!)
-        """
-        if self.client is None:
-            print("[WARN] Cannot delete collection - Qdrant not connected")
-            return
-
-        self.client.delete_collection(self.collection_name)
-
-    def get_collection_info(self) -> Dict[str, Any]:
-        """
-        Get information about the collection
-        """
-        if self.client is None:
-            print("[WARN] Cannot get collection info - Qdrant not connected")
-            return {
-                'name': self.collection_name,
-                'vector_size': self.vector_size,
-                'distance': 'COSINE',
-                'point_count': 0
-            }
-
-        info = self.client.get_collection(self.collection_name)
-        return {
-            'name': self.collection_name,
-            'vector_size': info.config.params.vectors.size,
-            'distance': info.config.params.vectors.distance,
-            'point_count': info.points_count
-        }
-
-
-# Global instance
-qdrant_client = QdrantVectorStore()

app/vector_store/vector_repository.py

@@ -1,49 +0,0 @@
-from typing import List, Dict, Any, Optional
-from app.vector_store.qdrant_client import QdrantVectorStore, qdrant_client
-from app.ingestion.chunker import TextChunk
-
-
-class VectorRepository:
-    """
-    Repository class for vector store operations
-    """
-
-    def __init__(self, vector_store: QdrantVectorStore):
-        self.vector_store = vector_store
-
-    def store_document_chunks(self, chunks_with_embeddings: List[Dict[str, Any]]):
-        """
-        Store document chunks with embeddings in the vector store
-        """
-        self.vector_store.store_embeddings(chunks_with_embeddings)
-
-    def search_relevant_chunks(
-        self,
-        query_embedding: List[float],
-        top_k: int = 5,
-        chapter_filter: Optional[str] = None
-    ) -> List[Dict[str, Any]]:
-        """
-        Search for relevant chunks based on query embedding
-        """
-        return self.vector_store.search_similar(
-            query_embedding=query_embedding,
-            top_k=top_k,
-            chapter_filter=chapter_filter
-        )
-
-    def get_document_by_id(self, doc_id: str) -> Optional[Dict[str, Any]]:
-        """
-        Retrieve a document by its ID
-        """
-        return self.vector_store.get_document_by_id(doc_id)
-
-    def get_collection_stats(self) -> Dict[str, Any]:
-        """
-        Get statistics about the vector collection
-        """
-        return self.vector_store.get_collection_info()
-
-
-# Global instance
-vector_repository = VectorRepository(qdrant_client)