Upload folder using huggingface_hub

app.py

@@ -1,4 +1,5 @@
-# from fastapi import FastAPI, File, UploadFile, HTTPException, Form, Request
+# perfect code
+# from fastapi import FastAPI, File, UploadFile, HTTPException, Request
 # from fastapi.responses import HTMLResponse, JSONResponse
 # from fastapi.staticfiles import StaticFiles
 # from fastapi.templating import Jinja2Templates
@@ -6,13 +7,14 @@
 # import os
 # import tempfile
 # import shutil
-# from typing import List, Dict, Any
+# from typing import List, Dict, Any, Optional, Union
 # import logging
+# from datetime import datetime
+# import mimetypes
 
+# # Import your RAG system
 # import sys
-# import os
 # sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-
 # try:
 #     from RAG import RAGSystem
 # except ImportError:
@@ -20,10 +22,21 @@
 #     print("Make sure RAG.py is in the same directory as app.py")
 #     sys.exit(1)
 
-# logging.basicConfig(level=logging.DEBUG)
+# # Configure logging
+# logging.basicConfig(
+#     level=logging.INFO,
+#     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+# )
 # logger = logging.getLogger(__name__)
 
-# app = FastAPI(title="RAG PDF QA System")
+# # Initialize FastAPI app
+# app = FastAPI(
+#     title="Scholar's Archive - Document Intelligence System",
+#     description="A sophisticated platform for intelligent document analysis and question answering using advanced retrieval-augmented generation technology",
+#     version="1.0.0",
+#     docs_url="/api/docs",
+#     redoc_url="/api/redoc"
+# )
 
 # # Setup templates directory
 # templates = Jinja2Templates(directory="templates")
@@ -37,56 +50,126 @@
 # # Initialize RAG System
 # try:
 #     rag_system = RAGSystem()
-#     logger.info("RAG System initialized successfully")
+#     logger.info("Scholar's Archive RAG System initialized successfully")
 # except Exception as e:
 #     logger.error(f"Failed to initialize RAG System: {e}")
 #     rag_system = None
 
+# # Pydantic models
 # class QuestionRequest(BaseModel):
 #     question: str
+#     top_k: int = 3
+
+# class DocumentInfo(BaseModel):
+#     title: str
+#     file_type: str
+#     upload_date: str
+#     chunk_count: int
+#     total_pages: Optional[int] = None
+
+# class AnswerResponse(BaseModel):
+#     answer: str
+#     sources: List[Dict[str, Any]]
+#     question_chunks: List[Union[str, Dict[str, Any]]] = []
+#     relevant_chunks: List[Union[str, Dict[str, Any]]] = []
+
+# class StatsResponse(BaseModel):
+#     total_documents: int
+#     total_chunks: int
+#     total_pages: int
+#     documents: List[DocumentInfo]
+
+# # Utility functions
+# def get_file_type_icon(filename: str) -> str:
+#     """Get appropriate icon for file type"""
+#     ext = os.path.splitext(filename)[1].lower()
+#     icons = {
+#         '.pdf': 'fas fa-file-pdf',
+#         '.docx': 'fas fa-file-word',
+#         '.txt': 'fas fa-file-alt',
+#         '.csv': 'fas fa-file-csv'
+#     }
+#     return icons.get(ext, 'fas fa-file')
+
+# def format_file_size(size_bytes: int) -> str:
+#     """Format file size in human readable format"""
+#     if size_bytes == 0:
+#         return "0 B"
+#     size_names = ["B", "KB", "MB", "GB"]
+#     i = 0
+#     while size_bytes >= 1024 and i < len(size_names) - 1:
+#         size_bytes /= 1024.0
+#         i += 1
+#     return f"{size_bytes:.1f} {size_names[i]}"
+
+# def extract_content_from_chunks(chunks):
+#     """Extract string content from chunk data structures"""
+#     if not chunks:
+#         return []
+    
+#     extracted = []
+#     for chunk in chunks:
+#         if isinstance(chunk, str):
+#             extracted.append(chunk)
+#         elif isinstance(chunk, dict):
+#             # Try different possible keys for text content
+#             content = chunk.get('text') or chunk.get('content') or chunk.get('document') or str(chunk)
+#             extracted.append(content)
+#         else:
+#             extracted.append(str(chunk))
+    
+#     return extracted
 
+
+
+# # Routes
 # @app.get("/", response_class=HTMLResponse)
 # async def read_root(request: Request):
+#     """Serve the main classical interface"""
 #     try:
 #         return templates.TemplateResponse("index.html", {"request": request})
 #     except Exception as e:
 #         logger.error(f"Error serving index.html from templates folder: {e}")
-#         return HTMLResponse(content=f"""
-#             <html>
-#                 <body>
-#                     <h1>RAG PDF QA System</h1>
-#                     <p>Error: Could not load index.html from templates folder</p>
-#                     <p>Error details: {str(e)}</p>
-#                     <p>Make sure you have:</p>
-#                     <ul>
-#                         <li>A 'templates' folder in the same directory as app.py</li>
-#                         <li>index.html file inside the templates folder</li>
-#                         <li>Installed jinja2: pip install jinja2</li>
-#                     </ul>
-#                 </body>
-#             </html>
-#         """)
+#         # Return the embedded HTML if templates folder is not available
+#         with open("scholar_archive.html", "r", encoding="utf-8") as f:
+#             html_content = f.read()
+#         return HTMLResponse(content=html_content)
 
 # @app.post("/upload")
 # async def upload_document(file: UploadFile = File(...)):
+#     """Upload and process a document"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         if not file.filename:
 #             raise HTTPException(status_code=400, detail="No file selected")
         
+#         # Validate file type
 #         allowed_extensions = ['.pdf', '.docx', '.txt', '.csv']
 #         file_extension = os.path.splitext(file.filename)[1].lower()
         
 #         if file_extension not in allowed_extensions:
-#             raise HTTPException(status_code=400, detail=f"File type {file_extension} not supported. Supported types: {', '.join(allowed_extensions)}")
+#             raise HTTPException(
+#                 status_code=400, 
+#                 detail=f"File type {file_extension} not supported. Supported formats: {', '.join(allowed_extensions)}"
+#             )
+        
+#         # Check file size (limit to 50MB)
+#         file_size = 0
+#         content = await file.read()
+#         file_size = len(content)
+        
+#         if file_size > 50 * 1024 * 1024:  # 50MB limit
+#             raise HTTPException(status_code=400, detail="File size too large. Maximum size is 50MB")
         
 #         # Create temporary file
 #         with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
-#             shutil.copyfileobj(file.file, temp_file)
+#             temp_file.write(content)
 #             temp_path = temp_file.name
         
+#         logger.info(f"Processing document: {file.filename} ({format_file_size(file_size)})")
+        
 #         # Process document
 #         success = rag_system.add_document(temp_path)
         
@@ -97,7 +180,13 @@
 #             logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
         
 #         if success:
-#             return JSONResponse(content={"message": f"Document '{file.filename}' uploaded and processed successfully"})
+#             logger.info(f"Successfully processed document: {file.filename}")
+#             return JSONResponse(content={
+#                 "message": f"Document '{file.filename}' has been successfully added to the Scholar's Archive",
+#                 "filename": file.filename,
+#                 "size": format_file_size(file_size),
+#                 "type": file_extension
+#             })
 #         else:
 #             raise HTTPException(status_code=500, detail="Failed to process document")
             
@@ -107,84 +196,292 @@
 #         logger.error(f"Upload error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
-# @app.post("/ask")
+# @app.post("/ask", response_model=AnswerResponse)
 # async def ask_question(request: QuestionRequest):
+#     """Ask a question about the uploaded documents"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         if not request.question.strip():
 #             raise HTTPException(status_code=400, detail="Question cannot be empty")
         
-#         result = rag_system.ask_question(request.question)
+#         logger.info(f"Processing question: {request.question[:100]}...")
         
-#         return JSONResponse(content={
-#             "answer": result["answer"],
-#             "sources": result["sources"]
-#         })
+#         # Get answer from RAG system
+#         result = rag_system.ask_question(request.question, top_k=request.top_k)
+        
+#         # Handle the chunks data properly
+#         question_chunks = result.get("question_chunks", [])
+#         relevant_chunks = result.get("relevant_chunks", [])
+        
+#         # Add page numbers to sources
+#         sources = result.get("sources", [])
+#         for source in sources:
+#             if isinstance(source, dict):
+#                 page_num = source.get('page')
+#                 if page_num:
+#                     source['page_reference'] = f"Page {page_num}"
+        
+#         # Log the structure to understand what we're getting
+#         logger.info(f"Question chunks type: {type(question_chunks)}")
+#         logger.info(f"Relevant chunks type: {type(relevant_chunks)}")
+#         if question_chunks:
+#             logger.info(f"First question chunk type: {type(question_chunks[0])}")
+#         if relevant_chunks:
+#             logger.info(f"First relevant chunk type: {type(relevant_chunks[0])}")
+        
+#         # Format the response - keep original structure but ensure it's serializable
+#         response = AnswerResponse(
+#             answer=result["answer"],
+#             sources=sources,
+#             question_chunks=question_chunks,
+#             relevant_chunks=relevant_chunks
+#         )
+        
+#         logger.info(f"Successfully answered question with {len(sources)} sources")
+        
+#         return response
         
 #     except HTTPException:
 #         raise
 #     except Exception as e:
-#         logger.error(f"Question error: {e}", exc_info=True)
+#         logger.error(f"Question processing error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
 # @app.get("/documents")
 # async def get_documents():
+#     """Get list of all uploaded documents"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         docs = rag_system.list_documents()
-#         return JSONResponse(content={"documents": docs})
+        
+#         # Format documents with additional metadata
+#         formatted_docs = []
+#         for doc in docs:
+#             formatted_doc = {
+#                 "title": doc.get("title", "Unknown Document"),
+#                 "chunk_count": doc.get("chunk_count", 0),
+#                 "total_pages": doc.get("total_pages"),
+#                 "file_type": os.path.splitext(doc.get("title", ""))[1].lower(),
+#                 "upload_date": doc.get("upload_date", datetime.now().isoformat()),
+#                 "icon": get_file_type_icon(doc.get("title", ""))
+#             }
+#             formatted_docs.append(formatted_doc)
+        
+#         return JSONResponse(content={"documents": formatted_docs})
+        
 #     except HTTPException:
 #         raise
 #     except Exception as e:
 #         logger.error(f"Documents list error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
+# @app.get("/stats", response_model=StatsResponse)
+# async def get_stats():
+#     """Get statistics about the document collection"""
+#     try:
+#         if rag_system is None:
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
+            
+#         docs = rag_system.list_documents()
+        
+#         total_chunks = sum(doc.get("chunk_count", 0) for doc in docs)
+#         total_pages = sum(doc.get("total_pages", 1) for doc in docs if doc.get("total_pages"))
+        
+#         # Format documents
+#         formatted_docs = []
+#         for doc in docs:
+#             formatted_doc = DocumentInfo(
+#                 title=doc.get("title", "Unknown Document"),
+#                 file_type=os.path.splitext(doc.get("title", ""))[1].lower(),
+#                 upload_date=doc.get("upload_date", datetime.now().isoformat()),
+#                 chunk_count=doc.get("chunk_count", 0),
+#                 total_pages=doc.get("total_pages")
+#             )
+#             formatted_docs.append(formatted_doc)
+        
+#         stats = StatsResponse(
+#             total_documents=len(docs),
+#             total_chunks=total_chunks,
+#             total_pages=total_pages,
+#             documents=formatted_docs
+#         )
+        
+#         return stats
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         logger.error(f"Stats error: {e}", exc_info=True)
+#         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
 # @app.delete("/clear")
 # async def clear_documents():
+#     """Clear all documents from the archive"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
+#         logger.info("Clearing all documents from Scholar's Archive")
+        
 #         success = rag_system.clear_index()
 #         if success:
-#             return JSONResponse(content={"message": "All documents cleared successfully"})
+#             logger.info("Successfully cleared all documents")
+#             return JSONResponse(content={
+#                 "message": "All documents have been successfully removed from the Scholar's Archive"
+#             })
 #         else:
 #             raise HTTPException(status_code=500, detail="Failed to clear documents")
+            
 #     except HTTPException:
 #         raise
 #     except Exception as e:
 #         logger.error(f"Clear error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
+# @app.post("/search")
+# async def search_chunks(request: QuestionRequest):
+#     """Search for relevant document chunks"""
+#     try:
+#         if rag_system is None:
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
+            
+#         if not request.question.strip():
+#             raise HTTPException(status_code=400, detail="Search query cannot be empty")
+        
+#         logger.info(f"Searching chunks for: {request.question[:100]}...")
+        
+#         chunks = rag_system.doc_processor.search_chunks(request.question, top_k=request.top_k)
+        
+#         # Format chunks with additional metadata
+#         formatted_chunks = []
+#         for chunk in chunks:
+#             formatted_chunk = {
+#                 "content": chunk.get("content", ""),
+#                 "document": chunk.get("document", "Unknown"),
+#                 "similarity": chunk.get("similarity", 0.0),
+#                 "page": chunk.get("page"),
+#                 "chunk_index": chunk.get("chunk_index")
+#             }
+#             formatted_chunks.append(formatted_chunk)
+        
+#         return JSONResponse(content={
+#             "query": request.question,
+#             "chunks": formatted_chunks,
+#             "total_found": len(formatted_chunks)
+#         })
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         logger.error(f"Search error: {e}", exc_info=True)
+#         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
 # @app.get("/health")
 # async def health_check():
-#     return {
-#         "status": "healthy",
-#         "rag_system_initialized": rag_system is not None,
-#         "message": "RAG PDF QA System is running"
-#     }
-
-# if __name__ == "__main__":
-#     import uvicorn
-#     logger.info("Starting FastAPI server...")
-#     uvicorn.run(app, host="0.0.0.0", port=8000, log_level="debug")
-
-
-
-
+#     """Health check endpoint"""
+#     try:
+#         doc_count = len(rag_system.list_documents()) if rag_system else 0
+        
+#         return {
+#             "status": "healthy",
+#             "service": "Scholar's Archive - Document Intelligence System",
+#             "version": "1.0.0",
+#             "rag_system_initialized": rag_system is not None,
+#             "indexed_documents": doc_count,
+#             "timestamp": datetime.now().isoformat(),
+#             "message": "Scholar's Archive is operational and ready to serve"
+#         }
+#     except Exception as e:
+#         logger.error(f"Health check error: {e}")
+#         return {
+#             "status": "degraded",
+#             "service": "Scholar's Archive - Document Intelligence System",
+#             "error": str(e),
+#             "timestamp": datetime.now().isoformat()
+#         }
 
+# @app.get("/api/info")
+# async def api_info():
+#     """Get API information"""
+#     return {
+#         "name": "Scholar's Archive API",
+#         "description": "Document Intelligence System API",
+#         "version": "1.0.0",
+#         "endpoints": {
+#             "upload": "POST /upload - Upload documents",
+#             "ask": "POST /ask - Ask questions",
+#             "documents": "GET /documents - List documents",
+#             "stats": "GET /stats - Get statistics",
+#             "search": "POST /search - Search chunks",
+#             "clear": "DELETE /clear - Clear all documents",
+#             "health": "GET /health - Health check"
+#         },
+#         "supported_formats": [".pdf", ".docx", ".txt", ".csv"],
+#         "max_file_size": "50MB"
+#     }
 
+# # Event handlers
+# @app.on_event("startup")
+# async def startup_event():
+#     """Application startup event"""
+#     logger.info("Starting Scholar's Archive - Document Intelligence System")
+#     logger.info("System initialized and ready to serve scholarly inquiries")
 
+# @app.on_event("shutdown")
+# async def shutdown_event():
+#     """Application shutdown event"""
+#     if rag_system:
+#         rag_system.close()
+#         logger.info("Scholar's Archive system closed gracefully")
+#     logger.info("Scholar's Archive shutdown complete")
 
+# # Error handlers
+# @app.exception_handler(404)
+# async def not_found_handler(request: Request, exc):
+#     """Custom 404 handler"""
+#     return JSONResponse(
+#         status_code=404,
+#         content={
+#             "detail": "The requested resource was not found in the Scholar's Archive",
+#             "path": str(request.url.path)
+#         }
+#     )
 
+# @app.exception_handler(500)
+# async def internal_error_handler(request: Request, exc):
+#     """Custom 500 handler"""
+#     logger.error(f"Internal server error: {exc}")
+#     return JSONResponse(
+#         status_code=500,
+#         content={
+#             "detail": "An internal error occurred in the Scholar's Archive system",
+#             "message": "Please try again later or contact support"
+#         }
+#     )
 
+# # Main execution
+# if __name__ == "__main__":
+#     import uvicorn
+    
+#     logger.info("Launching Scholar's Archive - Document Intelligence System")
+#     logger.info("Access the interface at: http://localhost:8000")
+#     logger.info("API documentation at: http://localhost:8000/api/docs")
+    
+#     uvicorn.run(
+#         app, 
+#         host="0.0.0.0", 
+#         port=7860, 
+#         log_level="info",
+#         reload=False,
+#         access_log=True
+#     )
 
 
-# # second code
+# complete worked code 
 # from fastapi import FastAPI, File, UploadFile, HTTPException, Request
 # from fastapi.responses import HTMLResponse, JSONResponse
 # from fastapi.staticfiles import StaticFiles
@@ -193,13 +490,14 @@
 # import os
 # import tempfile
 # import shutil
-# from typing import List, Dict, Any
+# from typing import List, Dict, Any, Optional, Union
 # import logging
+# from datetime import datetime
+# import mimetypes
 
+# # Import your RAG system
 # import sys
-# import os
 # sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-
 # try:
 #     from RAG import RAGSystem
 # except ImportError:
@@ -207,10 +505,21 @@
 #     print("Make sure RAG.py is in the same directory as app.py")
 #     sys.exit(1)
 
-# logging.basicConfig(level=logging.INFO)
+# # Configure logging
+# logging.basicConfig(
+#     level=logging.INFO,
+#     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+# )
 # logger = logging.getLogger(__name__)
 
-# app = FastAPI(title="RAG PDF QA System")
+# # Initialize FastAPI app
+# app = FastAPI(
+#     title="Scholar's Archive - Document Intelligence System",
+#     description="A sophisticated platform for intelligent document analysis and question answering using advanced retrieval-augmented generation technology",
+#     version="1.0.0",
+#     docs_url="/api/docs",
+#     redoc_url="/api/redoc"
+# )
 
 # # Setup templates directory
 # templates = Jinja2Templates(directory="templates")
@@ -224,57 +533,126 @@
 # # Initialize RAG System
 # try:
 #     rag_system = RAGSystem()
-#     logger.info("RAG System initialized successfully")
+#     logger.info("Scholar's Archive RAG System initialized successfully")
 # except Exception as e:
 #     logger.error(f"Failed to initialize RAG System: {e}")
 #     rag_system = None
 
+# # Pydantic models
 # class QuestionRequest(BaseModel):
 #     question: str
 #     top_k: int = 3
 
+# class DocumentInfo(BaseModel):
+#     title: str
+#     file_type: str
+#     upload_date: str
+#     chunk_count: int
+#     total_pages: Optional[int] = None
+
+# class AnswerResponse(BaseModel):
+#     answer: str
+#     sources: List[Dict[str, Any]]
+#     question_chunks: List[Union[str, Dict[str, Any]]] = []
+#     relevant_chunks: List[Union[str, Dict[str, Any]]] = []
+
+# class StatsResponse(BaseModel):
+#     total_documents: int
+#     total_chunks: int
+#     total_pages: int
+#     documents: List[DocumentInfo]
+
+# # Utility functions
+# def get_file_type_icon(filename: str) -> str:
+#     """Get appropriate icon for file type"""
+#     ext = os.path.splitext(filename)[1].lower()
+#     icons = {
+#         '.pdf': 'fas fa-file-pdf',
+#         '.docx': 'fas fa-file-word',
+#         '.txt': 'fas fa-file-alt',
+#         '.csv': 'fas fa-file-csv'
+#     }
+#     return icons.get(ext, 'fas fa-file')
+
+# def format_file_size(size_bytes: int) -> str:
+#     """Format file size in human readable format"""
+#     if size_bytes == 0:
+#         return "0 B"
+#     size_names = ["B", "KB", "MB", "GB"]
+#     i = 0
+#     while size_bytes >= 1024 and i < len(size_names) - 1:
+#         size_bytes /= 1024.0
+#         i += 1
+#     return f"{size_bytes:.1f} {size_names[i]}"
+
+# def extract_content_from_chunks(chunks):
+#     """Extract string content from chunk data structures"""
+#     if not chunks:
+#         return []
+    
+#     extracted = []
+#     for chunk in chunks:
+#         if isinstance(chunk, str):
+#             extracted.append(chunk)
+#         elif isinstance(chunk, dict):
+#             # Try different possible keys for text content
+#             content = chunk.get('text') or chunk.get('content') or chunk.get('document') or str(chunk)
+#             extracted.append(content)
+#         else:
+#             extracted.append(str(chunk))
+    
+#     return extracted
+
+
+
+# # Routes
 # @app.get("/", response_class=HTMLResponse)
 # async def read_root(request: Request):
+#     """Serve the main classical interface"""
 #     try:
 #         return templates.TemplateResponse("index.html", {"request": request})
 #     except Exception as e:
 #         logger.error(f"Error serving index.html from templates folder: {e}")
-#         return HTMLResponse(content=f"""
-#             <html>
-#                 <body>
-#                     <h1>RAG PDF QA System</h1>
-#                     <p>Error: Could not load index.html from templates folder</p>
-#                     <p>Error details: {str(e)}</p>
-#                     <p>Make sure you have:</p>
-#                     <ul>
-#                         <li>A 'templates' folder in the same directory as app.py</li>
-#                         <li>index.html file inside the templates folder</li>
-#                         <li>Installed jinja2: pip install jinja2</li>
-#                     </ul>
-#                 </body>
-#             </html>
-#         """)
+#         # Return the embedded HTML if templates folder is not available
+#         with open("scholar_archive.html", "r", encoding="utf-8") as f:
+#             html_content = f.read()
+#         return HTMLResponse(content=html_content)
 
 # @app.post("/upload")
 # async def upload_document(file: UploadFile = File(...)):
+#     """Upload and process a document"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         if not file.filename:
 #             raise HTTPException(status_code=400, detail="No file selected")
         
+#         # Validate file type
 #         allowed_extensions = ['.pdf', '.docx', '.txt', '.csv']
 #         file_extension = os.path.splitext(file.filename)[1].lower()
         
 #         if file_extension not in allowed_extensions:
-#             raise HTTPException(status_code=400, detail=f"File type {file_extension} not supported. Supported types: {', '.join(allowed_extensions)}")
+#             raise HTTPException(
+#                 status_code=400, 
+#                 detail=f"File type {file_extension} not supported. Supported formats: {', '.join(allowed_extensions)}"
+#             )
+        
+#         # Check file size (limit to 50MB)
+#         file_size = 0
+#         content = await file.read()
+#         file_size = len(content)
+        
+#         if file_size > 50 * 1024 * 1024:  # 50MB limit
+#             raise HTTPException(status_code=400, detail="File size too large. Maximum size is 50MB")
         
 #         # Create temporary file
 #         with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file:
-#             shutil.copyfileobj(file.file, temp_file)
+#             temp_file.write(content)
 #             temp_path = temp_file.name
         
+#         logger.info(f"Processing document: {file.filename} ({format_file_size(file_size)})")
+        
 #         # Process document
 #         success = rag_system.add_document(temp_path)
         
@@ -285,7 +663,13 @@
 #             logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
         
 #         if success:
-#             return JSONResponse(content={"message": f"Document '{file.filename}' uploaded and processed successfully"})
+#             logger.info(f"Successfully processed document: {file.filename}")
+#             return JSONResponse(content={
+#                 "message": f"Document '{file.filename}' has been successfully added to the Scholar's Archive",
+#                 "filename": file.filename,
+#                 "size": format_file_size(file_size),
+#                 "type": file_extension
+#             })
 #         else:
 #             raise HTTPException(status_code=500, detail="Failed to process document")
             
@@ -295,55 +679,146 @@
 #         logger.error(f"Upload error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
-# @app.post("/ask")
+# @app.post("/ask", response_model=AnswerResponse)
 # async def ask_question(request: QuestionRequest):
+#     """Ask a question about the uploaded documents"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         if not request.question.strip():
 #             raise HTTPException(status_code=400, detail="Question cannot be empty")
         
+#         logger.info(f"Processing question: {request.question[:100]}...")
+        
+#         # Get answer from RAG system
 #         result = rag_system.ask_question(request.question, top_k=request.top_k)
         
-#         return JSONResponse(content={
-#             "answer": result["answer"],
-#             "sources": result["sources"],
-#             "question_chunks": result.get("question_chunks", []),
-#             "relevant_chunks": result.get("relevant_chunks", [])
-#         })
+#         # Handle the chunks data properly
+#         question_chunks = result.get("question_chunks", [])
+#         relevant_chunks = result.get("relevant_chunks", [])
+        
+#         # Add page numbers to sources
+#         sources = result.get("sources", [])
+#         for source in sources:
+#             if isinstance(source, dict):
+#                 page_num = source.get('page_number') # Changed from 'page' to 'page_number'
+#                 if page_num:
+#                     source['page_reference'] = f"Page {page_num}"
+        
+#         # Log the structure to understand what we're getting
+#         logger.info(f"Question chunks type: {type(question_chunks)}")
+#         logger.info(f"Relevant chunks type: {type(relevant_chunks)}")
+#         if question_chunks:
+#             logger.info(f"First question chunk type: {type(question_chunks[0])}")
+#         if relevant_chunks:
+#             logger.info(f"First relevant chunk type: {type(relevant_chunks[0])}")
+        
+#         # Format the response - keep original structure but ensure it's serializable
+#         response = AnswerResponse(
+#             answer=result["answer"],
+#             sources=sources,
+#             question_chunks=question_chunks,
+#             relevant_chunks=relevant_chunks
+#         )
+        
+#         logger.info(f"Successfully answered question with {len(sources)} sources")
+        
+#         return response
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         logger.error(f"Question processing error: {e}", exc_info=True)
+#         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+# @app.get("/documents")
+# async def get_documents():
+#     """Get list of all uploaded documents"""
+#     try:
+#         if rag_system is None:
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
+            
+#         docs = rag_system.list_documents()
+        
+#         # Format documents with additional metadata
+#         formatted_docs = []
+#         for doc in docs:
+#             formatted_doc = {
+#                 "title": doc.get("title", "Unknown Document"),
+#                 "chunk_count": doc.get("chunk_count", 0),
+#                 "total_pages": doc.get("total_pages"),
+#                 "file_type": os.path.splitext(doc.get("title", ""))[1].lower(),
+#                 "upload_date": doc.get("upload_date", datetime.now().isoformat()),
+#                 "icon": get_file_type_icon(doc.get("title", ""))
+#             }
+#             formatted_docs.append(formatted_doc)
+        
+#         return JSONResponse(content={"documents": formatted_docs})
         
 #     except HTTPException:
 #         raise
 #     except Exception as e:
-#         logger.error(f"Question error: {e}", exc_info=True)
+#         logger.error(f"Documents list error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
-# @app.get("/documents")
-# async def get_documents():
+# @app.get("/stats", response_model=StatsResponse)
+# async def get_stats():
+#     """Get statistics about the document collection"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         docs = rag_system.list_documents()
-#         return JSONResponse(content={"documents": docs})
+        
+#         total_chunks = sum(doc.get("chunk_count", 0) for doc in docs)
+#         total_pages = sum(doc.get("total_pages", 1) for doc in docs if doc.get("total_pages"))
+        
+#         # Format documents
+#         formatted_docs = []
+#         for doc in docs:
+#             formatted_doc = DocumentInfo(
+#                 title=doc.get("title", "Unknown Document"),
+#                 file_type=os.path.splitext(doc.get("title", ""))[1].lower(),
+#                 upload_date=doc.get("upload_date", datetime.now().isoformat()),
+#                 chunk_count=doc.get("chunk_count", 0),
+#                 total_pages=doc.get("total_pages")
+#             )
+#             formatted_docs.append(formatted_doc)
+        
+#         stats = StatsResponse(
+#             total_documents=len(docs),
+#             total_chunks=total_chunks,
+#             total_pages=total_pages,
+#             documents=formatted_docs
+#         )
+        
+#         return stats
+        
 #     except HTTPException:
 #         raise
 #     except Exception as e:
-#         logger.error(f"Documents list error: {e}", exc_info=True)
+#         logger.error(f"Stats error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
 # @app.delete("/clear")
 # async def clear_documents():
+#     """Clear all documents from the archive"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
+#         logger.info("Clearing all documents from Scholar's Archive")
+        
 #         success = rag_system.clear_index()
 #         if success:
-#             return JSONResponse(content={"message": "All documents cleared successfully"})
+#             logger.info("Successfully cleared all documents")
+#             return JSONResponse(content={
+#                 "message": "All documents have been successfully removed from the Scholar's Archive"
+#             })
 #         else:
 #             raise HTTPException(status_code=500, detail="Failed to clear documents")
+            
 #     except HTTPException:
 #         raise
 #     except Exception as e:
@@ -352,19 +827,34 @@
 
 # @app.post("/search")
 # async def search_chunks(request: QuestionRequest):
+#     """Search for relevant document chunks"""
 #     try:
 #         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
             
 #         if not request.question.strip():
 #             raise HTTPException(status_code=400, detail="Search query cannot be empty")
         
+#         logger.info(f"Searching chunks for: {request.question[:100]}...")
+        
 #         chunks = rag_system.doc_processor.search_chunks(request.question, top_k=request.top_k)
         
+#         # Format chunks with additional metadata
+#         formatted_chunks = []
+#         for chunk in chunks:
+#             formatted_chunk = {
+#                 "content": chunk.get("text", ""),
+#                 "document": chunk.get("document", "Unknown"),
+#                 "similarity": chunk.get("similarity", 0.0),
+#                 "page": chunk.get("page_number"), # Changed from 'page' to 'page_number'
+#                 "chunk_index": chunk.get("chunk_index")
+#             }
+#             formatted_chunks.append(formatted_chunk)
+        
 #         return JSONResponse(content={
 #             "query": request.question,
-#             "chunks": chunks,
-#             "total_found": len(chunks)
+#             "chunks": formatted_chunks,
+#             "total_found": len(formatted_chunks)
 #         })
         
 #     except HTTPException:
@@ -375,51 +865,107 @@
 
 # @app.get("/health")
 # async def health_check():
-#     return {
-#         "status": "healthy",
-#         "rag_system_initialized": rag_system is not None,
-#         "message": "RAG PDF QA System is running",
-#         "indexed_documents": len(rag_system.list_documents()) if rag_system else 0
-#     }
-
-# @app.get("/stats")
-# async def get_stats():
+#     """Health check endpoint"""
 #     try:
-#         if rag_system is None:
-#             raise HTTPException(status_code=500, detail="RAG System not initialized")
-            
-#         docs = rag_system.list_documents()
-        
-#         total_chunks = sum(doc.get("chunk_count", 0) for doc in docs)
-#         total_pages = sum(doc.get("total_pages", 1) for doc in docs)
-        
-#         return JSONResponse(content={
-#             "total_documents": len(docs),
-#             "total_chunks": total_chunks,
-#             "total_pages": total_pages,
-#             "documents": docs
-#         })
+#         doc_count = len(rag_system.list_documents()) if rag_system else 0
         
-#     except HTTPException:
-#         raise
+#         return {
+#             "status": "healthy",
+#             "service": "Scholar's Archive - Document Intelligence System",
+#             "version": "1.0.0",
+#             "rag_system_initialized": rag_system is not None,
+#             "indexed_documents": doc_count,
+#             "timestamp": datetime.now().isoformat(),
+#             "message": "Scholar's Archive is operational and ready to serve"
+#         }
 #     except Exception as e:
-#         logger.error(f"Stats error: {e}", exc_info=True)
-#         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+#         logger.error(f"Health check error: {e}")
+#         return {
+#             "status": "degraded",
+#             "service": "Scholar's Archive - Document Intelligence System",
+#             "error": str(e),
+#             "timestamp": datetime.now().isoformat()
+#         }
+
+# @app.get("/api/info")
+# async def api_info():
+#     """Get API information"""
+#     return {
+#         "name": "Scholar's Archive API",
+#         "description": "Document Intelligence System API",
+#         "version": "1.0.0",
+#         "endpoints": {
+#             "upload": "POST /upload - Upload documents",
+#             "ask": "POST /ask - Ask questions",
+#             "documents": "GET /documents - List documents",
+#             "stats": "GET /stats - Get statistics",
+#             "search": "POST /search - Search chunks",
+#             "clear": "DELETE /clear - Clear all documents",
+#             "health": "GET /health - Health check"
+#         },
+#         "supported_formats": [".pdf", ".docx", ".txt", ".csv"],
+#         "max_file_size": "50MB"
+#     }
+
+# # Event handlers
+# @app.on_event("startup")
+# async def startup_event():
+#     """Application startup event"""
+#     logger.info("Starting Scholar's Archive - Document Intelligence System")
+#     logger.info("System initialized and ready to serve scholarly inquiries")
 
 # @app.on_event("shutdown")
 # async def shutdown_event():
+#     """Application shutdown event"""
 #     if rag_system:
 #         rag_system.close()
-#         logger.info("RAG System closed gracefully")
+#         logger.info("Scholar's Archive system closed gracefully")
+#     logger.info("Scholar's Archive shutdown complete")
+
+# # Error handlers
+# @app.exception_handler(404)
+# async def not_found_handler(request: Request, exc):
+#     """Custom 404 handler"""
+#     return JSONResponse(
+#         status_code=404,
+#         content={
+#             "detail": "The requested resource was not found in the Scholar's Archive",
+#             "path": str(request.url.path)
+#         }
+#     )
+
+# @app.exception_handler(500)
+# async def internal_error_handler(request: Request, exc):
+#     """Custom 500 handler"""
+#     logger.error(f"Internal server error: {exc}")
+#     return JSONResponse(
+#         status_code=500,
+#         content={
+#             "detail": "An internal error occurred in the Scholar's Archive system",
+#             "message": "Please try again later or contact support"
+#         }
+#     )
 
+# # Main execution
 # if __name__ == "__main__":
 #     import uvicorn
-#     logger.info("Starting FastAPI server...")
-#     uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info")
+    
+#     logger.info("Launching Scholar's Archive - Document Intelligence System")
+#     logger.info("Access the interface at: http://localhost:8000")
+#     logger.info("API documentation at: http://localhost:8000/api/docs")
+    
+#     uvicorn.run(
+#         app, 
+#         host="0.0.0.0", 
+#         port=7860, 
+#         log_level="info",
+#         reload=False,
+#         access_log=True
+#     )
 
 
 
-# complate code 
+# complatete code
 # from fastapi import FastAPI, File, UploadFile, HTTPException, Request
 # from fastapi.responses import HTMLResponse, JSONResponse
 # from fastapi.staticfiles import StaticFiles
@@ -460,13 +1006,13 @@
 # )
 
 # # Setup templates directory
-# templates = Jinja2Templates(directory="templates")
+# templates = Jinja2Templates(directory="templates") # Changed to 'static' to serve index.html directly
 
 # # Try to mount static files directory
 # try:
-#     app.mount("/static", StaticFiles(directory="static"), name="static")
+#     app.mount("/templates", StaticFiles(directory="templates"), name="templates")
 # except Exception as e:
-#     logger.warning(f"Static files directory not found: {e}")
+#     logger.warning(f"templates files directory not found: {e}")
 
 # # Initialize RAG System
 # try:
@@ -488,12 +1034,19 @@
 #     chunk_count: int
 #     total_pages: Optional[int] = None
 
-# # Fixed AnswerResponse model to handle both strings and dictionaries
+# class ExplanationStep(BaseModel):
+#     title: str
+#     content: str
+#     details: Optional[str] = None
+#     chunks: Optional[List[Dict[str, Any]]] = None # For retrieved chunks info
+#     error: Optional[bool] = False
+
 # class AnswerResponse(BaseModel):
 #     answer: str
 #     sources: List[Dict[str, Any]]
 #     question_chunks: List[Union[str, Dict[str, Any]]] = []
 #     relevant_chunks: List[Union[str, Dict[str, Any]]] = []
+#     explanation: Optional[List[ExplanationStep]] = None # Added for explanation
 
 # class StatsResponse(BaseModel):
 #     total_documents: int
@@ -542,6 +1095,7 @@
     
 #     return extracted
 
+
 # # Routes
 # @app.get("/", response_class=HTMLResponse)
 # async def read_root(request: Request):
@@ -550,10 +1104,8 @@
 #         return templates.TemplateResponse("index.html", {"request": request})
 #     except Exception as e:
 #         logger.error(f"Error serving index.html from templates folder: {e}")
-#         # Return the embedded HTML if templates folder is not available
-#         with open("scholar_archive.html", "r", encoding="utf-8") as f:
-#             html_content = f.read()
-#         return HTMLResponse(content=html_content)
+#         # Fallback if templates directory is not set up correctly
+#         return HTMLResponse(content="<h1>Error: Frontend not found.</h1><p>Please ensure 'index.html' is in the 'templates' directory.</p>")
 
 # @app.post("/upload")
 # async def upload_document(file: UploadFile = File(...)):
@@ -631,27 +1183,24 @@
 #         # Get answer from RAG system
 #         result = rag_system.ask_question(request.question, top_k=request.top_k)
         
-#         # Handle the chunks data properly
-#         question_chunks = result.get("question_chunks", [])
-#         relevant_chunks = result.get("relevant_chunks", [])
-        
-#         # Log the structure to understand what we're getting
-#         logger.info(f"Question chunks type: {type(question_chunks)}")
-#         logger.info(f"Relevant chunks type: {type(relevant_chunks)}")
-#         if question_chunks:
-#             logger.info(f"First question chunk type: {type(question_chunks[0])}")
-#         if relevant_chunks:
-#             logger.info(f"First relevant chunk type: {type(relevant_chunks[0])}")
+#         # Add page numbers to sources
+#         sources = result.get("sources", [])
+#         for source in sources:
+#             if isinstance(source, dict):
+#                 page_num = source.get('page_number')
+#                 if page_num:
+#                     source['page_reference'] = f"Page {page_num}"
         
 #         # Format the response - keep original structure but ensure it's serializable
 #         response = AnswerResponse(
 #             answer=result["answer"],
-#             sources=result["sources"],
-#             question_chunks=question_chunks,
-#             relevant_chunks=relevant_chunks
+#             sources=sources,
+#             question_chunks=result.get("question_chunks", []),
+#             relevant_chunks=result.get("relevant_chunks", []),
+#             explanation=None # No explanation for standard ask
 #         )
         
-#         logger.info(f"Successfully answered question with {len(result['sources'])} sources")
+#         logger.info(f"Successfully answered question with {len(sources)} sources")
         
 #         return response
         
@@ -661,6 +1210,51 @@
 #         logger.error(f"Question processing error: {e}", exc_info=True)
 #         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
+# @app.post("/explain_ask", response_model=AnswerResponse)
+# async def explain_ask_question(request: QuestionRequest):
+#     """Ask a question and get a detailed explanation of the RAG process."""
+#     try:
+#         if rag_system is None:
+#             raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
+            
+#         if not request.question.strip():
+#             raise HTTPException(status_code=400, detail="Question cannot be empty")
+        
+#         logger.info(f"Processing question for explanation: {request.question[:100]}...")
+        
+#         # Get detailed explanation from RAG system
+#         explanation_result = rag_system.explain_retrieval(request.question, top_k=request.top_k)
+        
+#         # Add page numbers to sources
+#         sources = explanation_result.get("sources", [])
+#         for source in sources:
+#             if isinstance(source, dict):
+#                 page_num = source.get('page_number')
+#                 if page_num:
+#                     source['page_reference'] = f"Page {page_num}"
+
+#         # Ensure explanation steps are correctly typed for Pydantic
+#         explanation_steps_typed = [ExplanationStep(**step) for step in explanation_result.get("explanation", [])]
+
+#         response = AnswerResponse(
+#             answer=explanation_result["answer"],
+#             sources=sources,
+#             question_chunks=[], # Not directly used for explanation display in frontend
+#             relevant_chunks=[], # Not directly used for explanation display in frontend
+#             explanation=explanation_steps_typed
+#         )
+        
+#         logger.info(f"Successfully generated explanation for question: {request.question[:50]}")
+        
+#         return response
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         logger.error(f"Explanation processing error: {e}", exc_info=True)
+#         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+
 # @app.get("/documents")
 # async def get_documents():
 #     """Get list of all uploaded documents"""
@@ -772,10 +1366,10 @@
 #         formatted_chunks = []
 #         for chunk in chunks:
 #             formatted_chunk = {
-#                 "content": chunk.get("content", ""),
-#                 "document": chunk.get("document", "Unknown"),
+#                 "content": chunk.get("text", ""),
+#                 "document": chunk.get("doc_title", "Unknown"),
 #                 "similarity": chunk.get("similarity", 0.0),
-#                 "page": chunk.get("page"),
+#                 "page": chunk.get("page_number"),
 #                 "chunk_index": chunk.get("chunk_index")
 #             }
 #             formatted_chunks.append(formatted_chunk)
@@ -826,6 +1420,7 @@
 #         "endpoints": {
 #             "upload": "POST /upload - Upload documents",
 #             "ask": "POST /ask - Ask questions",
+#             "explain_ask": "POST /explain_ask - Ask questions with detailed RAG process explanation",
 #             "documents": "GET /documents - List documents",
 #             "stats": "GET /stats - Get statistics",
 #             "search": "POST /search - Search chunks",
@@ -896,7 +1491,8 @@
 
 
 
-# perfect code
+
+
 from fastapi import FastAPI, File, UploadFile, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
@@ -937,13 +1533,15 @@ app = FastAPI(
 )
 
 # Setup templates directory
-templates = Jinja2Templates(directory="templates")
+# This line is changed to point to the 'templates' directory as per your setup.
+templates = Jinja2Templates(directory="templates") 
 
-# Try to mount static files directory
+# Mount static files directory
+# This remains 'static' for other static assets like CSS, JS, images if they exist in a 'static' folder.
 try:
     app.mount("/static", StaticFiles(directory="static"), name="static")
 except Exception as e:
-    logger.warning(f"Static files directory not found: {e}")
+    logger.warning(f"Static files directory not found or could not be mounted: {e}")
 
 # Initialize RAG System
 try:
@@ -965,11 +1563,19 @@ class DocumentInfo(BaseModel):
     chunk_count: int
     total_pages: Optional[int] = None
 
+class ExplanationStep(BaseModel):
+    title: str
+    content: str
+    details: Optional[str] = None
+    chunks: Optional[List[Dict[str, Any]]] = None # For retrieved chunks info
+    error: Optional[bool] = False
+
 class AnswerResponse(BaseModel):
     answer: str
     sources: List[Dict[str, Any]]
     question_chunks: List[Union[str, Dict[str, Any]]] = []
     relevant_chunks: List[Union[str, Dict[str, Any]]] = []
+    explanation: Optional[List[ExplanationStep]] = None # Added for explanation
 
 class StatsResponse(BaseModel):
     total_documents: int
@@ -1000,38 +1606,17 @@ def format_file_size(size_bytes: int) -> str:
         i += 1
     return f"{size_bytes:.1f} {size_names[i]}"
 
-def extract_content_from_chunks(chunks):
-    """Extract string content from chunk data structures"""
-    if not chunks:
-        return []
-    
-    extracted = []
-    for chunk in chunks:
-        if isinstance(chunk, str):
-            extracted.append(chunk)
-        elif isinstance(chunk, dict):
-            # Try different possible keys for text content
-            content = chunk.get('text') or chunk.get('content') or chunk.get('document') or str(chunk)
-            extracted.append(content)
-        else:
-            extracted.append(str(chunk))
-    
-    return extracted
-
-
-
 # Routes
 @app.get("/", response_class=HTMLResponse)
 async def read_root(request: Request):
     """Serve the main classical interface"""
     try:
+        # This will now correctly look for 'index.html' inside the 'templates' directory
         return templates.TemplateResponse("index.html", {"request": request})
     except Exception as e:
-        logger.error(f"Error serving index.html from templates folder: {e}")
-        # Return the embedded HTML if templates folder is not available
-        with open("scholar_archive.html", "r", encoding="utf-8") as f:
-            html_content = f.read()
-        return HTMLResponse(content=html_content)
+        logger.error(f"Error serving index.html from templates directory: {e}") # Updated log message
+        # Updated fallback message to guide the user correctly
+        return HTMLResponse(content="<h1>Error: Frontend not found.</h1><p>Please ensure 'index.html' is in the 'templates' directory.</p>")
 
 @app.post("/upload")
 async def upload_document(file: UploadFile = File(...)):
@@ -1109,32 +1694,21 @@ async def ask_question(request: QuestionRequest):
         # Get answer from RAG system
         result = rag_system.ask_question(request.question, top_k=request.top_k)
         
-        # Handle the chunks data properly
-        question_chunks = result.get("question_chunks", [])
-        relevant_chunks = result.get("relevant_chunks", [])
-        
         # Add page numbers to sources
         sources = result.get("sources", [])
         for source in sources:
             if isinstance(source, dict):
-                page_num = source.get('page')
+                page_num = source.get('page_number')
                 if page_num:
                     source['page_reference'] = f"Page {page_num}"
         
-        # Log the structure to understand what we're getting
-        logger.info(f"Question chunks type: {type(question_chunks)}")
-        logger.info(f"Relevant chunks type: {type(relevant_chunks)}")
-        if question_chunks:
-            logger.info(f"First question chunk type: {type(question_chunks[0])}")
-        if relevant_chunks:
-            logger.info(f"First relevant chunk type: {type(relevant_chunks[0])}")
-        
         # Format the response - keep original structure but ensure it's serializable
         response = AnswerResponse(
             answer=result["answer"],
             sources=sources,
-            question_chunks=question_chunks,
-            relevant_chunks=relevant_chunks
+            question_chunks=result.get("question_chunks", []),
+            relevant_chunks=result.get("relevant_chunks", []),
+            explanation=None # No explanation for standard ask
         )
         
         logger.info(f"Successfully answered question with {len(sources)} sources")
@@ -1147,6 +1721,51 @@ async def ask_question(request: QuestionRequest):
         logger.error(f"Question processing error: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
+@app.post("/explain_ask", response_model=AnswerResponse)
+async def explain_ask_question(request: QuestionRequest):
+    """Ask a question and get a detailed explanation of the RAG process."""
+    try:
+        if rag_system is None:
+            raise HTTPException(status_code=500, detail="Scholar's Archive system not initialized")
+            
+        if not request.question.strip():
+            raise HTTPException(status_code=400, detail="Question cannot be empty")
+        
+        logger.info(f"Processing question for explanation: {request.question[:100]}...")
+        
+        # Get detailed explanation from RAG system
+        explanation_result = rag_system.explain_retrieval(request.question, top_k=request.top_k)
+        
+        # Add page numbers to sources
+        sources = explanation_result.get("sources", [])
+        for source in sources:
+            if isinstance(source, dict):
+                page_num = source.get('page_number')
+                if page_num:
+                    source['page_reference'] = f"Page {page_num}"
+
+        # Ensure explanation steps are correctly typed for Pydantic
+        explanation_steps_typed = [ExplanationStep(**step) for step in explanation_result.get("explanation", [])]
+
+        response = AnswerResponse(
+            answer=explanation_result["answer"],
+            sources=sources,
+            question_chunks=[], # Not directly used for explanation display in frontend
+            relevant_chunks=[], # Not directly used for explanation display in frontend
+            explanation=explanation_steps_typed
+        )
+        
+        logger.info(f"Successfully generated explanation for question: {request.question[:50]}")
+        
+        return response
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Explanation processing error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+
 @app.get("/documents")
 async def get_documents():
     """Get list of all uploaded documents"""
@@ -1163,7 +1782,7 @@ async def get_documents():
                 "title": doc.get("title", "Unknown Document"),
                 "chunk_count": doc.get("chunk_count", 0),
                 "total_pages": doc.get("total_pages"),
-                "file_type": os.path.splitext(doc.get("title", ""))[1].lower(),
+                "file_type": os.path.splitext(doc.get("title", "") if doc.get("title") else "")[1].lower(), # Handle None title
                 "upload_date": doc.get("upload_date", datetime.now().isoformat()),
                 "icon": get_file_type_icon(doc.get("title", ""))
             }
@@ -1194,7 +1813,7 @@ async def get_stats():
         for doc in docs:
             formatted_doc = DocumentInfo(
                 title=doc.get("title", "Unknown Document"),
-                file_type=os.path.splitext(doc.get("title", ""))[1].lower(),
+                file_type=os.path.splitext(doc.get("title", "") if doc.get("title") else "")[1].lower(), # Handle None title
                 upload_date=doc.get("upload_date", datetime.now().isoformat()),
                 chunk_count=doc.get("chunk_count", 0),
                 total_pages=doc.get("total_pages")
@@ -1258,10 +1877,10 @@ async def search_chunks(request: QuestionRequest):
         formatted_chunks = []
         for chunk in chunks:
             formatted_chunk = {
-                "content": chunk.get("content", ""),
-                "document": chunk.get("document", "Unknown"),
+                "content": chunk.get("text", ""),
+                "document": chunk.get("doc_title", "Unknown"), 
                 "similarity": chunk.get("similarity", 0.0),
-                "page": chunk.get("page"),
+                "page": chunk.get("page_number"),
                 "chunk_index": chunk.get("chunk_index")
             }
             formatted_chunks.append(formatted_chunk)
@@ -1312,6 +1931,7 @@ async def api_info():
         "endpoints": {
             "upload": "POST /upload - Upload documents",
             "ask": "POST /ask - Ask questions",
+            "explain_ask": "POST /explain_ask - Ask questions with detailed RAG process explanation",
             "documents": "GET /documents - List documents",
             "stats": "GET /stats - Get statistics",
             "search": "POST /search - Search chunks",
@@ -1376,4 +1996,7 @@ if __name__ == "__main__":
         log_level="info",
         reload=False,
         access_log=True
-    )
+    )
+
+
+

rag_storage/metadata.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea47f16e743fe3b0236ad60c5bf23262ac2a654bf3471d0ef3da50af2813bd3f
-size 53947
+oid sha256:914888c9de5250152457f0bd7c4c18253a6769a99c5ce8e275ee05ad32e6b448
+size 44