Spaces:

kkthyagharajan
/

Secured-Document-Retrieval-System-SDRS

Sleeping

App Files Files Community

kkthyagharajan commited on Apr 18

Commit

46f8a04

verified ·

1 Parent(s): a776e70

Upload 22 files

Browse files

Files changed (23) hide show

.gitattributes +4 -0
Dockerfile +21 -0
FastAPI_Client/admin.html +52 -0
FastAPI_Client/admin.js +157 -0
FastAPI_Client/index.html +88 -0
FastAPI_Client/script.js +383 -0
FastAPI_Client/style.css +145 -0
READMEmodify.md +172 -0
app.py +1019 -0
config.py +25 -0
doc_ingestion/AIML_Unit1_RMD_ECE.pdf +3 -0
doc_ingestion/AIML_Unit2_RMD_ECE.pdf +3 -0
faiss.index +3 -0
kkt_SQLite_DB.db +3 -0
rag/__init__.py +7 -0
rag/chunker.py +160 -0
rag/qdrant_retriever.py +10 -0
requirements.txt +20 -0
utils/__init__.py +7 -0
utils/admin_fns.py +142 -0
utils/core_imports.py +39 -0
utils/text_cleanerV1.py +37 -0
utils/text_cleanerV2.py +97 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+doc_ingestion/AIML_Unit1_RMD_ECE.pdf filter=lfs diff=lfs merge=lfs -text
+doc_ingestion/AIML_Unit2_RMD_ECE.pdf filter=lfs diff=lfs merge=lfs -text
+faiss.index filter=lfs diff=lfs merge=lfs -text
+kkt_SQLite_DB.db filter=lfs diff=lfs merge=lfs -text

Dockerfile ADDED Viewed

	@@ -0,0 +1,21 @@

+FROM python:3.11-slim
+WORKDIR /app
+# Install system dependencies (OCR support)
+RUN apt-get update && apt-get install -y \
+    tesseract-ocr \
+    libgl1 \
+    && rm -rf /var/lib/apt/lists/*
+# Copy all files
+COPY . .
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+# Expose Hugging Face port
+EXPOSE 7860
+# Run FastAPI
+CMD ["uvicorn", "kkt_FastAPI_serverV22:app", "--host", "0.0.0.0", "--port", "7860"]

FastAPI_Client/admin.html ADDED Viewed

	@@ -0,0 +1,52 @@

+<!DOCTYPE html>
+<html>
+<head>
+<title>KKT RAG Admin</title>
+<style>
+body{font-family:Arial;padding:30px}
+table{border-collapse:collapse;width:70%}
+th,td{border:1px solid #ccc;padding:8px}
+button{padding:5px 10px;margin:4px}
+</style>
+</head>
+<body>
+<h2>KKT Secure Modular RAG Engine — Admin</h2>
+<!-- LOGIN FOR ADMIN -->
+<div id="authSection">
+    <h3>Admin Login</h3>
+    <input type="text" id="adminUser" placeholder="Username">
+    <input type="password" id="adminPass" placeholder="Password">
+    <button onclick="loginAdmin()">Login</button>
+</div>
+<!-- ADMIN PANEL -->
+<div id="adminPanel" style="display:none;">
+<h3>Upload and Index Document</h3>
+<input type="file" id="fileInput">
+<button onclick="uploadDocument()">Upload & Index</button>
+<br><br>
+<button onclick="loadDocs()">Refresh Documents</button>
+<button onclick="deleteFolder()">Delete Knowledge</button>
+<button onclick="resetIndex()">Reset Index</button>
+<br><br>
+<table id="docTable">
+<tr>
+<th>Document</th>
+<th>Chunks</th>
+<th>Action</th>
+</tr>
+</table>
+</div>
+<script src="/static/admin.js?v=1"></script>
+</body>
+</html>

FastAPI_Client/admin.js ADDED Viewed

	@@ -0,0 +1,157 @@

+let token = "";
+// ------------------ ADMIN LOGIN ------------------
+async function loginAdmin() {
+    const username = document.getElementById("adminUser").value;
+    const password = document.getElementById("adminPass").value;
+    if (!username || !password) {
+        alert("Please enter username and password");
+        return;
+    }
+    const formData = new URLSearchParams();
+    formData.append("username", username);
+    formData.append("password", password);
+    try {
+        const response = await fetch("/login", {
+            method: "POST",
+            headers: {"Content-Type":"application/x-www-form-urlencoded"},
+            body: formData
+        });
+        const data = await response.json();
+        if (response.ok) {
+            token = data.access_token;
+            localStorage.setItem("token", token);
+            alert("Login Successful");
+            document.getElementById("authSection").style.display = "none";
+            document.getElementById("adminPanel").style.display = "block";
+            loadDocs();
+        } else {
+            alert(data.detail || "Login Failed");
+        }
+    } catch (err) {
+        console.error(err);
+        alert("Login request failed");
+    }
+}
+// ------------------ UPLOAD DOCUMENT ------------------
+async function uploadDocument() {
+    const fileInput = document.getElementById("fileInput");
+    const file = fileInput.files[0];
+    if (!file) {
+        alert("Please select a file");
+        return;
+    }
+    alert("File is being uploaded and indexed. Please wait ...");
+    const formData = new FormData();
+    formData.append("file", file);
+    try {
+        const res = await fetch("/admin/upload-document", {
+            headers: {"Authorization": "Bearer " + token},
+            method: "POST",
+            body: formData
+        });
+        const data = await res.json();
+        if (data.message) {
+            alert(data.message);
+        } else if (data.error) {
+            alert(data.error);
+        }
+        loadDocs();
+    } catch (err) {
+        console.error(err);
+        alert("Upload failed");
+    }
+}
+// ------------------ OTHER EXISTING FUNCTIONS ------------------
+async function loadDocs() {
+    try {
+        const response = await fetch("/admin/list-documents", {
+            headers: {"Authorization": "Bearer " + token}
+        });
+        if (!response.ok) throw new Error("Failed to fetch documents");
+        const data = await response.json();
+        const table = document.getElementById("docTable");
+        if (!table) return;
+        let rows = `<tr><th>Document</th><th>Chunks</th><th>Action</th></tr>`;
+        if (data.documents.length === 0) {
+            rows += `<tr><td colspan="3">No documents found</td></tr>`;
+        } else {
+            data.documents.forEach(doc => {
+                rows += `<tr>
+                    <td><a href="/uploads/${encodeURIComponent(doc.document)}" target="_blank">${doc.document}</a></td>
+                    <td>${doc.chunks}</td>
+                    <td><button onclick='deleteDocument(${JSON.stringify(doc.document)}, event)'>Delete</button></td>
+                </tr>`;
+            });
+        }
+        table.innerHTML = rows;
+    } catch (err) {
+        console.error(err);
+    }
+}
+async function deleteDocument(name, event) {
+    if (!confirm("Delete " + name + " ?")) return;
+    try {
+        const btn = event.target;
+        btn.disabled = true;
+        btn.innerText = "Deleting...";
+        const params = new URLSearchParams({ filename: name });
+        const res = await fetch(`/admin/delete-document?${params.toString()}`, {
+            headers: {"Authorization": "Bearer " + token},
+            method: "DELETE"
+        });
+        const data = await res.json();
+        alert(data.message || "Deleted successfully");
+        await loadDocs();
+    } catch (err) {
+        console.error(err);
+        alert("Error deleting file");
+    }
+}
+async function deleteFolder() {
+    const folder = prompt("Enter folder name to delete");
+    if (!folder) return;
+    await fetch(`/admin/delete-folder?folder=${encodeURIComponent(folder)}`, {
+        method: "DELETE",
+        headers: {"Authorization": "Bearer " + token}
+    });
+    loadDocs();
+}
+async function resetIndex() {
+    if (!confirm("Reset entire index?")) return;
+    await fetch("/admin/reset-index?confirm=true", {
+        headers: {"Authorization": "Bearer " + token},
+        method: "DELETE"
+    });
+    loadDocs();
+}

FastAPI_Client/index.html ADDED Viewed

	@@ -0,0 +1,88 @@

+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>KKT Secure Modular RAG Engine</title>
+<link rel="stylesheet" href="/static/style.css">
+</head>
+<body>
+<h1>KKT Secure Modular RAG Engine</h1>
+<!-- LOGIN / REGISTER -->
+<div id="authSection">
+<h2>Register</h2>
+<input type="text" id="regUser" placeholder="Username">
+<input type="password" id="regPass" placeholder="Password">
+<button onclick="register()">Register</button>
+<h2>Login</h2>
+<input type="text" id="loginUser" placeholder="Username">
+<input type="password" id="loginPass" placeholder="Password">
+<button onclick="login()">Login</button>
+</div>
+<!-- MAIN APP -->
+<div id="mainApp" style="display:none;">
+<h2>Select Model</h2>
+<div class="model-row">
+    <select id="modelSelect"></select>
+    <button onclick="loadModels()">Refresh Models</button>
+</div>
+<!-- NEW: Model change confirmation message -->
+<div id="modelInfo" style="margin-top:8px;color:green;font-weight:bold;"></div>
+<div class="ref-section">
+    <label>
+        <input type="checkbox" id="refInline">
+        <span>Inline Citation</span>
+    </label>
+    <label>
+        <input type="checkbox" id="refList">
+        <span>Bibliography</span>
+    </label>
+</div>
+<h2>Chat</h2>
+<div id="chatWindow"></div>
+<div id="chatControls">
+<input type="text" id="questionInput" placeholder="Ask a question">
+<button onclick="sendQuestion()">Send</button>
+<button onclick="startVoice()">🎤 Voice</button>
+<button onclick="readChat()">🔈Read</button>
+<button onclick="stopVoice()">Stop</button>
+<button onclick="saveChatAsPDF()">📄 Save as PDF</button>
+</div>
+</div>
+<script src="/static/script.js"></script>
+</body>
+</html>

FastAPI_Client/script.js ADDED Viewed

	@@ -0,0 +1,383 @@

+let token = "";
+let recognition;
+let previousModel = "";
+/* ------------------ REGISTER ------------------ */
+async function register()
+{
+const username = document.getElementById("regUser").value;
+const password = document.getElementById("regPass").value;
+const response = await fetch("/register",
+{
+method: "POST",
+headers:
+{
+"Content-Type":"application/json"
+},
+body: JSON.stringify(
+{
+username:username,
+password:password
+})
+});
+const data = await response.json();
+if(response.ok)
+{
+alert("Successfully Registered");
+}
+else
+{
+alert(data.detail || "Registration failed");
+}
+}
+/* ------------------ LOGIN ------------------ */
+async function login()
+{
+const username = document.getElementById("loginUser").value;
+const password = document.getElementById("loginPass").value;
+const formData = new URLSearchParams();
+formData.append("username",username);
+formData.append("password",password);
+const response = await fetch("/login",
+{
+method:"POST",
+headers:
+{
+"Content-Type":"application/x-www-form-urlencoded"
+},
+body:formData
+});
+const data = await response.json();
+if(response.ok)
+{
+token = data.access_token;
+localStorage.setItem("token", token);
+alert("Login Successful");
+document.getElementById("authSection").style.display="none";
+document.getElementById("mainApp").style.display="block";
+loadModels();
+}
+else
+{
+alert("Login Failed");
+}
+}
+/* ------------------ LOAD MODELS ------------------ */
+async function loadModels()
+{
+const select = document.getElementById("modelSelect");
+// 🧠 Refresh warning message
+const modelInfo = document.getElementById("modelInfo");
+if (modelInfo) {
+    modelInfo.innerText =
+        "🤖 If you click Refresh Models button, Default LLM model will be selected.";
+    modelInfo.style.color = "blue";
+}
+const response = await fetch("/v1/models");
+const data = await response.json();
+select.innerHTML="";
+data.data.forEach(model =>
+{
+let option = document.createElement("option");
+option.value = model.id;
+option.text = model.id;
+select.appendChild(option);
+});
+// ✅ SAFE default selection AFTER population
+if (select.options.length > 0) {
+    select.selectedIndex = 0;
+    previousModel = select.value;
+}
+// model change handler
+select.onchange = function () {
+    const modelInfo = document.getElementById("modelInfo");
+    if (!modelInfo) return;
+    const message =
+        "You switched to model " + select.value +
+        ". This model will be used for all new questions.";
+    modelInfo.innerText =
+        "🤖 You switched to: " + select.value +
+        ". This model will be used for ALL new questions.";
+    modelInfo.style.color = "green";
+    // 🔊 SPEAK MODEL CHANGE
+    speak(message);
+};
+}
+/* ------------------ FILE UPLOAD ------------------ */
+async function uploadFile()
+{
+const file = document.getElementById("fileInput").files[0];
+const formData = new FormData();
+formData.append("file",file);
+const response = await fetch("/upload",
+{
+method:"POST",
+headers:
+{
+Authorization:"Bearer "+token
+},
+body:formData
+});
+const data = await response.json();
+alert(data.message);  // shows only the message string
+}
+/* ------------------ CHAT ------------------ */
+async function sendQuestion()
+{
+const inputBox = document.getElementById("questionInput");
+const question = inputBox.value;
+if(!question.trim()) return;
+addUserMessage(question);
+inputBox.value = "";   // clears the input box
+const model = document.getElementById("modelSelect").value;
+const inline = document.getElementById("refInline").checked;
+const list = document.getElementById("refList").checked;
+let refStyle = "both";
+if (inline && !list) refStyle = "inline";
+else if (!inline && list) refStyle = "list";
+else if (!inline && !list) refStyle = "none";
+const response = await fetch("/v1/chat/completions",
+{
+method:"POST",
+headers:
+{
+"Content-Type":"application/json",
+"Authorization":"Bearer " + token
+},
+body:JSON.stringify(
+{
+model:model,
+reference_style: refStyle,
+messages:[
+{role:"user",content:question}
+]
+})
+});
+const data = await response.json();
+const answer = data.choices[0].message.content;
+//Clean unwanted model artifacts
+let displayAnswer = answer
+    .replace(/\[\d+\]/g, "")                         // remove [1], [2]
+    .replace(/Note: Citation marker.*$/s, "");
+addBotMessage(displayAnswer);
+let cleanAnswer = answer
+    .replace(/<[^>]*>/g, "")               //
+    .replace(/\([^)]*\.pdf[^)]*\)/gi, "")
+    .replace(/\[\d+\]/g, "")
+    .replace(/Note: Citation marker.*$/s, "")
+    .replace(/\s+/g, " ")                 //
+    .trim();
+cleanAnswer = cleanAnswer.split("References:")[0];
+speak(cleanAnswer);
+}
+document.addEventListener("DOMContentLoaded", function() {
+    document.getElementById("questionInput").addEventListener("keydown", function(event) {
+        if (event.key === "Enter") {
+            event.preventDefault();
+            sendQuestion();
+        }
+    });
+});
+/* ------------------ CHAT DISPLAY ------------------ */
+function addUserMessage(text)
+{
+const chat = document.getElementById("chatWindow");
+const div = document.createElement("div");
+div.className="userMessage";
+div.innerText=text;
+chat.appendChild(div);
+}
+function addBotMessage(text)
+{
+const chat = document.getElementById("chatWindow");
+const div = document.createElement("div");
+div.className="botMessage";
+div.innerHTML = text;
+chat.appendChild(div);
+}
+/* ------------------ VOICE INPUT ------------------ */
+function startVoice()
+{
+recognition = new webkitSpeechRecognition();
+recognition.lang="en-US";
+recognition.onresult=function(event)
+{
+document.getElementById("questionInput").value = event.results[0][0].transcript;
+};
+recognition.start();
+}
+function stopVoice()
+{
+if(recognition)
+{
+recognition.stop();
+}
+speechSynthesis.cancel();
+}
+/* ------------------ VOICE OUTPUT ------------------ */
+function speak(text)
+{
+//STOP any previous speech to avoid vrvrvr noise
+speechSynthesis.cancel();
+const speech = new SpeechSynthesisUtterance(text);
+speech.lang="en-US";
+speechSynthesis.speak(speech);
+}
+function readChat()
+{
+const messages = document.getElementsByClassName("botMessage");
+if(messages.length === 0) return;
+const lastMessage = messages[messages.length - 1];
+const chatText = lastMessage.innerText;
+//Remove inline references from speaking
+let cleanText = chatText
+    .replace(/\([^)]*\.pdf[^)]*\)/gi, "")   // remove full inline refs
+    .replace(/\[\d+\]/g, "");               // remove [1], [2]
+//Remove References section from speaking
+cleanText = cleanText.split("References:")[0];
+speak(cleanText);
+}
+function saveChatAsPDF()
+{
+    const chatWindow = document.getElementById("chatWindow");
+    const originalContent = chatWindow.innerHTML;
+    const printWindow = window.open('', '', 'width=800,height=600');
+    printWindow.document.write(`
+        <html>
+        <head>
+            <title>Chat Export</title>
+            <style>
+                body { font-family: Arial; padding: 20px; }
+                .userMessage { color: blue; margin: 5px 0; }
+                .botMessage { color: black; margin: 5px 0; }
+            </style>
+        </head>
+        <body>
+            <h2>Chat History</h2>
+            ${originalContent}
+        </body>
+        </html>
+    `);
+    printWindow.document.close();
+    printWindow.print();
+}

FastAPI_Client/style.css ADDED Viewed

	@@ -0,0 +1,145 @@

+body
+{
+font-family: Arial;
+background: #f4f4f4;
+margin: 40px;
+}
+h1
+{
+text-align: center;
+}
+#authSection
+{
+background: white;
+padding: 20px;
+width: 400px;
+margin: auto;
+border-radius: 8px;
+}
+#mainApp
+{
+margin-top: 30px;
+}
+input
+{
+width: 100%;
+padding: 10px;
+margin-top: 10px;
+font-size: 18px;   /* 🔥 increase input text size */
+}
+button
+{
+padding: 10px;
+margin-top: 10px;
+cursor: pointer;
+}
+#chatWindow
+{
+height: 400px;
+background: white;
+overflow-y: auto;
+padding: 10px;
+border-radius: 8px;
+}
+.userMessage
+{
+text-align:left;
+color:blue;
+margin:5px;
+font-family:Calibri;
+font-size:28px;
+}
+.botMessage
+{
+text-align: left;
+color:black;
+margin:5px;
+font-family:Arial;
+font-size:20px;
+}
+#chatControls
+{
+margin-top: 10px;
+display: flex;
+gap: 10px;
+}
+#chatControls
+{
+margin-top: 10px;
+display: flex;
+gap: 10px;
+}
+/* ===== Reference checkbox alignment fix ===== */
+.ref-section
+{
+    display: flex;
+    align-items: center;
+    gap: 20px;
+    margin-top: 25px;   /* 🔥 move downward */
+    flex-wrap: nowrap;   /* 🔥 force single row */
+}
+.ref-section label
+{
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    white-space: nowrap;  /* ===== Prevent label text from wrapping ===== */
+}
+/* ===== Fine-tune checkbox vertical position ===== */
+.ref-section input[type="checkbox"]
+{
+    margin: 0;
+    transform: translateY(-1px);   /* 🔥 move checkbox UP */
+}
+.model-row
+{
+    display: flex;
+    gap: 10px;
+    align-items: center;   /* 🔥 fix alignment */
+    margin-top: -15px;
+}
+.model-row select
+{
+    width: 250px;           /* 🔥 fixed reasonable width */
+    height: 36px;        /* 🔥 controls actual height */
+    font-size: 16px;
+}
+.model-row button
+{
+    height: 36px;
+    padding: 0 15px;
+    white-space: nowrap;
+    display: flex;              /* 🔥 important */
+    align-items: center;        /* 🔥 vertical centering */
+}
+/* ===== Fine-tune dropdown vertical position ===== */
+#modelSelect
+{
+    transform: translateY(2px);   /* 🔥 move DOWN */
+}
+/* ===== Slightly bigger checkbox ===== */
+.ref-section input[type="checkbox"]
+{
+    transform: scale(1.5);   /* 🔥 increase size */
+    margin: 0;
+}

READMEmodify.md ADDED Viewed

	@@ -0,0 +1,172 @@

+# 🚀 FastAPI RAG Server (V22)
+A production-ready Retrieval-Augmented Generation (RAG) API built using FastAPI. This project enables document ingestion, semantic search using embeddings, and LLM-based question answering.
+---
+## 📌 Features
+- 📄 PDF document ingestion and chunking
+- 🧹 Text cleaning pipeline
+- 🔎 Semantic search using FAISS
+- 🤖 LLM integration (Groq / OpenAI compatible API)
+- 🔐 Authentication with hashed passwords (Passlib + JWT)
+- ⚡ FastAPI async endpoints
+---
+## 🏗️ Project Structure
+```
+.
+├── kkt_FastAPI_serverV22.py   # Main FastAPI server
+├── chunker.py                # Document loading & chunking
+├── text_cleanerV2.py         # Text preprocessing pipeline
+├── indexer.py                # Embedding + FAISS index
+├── requirements.txt          # Dependencies
+└── README.md
+```
+---
+## ⚙️ Installation
+### 1. Clone the repository
+```bash
+git clone <your-repo-url>
+cd <repo-folder>
+```
+### 2. Create environment (recommended)
+```bash
+conda create -n rag_env python=3.11
+conda activate rag_env
+```
+### 3. Install dependencies
+```bash
+pip install -r requirements.txt
+```
+### Required dependencies
+Ensure the following are present:
+```
+fastapi
+uvicorn
+passlib[bcrypt]
+python-jose[cryptography]
+sentence-transformers
+faiss-cpu
+httpx
+```
+---
+## 🔑 Environment Variables
+Set your API keys before running:
+```bash
+export GROQ_API_KEY="your_api_key_here"
+```
+On Windows PowerShell:
+```powershell
+$env:GROQ_API_KEY="your_api_key_here"
+```
+---
+## ▶️ Running the Server
+```bash
+uvicorn kkt_FastAPI_serverV22:app --host 0.0.0.0 --port 8000
+```
+Access API docs:
+👉 http://localhost:8000/docs
+---
+## 🔄 Workflow
+1. Upload documents (PDF)
+2. Chunk and clean text
+3. Generate embeddings using SentenceTransformers
+4. Store vectors in FAISS
+5. Query → retrieve relevant chunks
+6. Send context to LLM → generate answer
+---
+## 🔐 Authentication
+- Password hashing: Passlib (bcrypt)
+- Token system: JWT (python-jose)
+Example flow:
+1. Register user
+2. Login → receive token
+3. Use token in protected endpoints
+---
+## 🐳 Docker (Optional)
+### Build image
+```bash
+docker build -t rag-fastapi .
+```
+### Run container
+```bash
+docker run -p 8000:8000 rag-fastapi
+```
+---
+## ⚠️ Common Issues
+### ❌ ModuleNotFoundError: passlib
+Fix:
+```bash
+pip install passlib[bcrypt]
+```
+### ❌ API timeout issues
+- Increase timeout in `httpx.AsyncClient`
+- Check API key validity
+---
+## 📈 Future Improvements
+- Streaming responses
+- Multi-document indexing
+- Role-based authentication
+- UI integration (React/Streamlit)
+---
+## 🧠 Tech Stack
+- FastAPI
+- FAISS
+- SentenceTransformers
+- Passlib
+- JWT (python-jose)
+- httpx
+---
+## 📄 License
+MIT License
+---
+## 👨‍💻 Author
+Developed by Thyagharajan K K

app.py ADDED Viewed

	@@ -0,0 +1,1019 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Fri Feb 20 13:39:23 2026
+@author: THYAGHARAJAN
+"""
+# In[]
+#NOTE
+#if you change the file name kkt_FastAPI_serverV21 change the same in core_imports.py file
+#All uploaded files are saved in the doc_ingestion folder where the server file exixsts
+# In[]
+"""
+orchestrator layer
+(torch26_cu124_trans_unsloth) D: cd kkt_secure_modular_rag_engine
+uvicorn kkt_FastAPI_server:app --host 0.0.0.0 --port 8000
+on Browser: http://localhost:3009
+V1 works well when selected llama3:8b
+V2 works with WebUI, LLM Model selecttion, to add pdf files, system prompt, RAG prompt at WebUI are intergrated
+but Halusination could not be controlled and some layers could not communicate with WebUI
+Don't delete V2'
+V3 server uses separate FastAPI Swagard client. Working
+V4 HTML client is used. Ollama output is not obtained
+V5 LLM provides output without references.
+V6 References included
+V7 References and Citations improved
+V8 User interface was improved and tested
+V9 doc_ingestion folder and its functionality was changed for opening the reference files on the browser.
+   But chat window didn't show any link
+V10 Inline reference was modified. working
+V11 facility needed to delete the knowledge were added but not tested completely
+V12 same upload_file used for admin. Single ingestion system. does chunking,embedding,FAISS update,DB storage. Many modifications done.
+V13 Changes made but not checked
+V14 checked working well
+V15 Reference modes are working well. Still at the end it reads vr vr vr
+V16 speaking vr vr vr at the end was removed in the script.js and working fine.
+Bibiliography refrences may be different from inline citation, because all bibiliography are not used by LLM. Checked working
+V17 References are properly displayed on all the three modes. Checked OK
+V18 File upload UI for user (in index.html) was removed.
+V19 UI was corrected. Refrences were corrected. working fine.
+V20 Save as PDF button was added in the index and script files
+V21 if you change the file name kkt_FastAPI_serverV21 change the same in core_imports.py file
+V22 Uses Groq, API key and cloud for deploying on Hugging Face
+"""
+# In[]
+import os
+import re
+import httpx
+import threading
+from fastapi import FastAPI, HTTPException, UploadFile, File
+#FastAPI application serves as a server for webui and client for local Ollama. This is to be done asynchronouly
+#So, the app acts as a proxy to Ollama
+from pydantic import BaseModel
+from typing import List, Optional
+from datetime import datetime
+import sqlite3
+from passlib.context import CryptContext
+from jose import JWTError, jwt
+from fastapi import Depends
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+import numpy as np
+import faiss
+from sentence_transformers import SentenceTransformer
+from rag.chunker import DocChunker   #import class
+#from rag.indexer import build_vector_index   #imports function
+import requests
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+import shutil
+from pathlib import Path
+from utils.text_cleanerV2 import clean_text
+from utils.admin_fns import router as admin_router
+#from rag.retriever_factory import get_retriever
+'''
+from rag.citation_validator import validate_citations
+from rag.hallucination_control import apply_confidence_filter
+from rag.permission_gate import check_external_access
+'''
+from models.model_config import VECTOR_BACKEND, INDEX_PATH
+# JWT Configuration
+SECRET_KEY = "KKT_SUPER_SECRET_KEY_CHANGE_THIS"
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 60
+#UPLOAD_FOLDER = "uploads"
+EMBEDDING_MODEL = SentenceTransformer("all-MiniLM-L6-v2")
+#EMBEDDING_MODEL →  encodes text to vector
+VECTOR_INDEX = None
+#VECTOR_INDEX → FAISS index storing vectors
+INDEX_READY = False  #Used if query is given before the vector rebuilding is not completed after the app starts
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="login")
+# In[]
+'''
+# =====================================
+# 🔐 Allowed Models (Ollama)
+# =====================================
+ALLOWED_MODELS = [
+    "llama3:8b",
+    "llama3.1:8b",
+    "phi3:mini",
+    "Phi-3 Medium",
+    "mistral",
+    "qwen2.5:7b",
+    "deepseek-r1:7b",
+    "llama3.2:3b"
+]
+#
+# Default model (if WebUI sends None)
+DEFAULT_MODEL = "deepseek-r1:7b"
+'''
+# =====================================
+# 🔐 Allowed Models (Groq)
+# =====================================
+ALLOWED_MODELS = [
+    "llama-3.1-8b-instant",
+    "llama-3.3-70b-versatile",
+    "mixtral-8x7b-32768",
+    "gemma-7b-it"
+]
+DEFAULT_MODEL = "llama-3.1-8b-instant"
+#GROQ_API_KEY -- Environment variable was set using setx GROQ_API_KEY "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"
+#in Windows Powershell
+#Ollama endpoint (since FastAPI runs on Windows host)
+#OLLAMA_URL = "http://localhost:11434/api/chat"      #Hugging Face Spaces does NOT support Ollama Delete this not needed
+# In[] SERVER-SIDE DOCUMNET REPOSITORY (FOR REFERENCE)
+from config import UPLOAD_FOLDER, DB_PATH_FILE, FAISS_INDEX_PATH
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+FAISS_LOCK = threading.Lock() #Ensure safe concurrent access. Prevent index corruption.
+# In[] # FASTAPI SERVER
+app = FastAPI(
+    title="KKT Secure Modular RAG Engine V11",
+    version="11.0.0",
+    description="Secure modular RAG pipeline with authentication, chunking, FAISS retrieval"
+)
+#returns an ASGI (Asynchronous Server Gateway Interface) compatibe app instance
+#This app can be run by uvicorn server
+#Title will be shown in the doc page of http://localhost:8000/docs
+#app = FastAPI() #without metadata
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+#configures CORS (Cross-Origin Resource Sharing) of the app
+# UI files. Maps URL prefix /static → local folder FastAPI_Client
+app.mount("/static", StaticFiles(directory="FastAPI_Client"), name="static")
+#URL: /static/app.js will be mapped to the File: FastAPI_Client/app.js
+# uploaded documents
+app.mount("/uploads", StaticFiles(directory=UPLOAD_FOLDER), name="uploads")   #uploaded files will be saved in doc_ingestion
+app.include_router(admin_router) #registeres all admin endpoints with FastAPI
+#UPLOAD_FOLDER = /kkt_secure_modular_rag_engine/doc_ingestion
+#URL: /uploads/file1.pdf
+#File: /kkt_secure_modular_rag_engine/doc_ingestion/file1.pdf
+#uploaded documents are saved in the doc_ingestion folder but , when they are opened in the browser with folder name defined here
+#It will not expose the actual folder in the server   http://localhost:8000/uploads/AI_book.pdf
+# In[] Admin Credentials
+# Fixed admin user
+ADMIN_USERNAME = "admin"
+ADMIN_PASSWORD = "Tnivedha@123"  # plain password (for testing)
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+ADMIN_PASSWORD_HASH = pwd_context.hash(ADMIN_PASSWORD)
+def authenticate_admin(username: str, password: str):
+    if username == ADMIN_USERNAME and verify_password(password, ADMIN_PASSWORD_HASH):
+        return {"username": username}  # returns user info for token
+    return None
+# In[]
+# =====================================
+# 🔐 Current User Dependency
+# =====================================
+def get_current_user(token: str = Depends(oauth2_scheme)):
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        username: str = payload.get("sub")
+        if username is None:
+            raise HTTPException(status_code=401, detail="Invalid token")
+        return username
+    except JWTError:
+        raise HTTPException(status_code=401, detail="Invalid token")
+# In[] Data Base Tables
+#Create SQLite connectection to the database file. Used in many places to connect to database
+def get_db():
+    return sqlite3.connect(DB_PATH_FILE, check_same_thread=False)
+# =====================================
+# 🗄 SQLite Database Setup
+# =====================================
+def init_db():
+    conn = get_db() #connect DB file
+    cursor = conn.cursor()
+    # Users table (existing)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS users (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            username TEXT UNIQUE NOT NULL,
+            hashed_password TEXT NOT NULL,
+            created_at TEXT NOT NULL
+        )
+    """)
+    # NEW: Chunk metadata table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS document_chunks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            faiss_id INTEGER,
+            source TEXT NOT NULL,
+            path TEXT,
+            page INTEGER NOT NULL,
+            text TEXT NOT NULL,
+            created_at TEXT NOT NULL
+        )
+    """)
+    conn.commit()
+    conn.close()
+init_db()     #Database is initialized
+'''
+Explanation
+faiss_id : 525
+source   : "Unit 1 AI & Python Complete.pdf"  only file name is stored
+path     : "D:/.../doc_ingestion/Unit 1 AI & Python Complete.pdf"
+text     : "Artificial Intelligence is..."
+page     : 7
+created_at : timestamp
+Each real chunk will look like
+{
+"source": "Unit 1 AI & Python Complete.pdf",
+"text": "Artificial Intelligence is...",
+"page": 7
+}
+After embedding, FAISS contains
+FAISS ID --> vector(one chunk text)
+'''
+# In[]
+# ========================================================
+# 🔐Store and Retrieve Chunks via SQLite
+# ========================================================
+def store_chunks_in_db(chunks, faiss_ids):
+    conn = get_db()      #connect DB file
+    cursor = conn.cursor()
+    for chunk, fid in zip(chunks, faiss_ids):
+        #chunks - list of dictionories with [{"source": "file1.pdf", "text": "...",  "page": 3}]
+        #faiss_ids - each ID corresponds to one embedding vector.
+        #zip creates a pairing between the two lists chunks and faiss_ids:
+        cursor.execute("""
+            INSERT INTO document_chunks
+            (faiss_id, source, path, text, page, created_at)
+            VALUES (?, ?, ?, ?, ?, datetime('now'))
+        """, (
+            fid,
+            chunk["source"],
+            f"{UPLOAD_FOLDER}/{chunk['source']}",
+            chunk["text"],
+            chunk.get("page", 0)
+        ))
+    conn.commit()
+    conn.close()
+def get_next_faiss_id():
+    #generates the next unique FAISS ID based on what’s already stored in the database.
+    #MAX(faiss_id)- Scans column faiss_id and Returns the largest value
+    #returns 0 if table is empty otherwise max_id + 1
+    conn = get_db()     #connect DB file
+    cursor = conn.cursor()
+    cursor.execute("SELECT MAX(faiss_id) FROM document_chunks")
+    result = cursor.fetchone()[0]
+    #fetchone() returns a tuple (max_value,)
+    conn.close()
+    if result is None:
+        return 0
+    return result + 1
+def fetch_chunks_by_faiss_ids(faiss_ids):
+    conn = get_db()     #connect DB file
+    cursor = conn.cursor()
+    placeholders = ",".join(["?"] * len(faiss_ids))
+    #if faiss_ids = [101, 205, 87]  placeholders = "?,?,?" this is needed because SQL expects 3 values
+    #FAISS returns IDs in ranked order (most relevant first). But SQL does NOT preserve order. It may return wrong oreder [101, 205, 87]
+    query = f"""
+        SELECT faiss_id, text, source, page
+        FROM document_chunks
+        WHERE faiss_id IN ({placeholders})
+    """
+    cursor.execute(query, faiss_ids)
+    rows = cursor.fetchall()
+    '''
+    rows will be list of tuples as given below
+    [
+        (101, "text1", "file1.pdf", 2),
+        (205, "text2", "file2.pdf", 5),
+    ]
+    '''
+    conn.close()
+    results = []
+    #{ faiss_id → row_data }
+    id_to_row = {
+        row[0]: {
+            "faiss_id": row[0],
+            "text": row[1],
+            "source": row[2],
+            "page": row[3]
+        }
+        for row in rows
+    }
+    results = [id_to_row[fid] for fid in faiss_ids if fid in id_to_row]  #reorder to preserve FAISS order
+    return results
+def retrieve_relevant_chunks(query, top_k=5):
+    #FAISS retrives faiss id and SQLite fetches metadata from that id
+    global VECTOR_INDEX, EMBEDDING_MODEL, INDEX_READY
+    #EMBEDDING_MODEL → encodes text to vector
+    #VECTOR_INDEX → FAISS index storing vectors
+    #INDEX_READY -> used to handle thesituation when vectors are unbulit at the starting of the app and query is given
+    if not INDEX_READY or VECTOR_INDEX is None:
+        raise HTTPException(status_code=503, detail="Index is still building")   # raise HTTPException
+    if EMBEDDING_MODEL is None:
+        raise HTTPException(status_code=500, detail="Embedding model not loaded")
+    # Encode query. Does not normalize embeddings. So, FAISS uses L2 distance normally not cosine similarity equivalent
+    query_embedding = EMBEDDING_MODEL.encode([query])
+    query_embedding = np.array(query_embedding).astype("float32")
+    # FAISS search
+    with FAISS_LOCK:
+        k = min(top_k, VECTOR_INDEX.ntotal)
+        distances, indices = VECTOR_INDEX.search(query_embedding, top_k)
+    print("FAISS distances:", distances)
+    print("FAISS indices:", indices)
+    #To avoid potential crash in empty FAISS search
+    if indices is None or len(indices[0]) == 0:
+        return []
+    faiss_ids = [int(i) for i in indices[0] if i != -1]
+    # Fetch metadata from SQLite
+    retrieved_chunks = fetch_chunks_by_faiss_ids(faiss_ids)
+    return retrieved_chunks
+# ========================================================
+# 🔄 Rebuild FAISS Index From SQLite on Server Start
+# ========================================================
+def rebuild_faiss_index():
+    global VECTOR_INDEX, EMBEDDING_MODEL, INDEX_READY
+    with FAISS_LOCK:
+        INDEX_READY = False
+        conn = get_db()
+        cursor = conn.cursor()
+        cursor.execute("""
+            SELECT faiss_id, text
+            FROM document_chunks
+            ORDER BY faiss_id
+        """)
+        rows = cursor.fetchall()
+        conn.close()
+        if not rows:
+            VECTOR_INDEX = None
+            INDEX_READY = True
+            if os.path.exists(FAISS_INDEX_PATH):
+                os.remove(FAISS_INDEX_PATH)
+            print("No documents found. FAISS cleared and file removed.")
+            return
+        texts = [row[1] for row in rows]
+        ids = [row[0] for row in rows]
+        embeddings = EMBEDDING_MODEL.encode(texts)
+        embeddings = np.array(embeddings).astype("float32")
+        dimension = embeddings.shape[1]
+        base_index = faiss.IndexFlatL2(dimension)
+        VECTOR_INDEX = faiss.IndexIDMap(base_index)
+        VECTOR_INDEX.add_with_ids(
+            embeddings,
+            np.array(ids, dtype="int64")
+        )
+        INDEX_READY = True
+        print(f"FAISS index rebuilt with {len(texts)} chunks.")
+        faiss.write_index(VECTOR_INDEX, FAISS_INDEX_PATH)
+        print("FAISS index saved to disk.")
+        INDEX_READY = True
+# In[]
+@app.on_event("startup")
+def startup_event():
+    print("Server started successfully")
+    os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+    threading.Thread(target=rebuild_faiss_index).start()
+# In[]
+# ============================================
+# Save uploaded file
+# ============================================
+async def save_file(file: UploadFile):
+    filename = Path(file.filename).name
+    file_path = os.path.join(UPLOAD_FOLDER, filename)
+    if os.path.exists(file_path):
+        raise HTTPException(
+            status_code=400,
+            detail="File already exists. Please rename or delete the existing file."
+        )
+    with open(file_path, "wb") as buffer:
+        while chunk := await file.read(1024 * 1024):
+            buffer.write(chunk)
+    await file.seek(0)   # reset pointer (important)
+    return file_path
+# In[]
+# =====================================
+# 👤 User Registration
+# =====================================
+class UserRegister(BaseModel):
+    username: str
+    password: str
+@app.post("/register")
+def register(user: UserRegister):
+    conn = get_db()
+    cursor = conn.cursor()
+    hashed_pw = hash_password(user.password)
+    try:
+        cursor.execute(
+            "INSERT INTO users (username, hashed_password, created_at) VALUES (?, ?, ?)",
+            (user.username, hashed_pw, datetime.now().isoformat())
+        )
+        conn.commit()
+    except sqlite3.IntegrityError:
+        conn.close()
+        raise HTTPException(status_code=400, detail="Username already exists")
+    conn.close()
+    return {"message": "User registered successfully"}
+# =====================================
+# 🔑 Login Endpoint
+# =====================================
+@app.post("/login")
+def login(form_data: OAuth2PasswordRequestForm = Depends()):
+    # --- Check hardcoded admin first ---
+    if form_data.username == ADMIN_USERNAME and verify_password(form_data.password, ADMIN_PASSWORD_HASH):
+        access_token = create_access_token(data={"sub": ADMIN_USERNAME})
+        return {"access_token": access_token, "token_type": "bearer"}
+    # --- Otherwise fallback to database users ---
+    conn = get_db()
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT id, username, hashed_password FROM users WHERE username = ?",
+        (form_data.username,)
+    )
+    user = cursor.fetchone()
+    conn.close()
+    if not user:
+        raise HTTPException(status_code=400, detail="Invalid credentials")
+    user_id, username, hashed_password = user
+    if not verify_password(form_data.password, hashed_password):
+        raise HTTPException(status_code=400, detail="Invalid credentials")
+    access_token = create_access_token(data={"sub": username})
+    return {"access_token": access_token, "token_type": "bearer"}
+# =======================================================================
+# 🔐 Upload Files Using FastAPI User Interface and Split in to Chunks
+# =======================================================================
+@app.post("/upload")
+async def upload_file(file: UploadFile = File(...),current_user: str = Depends(get_current_user)):
+    #registers the endpoint, upload → upload_file() in a routing table
+    #Uses FAISS logic,SQLite logic, chunking logic
+    #Single ingestion system. does chunking,embedding,FAISS update,DB storage
+    global VECTOR_INDEX, EMBEDDING_MODEL, INDEX_READY
+    file_path = await save_file(file)
+    filename = Path(file.filename).name.strip() #new_chunks = [c for c in all_chunks if c["source"].strip() == filename]handle & space etc
+    # Chunk documents
+    chunker = DocChunker(doc_folder=UPLOAD_FOLDER)
+    all_chunks = chunker.chunk_documents()
+    # Only new file chunks
+    new_chunks = [c for c in all_chunks if c["source"].strip() == filename]
+    if not new_chunks:
+        return {"message": "No text extracted from document."}
+    #return the above to client which called this function as a JSON with message
+    new_texts = [clean_text(chunk["text"]) for chunk in new_chunks]
+    # Encode new chunks
+    new_embeddings = EMBEDDING_MODEL.encode(new_texts)
+    new_embeddings = np.array(new_embeddings).astype("float32")
+    # Determine FAISS ids using SQLite
+    start_id = get_next_faiss_id()
+    faiss_ids = list(range(start_id, start_id + len(new_embeddings)))
+    #Update FAISS FIRST
+    with FAISS_LOCK:
+        if VECTOR_INDEX is None:
+            dimension = new_embeddings.shape[1]
+            base_index = faiss.IndexFlatL2(dimension)
+            VECTOR_INDEX = faiss.IndexIDMap(base_index)
+        VECTOR_INDEX.add_with_ids(
+            new_embeddings,
+            np.array(faiss_ids, dtype="int64")
+        )
+        faiss.write_index(VECTOR_INDEX, FAISS_INDEX_PATH)
+        INDEX_READY = True
+        #Store metadata AFTER FAISS succeeds.
+        store_chunks_in_db(new_chunks, faiss_ids)
+        #return the following to client which called this function as a JSON with message
+        return {
+            "message": f"{file.filename} uploaded and indexed successfully",
+            "chunks_added": len(new_chunks)
+        }
+@app.post("/admin/upload-document")
+async def upload_document(
+    file: UploadFile = File(...),
+    current_user: str = Depends(get_current_user)
+):
+    try:
+        return await upload_file(file, current_user)
+    except Exception as e:
+        # Always return a JSON with 'message' so client alert works
+        return {"message": f"Upload failed: {str(e)}"}
+# In[]
+# =====================================
+# 📦 Request Models
+# =====================================
+class Message(BaseModel):
+    role: str
+    content: str
+class ChatRequest(BaseModel):
+    model: Optional[str] = None
+    messages: List[Message]
+    temperature: Optional[float] = 0.7
+    reference_style: Optional[str] = "both"
+# =====================================
+# 🌍 Tamil Detection
+# =====================================
+def contains_tamil(text: str) -> bool:
+    return bool(re.search(r'[\u0B80-\u0BFF]', text))
+# In[]
+# =====================================
+# 🛡 Retrieval Evidence Detection
+# =====================================
+def has_retrieved_context(messages: List[Message]) -> bool:
+    """
+    Detects whether WebUI injected retrieved document context.
+    Looks for common RAG markers like 'Source', 'Page', etc.
+    """
+    for m in messages:
+        content = m.content.lower()
+        if "source:" in content or "page" in content or "document:" in content:
+            return True
+    return False
+def refusal_response(reason: str):
+    return {
+        "id": "chatcmpl-local",
+        "object": "chat.completion",
+        "created": int(datetime.now().timestamp()),
+        "model": "control-layer",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": reason
+                },
+                "finish_reason": "stop"
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0
+        }
+    }
+def apply_reference_style(assistant_message, references_map, style):
+    # REMOVE existing inline references first
+    assistant_message = re.sub(r"\([^)]*\.pdf[^)]*\)", "", assistant_message, flags=re.IGNORECASE)
+    # REMOVE existing bibliography
+    assistant_message = re.sub(r"References:.*", "", assistant_message, flags=re.IGNORECASE | re.DOTALL)
+    # INLINE ONLY
+    if style == "inline":
+        for doc_marker, ref_text in references_map.items():
+            assistant_message = assistant_message.replace(
+                doc_marker, f"({ref_text})"
+            )
+    # LIST ONLY
+    elif style == "list":
+        used_markers = re.findall(r"\[Doc\d+\]", assistant_message)
+        assistant_message = re.sub(r"\[Doc\d+\]", "", assistant_message)   # REMOVE INLINE MARKERS
+        refs_list = []
+        for doc_marker in used_markers:
+            if doc_marker in references_map:
+                ref = references_map[doc_marker]
+                if ref not in refs_list:
+                    refs_list.append(ref)
+        if not refs_list:
+            refs_list = list(references_map.values())
+        if refs_list:
+            assistant_message = assistant_message.replace("References:", "")
+            assistant_message += "<br><br><br><b>References:</b><br>"
+            assistant_message += "<br>".join(f"- {r}" for r in refs_list)
+    # BOTH
+    elif style == "both":
+        used_markers = re.findall(r"\[Doc\d+\]", assistant_message)
+        for doc_marker, ref_text in references_map.items():
+            assistant_message = assistant_message.replace(
+                doc_marker, f"({ref_text})"
+            )
+        refs_list = []
+        for doc_marker in used_markers:
+            if doc_marker in references_map:
+                ref = references_map[doc_marker]
+                if ref not in refs_list:
+                    refs_list.append(ref)
+        if refs_list:
+            assistant_message += "<br><br><br><b>References:</b><br>"
+            assistant_message += "<br>".join(f"- {r}" for r in refs_list)
+    # NONE
+    elif style == "none":
+        assistant_message = assistant_message.replace("References:", "")
+    return assistant_message
+# =====================================
+# 🔐 Password Utilities
+# =====================================
+def hash_password(password: str) -> str:
+    return pwd_context.hash(password)
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    return pwd_context.verify(plain_password, hashed_password)
+def create_access_token(data: dict):
+    if "sub" not in data:
+        raise ValueError("Token data must include 'sub'")
+    return jwt.encode(data, SECRET_KEY, algorithm=ALGORITHM)
+@app.get("/protected")
+def protected_route(current_user: str = Depends(get_current_user)):
+    return {"message": f"Hello {current_user}"}
+#root endpoint
+@app.get("/")
+def serve_ui():
+    return FileResponse("FastAPI_Client/index.html")
+@app.get("/v1/models")
+def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": model,
+                "object": "model",
+                "created": 0,
+                "owned_by": "local"
+            }
+            for model in ALLOWED_MODELS
+        ]
+    }
+# In[]
+# =====================================
+# 💬 Chat Endpoint
+# =====================================
+@app.post("/v1/chat/completions")
+async def chat_completion(request: ChatRequest):
+    #Model selection from WebUI
+    selected_model = request.model or DEFAULT_MODEL
+    if selected_model not in ALLOWED_MODELS:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Model '{selected_model}' is not allowed."
+        )
+    user_message = request.messages[-1].content
+    # 🔎 Step 0: Retrieve relevant chunks from FAISS + SQLite
+    retrieved_chunks = retrieve_relevant_chunks(user_message, top_k=5)  # returns list of dicts
+    print("Number of chunks:", len(retrieved_chunks))
+    #Build context for LLM with inline references (source attribution)
+    context_parts = []
+    for i, c in enumerate(retrieved_chunks, start=1):
+        chunk_text = clean_text(c["text"])
+        context_parts.append(
+            f"""
+            [Doc{i}]
+            Document: {c['source']}
+            Page: {c['page']}
+            Content:
+            {chunk_text}
+            """
+        )
+    '''
+    for i, c in enumerate(retrieved_chunks, start=1):
+        chunk_text = c['text'].replace("[", "").replace("]", "")
+        context_parts.append(
+            f"[Doc{i}] {chunk_text}"
+            #f"Source [Doc{i}] | Document: {c['source']} | Page: {c['page']}\n{chunk_text}"
+        )
+    '''
+    rag_context = "\n\n".join(context_parts)
+    references_map = {}
+    for i, c in enumerate(retrieved_chunks, start=1):
+        #references_map[f"[Doc{i}]"] = f"{c['source']} (Page {c['page']})"
+        #references_map[f"[Doc{i}]"] = f"<a href='/uploads/{c['source']}#page={c['page']}' target='_blank'>{c['source']} (Page {c['page']})</a>"
+        references_map[f"[Doc{i}]"] = (
+            f"<a href='/uploads/{c['source']}#page={c['page']}' target='_blank'>"
+            f"{c['source']} — Page {c['page']}</a>"
+        )
+    #Hallucination Control: Evidence Gate (Pre-LLM)
+    if not retrieved_chunks:
+        print("🚫 BLOCKED BEFORE LLM CALL — No retrieved evidence detected.")
+        return refusal_response(
+            "The answer is not found in the local documents provided by KKT."
+        )
+    #Language handling
+    if contains_tamil(user_message):
+        system_prompt = "You are a helpful AI assistant. Always respond only in Tamil."
+    else:
+        system_prompt = "You are a helpful AI assistant. Answer ONLY using the provided document context. If the answer is not in the context, say the information is not available in the documents."
+    #style = request.dict().get("reference_style", "both")
+    style = request.reference_style or "both"  #✅ Included newly
+    if style == "none":
+        citation_instruction = "STRICTLY DO NOT include any citations or markers."
+        rules_text = """
+- Do NOT include any citation markers like [Doc1].
+- Do NOT include any References section.
+"""
+    elif style == "inline":
+        citation_instruction = "Include inline citation markers like [Doc1]."
+        rules_text = """
+- Use ONLY the markers [Doc1], [Doc2], etc.
+- Do NOT write document names yourself.
+- Do NOT invent citations.
+- Do NOT include any References section.
+"""
+    elif style == "list":
+        citation_instruction = "STRICTLY DO NOT include any inline citation markers like [Doc1]."
+        rules_text = """
+- Do NOT include any inline citation markers like [Doc1].
+- Do NOT write document names yourself.
+- Do NOT invent citations.
+- Do NOT include any References section.
+"""
+    elif style == "both":
+        citation_instruction = "STRICTLY include citation markers like [Doc1], [Doc2] in every factual sentence."
+        rules_text = """
+- Use ONLY the markers [Doc1], [Doc2], etc.
+- Do NOT write document names yourself.
+- Do NOT invent citations.
+"""
+    #Inject retrieved context as system message
+    system_prompt = f"""
+    You are a document-grounded AI assistant.
+    Answer the question ONLY using the provided context.
+    {citation_instruction}
+    Rules:
+    {rules_text}
+    If the answer is not present in the context, say the information is not available.
+    Context:
+    {rag_context}
+    """
+    '''
+    #old prompt where citations were not displayed as per check box selection
+    system_prompt = f"""
+    You are a document-grounded AI assistant.
+    Answer the question ONLY using the provided context.
+    Citation Rules:
+    1. Every factual statement MUST include a citation marker.
+    2. Use ONLY the markers [Doc1], [Doc2], etc.
+    3. Copy the marker EXACTLY as written.
+    4. Do NOT write document names yourself.
+    5. Do NOT invent citations.
+    If the answer is not present in the context, say the information is not available.
+    Context:
+    {rag_context}
+    """
+    '''
+    # 3 Forward request to Ollama
+    #final_messages = [{"role": "system", "content": system_prompt}]
+    final_messages = [
+        {
+            "role": "system",
+            "content": system_prompt
+        }
+    ]
+    for m in request.messages:
+        final_messages.append({
+            "role": m.role.lower(),
+            "content": m.content
+        })
+    print("MODEL:", selected_model)
+    print("AVAILABLE MODELS:", ALLOWED_MODELS)
+    print("SENDING TO GROQ:", final_messages)
+    print("GROQ KEY:", os.getenv("GROQ_API_KEY"))
+    try:
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            response = await client.post(
+                url="https://api.groq.com/openai/v1/chat/completions",  #uses cloud server
+                headers={
+                    "Authorization": f"Bearer {os.getenv('GROQ_API_KEY')}",   #environment variable set using Windows Power Shell
+                    "Content-Type": "application/json"
+                },
+                json={
+                    "model": selected_model,
+                    "messages": final_messages,
+                    "temperature": request.temperature,
+                    "stream": False
+                }
+            )
+        if response.status_code != 200:
+            print("STATUS:", response.status_code)
+            print("ERROR:", response.text)
+            raise HTTPException(status_code=500, detail="Groq API Error")
+        result = response.json()
+        #assistant_message = result["message"]["content"] #This works for Ollama not for Groq
+        #assistant_message = result["choices"][0]["message"]["content"]
+        assistant_message = result.get("choices", [{}])[0].get("message", {}).get("content", "")
+        # Determine reference style: inline, list, or both
+        #style = request.dict().get("reference_style", "both").lower()
+        style = (request.reference_style or "both").lower()
+        if style not in ["inline", "list", "both", "none"]:
+            style = "both"
+        assistant_message = apply_reference_style(assistant_message, references_map, style)
+    except Exception as e:
+        print("FULL ERROR:", e)
+        raise
+    #Return OpenAI-compatible response
+    return {
+        "id": "chatcmpl-local",
+        "object": "chat.completion",
+        "created": int(datetime.now().timestamp()),
+        "model": selected_model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": assistant_message
+                },
+                "finish_reason": "stop"
+            }
+        ],
+        "usage": {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0
+        }
+    }

config.py ADDED Viewed

	@@ -0,0 +1,25 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Mon Apr 13 19:22:03 2026
+@author: THYAGHARAJAN
+"""
+#NOTE
+# All uploaded files are saved in the doc_ingestion folder where the server file exixsts
+#BASE_DIR -- full absolute directory path of the current script file
+#UPLOAD_FOLDER -- This folder is created if not available see startup_event() in the server file
+#DB_FILE = "kkt_SQLite_DB.db"
+#DB_PATH_FILE -- database file config.db in the same directory as this script (BASE_DIR)
+#FAISS_INDEX_PATH = os.path.join(BASE_DIR, "faiss.index")
+import os
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+UPLOAD_FOLDER = os.path.join(BASE_DIR, "doc_ingestion")  #Folder is created in the startup_event() in the server file
+DB_FILE = "kkt_SQLite_DB.db"
+DB_PATH_FILE = os.path.join(BASE_DIR, DB_FILE)  #this path is used in the server file to connect the dtata base
+FAISS_INDEX_PATH = os.path.join(BASE_DIR, "faiss.index")

doc_ingestion/AIML_Unit1_RMD_ECE.pdf ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e12a289078f97daafeade764a58c9c217b5d25d2ba69c056bbf4f338046cad46
+size 2663371

doc_ingestion/AIML_Unit2_RMD_ECE.pdf ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d68a2034ebc91200784fd39eae4818f407811e26e9d134183b710d2f3c28f663
+size 3598628

faiss.index ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e5420cf84512c0c5438b7c21831c098ba4a155f1c5b47ae2013f538732472e
+size 1085522

kkt_SQLite_DB.db ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c4cd7fa92c9fec7ae91c60ac2c43f7e039dc8a9a8fb1498602987035caf4158c
+size 389120

rag/__init__.py ADDED Viewed

	@@ -0,0 +1,7 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar  4 12:55:04 2026
+@author: THYAGHARAJAN
+"""

rag/chunker.py ADDED Viewed

	@@ -0,0 +1,160 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar  3 16:41:30 2026
+@author: THYAGHARAJAN
+Reads PDFs from kkt_AIML_PDFs/
+Chunk into fixed size segments
+Return list of chunks with metadata
+"""
+import os
+from typing import List, Dict
+from pypdf import PdfReader
+from docx import Document
+import pandas as pd
+from PIL import Image
+import pytesseract
+import cv2
+from pytesseract import Output
+from utils.text_cleanerV2 import clean_text
+import shutil
+tesseract_path = shutil.which("tesseract")
+if tesseract_path:
+    pytesseract.pytesseract.tesseract_cmd = tesseract_path
+class DocChunker:
+    """
+    Handles document ingestion and text chunking.
+    """
+    def __init__(self, doc_folder: str, chunk_size: int = 500, overlap: int = 50):
+        self.doc_folder = doc_folder
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+    # ---------------------------------------------------
+    # Load and Parse Documents (PDF, DOCX, Excel, Images)
+    # ---------------------------------------------------
+    def load_pdfs(self) -> List[Dict]:
+        """
+        Reads all supported documents and returns page-level texts with metadata.
+        (Method name preserved for compatibility.)
+        """
+        documents = []
+        for filename in os.listdir(self.doc_folder):
+            file_path = os.path.join(self.doc_folder, filename)
+            ext = filename.lower().split(".")[-1]
+            try:
+                # ---------------- PDF ----------------
+                if ext == "pdf":
+                    reader = PdfReader(file_path)
+                    for page_number, page in enumerate(reader.pages, start=1):
+                        text = page.extract_text()
+                        if text:
+                            documents.append({
+                                "text": text.strip(),
+                                "source": filename,
+                                "page": page_number
+                            })
+                # ---------------- DOCX ----------------
+                elif ext == "docx":
+                    doc = Document(file_path)
+                    full_text = "\n".join([p.text for p in doc.paragraphs])
+                    documents.append({
+                        "text": full_text.strip(),
+                        "source": filename,
+                        "page": 1
+                    })
+                # ---------------- Excel ----------------
+                elif ext in ["xlsx", "xls"]:
+                    df = pd.read_excel(file_path)
+                    documents.append({
+                        "text": df.to_string(),
+                        "source": filename,
+                        "page": 1
+                    })
+                # ---------------- Image (OCR) ----------------
+                elif ext in ["png", "jpg", "jpeg"]:
+                    # Read image with OpenCV
+                    img = cv2.imread(file_path)
+                    # Detect orientation
+                    osd = pytesseract.image_to_osd(img, output_type=Output.DICT)
+                    angle = osd["rotate"]
+                    if angle == 90:
+                        img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
+                    elif angle == 180:
+                        img = cv2.rotate(img, cv2.ROTATE_180)
+                    elif angle == 270:
+                        img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)
+                    # Convert to grayscale
+                    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+                    # Resize for better OCR
+                    gray = cv2.resize(gray, None, fx=2, fy=2, interpolation=cv2.INTER_CUBIC)
+                    # Apply threshold
+                    thresh = cv2.threshold(
+                        gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU
+                    )[1]
+                    # OCR
+                    text = pytesseract.image_to_string(thresh, config="--psm 6")
+                    documents.append({
+                        "text": text.strip(),
+                        "source": filename,
+                        "page": 1
+                    })
+            except Exception as e:
+                print(f"Error processing {filename}: {e}")
+        return documents
+    # ---------------------------------------------------
+    # Chunk Text
+    # ---------------------------------------------------
+    def chunk_documents(self) -> List[Dict]:
+        """
+        Splits document text into smaller chunks.
+        Returns list of chunks with metadata.
+        """
+        pages = self.load_pdfs()
+        chunks = []
+        for page in pages:
+            raw_text = page["text"]
+            cleaned_text = clean_text(raw_text)
+            start = 0
+            while start < len(cleaned_text):
+                end = start + self.chunk_size
+                chunk_text = cleaned_text[start:end]
+                chunks.append({
+                    "text": chunk_text,
+                    "source": page["source"],
+                    "page": page["page"]
+                })
+                start += self.chunk_size - self.overlap
+        return chunks

rag/qdrant_retriever.py ADDED Viewed

	@@ -0,0 +1,10 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar  3 14:40:03 2026
+@author: THYAGHARAJAN
+"""
+from rag.base_retriever import BaseRetriever
+class QdrantRetriever(BaseRetriever):

requirements.txt ADDED Viewed

	@@ -0,0 +1,20 @@

+fastapi
+uvicorn
+httpx
+pydantic
+python-multipart
+sentence-transformers
+faiss-cpu
+pypdf
+python-docx
+pandas
+pytesseract
+opencv-python-headless
+Pillow
+passlib[bcrypt]
+python-jose[cryptography]
+requests

utils/__init__.py ADDED Viewed

	@@ -0,0 +1,7 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar  4 12:55:04 2026
+@author: THYAGHARAJAN
+"""

utils/admin_fns.py ADDED Viewed

	@@ -0,0 +1,142 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Sat Apr 11 15:03:04 2026
+@author: THYAGHARAJAN
+"""
+#line 20 import avoids circular error
+from utils.core_imports import get_current_user
+from fastapi import APIRouter, Depends, HTTPException
+import os
+import shutil
+import threading   #used for rebuild_faiss_index line 54
+router = APIRouter()  #the app acted as a proxy to Ollama in main file
+@router.delete("/admin/delete-document")
+def delete_document(filename: str,current_user: str = Depends(get_current_user)):
+    from utils.core_imports import get_db, rebuild_faiss_index, get_upload_folder  #lazy import
+    filename = filename.strip()
+    conn = get_db()
+    cursor = conn.cursor()
+    # Check existence
+    cursor.execute(
+        "SELECT faiss_id FROM document_chunks WHERE TRIM(source)=?",
+        (filename,)
+    )
+    rows = cursor.fetchall()
+    if not rows:
+        conn.close()
+        raise HTTPException(status_code=404, detail="Document not found")
+    # ✅ DELETE (this was missing in your code)
+    cursor.execute(
+        "DELETE FROM document_chunks WHERE TRIM(source)=?",
+        (filename,)
+    )
+    conn.commit()
+    conn.close()
+    # Delete physical file
+    file_path = os.path.join(get_upload_folder(), filename)
+    if os.path.exists(file_path):
+        os.remove(file_path)
+        rebuild_faiss_index()
+    # Rebuild FAISS in background
+    threading.Thread(target=rebuild_faiss_index).start()
+    '''
+    #used for debugging. Found & was not converted to %26
+    print(f"Incoming filename: [{filename}]")
+    cursor.execute("SELECT DISTINCT source FROM document_chunks")
+    all_sources = cursor.fetchall()
+    print("DB sources:")
+    for s in all_sources:
+        print(f"[{s[0]}]")
+    '''
+    return {"message": f"{filename} removed from index"}
+@router.delete("/admin/delete-folder")
+def delete_folder(folder: str,current_user: str = Depends(get_current_user)):
+    from utils.core_imports import get_db, rebuild_faiss_index
+    conn = get_db()
+    cursor = conn.cursor()
+    cursor.execute(
+        "DELETE FROM document_chunks WHERE source LIKE ?",
+        (f"%{folder}%",)
+    )
+    deleted_count = cursor.rowcount
+    conn.commit()
+    conn.close()
+    if deleted_count == 0:
+        raise HTTPException(status_code=404, detail="Folder not found")
+    threading.Thread(target=rebuild_faiss_index).start()
+    return {"message": f"{folder} folder removed from index"}
+@router.delete("/admin/reset-index")
+def reset_index(confirm: bool = False,current_user: str = Depends(get_current_user)):
+    from utils.core_imports import get_db, rebuild_faiss_index, get_upload_folder   #lazy import to avoid circular import
+    #confirm button will be displayed
+    if not confirm:
+        return {"message": "Set confirm=true to reset index"}
+    conn = get_db()
+    cursor = conn.cursor()
+    cursor.execute("DELETE FROM document_chunks")
+    # delete ALL rows in document_chunks table
+    conn.commit()
+    conn.close()
+    #delete the files in the UPLOAD dir doc_ingestion folder
+    upload_dir = get_upload_folder()
+    shutil.rmtree(upload_dir)
+    os.makedirs(upload_dir, exist_ok=True)
+    threading.Thread(target=rebuild_faiss_index).start()
+    return {"message": "Index reset completed"}
+@router.get("/admin/list-documents")
+def list_documents(current_user: str = Depends(get_current_user)):
+    from utils.core_imports import get_db
+    conn = get_db()
+    cursor = conn.cursor()
+    cursor.execute("""
+        SELECT source, COUNT(*) as chunks
+        FROM document_chunks
+        GROUP BY source
+    """)
+    rows = cursor.fetchall()
+    conn.close()
+    docs = [{"document": r[0], "chunks": r[1]} for r in rows]
+    return {"documents": docs}

utils/core_imports.py ADDED Viewed

	@@ -0,0 +1,39 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Sun Apr 12 22:31:05 2026
+@author: THYAGHARAJAN
+"""
+#NOTE
+#If you chnage the kkt_FastAPI_server file name, then change the SERVER_MODULE name here
+# =====================================
+# 🔁 Centralized Server Module Import
+# =====================================
+SERVER_MODULE = "app"
+import importlib
+def _get_server():
+    return importlib.import_module(SERVER_MODULE)
+# =====================================
+# 🔁 Lazy Re-export functions
+# =====================================
+def get_db():
+    return _get_server().get_db()
+def rebuild_faiss_index():
+    return _get_server().rebuild_faiss_index()
+def get_current_user():
+    return _get_server().get_current_user
+def get_upload_folder():
+    return _get_server().UPLOAD_FOLDER

utils/text_cleanerV1.py ADDED Viewed

	@@ -0,0 +1,37 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar  4 12:25:04 2026
+@author: THYAGHARAJAN
+"""
+import re
+def clean_text(text: str) -> str:
+    """
+    Basic PDF text cleaning for RAG.
+    Removes URLs, repeated lines, extra whitespace, and noise.
+    """
+    # Remove URLs
+    text = re.sub(r"http\S+", "", text)
+    # Remove standalone dates like 02-03-2026
+    text = re.sub(r"\b\d{2}-\d{2}-\d{4}\b", "", text)
+    # Remove QR instruction lines
+    text = re.sub(r"Scan the QR code.*", "", text, flags=re.IGNORECASE)
+    # Remove extra spaces
+    text = re.sub(r"\s+", " ", text)
+    # Remove duplicate consecutive words
+    words = text.split()
+    cleaned_words = []
+    prev_word = None
+    for word in words:
+        if word != prev_word:
+            cleaned_words.append(word)
+        prev_word = word
+    return " ".join(cleaned_words).strip()

utils/text_cleanerV2.py ADDED Viewed

	@@ -0,0 +1,97 @@

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar  4 12:25:04 2026
+@author: THYAGHARAJAN
+"""
+import re
+import unicodedata
+def clean_text(text: str) -> str:
+    """
+    Main cleaning pipeline.
+    Order matters.
+    """
+    if not text:
+        return ""
+    text = normalize_unicode(text)
+    text = remove_non_printable(text)
+    text = remove_headers_footers(text)
+    text = remove_page_numbers(text)
+    text = remove_extra_whitespace(text)
+    text = remove_duplicate_words(text)
+    return text
+def normalize_unicode(text: str) -> str:
+    """
+    Normalize unicode characters to a consistent form.
+    Prevents strange PDF extraction artifacts.
+    """
+    return unicodedata.normalize("NFKC", text)
+def remove_extra_whitespace(text: str) -> str:
+    """
+    Remove excessive spaces, tabs, and line breaks.
+    """
+    text = re.sub(r"[ \t]+", " ", text)         # collapse spaces
+    text = re.sub(r"\n\s*\n+", "\n\n", text)    # max 2 newlines
+    return text.strip()
+def remove_page_numbers(text: str) -> str:
+    """
+    Remove standalone page numbers.
+    Example: '12', '- 23 -', 'Page 5'
+    """
+    text = re.sub(r"\n\s*[-–]?\s*\d+\s*[-–]?\s*\n", "\n", text)
+    text = re.sub(r"Page\s*\d+", "", text, flags=re.IGNORECASE)
+    return text
+def remove_headers_footers(text: str) -> str:
+    """
+    Remove common repeating header/footer patterns.
+    Customize if needed.
+    """
+    patterns = [
+        r"Copyright\s.*",
+        r"All rights reserved.*",
+        r"www\.[^\s]+",
+        r"http[s]?://[^\s]+",
+    ]
+    for pattern in patterns:
+        text = re.sub(pattern, "", text, flags=re.IGNORECASE)
+    return text
+def remove_non_printable(text: str) -> str:
+    """
+    Remove non-printable characters from PDF extraction.
+    """
+    return "".join(ch for ch in text if ch.isprintable())
+def remove_duplicate_words(text: str) -> str:
+    words = text.split()
+    cleaned_words = []
+    prev_word = None
+    for word in words:
+        if word != prev_word:
+            cleaned_words.append(word)
+        prev_word = word
+    return " ".join(cleaned_words)