Spaces:
Sleeping
Sleeping
Commit Β·
768fcad
1
Parent(s): e193e4d
Switch to Pinecone, add Dockerfile for HuggingFace
Browse files- .gitignore +2 -0
- Dockerfile +6 -2
- app.py +51 -70
- ingest.py +36 -8
- requirements.txt +3 -2
.gitignore
CHANGED
|
@@ -4,3 +4,5 @@ __pycache__/
|
|
| 4 |
*.pyc
|
| 5 |
feedback.json
|
| 6 |
chat_log.json
|
|
|
|
|
|
|
|
|
| 4 |
*.pyc
|
| 5 |
feedback.json
|
| 6 |
chat_log.json
|
| 7 |
+
db/
|
| 8 |
+
data/
|
Dockerfile
CHANGED
|
@@ -2,12 +2,16 @@ FROM python:3.12-slim
|
|
| 2 |
|
| 3 |
WORKDIR /app
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
COPY requirements.txt .
|
| 6 |
|
| 7 |
-
RUN pip install --no-cache-dir
|
| 8 |
|
| 9 |
COPY . .
|
| 10 |
|
| 11 |
EXPOSE 7860
|
| 12 |
|
| 13 |
-
CMD ["
|
|
|
|
| 2 |
|
| 3 |
WORKDIR /app
|
| 4 |
|
| 5 |
+
RUN apt-get update && apt-get install -y \
|
| 6 |
+
build-essential \
|
| 7 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 8 |
+
|
| 9 |
COPY requirements.txt .
|
| 10 |
|
| 11 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 12 |
|
| 13 |
COPY . .
|
| 14 |
|
| 15 |
EXPOSE 7860
|
| 16 |
|
| 17 |
+
CMD ["python", "app.py"]
|
app.py
CHANGED
|
@@ -1,17 +1,17 @@
|
|
| 1 |
import os
|
| 2 |
import json
|
| 3 |
-
import shutil
|
| 4 |
import warnings
|
| 5 |
warnings.filterwarnings("ignore")
|
| 6 |
|
| 7 |
from dotenv import load_dotenv
|
| 8 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 9 |
-
from langchain_community.vectorstores import Chroma
|
| 10 |
from langchain_groq import ChatGroq
|
| 11 |
from langchain_core.prompts import PromptTemplate
|
| 12 |
from langchain_core.output_parsers import StrOutputParser
|
| 13 |
from langchain_community.document_loaders import PyPDFLoader
|
| 14 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
|
|
|
| 15 |
from fastapi import FastAPI, UploadFile, File, Form, Request
|
| 16 |
from fastapi.responses import HTMLResponse, RedirectResponse
|
| 17 |
from pydantic import BaseModel
|
|
@@ -22,15 +22,24 @@ import uvicorn
|
|
| 22 |
|
| 23 |
load_dotenv()
|
| 24 |
|
| 25 |
-
# ββ
|
| 26 |
-
ADMIN_PASSWORD
|
| 27 |
-
CHAT_LOG_FILE
|
| 28 |
-
FEEDBACK_FILE
|
|
|
|
|
|
|
| 29 |
|
| 30 |
-
print("π Loading
|
| 31 |
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
|
| 32 |
-
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
print("π€ Connecting to Groq LLM...")
|
| 36 |
llm = ChatGroq(
|
|
@@ -82,8 +91,7 @@ def ask_with_memory(question: str, history: list) -> str:
|
|
| 82 |
return chain.invoke({"context": context, "history": formatted_history, "question": question})
|
| 83 |
|
| 84 |
def rebuild_knowledge_base(pdf_path: str):
|
| 85 |
-
global
|
| 86 |
-
|
| 87 |
loader = PyPDFLoader(pdf_path)
|
| 88 |
documents = loader.load()
|
| 89 |
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
|
|
@@ -95,22 +103,21 @@ def rebuild_knowledge_base(pdf_path: str):
|
|
| 95 |
seen.add(text)
|
| 96 |
unique_chunks.append(chunk)
|
| 97 |
|
| 98 |
-
# ββ
|
| 99 |
try:
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
|
|
|
| 103 |
|
| 104 |
-
|
| 105 |
-
from langchain_community.vectorstores import Chroma as ChromaStore
|
| 106 |
-
vectordb = ChromaStore.from_documents(
|
| 107 |
documents=unique_chunks,
|
| 108 |
embedding=embeddings,
|
| 109 |
-
|
|
|
|
| 110 |
)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
print(f"β
Knowledge base rebuilt with {len(unique_chunks)} unique chunks")
|
| 114 |
|
| 115 |
def load_feedback():
|
| 116 |
if os.path.exists(FEEDBACK_FILE):
|
|
@@ -163,7 +170,6 @@ async def feedback(payload: Feedback):
|
|
| 163 |
print(f"{'π' if payload.rating == 'up' else 'π'} Feedback: {payload.question[:50]}")
|
| 164 |
return {"status": "saved"}
|
| 165 |
|
| 166 |
-
# ββ Admin login page ββ
|
| 167 |
@app.get("/admin", response_class=HTMLResponse)
|
| 168 |
async def admin_login():
|
| 169 |
return HTMLResponse(content="""<!DOCTYPE html>
|
|
@@ -184,7 +190,6 @@ async def admin_login():
|
|
| 184 |
input:focus{border-color:#22c55e}
|
| 185 |
button{width:100%;background:#16a34a;border:none;border-radius:12px;color:white;font-family:'Plus Jakarta Sans',sans-serif;font-weight:700;font-size:.9rem;padding:.75rem;cursor:pointer;transition:all .2s}
|
| 186 |
button:hover{background:#15803d;transform:translateY(-1px)}
|
| 187 |
-
.err{color:#dc2626;font-size:.8rem;margin-top:.5rem;display:none}
|
| 188 |
.back{display:inline-block;margin-top:1rem;font-size:.78rem;color:#16a34a;text-decoration:none;font-weight:600}
|
| 189 |
</style>
|
| 190 |
</head>
|
|
@@ -197,7 +202,6 @@ async def admin_login():
|
|
| 197 |
<input type="password" name="password" placeholder="Enter admin password" required autofocus/>
|
| 198 |
<button type="submit">Login β</button>
|
| 199 |
</form>
|
| 200 |
-
<div class="err" id="err">Incorrect password. Try again.</div>
|
| 201 |
<a href="/" class="back">β Back to Chatbot</a>
|
| 202 |
</div>
|
| 203 |
</body>
|
|
@@ -208,8 +212,7 @@ async def admin_login_post(request: Request, password: str = Form(...)):
|
|
| 208 |
if password != ADMIN_PASSWORD:
|
| 209 |
return HTMLResponse(content="""<!DOCTYPE html>
|
| 210 |
<html><head><meta charset="UTF-8"/>
|
| 211 |
-
<
|
| 212 |
-
<style>*{box-sizing:border-box;margin:0;padding:0}body{background:#f0faf4;font-family:'Plus Jakarta Sans',sans-serif;min-height:100vh;display:flex;align-items:center;justify-content:center}.card{background:white;border:1.5px solid #fecaca;border-radius:24px;padding:2.5rem 2rem;width:100%;max-width:380px;text-align:center}.icon{font-size:2.5rem;margin-bottom:1rem}h1{color:#dc2626;font-size:1.2rem;margin-bottom:1rem}a{display:inline-block;background:#16a34a;color:white;padding:.6rem 1.4rem;border-radius:10px;text-decoration:none;font-weight:700;font-size:.85rem}</style>
|
| 213 |
</head><body><div class="card"><div class="icon">β</div><h1>Incorrect Password</h1><a href="/admin">Try Again</a></div></body></html>""")
|
| 214 |
return RedirectResponse(url=f"/admin/panel?pwd={password}", status_code=303)
|
| 215 |
|
|
@@ -235,10 +238,13 @@ async def admin_panel(pwd: str = ""):
|
|
| 235 |
|
| 236 |
chat_table = f"<table><thead><tr><th>Time</th><th>Question</th><th>Answer</th></tr></thead><tbody>{chat_rows}</tbody></table>" if chat_rows else "<div class='empty'>No chats yet!</div>"
|
| 237 |
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
|
|
|
|
|
|
|
|
|
| 242 |
|
| 243 |
return HTMLResponse(content=f"""<!DOCTYPE html>
|
| 244 |
<html lang="en">
|
|
@@ -253,7 +259,7 @@ async def admin_panel(pwd: str = ""):
|
|
| 253 |
.header{{background:#0a2e1a;color:#4ade80;padding:1.5rem 2rem;border-radius:16px;margin-bottom:1.5rem;display:flex;justify-content:space-between;align-items:center}}
|
| 254 |
.header h1{{font-size:1.4rem;font-weight:800}}
|
| 255 |
.header-links{{display:flex;gap:.6rem}}
|
| 256 |
-
.hlink{{background:rgba(74,222,128,.15);border:1px solid rgba(74,222,128,.3);border-radius:8px;color:#4ade80;font-size:.72rem;font-weight:700;padding:.35rem .8rem;text-decoration:none
|
| 257 |
.hlink:hover{{background:rgba(74,222,128,.25)}}
|
| 258 |
.grid{{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:1rem;margin-bottom:1.5rem}}
|
| 259 |
.stat{{background:white;border:1.5px solid #bbf7d0;border-radius:16px;padding:1.2rem;text-align:center;box-shadow:0 2px 12px rgba(34,197,94,.08)}}
|
|
@@ -263,9 +269,6 @@ async def admin_panel(pwd: str = ""):
|
|
| 263 |
.section h2{{font-size:.85rem;font-weight:700;color:#16a34a;text-transform:uppercase;letter-spacing:.08em;margin-bottom:1rem}}
|
| 264 |
.status-row{{display:flex;gap:1rem;flex-wrap:wrap;margin-bottom:1rem}}
|
| 265 |
.status-badge{{background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;padding:.4rem .8rem;font-size:.8rem;color:#14532d;font-weight:600}}
|
| 266 |
-
.upload-form{{display:flex;flex-direction:column;gap:.8rem}}
|
| 267 |
-
.file-input{{border:2px dashed #bbf7d0;border-radius:12px;padding:1.5rem;text-align:center;cursor:pointer;color:#16a34a;font-size:.85rem;font-weight:600;transition:all .2s}}
|
| 268 |
-
.file-input:hover{{border-color:#22c55e;background:#f0fdf4}}
|
| 269 |
.btn{{border:none;border-radius:12px;font-family:'Plus Jakarta Sans',sans-serif;font-weight:700;font-size:.85rem;padding:.65rem 1.3rem;cursor:pointer;transition:all .2s;white-space:nowrap}}
|
| 270 |
.btn-green{{background:#16a34a;color:white;box-shadow:0 2px 10px rgba(22,163,74,.3)}}
|
| 271 |
.btn-green:hover{{background:#15803d;transform:translateY(-1px)}}
|
|
@@ -284,39 +287,31 @@ async def admin_panel(pwd: str = ""):
|
|
| 284 |
<div class="header">
|
| 285 |
<h1>π οΈ Admin Panel</h1>
|
| 286 |
<div class="header-links">
|
| 287 |
-
<a href="/dashboard
|
| 288 |
<a href="/" class="hlink">β Chatbot</a>
|
| 289 |
</div>
|
| 290 |
</div>
|
| 291 |
-
|
| 292 |
-
<!-- Stats -->
|
| 293 |
<div class="grid">
|
| 294 |
<div class="stat"><div class="val">{total_chats}</div><div class="lbl">Total Chats</div></div>
|
| 295 |
<div class="stat"><div class="val" style="color:#16a34a">π {thumbs_up}</div><div class="lbl">Helpful</div></div>
|
| 296 |
<div class="stat"><div class="val" style="color:#dc2626">π {thumbs_down}</div><div class="lbl">Not Helpful</div></div>
|
| 297 |
<div class="stat"><div class="val">{satisfaction}%</div><div class="lbl">Satisfaction</div></div>
|
| 298 |
</div>
|
| 299 |
-
|
| 300 |
-
<!-- Knowledge Base Management -->
|
| 301 |
<div class="section">
|
| 302 |
<h2>π Knowledge Base Management</h2>
|
| 303 |
<div class="status-row">
|
| 304 |
-
<div class="status-badge">
|
| 305 |
<div class="status-badge">{db_status}</div>
|
| 306 |
</div>
|
| 307 |
-
<
|
| 308 |
-
|
| 309 |
-
<
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
<
|
| 313 |
-
</
|
| 314 |
-
</
|
| 315 |
-
<div class="alert" id="upload-alert">β
PDF uploaded and knowledge base rebuilt successfully!</div>
|
| 316 |
-
</div>
|
| 317 |
</div>
|
| 318 |
-
|
| 319 |
-
<!-- Data Management -->
|
| 320 |
<div class="section">
|
| 321 |
<h2>ποΈ Data Management</h2>
|
| 322 |
<p style="font-size:.82rem;color:#6b7280;margin-bottom:1rem">Manage stored feedback and chat logs. These actions cannot be undone.</p>
|
|
@@ -329,15 +324,11 @@ async def admin_panel(pwd: str = ""):
|
|
| 329 |
</form>
|
| 330 |
</div>
|
| 331 |
</div>
|
| 332 |
-
|
| 333 |
-
<!-- Recent Chat Logs -->
|
| 334 |
<div class="section">
|
| 335 |
<h2>π¬ Recent Chat Logs</h2>
|
| 336 |
{chat_table}
|
| 337 |
</div>
|
| 338 |
-
|
| 339 |
<script>
|
| 340 |
-
// Show upload success if redirected with success param
|
| 341 |
if(window.location.search.includes('success=1')){{
|
| 342 |
const a=document.getElementById('upload-alert');a.classList.add('show');
|
| 343 |
setTimeout(()=>a.classList.remove('show'),4000);
|
|
@@ -366,7 +357,6 @@ async def clear_feedback(pwd: str = ""):
|
|
| 366 |
if pwd != ADMIN_PASSWORD:
|
| 367 |
return RedirectResponse(url="/admin")
|
| 368 |
save_feedback([])
|
| 369 |
-
print("ποΈ Feedback data cleared")
|
| 370 |
return RedirectResponse(url=f"/admin/panel?pwd={pwd}", status_code=303)
|
| 371 |
|
| 372 |
@app.post("/admin/clear-chats")
|
|
@@ -374,7 +364,6 @@ async def clear_chats(pwd: str = ""):
|
|
| 374 |
if pwd != ADMIN_PASSWORD:
|
| 375 |
return RedirectResponse(url="/admin")
|
| 376 |
save_chat_log([])
|
| 377 |
-
print("ποΈ Chat logs cleared")
|
| 378 |
return RedirectResponse(url=f"/admin/panel?pwd={pwd}", status_code=303)
|
| 379 |
|
| 380 |
@app.get("/dashboard", response_class=HTMLResponse)
|
|
@@ -414,7 +403,8 @@ async def dashboard():
|
|
| 414 |
.stat-card .value{{font-size:2.2rem;font-weight:800;color:#16a34a}}.stat-card .label{{font-size:.75rem;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:.08em;margin-top:.3rem}}
|
| 415 |
.section{{background:white;border:1.5px solid #bbf7d0;border-radius:16px;padding:1.4rem;margin-bottom:1.5rem}}
|
| 416 |
.section h2{{font-size:.85rem;font-weight:700;color:#16a34a;text-transform:uppercase;letter-spacing:.08em;margin-bottom:1rem}}
|
| 417 |
-
table{{width:100%;border-collapse:collapse;font-size:.85rem}}
|
|
|
|
| 418 |
td{{padding:.6rem .8rem;border-top:1px solid #f0fdf4;color:#374151;vertical-align:top}}
|
| 419 |
.top-q-list{{list-style:none;display:flex;flex-direction:column;gap:.6rem}}
|
| 420 |
.top-q-list li{{display:flex;justify-content:space-between;align-items:center;padding:.6rem .8rem;background:#f0fdf4;border-radius:10px;font-size:.85rem}}
|
|
@@ -581,13 +571,11 @@ async def home():
|
|
| 581 |
function updateMemoryBadge(){const n=conversationHistory.length;memoryBadge.textContent=langConfig[currentLang].memoryText(n);}
|
| 582 |
function handleKey(e){if(e.key==='Enter'&&!e.shiftKey){e.preventDefault();sendMessage();}}
|
| 583 |
function useExample(btn){msgInput.value=btn.textContent;msgInput.focus();}
|
| 584 |
-
|
| 585 |
function clearChat(){
|
| 586 |
chatWindow.innerHTML='';chatWindow.appendChild(emptyState);emptyState.style.display='flex';
|
| 587 |
conversationHistory=[];lastQuestion='';updateMemoryBadge();
|
| 588 |
window.speechSynthesis&&window.speechSynthesis.cancel();
|
| 589 |
}
|
| 590 |
-
|
| 591 |
function appendMessage(role,text){
|
| 592 |
emptyState.style.display='none';
|
| 593 |
const row=document.createElement('div');row.className=`msg-row ${role}`;
|
|
@@ -607,14 +595,12 @@ async def home():
|
|
| 607 |
}
|
| 608 |
row.appendChild(avatar);row.appendChild(wrap);chatWindow.appendChild(row);chatWindow.scrollTop=chatWindow.scrollHeight;
|
| 609 |
}
|
| 610 |
-
|
| 611 |
async function submitFeedback(question,answer,rating,upBtn,downBtn,thanks){
|
| 612 |
upBtn.disabled=true;downBtn.disabled=true;
|
| 613 |
upBtn.classList.toggle('selected',rating==='up');downBtn.classList.toggle('selected',rating==='down');
|
| 614 |
thanks.style.display='inline';
|
| 615 |
await fetch('/feedback',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({question,answer,rating})});
|
| 616 |
}
|
| 617 |
-
|
| 618 |
function showTyping(){
|
| 619 |
emptyState.style.display='none';
|
| 620 |
const row=document.createElement('div');row.className='msg-row bot';row.id='typing-row';
|
|
@@ -623,9 +609,7 @@ async def home():
|
|
| 623 |
typing.innerHTML='<div class="dot"></div><div class="dot"></div><div class="dot"></div>';
|
| 624 |
row.appendChild(avatar);row.appendChild(typing);chatWindow.appendChild(row);chatWindow.scrollTop=chatWindow.scrollHeight;
|
| 625 |
}
|
| 626 |
-
|
| 627 |
function removeTyping(){const t=document.getElementById('typing-row');if(t)t.remove();}
|
| 628 |
-
|
| 629 |
async function sendMessage(){
|
| 630 |
const text=msgInput.value.trim();if(!text)return;
|
| 631 |
lastQuestion=text;msgInput.value='';sendBtn.disabled=true;
|
|
@@ -640,7 +624,6 @@ async def home():
|
|
| 640 |
}catch(err){removeTyping();appendMessage('bot','β οΈ Something went wrong. / ΰ€ΰ₯ΰ€ ΰ€ΰ€²ΰ€€ ΰ€Ήΰ₯ ΰ€ΰ€―ΰ€Ύΰ₯€');}
|
| 641 |
finally{sendBtn.disabled=false;msgInput.focus();}
|
| 642 |
}
|
| 643 |
-
|
| 644 |
const SpeechRecognition=window.SpeechRecognition||window.webkitSpeechRecognition;
|
| 645 |
let recognition=null,isListening=false;
|
| 646 |
if(SpeechRecognition){
|
|
@@ -649,11 +632,9 @@ async def home():
|
|
| 649 |
recognition.onresult=function(e){const t=e.results[0][0].transcript;msgInput.value=t;stopListening();sendMessage();};
|
| 650 |
recognition.onerror=function(){stopListening();};recognition.onend=function(){stopListening();};
|
| 651 |
}
|
| 652 |
-
|
| 653 |
function toggleVoice(){if(!recognition){alert('Use Chrome or Edge for voice.');return;}isListening?stopListening():startListening();}
|
| 654 |
function startListening(){isListening=true;if(recognition)recognition.lang=langConfig[currentLang].speechLang;const b=document.getElementById('voice-btn');b.classList.add('listening');b.textContent='π΄';voiceStatus.classList.add('visible');msgInput.placeholder=langConfig[currentLang].voiceStatus;recognition.start();}
|
| 655 |
function stopListening(){isListening=false;const b=document.getElementById('voice-btn');b.classList.remove('listening');b.textContent='π€';voiceStatus.classList.remove('visible');msgInput.placeholder=langConfig[currentLang].placeholder;try{recognition.stop();}catch(e){}}
|
| 656 |
-
|
| 657 |
function speakAnswer(text){
|
| 658 |
if(!window.speechSynthesis)return;window.speechSynthesis.cancel();
|
| 659 |
const u=new SpeechSynthesisUtterance(text);
|
|
@@ -665,5 +646,5 @@ async def home():
|
|
| 665 |
</body>
|
| 666 |
</html>""")
|
| 667 |
|
| 668 |
-
print("π Launching at http://
|
| 669 |
uvicorn.run(app, host="0.0.0.0", port=7860)
|
|
|
|
| 1 |
import os
|
| 2 |
import json
|
|
|
|
| 3 |
import warnings
|
| 4 |
warnings.filterwarnings("ignore")
|
| 5 |
|
| 6 |
from dotenv import load_dotenv
|
| 7 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
|
|
|
| 8 |
from langchain_groq import ChatGroq
|
| 9 |
from langchain_core.prompts import PromptTemplate
|
| 10 |
from langchain_core.output_parsers import StrOutputParser
|
| 11 |
from langchain_community.document_loaders import PyPDFLoader
|
| 12 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 13 |
+
from langchain_pinecone import PineconeVectorStore
|
| 14 |
+
from pinecone import Pinecone, ServerlessSpec
|
| 15 |
from fastapi import FastAPI, UploadFile, File, Form, Request
|
| 16 |
from fastapi.responses import HTMLResponse, RedirectResponse
|
| 17 |
from pydantic import BaseModel
|
|
|
|
| 22 |
|
| 23 |
load_dotenv()
|
| 24 |
|
| 25 |
+
# ββ Config ββ
|
| 26 |
+
ADMIN_PASSWORD = "admin123"
|
| 27 |
+
CHAT_LOG_FILE = "chat_log.json"
|
| 28 |
+
FEEDBACK_FILE = "feedback.json"
|
| 29 |
+
INDEX_NAME = "college-chatbot"
|
| 30 |
+
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
|
| 31 |
|
| 32 |
+
print("π Loading embedding model...")
|
| 33 |
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
|
| 34 |
+
|
| 35 |
+
print("βοΈ Connecting to Pinecone...")
|
| 36 |
+
pc = Pinecone(api_key=PINECONE_API_KEY)
|
| 37 |
+
vectorstore = PineconeVectorStore(
|
| 38 |
+
index_name=INDEX_NAME,
|
| 39 |
+
embedding=embeddings,
|
| 40 |
+
pinecone_api_key=PINECONE_API_KEY
|
| 41 |
+
)
|
| 42 |
+
retriever = vectorstore.as_retriever(search_kwargs={"k": 6})
|
| 43 |
|
| 44 |
print("π€ Connecting to Groq LLM...")
|
| 45 |
llm = ChatGroq(
|
|
|
|
| 91 |
return chain.invoke({"context": context, "history": formatted_history, "question": question})
|
| 92 |
|
| 93 |
def rebuild_knowledge_base(pdf_path: str):
|
| 94 |
+
global vectorstore, retriever
|
|
|
|
| 95 |
loader = PyPDFLoader(pdf_path)
|
| 96 |
documents = loader.load()
|
| 97 |
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
|
|
|
|
| 103 |
seen.add(text)
|
| 104 |
unique_chunks.append(chunk)
|
| 105 |
|
| 106 |
+
# ββ Clear existing Pinecone index and re-upload ββ
|
| 107 |
try:
|
| 108 |
+
pc.Index(INDEX_NAME).delete(delete_all=True)
|
| 109 |
+
print("β
Old Pinecone data cleared")
|
| 110 |
+
except Exception as e:
|
| 111 |
+
print(f"β οΈ Could not clear index: {e}")
|
| 112 |
|
| 113 |
+
vectorstore = PineconeVectorStore.from_documents(
|
|
|
|
|
|
|
| 114 |
documents=unique_chunks,
|
| 115 |
embedding=embeddings,
|
| 116 |
+
index_name=INDEX_NAME,
|
| 117 |
+
pinecone_api_key=PINECONE_API_KEY
|
| 118 |
)
|
| 119 |
+
retriever = vectorstore.as_retriever(search_kwargs={"k": 6})
|
| 120 |
+
print(f"β
Knowledge base rebuilt with {len(unique_chunks)} chunks in Pinecone")
|
|
|
|
| 121 |
|
| 122 |
def load_feedback():
|
| 123 |
if os.path.exists(FEEDBACK_FILE):
|
|
|
|
| 170 |
print(f"{'π' if payload.rating == 'up' else 'π'} Feedback: {payload.question[:50]}")
|
| 171 |
return {"status": "saved"}
|
| 172 |
|
|
|
|
| 173 |
@app.get("/admin", response_class=HTMLResponse)
|
| 174 |
async def admin_login():
|
| 175 |
return HTMLResponse(content="""<!DOCTYPE html>
|
|
|
|
| 190 |
input:focus{border-color:#22c55e}
|
| 191 |
button{width:100%;background:#16a34a;border:none;border-radius:12px;color:white;font-family:'Plus Jakarta Sans',sans-serif;font-weight:700;font-size:.9rem;padding:.75rem;cursor:pointer;transition:all .2s}
|
| 192 |
button:hover{background:#15803d;transform:translateY(-1px)}
|
|
|
|
| 193 |
.back{display:inline-block;margin-top:1rem;font-size:.78rem;color:#16a34a;text-decoration:none;font-weight:600}
|
| 194 |
</style>
|
| 195 |
</head>
|
|
|
|
| 202 |
<input type="password" name="password" placeholder="Enter admin password" required autofocus/>
|
| 203 |
<button type="submit">Login β</button>
|
| 204 |
</form>
|
|
|
|
| 205 |
<a href="/" class="back">β Back to Chatbot</a>
|
| 206 |
</div>
|
| 207 |
</body>
|
|
|
|
| 212 |
if password != ADMIN_PASSWORD:
|
| 213 |
return HTMLResponse(content="""<!DOCTYPE html>
|
| 214 |
<html><head><meta charset="UTF-8"/>
|
| 215 |
+
<style>*{box-sizing:border-box;margin:0;padding:0}body{background:#f0faf4;font-family:sans-serif;min-height:100vh;display:flex;align-items:center;justify-content:center}.card{background:white;border:1.5px solid #fecaca;border-radius:24px;padding:2.5rem 2rem;width:100%;max-width:380px;text-align:center}.icon{font-size:2.5rem;margin-bottom:1rem}h1{color:#dc2626;font-size:1.2rem;margin-bottom:1rem}a{display:inline-block;background:#16a34a;color:white;padding:.6rem 1.4rem;border-radius:10px;text-decoration:none;font-weight:700;font-size:.85rem}</style>
|
|
|
|
| 216 |
</head><body><div class="card"><div class="icon">β</div><h1>Incorrect Password</h1><a href="/admin">Try Again</a></div></body></html>""")
|
| 217 |
return RedirectResponse(url=f"/admin/panel?pwd={password}", status_code=303)
|
| 218 |
|
|
|
|
| 238 |
|
| 239 |
chat_table = f"<table><thead><tr><th>Time</th><th>Question</th><th>Answer</th></tr></thead><tbody>{chat_rows}</tbody></table>" if chat_rows else "<div class='empty'>No chats yet!</div>"
|
| 240 |
|
| 241 |
+
# ββ Pinecone status ββ
|
| 242 |
+
try:
|
| 243 |
+
index_info = pc.Index(INDEX_NAME).describe_index_stats()
|
| 244 |
+
total_vectors = index_info.get("total_vector_count", 0)
|
| 245 |
+
db_status = f"β
Pinecone connected β {total_vectors} vectors"
|
| 246 |
+
except Exception:
|
| 247 |
+
db_status = "β οΈ Pinecone connection issue"
|
| 248 |
|
| 249 |
return HTMLResponse(content=f"""<!DOCTYPE html>
|
| 250 |
<html lang="en">
|
|
|
|
| 259 |
.header{{background:#0a2e1a;color:#4ade80;padding:1.5rem 2rem;border-radius:16px;margin-bottom:1.5rem;display:flex;justify-content:space-between;align-items:center}}
|
| 260 |
.header h1{{font-size:1.4rem;font-weight:800}}
|
| 261 |
.header-links{{display:flex;gap:.6rem}}
|
| 262 |
+
.hlink{{background:rgba(74,222,128,.15);border:1px solid rgba(74,222,128,.3);border-radius:8px;color:#4ade80;font-size:.72rem;font-weight:700;padding:.35rem .8rem;text-decoration:none}}
|
| 263 |
.hlink:hover{{background:rgba(74,222,128,.25)}}
|
| 264 |
.grid{{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:1rem;margin-bottom:1.5rem}}
|
| 265 |
.stat{{background:white;border:1.5px solid #bbf7d0;border-radius:16px;padding:1.2rem;text-align:center;box-shadow:0 2px 12px rgba(34,197,94,.08)}}
|
|
|
|
| 269 |
.section h2{{font-size:.85rem;font-weight:700;color:#16a34a;text-transform:uppercase;letter-spacing:.08em;margin-bottom:1rem}}
|
| 270 |
.status-row{{display:flex;gap:1rem;flex-wrap:wrap;margin-bottom:1rem}}
|
| 271 |
.status-badge{{background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;padding:.4rem .8rem;font-size:.8rem;color:#14532d;font-weight:600}}
|
|
|
|
|
|
|
|
|
|
| 272 |
.btn{{border:none;border-radius:12px;font-family:'Plus Jakarta Sans',sans-serif;font-weight:700;font-size:.85rem;padding:.65rem 1.3rem;cursor:pointer;transition:all .2s;white-space:nowrap}}
|
| 273 |
.btn-green{{background:#16a34a;color:white;box-shadow:0 2px 10px rgba(22,163,74,.3)}}
|
| 274 |
.btn-green:hover{{background:#15803d;transform:translateY(-1px)}}
|
|
|
|
| 287 |
<div class="header">
|
| 288 |
<h1>π οΈ Admin Panel</h1>
|
| 289 |
<div class="header-links">
|
| 290 |
+
<a href="/dashboard" class="hlink">π Dashboard</a>
|
| 291 |
<a href="/" class="hlink">β Chatbot</a>
|
| 292 |
</div>
|
| 293 |
</div>
|
|
|
|
|
|
|
| 294 |
<div class="grid">
|
| 295 |
<div class="stat"><div class="val">{total_chats}</div><div class="lbl">Total Chats</div></div>
|
| 296 |
<div class="stat"><div class="val" style="color:#16a34a">π {thumbs_up}</div><div class="lbl">Helpful</div></div>
|
| 297 |
<div class="stat"><div class="val" style="color:#dc2626">π {thumbs_down}</div><div class="lbl">Not Helpful</div></div>
|
| 298 |
<div class="stat"><div class="val">{satisfaction}%</div><div class="lbl">Satisfaction</div></div>
|
| 299 |
</div>
|
|
|
|
|
|
|
| 300 |
<div class="section">
|
| 301 |
<h2>π Knowledge Base Management</h2>
|
| 302 |
<div class="status-row">
|
| 303 |
+
<div class="status-badge">βοΈ Pinecone Cloud DB</div>
|
| 304 |
<div class="status-badge">{db_status}</div>
|
| 305 |
</div>
|
| 306 |
+
<p style="font-size:.82rem;color:#6b7280;margin-bottom:.8rem">Upload a new college PDF to update the Pinecone knowledge base in the cloud.</p>
|
| 307 |
+
<form method="POST" action="/admin/upload?pwd={pwd}" enctype="multipart/form-data">
|
| 308 |
+
<input type="file" name="file" accept=".pdf" required style="border:2px dashed #bbf7d0;border-radius:12px;padding:1rem;width:100%;font-size:.85rem;color:#14532d;background:#f0fdf4;cursor:pointer;margin-bottom:.8rem"/>
|
| 309 |
+
<div class="btn-row">
|
| 310 |
+
<button type="submit" class="btn btn-green">π€ Upload & Rebuild Knowledge Base</button>
|
| 311 |
+
</div>
|
| 312 |
+
</form>
|
| 313 |
+
<div class="alert" id="upload-alert">β
PDF uploaded and Pinecone knowledge base rebuilt successfully!</div>
|
|
|
|
|
|
|
| 314 |
</div>
|
|
|
|
|
|
|
| 315 |
<div class="section">
|
| 316 |
<h2>ποΈ Data Management</h2>
|
| 317 |
<p style="font-size:.82rem;color:#6b7280;margin-bottom:1rem">Manage stored feedback and chat logs. These actions cannot be undone.</p>
|
|
|
|
| 324 |
</form>
|
| 325 |
</div>
|
| 326 |
</div>
|
|
|
|
|
|
|
| 327 |
<div class="section">
|
| 328 |
<h2>π¬ Recent Chat Logs</h2>
|
| 329 |
{chat_table}
|
| 330 |
</div>
|
|
|
|
| 331 |
<script>
|
|
|
|
| 332 |
if(window.location.search.includes('success=1')){{
|
| 333 |
const a=document.getElementById('upload-alert');a.classList.add('show');
|
| 334 |
setTimeout(()=>a.classList.remove('show'),4000);
|
|
|
|
| 357 |
if pwd != ADMIN_PASSWORD:
|
| 358 |
return RedirectResponse(url="/admin")
|
| 359 |
save_feedback([])
|
|
|
|
| 360 |
return RedirectResponse(url=f"/admin/panel?pwd={pwd}", status_code=303)
|
| 361 |
|
| 362 |
@app.post("/admin/clear-chats")
|
|
|
|
| 364 |
if pwd != ADMIN_PASSWORD:
|
| 365 |
return RedirectResponse(url="/admin")
|
| 366 |
save_chat_log([])
|
|
|
|
| 367 |
return RedirectResponse(url=f"/admin/panel?pwd={pwd}", status_code=303)
|
| 368 |
|
| 369 |
@app.get("/dashboard", response_class=HTMLResponse)
|
|
|
|
| 403 |
.stat-card .value{{font-size:2.2rem;font-weight:800;color:#16a34a}}.stat-card .label{{font-size:.75rem;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:.08em;margin-top:.3rem}}
|
| 404 |
.section{{background:white;border:1.5px solid #bbf7d0;border-radius:16px;padding:1.4rem;margin-bottom:1.5rem}}
|
| 405 |
.section h2{{font-size:.85rem;font-weight:700;color:#16a34a;text-transform:uppercase;letter-spacing:.08em;margin-bottom:1rem}}
|
| 406 |
+
table{{width:100%;border-collapse:collapse;font-size:.85rem}}
|
| 407 |
+
th{{background:#f0fdf4;color:#16a34a;font-weight:700;padding:.6rem .8rem;text-align:left;font-size:.75rem;text-transform:uppercase}}
|
| 408 |
td{{padding:.6rem .8rem;border-top:1px solid #f0fdf4;color:#374151;vertical-align:top}}
|
| 409 |
.top-q-list{{list-style:none;display:flex;flex-direction:column;gap:.6rem}}
|
| 410 |
.top-q-list li{{display:flex;justify-content:space-between;align-items:center;padding:.6rem .8rem;background:#f0fdf4;border-radius:10px;font-size:.85rem}}
|
|
|
|
| 571 |
function updateMemoryBadge(){const n=conversationHistory.length;memoryBadge.textContent=langConfig[currentLang].memoryText(n);}
|
| 572 |
function handleKey(e){if(e.key==='Enter'&&!e.shiftKey){e.preventDefault();sendMessage();}}
|
| 573 |
function useExample(btn){msgInput.value=btn.textContent;msgInput.focus();}
|
|
|
|
| 574 |
function clearChat(){
|
| 575 |
chatWindow.innerHTML='';chatWindow.appendChild(emptyState);emptyState.style.display='flex';
|
| 576 |
conversationHistory=[];lastQuestion='';updateMemoryBadge();
|
| 577 |
window.speechSynthesis&&window.speechSynthesis.cancel();
|
| 578 |
}
|
|
|
|
| 579 |
function appendMessage(role,text){
|
| 580 |
emptyState.style.display='none';
|
| 581 |
const row=document.createElement('div');row.className=`msg-row ${role}`;
|
|
|
|
| 595 |
}
|
| 596 |
row.appendChild(avatar);row.appendChild(wrap);chatWindow.appendChild(row);chatWindow.scrollTop=chatWindow.scrollHeight;
|
| 597 |
}
|
|
|
|
| 598 |
async function submitFeedback(question,answer,rating,upBtn,downBtn,thanks){
|
| 599 |
upBtn.disabled=true;downBtn.disabled=true;
|
| 600 |
upBtn.classList.toggle('selected',rating==='up');downBtn.classList.toggle('selected',rating==='down');
|
| 601 |
thanks.style.display='inline';
|
| 602 |
await fetch('/feedback',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({question,answer,rating})});
|
| 603 |
}
|
|
|
|
| 604 |
function showTyping(){
|
| 605 |
emptyState.style.display='none';
|
| 606 |
const row=document.createElement('div');row.className='msg-row bot';row.id='typing-row';
|
|
|
|
| 609 |
typing.innerHTML='<div class="dot"></div><div class="dot"></div><div class="dot"></div>';
|
| 610 |
row.appendChild(avatar);row.appendChild(typing);chatWindow.appendChild(row);chatWindow.scrollTop=chatWindow.scrollHeight;
|
| 611 |
}
|
|
|
|
| 612 |
function removeTyping(){const t=document.getElementById('typing-row');if(t)t.remove();}
|
|
|
|
| 613 |
async function sendMessage(){
|
| 614 |
const text=msgInput.value.trim();if(!text)return;
|
| 615 |
lastQuestion=text;msgInput.value='';sendBtn.disabled=true;
|
|
|
|
| 624 |
}catch(err){removeTyping();appendMessage('bot','β οΈ Something went wrong. / ΰ€ΰ₯ΰ€ ΰ€ΰ€²ΰ€€ ΰ€Ήΰ₯ ΰ€ΰ€―ΰ€Ύΰ₯€');}
|
| 625 |
finally{sendBtn.disabled=false;msgInput.focus();}
|
| 626 |
}
|
|
|
|
| 627 |
const SpeechRecognition=window.SpeechRecognition||window.webkitSpeechRecognition;
|
| 628 |
let recognition=null,isListening=false;
|
| 629 |
if(SpeechRecognition){
|
|
|
|
| 632 |
recognition.onresult=function(e){const t=e.results[0][0].transcript;msgInput.value=t;stopListening();sendMessage();};
|
| 633 |
recognition.onerror=function(){stopListening();};recognition.onend=function(){stopListening();};
|
| 634 |
}
|
|
|
|
| 635 |
function toggleVoice(){if(!recognition){alert('Use Chrome or Edge for voice.');return;}isListening?stopListening():startListening();}
|
| 636 |
function startListening(){isListening=true;if(recognition)recognition.lang=langConfig[currentLang].speechLang;const b=document.getElementById('voice-btn');b.classList.add('listening');b.textContent='π΄';voiceStatus.classList.add('visible');msgInput.placeholder=langConfig[currentLang].voiceStatus;recognition.start();}
|
| 637 |
function stopListening(){isListening=false;const b=document.getElementById('voice-btn');b.classList.remove('listening');b.textContent='π€';voiceStatus.classList.remove('visible');msgInput.placeholder=langConfig[currentLang].placeholder;try{recognition.stop();}catch(e){}}
|
|
|
|
| 638 |
function speakAnswer(text){
|
| 639 |
if(!window.speechSynthesis)return;window.speechSynthesis.cancel();
|
| 640 |
const u=new SpeechSynthesisUtterance(text);
|
|
|
|
| 646 |
</body>
|
| 647 |
</html>""")
|
| 648 |
|
| 649 |
+
print("π Launching at http://0.0.0.0:7860")
|
| 650 |
uvicorn.run(app, host="0.0.0.0", port=7860)
|
ingest.py
CHANGED
|
@@ -5,10 +5,15 @@ from dotenv import load_dotenv
|
|
| 5 |
from langchain_community.document_loaders import PyPDFLoader
|
| 6 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 7 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 8 |
-
from
|
|
|
|
| 9 |
|
| 10 |
load_dotenv()
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
print("π Loading your college PDF...")
|
| 13 |
loader = PyPDFLoader("data/college_info.pdf")
|
| 14 |
documents = loader.load()
|
|
@@ -16,8 +21,8 @@ print(f"β
Loaded {len(documents)} pages")
|
|
| 16 |
|
| 17 |
print("βοΈ Splitting into chunks...")
|
| 18 |
splitter = RecursiveCharacterTextSplitter(
|
| 19 |
-
chunk_size=1000,
|
| 20 |
-
chunk_overlap=100
|
| 21 |
)
|
| 22 |
chunks = splitter.split_documents(documents)
|
| 23 |
print(f"π¦ Created {len(chunks)} chunks before deduplication")
|
|
@@ -37,11 +42,34 @@ chunks = unique_chunks
|
|
| 37 |
print("π§ Loading embedding model...")
|
| 38 |
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
|
| 39 |
|
| 40 |
-
print("
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
documents=chunks,
|
| 43 |
embedding=embeddings,
|
| 44 |
-
|
|
|
|
| 45 |
)
|
| 46 |
-
|
| 47 |
-
print("
|
|
|
|
| 5 |
from langchain_community.document_loaders import PyPDFLoader
|
| 6 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 7 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 8 |
+
from pinecone import Pinecone, ServerlessSpec
|
| 9 |
+
from langchain_pinecone import PineconeVectorStore
|
| 10 |
|
| 11 |
load_dotenv()
|
| 12 |
|
| 13 |
+
# ββ Pinecone config ββ
|
| 14 |
+
PINECONE_API_KEY = os.getenv("pcsk_b7nf9_BxgcNVXmQiJJ3t8WQrWkwo3CQCSjm2SjCuJExLSUGVycQP2ch3RLnbN8ToSxQsR")
|
| 15 |
+
INDEX_NAME = "college-chatbot"
|
| 16 |
+
|
| 17 |
print("π Loading your college PDF...")
|
| 18 |
loader = PyPDFLoader("data/college_info.pdf")
|
| 19 |
documents = loader.load()
|
|
|
|
| 21 |
|
| 22 |
print("βοΈ Splitting into chunks...")
|
| 23 |
splitter = RecursiveCharacterTextSplitter(
|
| 24 |
+
chunk_size=1000,
|
| 25 |
+
chunk_overlap=100
|
| 26 |
)
|
| 27 |
chunks = splitter.split_documents(documents)
|
| 28 |
print(f"π¦ Created {len(chunks)} chunks before deduplication")
|
|
|
|
| 42 |
print("π§ Loading embedding model...")
|
| 43 |
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
|
| 44 |
|
| 45 |
+
print("βοΈ Connecting to Pinecone...")
|
| 46 |
+
pc = Pinecone(api_key=PINECONE_API_KEY)
|
| 47 |
+
|
| 48 |
+
# ββ Create index if it doesn't exist ββ
|
| 49 |
+
existing_indexes = [index.name for index in pc.list_indexes()]
|
| 50 |
+
if INDEX_NAME not in existing_indexes:
|
| 51 |
+
print(f"π Creating new Pinecone index: {INDEX_NAME}")
|
| 52 |
+
pc.create_index(
|
| 53 |
+
name=INDEX_NAME,
|
| 54 |
+
dimension=384, # all-MiniLM-L6-v2 produces 384-dim vectors
|
| 55 |
+
metric="cosine",
|
| 56 |
+
spec=ServerlessSpec(
|
| 57 |
+
cloud="aws",
|
| 58 |
+
region="us-east-1"
|
| 59 |
+
)
|
| 60 |
+
)
|
| 61 |
+
print(f"β
Index '{INDEX_NAME}' created!")
|
| 62 |
+
else:
|
| 63 |
+
print(f"β
Index '{INDEX_NAME}' already exists β clearing old data...")
|
| 64 |
+
pc.Index(INDEX_NAME).delete(delete_all=True)
|
| 65 |
+
print("β
Old data cleared!")
|
| 66 |
+
|
| 67 |
+
print("πΎ Uploading chunks to Pinecone...")
|
| 68 |
+
vectorstore = PineconeVectorStore.from_documents(
|
| 69 |
documents=chunks,
|
| 70 |
embedding=embeddings,
|
| 71 |
+
index_name=INDEX_NAME,
|
| 72 |
+
pinecone_api_key=PINECONE_API_KEY
|
| 73 |
)
|
| 74 |
+
print(f"β
All done! {len(chunks)} chunks uploaded to Pinecone cloud.")
|
| 75 |
+
print(f"π Your knowledge base is now live in the cloud!")
|
requirements.txt
CHANGED
|
@@ -5,9 +5,10 @@ langchain-community
|
|
| 5 |
langchain-groq
|
| 6 |
langchain-core
|
| 7 |
langchain-text-splitters
|
| 8 |
-
|
|
|
|
| 9 |
sentence-transformers
|
| 10 |
pypdf
|
| 11 |
python-dotenv
|
| 12 |
pydantic
|
| 13 |
-
python-multipart
|
|
|
|
| 5 |
langchain-groq
|
| 6 |
langchain-core
|
| 7 |
langchain-text-splitters
|
| 8 |
+
langchain-pinecone
|
| 9 |
+
pinecone-client
|
| 10 |
sentence-transformers
|
| 11 |
pypdf
|
| 12 |
python-dotenv
|
| 13 |
pydantic
|
| 14 |
+
python-multipart
|