Spaces:
Running
Running
github-actions[bot] commited on
Commit ·
bd44d71
1
Parent(s): 15dd601
Deploy from GitHub Actions: 27118b812764ca9d2a18f390dbfd0affb5b9029d
Browse files- app/config.py +4 -4
- app/deps.py +6 -0
- app/main.py +2 -0
- app/rag/loader.py +3 -3
- app/rag/vectorstore.py +9 -4
- app/routes/ask.py +5 -3
- app/routes/auth.py +8 -0
- app/routes/chats.py +16 -8
- app/routes/clear.py +7 -6
- app/routes/delete.py +5 -3
- app/routes/upload.py +9 -3
- requirements.txt +2 -3
app/config.py
CHANGED
|
@@ -20,8 +20,8 @@ EMBED_MODEL = "models/gemini-embedding-001"
|
|
| 20 |
CHAT_MODEL = "gemini-2.5-flash-lite"
|
| 21 |
|
| 22 |
TOP_K = 10
|
| 23 |
-
CHUNK_SIZE =
|
| 24 |
-
CHUNK_OVERLAP =
|
| 25 |
UPLOAD_BATCH_SIZE = 100
|
| 26 |
|
| 27 |
MAX_FILE_COUNT = 6
|
|
@@ -49,6 +49,6 @@ PROMPT = (
|
|
| 49 |
)
|
| 50 |
|
| 51 |
CREATORS = [
|
| 52 |
-
{"
|
| 53 |
-
{"
|
| 54 |
]
|
|
|
|
| 20 |
CHAT_MODEL = "gemini-2.5-flash-lite"
|
| 21 |
|
| 22 |
TOP_K = 10
|
| 23 |
+
CHUNK_SIZE = 800
|
| 24 |
+
CHUNK_OVERLAP = 100
|
| 25 |
UPLOAD_BATCH_SIZE = 100
|
| 26 |
|
| 27 |
MAX_FILE_COUNT = 6
|
|
|
|
| 49 |
)
|
| 50 |
|
| 51 |
CREATORS = [
|
| 52 |
+
{"Krishnendu Das" : "https://itskdhere.com"},
|
| 53 |
+
{"Saptarshi Roy" : "https://hirishi.in"}
|
| 54 |
]
|
app/deps.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import Header, HTTPException
|
| 2 |
+
|
| 3 |
+
async def get_user_id(x_user_id: str = Header(None)) -> str:
|
| 4 |
+
if not x_user_id:
|
| 5 |
+
raise HTTPException(status_code=401, detail="User ID required")
|
| 6 |
+
return x_user_id
|
app/main.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from app.routes.ask import router as ask_router
|
| 2 |
from app.routes.delete import router as delete_router
|
| 3 |
from app.routes.clear import router as clear_router
|
|
@@ -27,6 +28,7 @@ app.add_middleware(
|
|
| 27 |
allow_headers=["*"],
|
| 28 |
)
|
| 29 |
|
|
|
|
| 30 |
app.include_router(ask_router)
|
| 31 |
app.include_router(upload_router)
|
| 32 |
app.include_router(delete_router)
|
|
|
|
| 1 |
+
from app.routes.auth import router as auth_router
|
| 2 |
from app.routes.ask import router as ask_router
|
| 3 |
from app.routes.delete import router as delete_router
|
| 4 |
from app.routes.clear import router as clear_router
|
|
|
|
| 28 |
allow_headers=["*"],
|
| 29 |
)
|
| 30 |
|
| 31 |
+
app.include_router(auth_router)
|
| 32 |
app.include_router(ask_router)
|
| 33 |
app.include_router(upload_router)
|
| 34 |
app.include_router(delete_router)
|
app/rag/loader.py
CHANGED
|
@@ -2,19 +2,19 @@ from langchain_community.document_loaders import (
|
|
| 2 |
CSVLoader,
|
| 3 |
Docx2txtLoader,
|
| 4 |
JSONLoader,
|
| 5 |
-
PDFPlumberLoader,
|
| 6 |
TextLoader,
|
| 7 |
UnstructuredExcelLoader,
|
| 8 |
UnstructuredMarkdownLoader,
|
| 9 |
UnstructuredPowerPointLoader,
|
| 10 |
)
|
|
|
|
| 11 |
from langchain_core.documents import Document
|
| 12 |
|
| 13 |
|
| 14 |
# PDF
|
| 15 |
-
# https://
|
| 16 |
def read_pdf(path: str) -> list[Document]:
|
| 17 |
-
loader =
|
| 18 |
docs = loader.load()
|
| 19 |
return docs
|
| 20 |
|
|
|
|
| 2 |
CSVLoader,
|
| 3 |
Docx2txtLoader,
|
| 4 |
JSONLoader,
|
|
|
|
| 5 |
TextLoader,
|
| 6 |
UnstructuredExcelLoader,
|
| 7 |
UnstructuredMarkdownLoader,
|
| 8 |
UnstructuredPowerPointLoader,
|
| 9 |
)
|
| 10 |
+
from langchain_pymupdf4llm import PyMuPDF4LLMLoader
|
| 11 |
from langchain_core.documents import Document
|
| 12 |
|
| 13 |
|
| 14 |
# PDF
|
| 15 |
+
# https://docs.langchain.com/oss/python/integrations/document_loaders/pymupdf4llm
|
| 16 |
def read_pdf(path: str) -> list[Document]:
|
| 17 |
+
loader = PyMuPDF4LLMLoader(path)
|
| 18 |
docs = loader.load()
|
| 19 |
return docs
|
| 20 |
|
app/rag/vectorstore.py
CHANGED
|
@@ -52,22 +52,27 @@ def delete_vectorstore(session_id: str) -> bool:
|
|
| 52 |
return False
|
| 53 |
|
| 54 |
|
| 55 |
-
def
|
| 56 |
try:
|
| 57 |
index = _get_index()
|
| 58 |
stats = index.describe_index_stats()
|
| 59 |
namespaces = list(stats.namespaces.keys())
|
| 60 |
failed: list[str] = []
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
try:
|
| 63 |
index.delete(delete_all=True, namespace=ns)
|
| 64 |
except Exception as e:
|
| 65 |
print(f"Failed to delete namespace '{ns}': {e}")
|
| 66 |
failed.append(ns)
|
|
|
|
| 67 |
if failed:
|
| 68 |
-
print(f"
|
| 69 |
return False
|
| 70 |
return True
|
| 71 |
except Exception as e:
|
| 72 |
-
print(f"
|
| 73 |
return False
|
|
|
|
| 52 |
return False
|
| 53 |
|
| 54 |
|
| 55 |
+
def delete_user_vectorstores(user_id: str) -> bool:
|
| 56 |
try:
|
| 57 |
index = _get_index()
|
| 58 |
stats = index.describe_index_stats()
|
| 59 |
namespaces = list(stats.namespaces.keys())
|
| 60 |
failed: list[str] = []
|
| 61 |
+
prefix = f"{user_id}_"
|
| 62 |
+
|
| 63 |
+
target_namespaces = [ns for ns in namespaces if ns.startswith(prefix)]
|
| 64 |
+
|
| 65 |
+
for ns in target_namespaces:
|
| 66 |
try:
|
| 67 |
index.delete(delete_all=True, namespace=ns)
|
| 68 |
except Exception as e:
|
| 69 |
print(f"Failed to delete namespace '{ns}': {e}")
|
| 70 |
failed.append(ns)
|
| 71 |
+
|
| 72 |
if failed:
|
| 73 |
+
print(f"delete_user_vectorstores: {len(failed)}/{len(target_namespaces)} namespaces failed: {failed}")
|
| 74 |
return False
|
| 75 |
return True
|
| 76 |
except Exception as e:
|
| 77 |
+
print(f"delete_user_vectorstores: unexpected error: {e}")
|
| 78 |
return False
|
app/routes/ask.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from app.config import CHAT_MODEL, GOOGLE_API_KEY, PROMPT, TOP_K
|
| 2 |
from app.rag.vectorstore import get_vectorstore
|
| 3 |
-
from fastapi import APIRouter, HTTPException
|
|
|
|
| 4 |
from langchain_core.messages import HumanMessage
|
| 5 |
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 6 |
from pydantic import BaseModel
|
|
@@ -20,8 +21,9 @@ class AskResponse(BaseModel):
|
|
| 20 |
|
| 21 |
|
| 22 |
@router.post("/ask")
|
| 23 |
-
async def ask(body: AskRequest) -> AskResponse:
|
| 24 |
-
|
|
|
|
| 25 |
docs = store.similarity_search(body.question, k=TOP_K)
|
| 26 |
|
| 27 |
if not docs:
|
|
|
|
| 1 |
from app.config import CHAT_MODEL, GOOGLE_API_KEY, PROMPT, TOP_K
|
| 2 |
from app.rag.vectorstore import get_vectorstore
|
| 3 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 4 |
+
from app.deps import get_user_id
|
| 5 |
from langchain_core.messages import HumanMessage
|
| 6 |
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 7 |
from pydantic import BaseModel
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
@router.post("/ask")
|
| 24 |
+
async def ask(body: AskRequest, user_id: str = Depends(get_user_id)) -> AskResponse:
|
| 25 |
+
prefixed_session_id = f"{user_id}_{body.session_id}"
|
| 26 |
+
store = get_vectorstore(prefixed_session_id)
|
| 27 |
docs = store.similarity_search(body.question, k=TOP_K)
|
| 28 |
|
| 29 |
if not docs:
|
app/routes/auth.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
+
from fastapi import APIRouter
|
| 3 |
+
|
| 4 |
+
router = APIRouter(prefix="/auth", tags=["auth"])
|
| 5 |
+
|
| 6 |
+
@router.get("/guest")
|
| 7 |
+
async def create_guest_id() -> dict:
|
| 8 |
+
return {"user_id": f"guest_{uuid.uuid4().hex[:12]}"}
|
app/routes/chats.py
CHANGED
|
@@ -1,28 +1,36 @@
|
|
| 1 |
import re
|
| 2 |
-
from datetime import datetime
|
| 3 |
from app.rag.vectorstore import _get_index
|
| 4 |
-
from fastapi import APIRouter, HTTPException
|
|
|
|
| 5 |
|
| 6 |
router = APIRouter()
|
| 7 |
|
| 8 |
|
| 9 |
@router.get("/chats")
|
| 10 |
-
async def get_chats() -> dict:
|
| 11 |
try:
|
| 12 |
index = _get_index()
|
| 13 |
stats = index.describe_index_stats()
|
| 14 |
|
| 15 |
namespaces = list(stats.namespaces.keys())
|
| 16 |
chats = []
|
|
|
|
|
|
|
| 17 |
for ns in namespaces:
|
| 18 |
-
if not
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
continue
|
| 20 |
|
| 21 |
-
timestamp = int(
|
| 22 |
-
dt = datetime.fromtimestamp(timestamp)
|
| 23 |
|
| 24 |
chats.append({
|
| 25 |
-
"id":
|
| 26 |
"title": f"Analysis {dt.strftime('%H:%M:%S')}",
|
| 27 |
"date": dt.strftime('%Y-%m-%d')
|
| 28 |
})
|
|
@@ -30,5 +38,5 @@ async def get_chats() -> dict:
|
|
| 30 |
chats.sort(key=lambda x: int(x["id"]), reverse=True)
|
| 31 |
return {"chats": chats}
|
| 32 |
except Exception as e:
|
| 33 |
-
print(f"Error fetching chats: {e}")
|
| 34 |
raise HTTPException(500, "Failed to fetch chats from Pinecone.")
|
|
|
|
| 1 |
import re
|
| 2 |
+
from datetime import datetime, timezone
|
| 3 |
from app.rag.vectorstore import _get_index
|
| 4 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 5 |
+
from app.deps import get_user_id
|
| 6 |
|
| 7 |
router = APIRouter()
|
| 8 |
|
| 9 |
|
| 10 |
@router.get("/chats")
|
| 11 |
+
async def get_chats(user_id: str = Depends(get_user_id)) -> dict:
|
| 12 |
try:
|
| 13 |
index = _get_index()
|
| 14 |
stats = index.describe_index_stats()
|
| 15 |
|
| 16 |
namespaces = list(stats.namespaces.keys())
|
| 17 |
chats = []
|
| 18 |
+
prefix = f"{user_id}_"
|
| 19 |
+
|
| 20 |
for ns in namespaces:
|
| 21 |
+
if not ns.startswith(prefix):
|
| 22 |
+
continue
|
| 23 |
+
|
| 24 |
+
session_id = ns[len(prefix):]
|
| 25 |
+
|
| 26 |
+
if not re.fullmatch(r'\d+', session_id):
|
| 27 |
continue
|
| 28 |
|
| 29 |
+
timestamp = int(session_id) / 1000.0
|
| 30 |
+
dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
|
| 31 |
|
| 32 |
chats.append({
|
| 33 |
+
"id": session_id,
|
| 34 |
"title": f"Analysis {dt.strftime('%H:%M:%S')}",
|
| 35 |
"date": dt.strftime('%Y-%m-%d')
|
| 36 |
})
|
|
|
|
| 38 |
chats.sort(key=lambda x: int(x["id"]), reverse=True)
|
| 39 |
return {"chats": chats}
|
| 40 |
except Exception as e:
|
| 41 |
+
print(f"Error fetching chats for user {user_id}: {e}")
|
| 42 |
raise HTTPException(500, "Failed to fetch chats from Pinecone.")
|
app/routes/clear.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
-
from app.rag.vectorstore import
|
| 2 |
-
from fastapi import APIRouter, HTTPException
|
|
|
|
| 3 |
|
| 4 |
router = APIRouter()
|
| 5 |
|
| 6 |
|
| 7 |
@router.delete("/clear")
|
| 8 |
-
async def clear_index() -> dict:
|
| 9 |
-
if not
|
| 10 |
-
raise HTTPException(500, "Failed to clear the vector
|
| 11 |
-
return {"message": "
|
|
|
|
| 1 |
+
from app.rag.vectorstore import delete_user_vectorstores
|
| 2 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 3 |
+
from app.deps import get_user_id
|
| 4 |
|
| 5 |
router = APIRouter()
|
| 6 |
|
| 7 |
|
| 8 |
@router.delete("/clear")
|
| 9 |
+
async def clear_index(user_id: str = Depends(get_user_id)) -> dict:
|
| 10 |
+
if not delete_user_vectorstores(user_id):
|
| 11 |
+
raise HTTPException(500, "Failed to clear the user's vector stores.")
|
| 12 |
+
return {"message": "User's vector stores cleared."}
|
app/routes/delete.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
| 1 |
from app.rag.vectorstore import delete_vectorstore
|
| 2 |
-
from fastapi import APIRouter, HTTPException
|
|
|
|
| 3 |
|
| 4 |
router = APIRouter()
|
| 5 |
|
| 6 |
|
| 7 |
@router.delete("/delete/{session_id}")
|
| 8 |
-
async def delete_specific_chat(session_id: str) -> dict:
|
| 9 |
-
|
|
|
|
| 10 |
raise HTTPException(404, f"No vector store found for session: {session_id}")
|
| 11 |
return {"message": f"Vector store for session {session_id} deleted."}
|
|
|
|
| 1 |
from app.rag.vectorstore import delete_vectorstore
|
| 2 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 3 |
+
from app.deps import get_user_id
|
| 4 |
|
| 5 |
router = APIRouter()
|
| 6 |
|
| 7 |
|
| 8 |
@router.delete("/delete/{session_id}")
|
| 9 |
+
async def delete_specific_chat(session_id: str, user_id: str = Depends(get_user_id)) -> dict:
|
| 10 |
+
prefixed_session_id = f"{user_id}_{session_id}"
|
| 11 |
+
if not delete_vectorstore(prefixed_session_id):
|
| 12 |
raise HTTPException(404, f"No vector store found for session: {session_id}")
|
| 13 |
return {"message": f"Vector store for session {session_id} deleted."}
|
app/routes/upload.py
CHANGED
|
@@ -1,13 +1,19 @@
|
|
| 1 |
import os
|
| 2 |
from app.config import ALLOWED_TYPES, MAX_FILE_COUNT, MAX_FILE_SIZE, UPLOAD_DIR
|
| 3 |
from app.rag.pipeline import process_file
|
| 4 |
-
from fastapi import APIRouter, File, Form, HTTPException, UploadFile
|
|
|
|
| 5 |
|
| 6 |
router = APIRouter()
|
| 7 |
|
| 8 |
|
| 9 |
@router.post("/upload")
|
| 10 |
-
async def upload_files(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
results = []
|
| 12 |
errors = []
|
| 13 |
total_chunks = 0
|
|
@@ -39,7 +45,7 @@ async def upload_files(files: list[UploadFile] = File(...), session_id: str = Fo
|
|
| 39 |
try:
|
| 40 |
with open(path, "wb") as f:
|
| 41 |
f.write(content)
|
| 42 |
-
chunks = process_file(path, ext, session_id=
|
| 43 |
total_chunks += chunks
|
| 44 |
results.append({"source": original_name, "chunks": chunks})
|
| 45 |
except Exception as e:
|
|
|
|
| 1 |
import os
|
| 2 |
from app.config import ALLOWED_TYPES, MAX_FILE_COUNT, MAX_FILE_SIZE, UPLOAD_DIR
|
| 3 |
from app.rag.pipeline import process_file
|
| 4 |
+
from fastapi import APIRouter, File, Form, HTTPException, UploadFile, Depends
|
| 5 |
+
from app.deps import get_user_id
|
| 6 |
|
| 7 |
router = APIRouter()
|
| 8 |
|
| 9 |
|
| 10 |
@router.post("/upload")
|
| 11 |
+
async def upload_files(
|
| 12 |
+
files: list[UploadFile] = File(...),
|
| 13 |
+
session_id: str = Form(...),
|
| 14 |
+
user_id: str = Depends(get_user_id)
|
| 15 |
+
) -> dict:
|
| 16 |
+
prefixed_session_id = f"{user_id}_{session_id}"
|
| 17 |
results = []
|
| 18 |
errors = []
|
| 19 |
total_chunks = 0
|
|
|
|
| 45 |
try:
|
| 46 |
with open(path, "wb") as f:
|
| 47 |
f.write(content)
|
| 48 |
+
chunks = process_file(path, ext, session_id=prefixed_session_id)
|
| 49 |
total_chunks += chunks
|
| 50 |
results.append({"source": original_name, "chunks": chunks})
|
| 51 |
except Exception as e:
|
requirements.txt
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
fastapi
|
| 2 |
uvicorn[standard]
|
| 3 |
python-multipart
|
|
|
|
| 4 |
|
| 5 |
langchain-community
|
| 6 |
langchain-core
|
|
@@ -12,12 +13,10 @@ langchain-pinecone
|
|
| 12 |
|
| 13 |
google-generativeai
|
| 14 |
|
| 15 |
-
|
| 16 |
docx2txt
|
| 17 |
openpyxl
|
| 18 |
python-pptx
|
| 19 |
unstructured
|
| 20 |
markdown
|
| 21 |
jq
|
| 22 |
-
|
| 23 |
-
python-dotenv
|
|
|
|
| 1 |
fastapi
|
| 2 |
uvicorn[standard]
|
| 3 |
python-multipart
|
| 4 |
+
python-dotenv
|
| 5 |
|
| 6 |
langchain-community
|
| 7 |
langchain-core
|
|
|
|
| 13 |
|
| 14 |
google-generativeai
|
| 15 |
|
| 16 |
+
langchain-pymupdf4llm
|
| 17 |
docx2txt
|
| 18 |
openpyxl
|
| 19 |
python-pptx
|
| 20 |
unstructured
|
| 21 |
markdown
|
| 22 |
jq
|
|
|
|
|
|