UPDATE: speed ups

app.py

@@ -2,12 +2,10 @@ import io
 import os
 import tempfile
 from functions import *
-from langchain_community.document_loaders import PDFMinerLoader
 import pandas as pd
 from fastapi import FastAPI, File, UploadFile, HTTPException
 from pydantic import BaseModel
 from fastapi.middleware.cors import CORSMiddleware
-from langchain_community.document_loaders import UnstructuredURLLoader
 from src.api.speech_api import speech_translator_router
 from functions import client as supabase
 from urllib.parse import urlparse
@@ -158,8 +156,7 @@ async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
     with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
         temp_file.write(pdf)
         temp_file_path = temp_file.name
-        loader = PDFMinerLoader(file_path = temp_file_path, concatenate_pages = True)
-    text = loader.load()[0].page_content
+    text = extractTextFromPdf(temp_file_path)
     os.remove(temp_file_path)
     username, chatbotname = vectorstore.split("$")[1], vectorstore.split("$")[2]
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
@@ -229,11 +226,7 @@ async def addText(addQaPair: AddQAPair):
 
 @app.post("/addWebsite")
 async def addWebsite(vectorstore: str, websiteUrls: list[str]):
-    loader = UnstructuredURLLoader(urls=websiteUrls)
-    docs = loader.load()
-    text = "\n\n".join(
-        [f"{docs[doc].page_content}" for doc in range(len(docs))]
-    )
+    text = extractTextFromUrlList(urls = websiteUrls)
     username, chatbotname = vectorstore.split("$")[1], vectorstore.split("$")[2]
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     currentCount = df[(df["user_id"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]

functions.py

@@ -1,3 +1,5 @@
+import pymupdf 
+from concurrent.futures import ThreadPoolExecutor
 from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
 from langchain_core.runnables import RunnablePassthrough, RunnableLambda
 from langchain_text_splitters import RecursiveCharacterTextSplitter
@@ -38,12 +40,12 @@ qdrantClient = QdrantClient(url=os.environ["QDRANT_URL"], api_key=os.environ["QD
 model_kwargs = {"device": "cuda"}
 encode_kwargs = {"normalize_embeddings": True}
 vectorEmbeddings = HuggingFaceEmbeddings(
-    model_name="BAAI/bge-m3",
+    model_name="sentence-transformers/all-MiniLM-L6-v2",
     model_kwargs=model_kwargs,
     encode_kwargs=encode_kwargs
 )
 reader = easyocr.Reader(['en'], gpu=True, model_storage_directory="/app/EasyOCRModels")
-sparseEmbeddings = FastEmbedSparse(model="Qdrant/BM25")
+sparseEmbeddings = FastEmbedSparse(model="Qdrant/BM25", threads = 20 , parallel = 0)
 prompt = """
 INSTRUCTIONS:
 =====================================
@@ -123,11 +125,11 @@ def addDocuments(text: str, source: str, vectorstore: str):
     global sparseEmbeddings
     global store
     parentSplitter = RecursiveCharacterTextSplitter(
-        chunk_size=2100,
+        chunk_size=2000,
         add_start_index=True
     )
     childSplitter = RecursiveCharacterTextSplitter(
-        chunk_size=300,
+        chunk_size=400,
         add_start_index=True
     )
     texts = [Document(page_content=text, metadata={"source": source})]
@@ -323,3 +325,28 @@ def analyzeData(query, dataframe):
         return f"data:image/png;base64,{b64string}"
     else:
         return response
+
+
+
+def extractTextFromPage(page):
+    return page.get_text()
+
+def extractTextFromPdf(pdf_path):
+    doc = pymupdf.open(pdf_path)
+    pages = [doc.load_page(i) for i in range(len(doc))]
+    with ThreadPoolExecutor() as executor:
+        texts = list(executor.map(extractTextFromPage, pages))
+    doc.close()    
+    return '.'.join(texts)
+
+def extractTextFromUrl(url):
+    response = requests.get(url)
+    response.raise_for_status()
+    html = response.text
+    soup = BeautifulSoup(html, 'lxml')
+    return soup.get_text(separator=' ', strip=True)
+
+def extractTextFromUrlList(urls):
+    with ThreadPoolExecutor() as executor:
+        texts = list(executor.map(extractTextFromUrl, urls))
+    return '.'.join(texts)

requirements.txt

@@ -73,6 +73,7 @@ fastembed-gpu
 nest_asyncio
 beautifulsoup4
 flashrank
+PyMuPDF
 langchain
 langchain-community
 langchain-cohere
@@ -80,7 +81,6 @@ langchain-huggingface
 langchain-qdrant
 langchain-groq
 lxml
-pdfminer.six
 python-dotenv
 pillow
 pandas