Update PDF_Reader.py

PDF_Reader.py

@@ -1,31 +1,45 @@
-import PyPDF2
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings import HuggingFaceBgeEmbeddings
-from langchain.vectorstores import FAISS
+from langchain_experimental.text_splitter import SemanticChunker
+from langchain_chroma import Chroma
+from langchain_community.document_loaders import PyPDFLoader
 
-def read_pdf(uploaded_file):
-    pdf_reader = PyPDF2.PdfReader(uploaded_file)
-    text = ""
-    for page in pdf_reader.pages:
-        text += page.extract_text()
-    return text
+embedding_modelPath = "sentence-transformers/all-MiniLM-l6-v2"
+embeddings = HuggingFaceEmbeddings(model_name=embedding_modelPath,model_kwargs = {'device':'cpu'},encode_kwargs = {'normalize_embeddings': False})
 
-def Chunks(docs):
-    text_splitter = RecursiveCharacterTextSplitter(
-    # Set a really small chunk size, just to show.
-    chunk_size = 1000,
-    chunk_overlap  = 100,
-)
-    doc = text_splitter.split_text(docs)
-    return doc
+def replace_t_with_space(list_of_documents):
+    """
+    Replaces all tab characters ('\t') with spaces in the page content of each document.
 
+    Args:
+        list_of_documents: A list of document objects, each with a 'page_content' attribute.
 
-def PDF_4_QA(file):
-    content = read_pdf(file)
-    pdf_chunks = Chunks(docs=content)
+    Returns:
+        The modified list of documents with tab characters replaced by spaces.
+    """
 
-    embeddings = HuggingFaceBgeEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2',
-                                          model_kwargs={'device': 'cpu'})
-    vectorstore_openai = FAISS.from_texts(pdf_chunks, embeddings)
+    for doc in list_of_documents:
+        doc.page_content = doc.page_content.replace('\t', ' ')  # Replace tabs with spaces
+    return list_of_documents
 
-    return vectorstore_openai
+def read_pdf(uploaded_file):
+    loader = PyPDFLoader(pdf_path)
+    docs = loader.load()
+    print("Total Documents :",len(docs))
+    return docs
+
+def Chunks(docs):
+    
+    text_splitter = SemanticChunker(embeddings,breakpoint_threshold_type='interquartile')
+    docs = text_splitter.split_documents(docs)
+    cleaned_docs = replace_t_with_space(docs)
+    return cleaned_docs
+    
+def PDF_4_QA(file):
+    docs = read_pdf(file)
+    cleaned_docs = Chunks(docs)
+    
+    vectordb = Chroma.from_documents(
+        documents=cleaned_docs,
+        embedding=local_embeddings,
+        persist_directory=persist_directory
+    )
+    return vectordb