Update PDF_Reader.py

PDF_Reader.py

@@ -1,8 +1,10 @@
 import os 
 from langchain_experimental.text_splitter import SemanticChunker
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_chroma import Chroma
 from langchain_community.document_loaders import PyPDFLoader
 from langchain.embeddings import HuggingFaceEmbeddings
+from PyPDF2 import PdfReader
 
 embedding_modelPath = "sentence-transformers/all-MiniLM-l6-v2"
 embeddings = HuggingFaceEmbeddings(model_name=embedding_modelPath,model_kwargs = {'device':'cpu'},encode_kwargs = {'normalize_embeddings': False})
@@ -22,6 +24,16 @@ def replace_t_with_space(list_of_documents):
         doc.page_content = doc.page_content.replace('\t', ' ')  # Replace tabs with spaces
     return list_of_documents
 
+def read_pdf_text(pdf_path):
+    text = ""
+    pdf_reader = PdfReader(pdf_path)
+        for page in pdf_reader.pages:
+            text += page.extract_text()
+    
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
+    text_chunks = text_splitter.split_text(text)
+    return text_chunks
+
 def read_pdf(pdf_path):
     loader = PyPDFLoader(pdf_path)
     docs = loader.load()
@@ -29,15 +41,15 @@ def read_pdf(pdf_path):
     return docs
 
 def Chunks(docs):
-    
     text_splitter = SemanticChunker(embeddings,breakpoint_threshold_type='interquartile')
     docs = text_splitter.split_documents(docs)
     cleaned_docs = replace_t_with_space(docs)
     return cleaned_docs
     
-def PDF_4_QA(file):
-    docs = read_pdf(file)
-    cleaned_docs = Chunks(docs)
+def PDF_4_QA(file_path):
+    #docs = read_pdf(file_path)
+    #cleaned_docs = Chunks(docs)
+    read_pdf_text(file_path)
     vectordb = Chroma.from_documents(
         documents=cleaned_docs,
         embedding=embeddings,