File size: 2,257 Bytes
04e75ed | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | from pypdf import PdfReader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from langchain_openai import OpenAIEmbeddings
from langchain_chroma import Chroma
import re
import os
def extract_text_from_pdf(file_path:str) -> str:
reader = PdfReader(file_path)
text = ""
for page in reader.pages:
text += page.extract_text() or ""
return text
def pdf_to_documents(file_path:str,database_name:str,collection_name:str,embeddings:OpenAIEmbeddings,chunk_size=1000,chunk_overlap=200,metadata:dict=None):
text = extract_text_from_pdf(file_path)
text = re.sub(r"[^a-zA-Z0-9.,!?;:'\"()\s]", "", text)
if not text.strip():
return []
splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap)
chunks = splitter.split_text(text)
docs = []
for i,chunk in enumerate(chunks):
#print(f"index: {i} , {chunk}")
meta = metadata.copy() if metadata else {}
meta.update({"chunk":i})
docs.append(Document(page_content=chunk, metadata=meta))
if os.path.exists(database_name):
Chroma(persist_directory=database_name, embedding_function=embeddings,collection_name=collection_name).delete_collection()
vectorstore = Chroma.from_documents(documents=docs, embedding=embeddings, persist_directory=database_name,collection_name=collection_name)
return docs,vectorstore
def store_data(text:str,database_name:str,collection_name:str,embeddings:OpenAIEmbeddings):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size = 1000,
chunk_overlap = 0,
separators = [" ", ",", "\n"]
)
#with open(file_path) as f:
# text = f.read()
texts = text_splitter.split_text(text)
#print(f"split: {texts}")
docs = [Document(page_content=t) for t in texts]
if os.path.exists(database_name):
Chroma(persist_directory=database_name, embedding_function=embeddings,collection_name=collection_name).delete_collection()
vectorstore = Chroma.from_documents(documents=docs, embedding=embeddings, persist_directory=database_name,collection_name=collection_name)
return vectorstore
|