Spaces:
Build error
Build error
File size: 2,366 Bytes
c9ba277 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | from langchain.text_splitter import TokenTextSplitter
from langchain.docstore.document import Document
from langchain_community.chat_models import ChatOpenAI
from langchain.chains.summarize import load_summarize_chain
from prompts import PROMPT_QUESTIONS, REFINE_PROMPT_QUESTIONS
from langchain.chains import RetrievalQA
from langchain_community.embeddings import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
from PyPDF2 import PdfReader
# Function to load data from PDF
def load_data(uploaded_file):
# Load data from PDF
pdf_reader = PdfReader(uploaded_file)
# Combine text from Document into one string
text = ""
for page in pdf_reader.pages:
text += page.extract_text()
return text
# Function to split text into chunks
def split_text(text, chunk_size, chunk_overlap):
# Initialize text splitter
text_splitter = TokenTextSplitter(
model_name="gpt-3.5-turbo", chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
# Split text into chunks
text_chunks = text_splitter.split_text(text)
# Convert chunks to documents
documents = [Document(page_content=t) for t in text_chunks]
return documents
# Function to initialize large language model
def initialize_llm(openai_api_key, model, temperature):
# Initialize large language model
llm = ChatOpenAI(
openai_api_key=openai_api_key, model=model, temperature=temperature
)
return llm
# Function to generate questions
def generate_questions(llm, chain_type, documents):
# Initialize question chain
question_chain = load_summarize_chain(
llm=llm,
chain_type=chain_type,
question_prompt=PROMPT_QUESTIONS,
refine_prompt=REFINE_PROMPT_QUESTIONS,
)
# Generate questions
questions = question_chain.run(documents)
return questions
# Function to create Retrieval QA Chain
def create_retrieval_qa_chain(openai_api_key, documents, llm):
# Set embeddings
embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
# Create vector database
vector_database = Chroma.from_documents(documents=documents, embedding=embeddings)
# Create Retrieval QA Chain
retrieval_qa_chain = RetrievalQA.from_chain_type(
llm=llm, chain_type="stuff", retriever=vector_database.as_retriever()
)
return retrieval_qa_chain
|