Spaces:
Build error
Build error
| from langchain.text_splitter import TokenTextSplitter | |
| from langchain.docstore.document import Document | |
| from langchain_community.chat_models import ChatOpenAI | |
| from langchain.chains.summarize import load_summarize_chain | |
| from prompts import PROMPT_QUESTIONS, REFINE_PROMPT_QUESTIONS | |
| from langchain.chains import RetrievalQA | |
| from langchain_community.embeddings import OpenAIEmbeddings | |
| from langchain_community.vectorstores import Chroma | |
| from PyPDF2 import PdfReader | |
| # Function to load data from PDF | |
| def load_data(uploaded_file): | |
| # Load data from PDF | |
| pdf_reader = PdfReader(uploaded_file) | |
| # Combine text from Document into one string | |
| text = "" | |
| for page in pdf_reader.pages: | |
| text += page.extract_text() | |
| return text | |
| # Function to split text into chunks | |
| def split_text(text, chunk_size, chunk_overlap): | |
| # Initialize text splitter | |
| text_splitter = TokenTextSplitter( | |
| model_name="gpt-3.5-turbo", chunk_size=chunk_size, chunk_overlap=chunk_overlap | |
| ) | |
| # Split text into chunks | |
| text_chunks = text_splitter.split_text(text) | |
| # Convert chunks to documents | |
| documents = [Document(page_content=t) for t in text_chunks] | |
| return documents | |
| # Function to initialize large language model | |
| def initialize_llm(openai_api_key, model, temperature): | |
| # Initialize large language model | |
| llm = ChatOpenAI( | |
| openai_api_key=openai_api_key, model=model, temperature=temperature | |
| ) | |
| return llm | |
| # Function to generate questions | |
| def generate_questions(llm, chain_type, documents): | |
| # Initialize question chain | |
| question_chain = load_summarize_chain( | |
| llm=llm, | |
| chain_type=chain_type, | |
| question_prompt=PROMPT_QUESTIONS, | |
| refine_prompt=REFINE_PROMPT_QUESTIONS, | |
| ) | |
| # Generate questions | |
| questions = question_chain.run(documents) | |
| return questions | |
| # Function to create Retrieval QA Chain | |
| def create_retrieval_qa_chain(openai_api_key, documents, llm): | |
| # Set embeddings | |
| embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key) | |
| # Create vector database | |
| vector_database = Chroma.from_documents(documents=documents, embedding=embeddings) | |
| # Create Retrieval QA Chain | |
| retrieval_qa_chain = RetrievalQA.from_chain_type( | |
| llm=llm, chain_type="stuff", retriever=vector_database.as_retriever() | |
| ) | |
| return retrieval_qa_chain | |