import streamlit as st from langchain_community.document_loaders import WebBaseLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.vectorstores import Chroma from langchain_openai import OpenAIEmbeddings, ChatOpenAI from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain.chains import create_history_aware_retriever,create_retrieval_chain from langchain.chains.combine_documents import create_stuff_documents_chain from dotenv import load_dotenv import os load_dotenv() KEY=os.getenv("MY_KEY") def get_vectorstore_from_url(url): # get the text in document form loader = WebBaseLoader(url) document = loader.load() # split the document into chunks text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=10, length_function=len) document_chunks = text_splitter.split_documents(document) # create a vectorstore from the chunks vector_store = Chroma.from_documents(document_chunks, OpenAIEmbeddings(openai_api_key=KEY)) return vector_store def get_response(user_input): retriever_chain = get_context_retriever_chain(st.session_state.vector_store) conversation_rag_chain = get_conversational_rag_chain(retriever_chain) response = conversation_rag_chain.invoke({ "chat_history": st.session_state.chat_history, "input": user_input }) return response['answer'] def get_context_retriever_chain(vector_store): llm = ChatOpenAI(openai_api_key=KEY) retriever = vector_store.as_retriever() prompt = ChatPromptTemplate.from_messages([ MessagesPlaceholder(variable_name="chat_history"), ("user", "{input}"), ("user", "Given the above conversation, generate a search query to look up in order to get information relevant to the conversation") ]) retriever_chain = create_history_aware_retriever(llm, retriever, prompt) return retriever_chain def get_conversational_rag_chain(retriever_chain): llm = ChatOpenAI(openai_api_key=KEY) prompt = ChatPromptTemplate.from_messages([ ("system", "Answer the user's questions based on the below context:\n\n{context}"), MessagesPlaceholder(variable_name="chat_history"), ("user", "{input}"), ]) stuff_documents_chain = create_stuff_documents_chain(llm,prompt) return create_retrieval_chain(retriever_chain, stuff_documents_chain)