add website option

app.py

@@ -5,7 +5,7 @@ import tempfile
 from langchain_openai import OpenAIEmbeddings
 from langchain_openai.chat_models import ChatOpenAI
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import PyPDFLoader, WebBaseLoader
 from langchain_community.document_loaders.generic import GenericLoader
 from langchain_community.document_loaders.parsers import OpenAIWhisperParser
 from langchain_community.document_loaders.blob_loaders.youtube_audio import (
@@ -13,7 +13,6 @@ from langchain_community.document_loaders.blob_loaders.youtube_audio import (
 )
 from langchain_community.vectorstores import Chroma
 from langchain_core.messages import HumanMessage, AIMessage
-from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain.chains import create_history_aware_retriever, create_retrieval_chain
 from langchain.chains.combine_documents import create_stuff_documents_chain
@@ -25,13 +24,20 @@ st.set_page_config(page_title="Chat with your data", page_icon="🤖")
 st.title("Chat with your data")
 st.header("Add your data for RAG")
 
-data_type = st.radio("Choose the type of data to add:", ("Text", "PDF", "YouTube URL"))
+data_type = st.radio(
+    "Choose the type of data to add:", ("Text", "PDF", "Website", "YouTube")
+)
+
+if data_type == "YouTube":
+    st.warning(
+        "Note: Processing YouTube videos can be quite costly for me in terms of money. Please use this option sparingly. Thank you for your understanding!"
+    )
 
 if "vectordb" not in st.session_state:
     st.session_state.vectordb = None
 
 
-def add_text_to_chroma(text):
+def get_vectordb_from_text(text):
     embeddings = OpenAIEmbeddings()
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
     texts = text_splitter.split_text(text)
@@ -42,7 +48,7 @@ def add_text_to_chroma(text):
     return vectordb
 
 
-def add_pdf_to_chroma(uploaded_pdf):
+def get_vectordb_from_pdf(uploaded_pdf):
     with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
         tmp_file.write(uploaded_pdf.read())
         tmp_file_path = tmp_file.name
@@ -58,7 +64,19 @@ def add_pdf_to_chroma(uploaded_pdf):
     return vectordb
 
 
-def add_youtube_to_chroma(youtube_url):
+def get_vectordb_from_website(website_url):
+    loader = WebBaseLoader(website_url)
+    pages = loader.load()
+    embeddings = OpenAIEmbeddings()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+    docs = text_splitter.split_documents(pages)
+    vectordb = Chroma.from_documents(
+        documents=docs,
+        embedding=embeddings,
+    )
+
+
+def get_vectordb_from_youtube(youtube_url):
     save_dir = "docs/youtube"
     loader = GenericLoader(
         YoutubeAudioLoader([youtube_url], save_dir), OpenAIWhisperParser()
@@ -76,21 +94,23 @@ def add_youtube_to_chroma(youtube_url):
 if data_type == "Text":
     user_text = st.text_area("Enter text data")
     if st.button("Add"):
-        st.session_state.vectordb = add_text_to_chroma(user_text)
+        st.session_state.vectordb = get_vectordb_from_text(user_text)
 
 elif data_type == "PDF":
     uploaded_pdf = st.file_uploader("Upload PDF", type="pdf")
     if st.button("Add"):
-        st.session_state.vectordb = add_pdf_to_chroma(uploaded_pdf)
+        st.session_state.vectordb = get_vectordb_from_pdf(uploaded_pdf)
 
+elif data_type == "Website":
+    website_url = st.text_input("Enter website URL")
+    if st.button("Add"):
+        st.session_state.vectordb = get_vectordb_from_website(website_url)
 else:
     youtube_url = st.text_input("Enter YouTube URL")
     if st.button("Add"):
-        st.session_state.vectordb = add_youtube_to_chroma(youtube_url)
+        st.session_state.vectordb = get_vectordb_from_youtube(youtube_url)
 
-llm = ChatOpenAI(
-    api_key=openai_api_key, temperature=0.2, model="gpt-3.5-turbo"
-)
+llm = ChatOpenAI(api_key=openai_api_key, temperature=0.2, model="gpt-3.5-turbo")
 
 
 def get_context_retreiver_chain(vectordb):
@@ -113,11 +133,16 @@ def get_context_retreiver_chain(vectordb):
 
 
 def get_conversational_rag_chain(retriever_chain):
-    prompt = ChatPromptTemplate.from_messages([
-      ("system", "Answer the user's questions based on the below context:\n\n{context}"),
-      MessagesPlaceholder(variable_name="chat_history"),
-      ("user", "{input}"),
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "Answer the user's questions based on the below context:\n\n{context}",
+            ),
+            MessagesPlaceholder(variable_name="chat_history"),
+            ("user", "{input}"),
+        ]
+    )
 
     stuff_domain_chain = create_stuff_documents_chain(llm, prompt)
 
@@ -127,16 +152,15 @@ def get_conversational_rag_chain(retriever_chain):
 def get_response(user_input):
     if st.session_state.vectordb is None:
         return "Please add data first"
-    
+
     retrieveal_chain = get_context_retreiver_chain(st.session_state.vectordb)
     converasational_rag_chain = get_conversational_rag_chain(retrieveal_chain)
 
-    response = converasational_rag_chain.invoke({
-        "chat_history": st.session_state.chat_history,
-        "input": user_input
-    })
+    response = converasational_rag_chain.invoke(
+        {"chat_history": st.session_state.chat_history, "input": user_input}
+    )
 
-    return response['answer']
+    return response["answer"]
 
 
 user_query = st.chat_input("Your message")

requirements.txt

@@ -3,6 +3,7 @@ langchain_community
 langchain_openai
 langchain_pinecone
 pypdf
+beautifulsoup4
 yt_dlp
 pydub
 chromadb