rag initial pipeline setup

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -1,7 +1,53 @@
 import gradio as gr
+from src.rag import RAG
 
-def greet(name):
-    return "Hello " + name + "!!"
+import bs4
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+from dotenv import load_dotenv
+
+# set the required env variables
+load_dotenv(".env")
+
+def rag_handler(web_paths, model_name, temperature, question):
+    print(web_paths)
+    web_paths = web_paths.split(',')
+    print(web_paths)
+    loader = WebBaseLoader(
+        web_paths=web_paths,
+        bs_kwargs=dict(
+            parse_only=bs4.SoupStrainer(
+                class_=("post-content", "post-title", "post-header")
+            )
+        ),
+    )
+
+    llm = ChatOpenAI(model_name=model_name, temperature=temperature)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) # TODO: Parameterize this
+    
+    rag_pipeline = RAG(llm, loader, text_splitter, OpenAIEmbeddings)
+
+    return rag_pipeline.invoke(question)
+
+def create_rag_interface():
+    return gr.Interface(
+        fn=rag_handler,
+        inputs=[
+            gr.Textbox(value="https://lilianweng.github.io/posts/2023-06-23-agent/"),
+            gr.Dropdown(["gpt-3.5-turbo"], type="value"),
+            gr.Slider(0, 1, step=0.1),
+            'text'
+        ],
+        outputs="text"
+    )
+
+
+if __name__ == '__main__':
+    interface_list = []
+    interface_list.append(create_rag_interface())
+
+    demo = gr.TabbedInterface(interface_list)
+
+    demo.launch()

building.py

@@ -0,0 +1,53 @@
+import getpass
+from dotenv import dotenv_values, load_dotenv
+
+# p = getpass.getpass() # for user input
+
+config = dict(dotenv_values(".env"))
+load_dotenv(".env")
+
+
+import bs4
+from langchain import hub
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.vectorstores import Chroma
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+# Load, chunk and index the contents of the blog.
+loader = WebBaseLoader(
+    web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
+    bs_kwargs=dict(
+        parse_only=bs4.SoupStrainer(
+            class_=("post-content", "post-title", "post-header")
+        )
+    ),
+)
+docs = loader.load()
+
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+splits = text_splitter.split_documents(docs)
+vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings())
+
+# Retrieve and generate using the relevant snippets of the blog.
+retriever = vectorstore.as_retriever()
+prompt = hub.pull("rlm/rag-prompt")
+llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+
+rag_chain = (
+    {"context": retriever | format_docs, "question": RunnablePassthrough()}
+    | prompt
+    | llm
+    | StrOutputParser()
+)
+
+print(rag_chain.invoke("What is Task Decomposition?"))
+
+

src/rag.py

@@ -0,0 +1,50 @@
+from langchain import hub
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_community.vectorstores import Chroma
+
+# class for `Retreival Augmented Generation Pipeline`
+class RAG:
+    def __init__(
+            self,
+            llm,
+            loader,
+            text_splitter,
+            embedding,
+            prompt = None
+        ):
+        self.llm = llm
+        self.embedding = embedding
+        self.loader = loader
+        self.text_splitter = text_splitter
+        self.prompt = prompt if prompt else hub.pull("rlm/rag-prompt")
+
+        self.docs = self.load_docs()
+        self.splits = self.create_splits()
+        self.vectorstore = Chroma.from_documents(documents=self.splits, embedding=self.embedding())
+        self.retriever = self.get_retreiver()
+        self.rag_chain = self.generate_rag_chain()
+
+    def create_splits(self):
+        return self.text_splitter.split_documents(self.docs)
+    
+    def load_docs(self):
+        return self.loader.load()
+
+    def get_retreiver(self):
+        return self.vectorstore.as_retriever()
+
+    def format_docs(self, docs):
+        return "\n\n".join(doc.page_content for doc in docs)
+
+    def generate_rag_chain(self):
+        return (
+            {"context": self.retriever | self.format_docs, "question": RunnablePassthrough()}
+            | self.prompt
+            | self.llm
+            | StrOutputParser()
+        )
+
+    def invoke(self, question):
+        return self.rag_chain.invoke(question)
+