End of Video 1

.gitignore

@@ -1 +1,2 @@
 __pycache__
+.env

.idea/misc.xml

@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Poetry (pdf_rag)" />
+  </component>
   <component name="ProjectRootManager" version="2" project-jdk-name="Poetry (pdf_rag)" project-jdk-type="Python SDK" />
 </project>

app/rag_chain.py

@@ -0,0 +1,45 @@
+import os
+from operator import itemgetter
+from typing import TypedDict
+
+from dotenv import load_dotenv
+from langchain_community.vectorstores.pgvector import PGVector
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+from config import PG_COLLECTION_NAME
+
+load_dotenv()
+
+vector_store = PGVector(
+    collection_name=PG_COLLECTION_NAME,
+    connection_string=os.getenv("POSTGRES_URL"),
+    embedding_function=OpenAIEmbeddings()
+)
+
+template = """
+Answer given the following context:
+{context}
+
+Question: {question}
+"""
+
+ANSWER_PROMPT = ChatPromptTemplate.from_template(template)
+
+llm = ChatOpenAI(temperature=0, model='gpt-4-1106-preview', streaming=True)
+
+
+class RagInput(TypedDict):
+    question: str
+
+
+final_chain = (
+        {
+            "context": itemgetter("question") | vector_store.as_retriever(),
+            "question": itemgetter("question")
+        }
+        | ANSWER_PROMPT
+        | llm
+        | StrOutputParser()
+).with_types(input_type=RagInput)

app/server.py

@@ -2,6 +2,8 @@ from fastapi import FastAPI
 from fastapi.responses import RedirectResponse
 from langserve import add_routes
 
+from app.rag_chain import final_chain
+
 app = FastAPI()
 
 
@@ -11,7 +13,7 @@ async def redirect_root_to_docs():
 
 
 # Edit this to add the chain you want to add
-add_routes(app, NotImplemented)
+add_routes(app, final_chain, path="/rag")
 
 if __name__ == "__main__":
     import uvicorn

config.py

@@ -0,0 +1,2 @@
+EMBEDDING_MODEL = 'text-embedding-ada-002'
+PG_COLLECTION_NAME = "pdf_rag"

importer/load_and_process.py

@@ -0,0 +1,37 @@
+import os
+
+from dotenv import load_dotenv
+from langchain_community.document_loaders import DirectoryLoader, UnstructuredPDFLoader
+from langchain_community.vectorstores.pgvector import PGVector
+from langchain_experimental.text_splitter import SemanticChunker
+from langchain_openai import OpenAIEmbeddings
+
+from config import EMBEDDING_MODEL, PG_COLLECTION_NAME
+
+load_dotenv()
+
+loader = DirectoryLoader(
+    os.path.abspath("../source_docs"),
+    glob="**/*.pdf",
+    use_multithreading=True,
+    show_progress=True,
+    max_concurrency=50,
+    loader_cls=UnstructuredPDFLoader,
+)
+docs = loader.load()
+
+embeddings = OpenAIEmbeddings(model=EMBEDDING_MODEL, )
+
+text_splitter = SemanticChunker(
+    embeddings=OpenAIEmbeddings()
+)
+
+chunks = text_splitter.split_documents(docs)
+
+PGVector.from_documents(
+    documents=chunks,
+    embedding=embeddings,
+    collection_name=PG_COLLECTION_NAME,
+    connection_string="postgresql+psycopg://postgres@localhost:5432/pdf_rag_vectors",
+    pre_delete_collection=True,
+)

pyproject.toml

@@ -9,10 +9,19 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.11"
+python = ">=3.11,<3.12"
 uvicorn = "^0.23.2"
 langserve = {extras = ["server"], version = ">=0.0.30"}
 pydantic = "<2"
+tqdm = "^4.66.1"
+unstructured = {extras = ["all-docs"], version = "^0.12.2"}
+langchain-experimental = "^0.0.49"
+python-dotenv = "^1.0.0"
+openai = "^1.9.0"
+tiktoken = "^0.5.2"
+langchain-openai = "^0.0.3"
+psycopg = "^3.1.17"
+pgvector = "^0.2.4"
 
 
 [tool.poetry.group.dev.dependencies]