Update rag_langchain.py
Browse files- rag_langchain.py +13 -13
rag_langchain.py
CHANGED
|
@@ -67,13 +67,13 @@ def split_documents(config, docs):
|
|
| 67 |
|
| 68 |
return text_splitter.split_documents(docs)
|
| 69 |
|
| 70 |
-
def
|
| 71 |
Chroma.from_documents(
|
| 72 |
documents = chunks,
|
| 73 |
embedding = OpenAIEmbeddings(disallowed_special = ()),
|
| 74 |
persist_directory = CHROMA_DIR)
|
| 75 |
|
| 76 |
-
def
|
| 77 |
client = MongoClient(MONGODB_ATLAS_CLUSTER_URI)
|
| 78 |
collection = client[MONGODB_DB_NAME][MONGODB_COLLECTION_NAME]
|
| 79 |
|
|
@@ -88,15 +88,15 @@ def rag_ingestion(config):
|
|
| 88 |
|
| 89 |
chunks = split_documents(config, docs)
|
| 90 |
|
| 91 |
-
#
|
| 92 |
-
|
| 93 |
|
| 94 |
-
def
|
| 95 |
return Chroma(
|
| 96 |
embedding_function = OpenAIEmbeddings(disallowed_special = ()),
|
| 97 |
persist_directory = CHROMA_DIR)
|
| 98 |
|
| 99 |
-
def
|
| 100 |
return MongoDBAtlasVectorSearch.from_connection_string(
|
| 101 |
MONGODB_ATLAS_CLUSTER_URI,
|
| 102 |
MONGODB_DB_NAME + "." + MONGODB_COLLECTION_NAME,
|
|
@@ -113,23 +113,23 @@ def llm_chain(config, prompt):
|
|
| 113 |
llm = get_llm(config),
|
| 114 |
prompt = LLM_CHAIN_PROMPT)
|
| 115 |
|
| 116 |
-
with get_openai_callback() as
|
| 117 |
completion = llm_chain.generate([{"question": prompt}])
|
| 118 |
|
| 119 |
-
return completion, llm_chain,
|
| 120 |
|
| 121 |
def rag_chain(config, prompt):
|
| 122 |
-
#
|
| 123 |
-
|
| 124 |
|
| 125 |
rag_chain = RetrievalQA.from_chain_type(
|
| 126 |
get_llm(config),
|
| 127 |
chain_type_kwargs = {"prompt": RAG_CHAIN_PROMPT,
|
| 128 |
"verbose": True},
|
| 129 |
-
retriever =
|
| 130 |
return_source_documents = True)
|
| 131 |
|
| 132 |
-
with get_openai_callback() as
|
| 133 |
completion = rag_chain({"query": prompt})
|
| 134 |
|
| 135 |
-
return completion, rag_chain,
|
|
|
|
| 67 |
|
| 68 |
return text_splitter.split_documents(docs)
|
| 69 |
|
| 70 |
+
def store_documents_chroma(chunks):
|
| 71 |
Chroma.from_documents(
|
| 72 |
documents = chunks,
|
| 73 |
embedding = OpenAIEmbeddings(disallowed_special = ()),
|
| 74 |
persist_directory = CHROMA_DIR)
|
| 75 |
|
| 76 |
+
def store_documents_mongodb(chunks):
|
| 77 |
client = MongoClient(MONGODB_ATLAS_CLUSTER_URI)
|
| 78 |
collection = client[MONGODB_DB_NAME][MONGODB_COLLECTION_NAME]
|
| 79 |
|
|
|
|
| 88 |
|
| 89 |
chunks = split_documents(config, docs)
|
| 90 |
|
| 91 |
+
#store_documents_chroma(chunks)
|
| 92 |
+
store_documents_mongodb(chunks)
|
| 93 |
|
| 94 |
+
def get_vector_store_chroma():
|
| 95 |
return Chroma(
|
| 96 |
embedding_function = OpenAIEmbeddings(disallowed_special = ()),
|
| 97 |
persist_directory = CHROMA_DIR)
|
| 98 |
|
| 99 |
+
def get_vector_store_mongodb():
|
| 100 |
return MongoDBAtlasVectorSearch.from_connection_string(
|
| 101 |
MONGODB_ATLAS_CLUSTER_URI,
|
| 102 |
MONGODB_DB_NAME + "." + MONGODB_COLLECTION_NAME,
|
|
|
|
| 113 |
llm = get_llm(config),
|
| 114 |
prompt = LLM_CHAIN_PROMPT)
|
| 115 |
|
| 116 |
+
with get_openai_callback() as callback:
|
| 117 |
completion = llm_chain.generate([{"question": prompt}])
|
| 118 |
|
| 119 |
+
return completion, llm_chain, callback
|
| 120 |
|
| 121 |
def rag_chain(config, prompt):
|
| 122 |
+
#vector_store = get_vector_store_chroma()
|
| 123 |
+
vector_store = get_vector_store_mongodb()
|
| 124 |
|
| 125 |
rag_chain = RetrievalQA.from_chain_type(
|
| 126 |
get_llm(config),
|
| 127 |
chain_type_kwargs = {"prompt": RAG_CHAIN_PROMPT,
|
| 128 |
"verbose": True},
|
| 129 |
+
retriever = vector_store.as_retriever(search_kwargs = {"k": config["k"]}),
|
| 130 |
return_source_documents = True)
|
| 131 |
|
| 132 |
+
with get_openai_callback() as callback:
|
| 133 |
completion = rag_chain({"query": prompt})
|
| 134 |
|
| 135 |
+
return completion, rag_chain, callback
|