back to recursive

app.py

@@ -12,8 +12,8 @@ from aimakerspace.vectordatabase import VectorDatabase
 from aimakerspace.openai_utils.chatmodel import ChatOpenAI
 import chainlit as cl
 from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_experimental.text_splitter import SemanticChunker
-from langchain_openai.embeddings import OpenAIEmbeddings
+# from langchain_experimental.text_splitter import SemanticChunker
+# from langchain_openai.embeddings import OpenAIEmbeddings
 
 system_template = """\
 Use the following context to answer a users question. If you cannot find the answer in the context, say you don't know the answer."""
@@ -49,12 +49,12 @@ class RetrievalAugmentedQAPipeline:
 
         return {"response": generate_response(), "context": context_list}
 
-# text_splitter = RecursiveCharacterTextSplitter()
-try:
-    api_key = os.environ["OPENAI_API_KEY"]
-except KeyError:
-    print("Environment variable OPENAI_API_KEY not found")
-text_splitter = SemanticChunker(OpenAIEmbeddings(api_key=api_key), breakpoint_threshold_type="standard_deviation")
+text_splitter = RecursiveCharacterTextSplitter()
+# try:
+#     api_key = os.environ["OPENAI_API_KEY"]
+# except KeyError:
+#     print("Environment variable OPENAI_API_KEY not found")
+# text_splitter = SemanticChunker(OpenAIEmbeddings(api_key=api_key), breakpoint_threshold_type="standard_deviation")
 
 def process_text_file(file: AskFileResponse):
     import tempfile
@@ -74,8 +74,8 @@ def process_text_file(file: AskFileResponse):
         documents = pdf_loader.load()
     else:
         raise ValueError("Provide a .txt or .pdf file")
-    # texts = [x.page_content for x in text_splitter.transform_documents(documents)]
-    texts = [x.page_content for x in text_splitter.split_documents(documents)]
+    texts = [x.page_content for x in text_splitter.transform_documents(documents)]
+    # texts = [x.page_content for x in text_splitter.split_documents(documents)]
     return texts