Update app.py

app.py

@@ -2,6 +2,8 @@ import streamlit as st
 from dotenv import load_dotenv
 from PyPDF2 import PdfReader
 from langchain.text_splitter import CharacterTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
 from langchain.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.chat_models import ChatOpenAI
@@ -18,18 +20,43 @@ def get_pdf_text(pdf_docs):
             text += page.extract_text()
     return text
 
+#@st.cache_resource
+def split_texts(text, chunk_size, overlap, split_method):
+
+    # Split texts
+    # IN: text, chunk size, overlap, split_method
+    # OUT: list of str splits
+
+    st.info("`Splitting doc ...`")
+
+    split_method = "RecursiveTextSplitter"
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=overlap)
+
+    splits = text_splitter.split_text(text)
+    if not splits:
+        st.error("Failed to split document")
+        st.stop()
+
+    return splits
+
 
 def get_text_chunks(text):
-    text_splitter = CharacterTextSplitter(
-        separator="\n",
-        chunk_size=1000,
-        chunk_overlap=200,
-        length_function=len
-    )
-    chunks = text_splitter.split_text(text)
+    # text_splitter = CharacterTextSplitter(
+    #     separator="\n",
+    #     chunk_size=1000,
+    #     chunk_overlap=200,
+    #     length_function=len
+    # )
+    # chunks = text_splitter.split_text(text)
+
+    chunks = split_texts(text, 1000, 200, "RecursiveCharacterTextSplitter")
+    
     return chunks
 
 
+
+
 def get_vectorstore(text_chunks):
     #embeddings = OpenAIEmbeddings()
     embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl")