Update app.py

app.py

@@ -6,7 +6,8 @@ import nltk
 import gradio as gr
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import FAISS
-from langchain_text_splitters import RecursiveCharacterTextSplitter, SentenceTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain.text_splitters import TokenTextSplitter
 from sentence_transformers import SentenceTransformer
 from transformers import AutoTokenizer
 from nltk import sent_tokenize
@@ -78,8 +79,8 @@ def process_files(model_name, split_strategy, chunk_size=500, overlap_size=50, m
         text += FileHandler.extract_text(file_path)
 
     # Split text
-    if split_strategy == 'sentence':
-        splitter = SentenceTextSplitter(chunk_size=chunk_size, chunk_overlap=overlap_size)
+    if split_strategy == 'token':
+        splitter = TokenTextSplitter(chunk_size=chunk_size, chunk_overlap=overlap_size)
     else:
         splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=overlap_size)