Update app.py

app.py

@@ -707,6 +707,10 @@ def sentence_splitter_instantiation(
     sentence_splitter_config,
 ):
     ### Chunker/Sentence Splitter 
+    def simple_whitespace_tokenizer(text):
+    """Tokenizer that considers each word as one token"""
+    return text.split()
+    
     if sentence_splitter_config.value is not None:
         sentence_splitter_config_values = sentence_splitter_config.value
         validated_chunk_overlap = min(sentence_splitter_config_values.get("chunk_overlap"), 
@@ -719,6 +723,7 @@ def sentence_splitter_instantiation(
             paragraph_separator=sentence_splitter_config_values.get("paragraph_separator"),
             secondary_chunking_regex=sentence_splitter_config_values.get("secondary_chunking_regex"),
             include_metadata=sentence_splitter_config_values.get("include_metadata"),
+            tokenizer=simple_whitespace_tokenizer
         )
 
     else:
@@ -729,6 +734,7 @@ def sentence_splitter_instantiation(
             paragraph_separator="\n\n\n",
             secondary_chunking_regex="[^,.;?!]+[,.;?!]?",
             include_metadata=True,
+            tokenizer=simple_whitespace_tokenizer
         )
     return (sentence_splitter,)