Update app.py
Browse files
app.py
CHANGED
|
@@ -709,7 +709,7 @@ def sentence_splitter_instantiation(
|
|
| 709 |
### Chunker/Sentence Splitter
|
| 710 |
def simple_whitespace_tokenizer(text):
|
| 711 |
"""Tokenizer that considers each word as one token"""
|
| 712 |
-
|
| 713 |
|
| 714 |
if sentence_splitter_config.value is not None:
|
| 715 |
sentence_splitter_config_values = sentence_splitter_config.value
|
|
|
|
| 709 |
### Chunker/Sentence Splitter
|
| 710 |
def simple_whitespace_tokenizer(text):
|
| 711 |
"""Tokenizer that considers each word as one token"""
|
| 712 |
+
return text.split()
|
| 713 |
|
| 714 |
if sentence_splitter_config.value is not None:
|
| 715 |
sentence_splitter_config_values = sentence_splitter_config.value
|