working on text splitting

app.py

@@ -98,20 +98,21 @@ def main() -> None:
     #         return tuple(summarizer.abstractive_summary(list(summary_sentence)))
 
     def split_text(text: str) -> list:
-        sentences = sent_tokenize(text)
-        # token_count = 0
-        # text_block = ""
-        # result = []
-        # for sentence in sentences:
-        #     tokens = word_tokenize(sentence)
-        #     if token_count + len(tokens) < 500:
-        #         token_count += len(tokens)
-        #         text_block += " ".join(sentence)
-        #     else:
-        #         result.append(text_block)
-        #         text_block = "".join(sentence)
-        #         token_count = len(tokens)
-        return sentences
+        sentences = sent_tokenize(text, language="english")
+
+        token_count = 0
+        text_block = ""
+        result = []
+        for sentence in sentences:
+            tokens = word_tokenize(sentence, language="english", preserve_line=True)
+            if token_count + len(tokens) < 500:
+                token_count += len(tokens)
+                text_block += " ".join(sentence)
+            else:
+                result.append(text_block)
+                text_block = "".join(sentence)
+                token_count = len(tokens)
+        return result
 
     pipe = create_pipeline()