improve text preprocess

base_model.py

@@ -44,8 +44,8 @@ class BaseEngine:
         except NotImplementedError:
             pass 
 
-        text = self.preprocess_text(text)
-        chunks = utils.split_text_into_sentences(text, min_chunk_size=150)
+        chunks = self.preprocess_text(text)
+        
         
         loop = asyncio.get_event_loop()
         

kokoro_model.py

@@ -3,6 +3,7 @@ import re
 import asyncio
 from kokoro import KPipeline
 import base_model
+import utils
 
 class StreamingEngine(base_model.BaseEngine):
     def __init__(self, name):
@@ -43,24 +44,9 @@ class StreamingEngine(base_model.BaseEngine):
 
     def preprocess_text(self, text):
         if not text:
-            return ""
-
-        is_valid, unsupported = True, []
-        if not is_valid:
-            print(f"   ⚠️  Contains {len(unsupported)} unsupported character(s): {unsupported[:5]}")
-            # Escape characters safe for regex usage
-            pattern = f"[{re.escape(''.join(unsupported))}]"
-            preprocessed = re.sub(pattern, "", text)
-            
-            if preprocessed != text:
-                print(f"   After preprocessing: {preprocessed[:50]}...")
-                text = preprocessed
-        else:
-            # Optional: Comment this out in production to reduce log spam
-            print("   ✓ All characters supported")
-            
-        return text
-
+            return []
+        return [text]
+        
     def generate(self, chunks: str, voice_name: str, speed: float):
         """
         Generates audio.

supertonic_model.py

@@ -3,7 +3,7 @@ import re
 import asyncio
 from supertonic import TTS
 import base_model
-
+import utils
 class StreamingEngine(base_model.BaseEngine):
     def __init__(self, name):
         # 1. Initialize configuration variables first
@@ -56,7 +56,10 @@ class StreamingEngine(base_model.BaseEngine):
 
     def preprocess_text(self, text):
         if not text:
-            return ""
+            return []
+        
+        split_pattern = r'\n+'
+
 
         is_valid, unsupported = self.text_processor.validate_text(text)
 
@@ -73,7 +76,8 @@ class StreamingEngine(base_model.BaseEngine):
             # Optional: Comment this out in production to reduce log spam
             print("   ✓ All characters supported")
             
-        return text
+        chunks = utils.split_text_into_sentences(text, min_chunk_size=150)
+        return chunks
 
     def generate(self, chunks: str, voice_name: str, speed: float):
         """

utils.py

@@ -3,7 +3,7 @@ import re
 import struct
 import lameenc
 
-def split_text_into_sentences(text: str, min_chunk_size: int = 150):
+def split_text_into_sentences(text: str, min_chunk_size: int = 150,split_pattern = r'\n+'):
     if not text:
         return []
 
@@ -17,6 +17,8 @@ def split_text_into_sentences(text: str, min_chunk_size: int = 150):
     current_atomic = ""
     
     for part in raw_parts:
+        if not part.strip():
+            continue
         if re.match(r'^[.?!:;]+$', part):
             current_atomic += part
             if current_atomic.strip():