Update app.py

app.py

@@ -25,27 +25,17 @@ class ConvertResponse(BaseModel):
     input_text: str
     theni_tamil_text: str
 
-def _split_into_chunks(text: str, chunk_size: int = 250) -> list:
-    """Safely splits a giant paragraph into smaller sentences based on periods/punctuation."""
+def _split_into_sentences(text: str) -> list:
+    """Forcefully splits a giant paragraph into true individual sentences aggressively."""
     text = text.strip()
     if not text:
         return []
-    if len(text) <= chunk_size:
-        return [text]
     
+    # Split text exactly at periods, exclamation marks, or question marks
     sentences = re.split(r"(?<=[.!?])\s+", text)
-    chunks = []
-    current = ""
-    for sentence in sentences:
-        if len(current) + len(sentence) + 1 <= chunk_size:
-            current = f"{current} {sentence}".strip()
-        else:
-            if current:
-                chunks.append(current)
-            current = sentence.strip()
-    if current:
-        chunks.append(current)
-    return chunks
+    
+    # Clean up any empty spaces and return the array of individual sentences
+    return [s.strip() for s in sentences if s.strip()]
 
 @app.on_event("startup")
 def load_model():
@@ -69,18 +59,18 @@ def convert(payload: ConvertRequest):
         raise HTTPException(400, "text is required")
 
     try:
-        # 1. Break paragraph into bite-sized sentences
-        chunks = _split_into_chunks(text, chunk_size=200)
+        # 1. Break paragraph forcefully into exact sentences
+        sentences = _split_into_sentences(text)
         translated_chunks = []
         
-        # 2. Translate each sentence one by one sequentially
-        for chunk in chunks:
-            encoded = _tokenizer(chunk, return_tensors="pt", truncation=True, max_length=MAX_LENGTH)
+        # 2. Translate each individual sentence one by one sequentially
+        for sentence in sentences:
+            encoded = _tokenizer(sentence, return_tensors="pt", truncation=True, max_length=MAX_LENGTH)
             output_ids = _model.generate(**encoded, max_length=MAX_LENGTH, num_beams=NUM_BEAMS)
             output_text = _tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
             translated_chunks.append(" ".join(str(output_text).strip().split()))
             
-        # 3. Stitch them back into a giant translated paragraph!
+        # 3. Stitch them all back into a giant translated paragraph!
         final_text = " ".join(translated_chunks).strip()
         
         return ConvertResponse(