Update app.py

app.py

@@ -1,7 +1,6 @@
 import spaces
 import gradio as gr
 from sacremoses import MosesPunctNormalizer
-from stopes.pipelines.monolingual.utils.sentence_split import get_split_algo
 from transformers import pipeline
 from cultural_model import CulturalM2M100
 from cultural_tokenizer import CulturalTokenizer
@@ -11,7 +10,9 @@ import nltk
 from functools import lru_cache
 from config import LANGUAGE_MAPPING, ENDANGERED_LANGS, MODEL_NAME
 
+# Download required NLTK data
 nltk.download("punkt_tab")
+nltk.download("punkt")
 
 # Device configuration
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -27,16 +28,24 @@ punct_normalizer = MosesPunctNormalizer(lang="en")
 
 @lru_cache(maxsize=202)
 def get_language_specific_sentence_splitter(language_code):
-    splitter = get_split_algo(language_code[:3], "default")
-    return splitter
+    """Return a sentence splitter function for the given language"""
+    # For endangered languages, use NLTK with language-specific tokenizer
+    if language_code in ["qu", "ay", "chr"]:  # Endangered language codes
+        return lambda text: nltk.sent_tokenize(text, language="english")
+    # For other languages, use NLTK with default tokenizer
+    return nltk.sent_tokenize
 
 @spaces.GPU
 def translate(text: str, src_lang: str, tgt_lang: str):
     if not text.strip():
         return ""
     
-    src_code = LANGUAGE_MAPPING[src_lang]["code"]
-    tgt_code = LANGUAGE_MAPPING[tgt_lang]["code"]
+    src_info = LANGUAGE_MAPPING.get(src_lang)
+    tgt_info = LANGUAGE_MAPPING.get(tgt_lang)
+    if not src_info or not tgt_info:
+        raise gr.Error("Invalid language selection")
+    src_code = src_info["code"]
+    tgt_code = tgt_info["code"]
     
     # Enable cultural preservation for endangered languages
     cultural_preservation = tgt_lang in ENDANGERED_LANGS
@@ -48,8 +57,12 @@ def translate(text: str, src_lang: str, tgt_lang: str):
     translated_paragraphs = []
     
     for paragraph in paragraphs:
+        if not paragraph.strip():
+            translated_paragraphs.append("")
+            continue
+            
         splitter = get_language_specific_sentence_splitter(src_code)
-        sentences = list(splitter(paragraph))
+        sentences = splitter(paragraph)
         translated_sentences = []
         
         for sentence in sentences: