[update]add sent_tokenize

examples/sent_tokenize/sent_tokenize.py

@@ -9,7 +9,6 @@ from project_settings import project_path
 
 os.environ['NLTK_DATA'] = (project_path / "thirdparty_data/nltk_data").as_posix()
 
-import jieba
 import nltk
 
 
@@ -58,8 +57,10 @@ def chinese_sent_tokenize(text: str):
 def sent_tokenize(text: str, language: str) -> List[str]:
     if language in ["chinese"]:
         sent_list = chinese_sent_tokenize(text)
-    else:
+    elif language in nltk_sent_tokenize_languages:
         sent_list = nltk.sent_tokenize(text, language)
+    else:
+        sent_list = [text]
     return sent_list
 
 

main.py

@@ -17,6 +17,7 @@ from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
 
 
 language_map = {
+    "Arabic": "ar",
     "Chinese": "zh",
     "Czech": "cs",
     "Danish": "da",
@@ -65,8 +66,10 @@ def chinese_sent_tokenize(text: str):
 def sent_tokenize(text: str, language: str) -> List[str]:
     if language in ["chinese"]:
         sent_list = chinese_sent_tokenize(text)
-    else:
+    elif language in nltk_sent_tokenize_languages:
         sent_list = nltk.sent_tokenize(text, language)
+    else:
+        sent_list = [text]
     return sent_list
 
 
@@ -159,8 +162,6 @@ It was introduced in this [paper](https://arxiv.org/abs/2010.11125) and first re
         gr.Textbox(lines=4, placeholder="text", label="Input Text"),
         gr.Dropdown(choices=list(language_map.keys()), value="English", label="Source Language"),
         gr.Dropdown(choices=list(language_map.keys()), value="Chinese", label="Target Language"),
-        # gr.Textbox(lines=1, value="en", label="Source Language"),
-        # gr.Textbox(lines=1, value="zh", label="Target Language"),
         gr.Dropdown(choices=model_choices, value="facebook/m2m100_418M", label="model_name")
     ]