Update tokenization_orion.py

tokenization_orion.py

@@ -3,6 +3,7 @@
 import os
 from shutil import copyfile
 from typing import Any, Dict, List, Optional, Tuple
+import re
 
 import sentencepiece as spm
 from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
@@ -70,6 +71,7 @@ class OrionTokenizer(PreTrainedTokenizer):
         self.add_eos_token = add_eos_token
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(vocab_file)
+        
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -118,6 +120,8 @@ class OrionTokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
+        zhPattern = re.compile(u'[\u4e00-\u9fa5]+')
+        need_convert_punctuation=(",",";","!","?",":","(",")")
         current_sub_tokens = []
         out_string = ""
         prev_is_special = False
@@ -129,12 +133,22 @@ class OrionTokenizer(PreTrainedTokenizer):
                 out_string += self.sp_model.decode(current_sub_tokens) + token
                 prev_is_special = True
                 current_sub_tokens = []
+            if any([True if punctuation in token else False for punctuation in need_convert_punctuation]):
+                out_string += self.sp_model.decode(current_sub_tokens)
+                token=self.sp_model.decode(token)
+                if zhPattern.search(out_string[-20:]):
+                    token = self.to_zh_punctuation(token)
+                out_string += token
+                current_sub_tokens = []
             else:
                 current_sub_tokens.append(token)
                 prev_is_special = False
         out_string += self.sp_model.decode(current_sub_tokens)
         return out_string
 
+    def to_zh_punctuation(self, token):
+        return token.replace(",","，").replace(";","；").replace("!","！").replace("?","？").replace(":","：").replace("(","（").replace(")","）")
+
     def save_vocabulary(
         self, save_directory, filename_prefix: Optional[str] = None
     ) -> Tuple[str]: