update chat template

config.json

@@ -28,4 +28,4 @@
   "transformers_version": "4.34.0",
   "use_cache": true,
   "vocab_size": 84608
-}
+}

configuration.json

@@ -0,0 +1 @@
+{"framework":"Pytorch","task":"text-generation"}

generation_utils.py

@@ -3,6 +3,10 @@ from queue import Queue
 
 # build chat input prompt
 def build_chat_input(tokenizer, messages: List[dict]):
+    # chat format:
+    # single-turn: <s>Human: Hello!\n\nAssistant: </s>
+    # multi-turn:  <s>Human: Hello!\n\nAssistant: </s>Hi!</s>Human: How are you?\n\nAssistant: </s>I'm fine</s>
+
     prompt = "<s>"
     for msg in messages:
         role = msg["role"]
@@ -10,7 +14,7 @@ def build_chat_input(tokenizer, messages: List[dict]):
         if message is None :
             continue
         if role == "user":
-            prompt += "Human: " + message + "\nAssistant: "
+            prompt += "Human: " + message + "\n\nAssistant: </s>"
         if role == "assistant":
             prompt += message + "</s>"
 

pytorch_model-00001-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27160770c0c1ebef57df9555cd170953f83ee5095c4b14472b0a8ca255a2f29a
+oid sha256:50ad84420f47d71980877bb76d3320bd1346374370c79a04ed634f893fc8c333
 size 9937152090

pytorch_model-00002-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19323573478cbd880ed78542c07fed3a7a16192e60e0537bf60daca76f17df5c
+oid sha256:f11df7ddc630b02893f71e9a2cfdb4035cd3ac884cec74dbc38a19f592b862e0
 size 9857241994

pytorch_model-00003-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8892a94ff35e7adb8731bc22da47d535a69f3eaf138ab99925926d811ee0e3c0
+oid sha256:074a2e42d9ab0024293c7bb4d11c8ebdc689b404f3dc42b2c45f58ebf5f15e76
 size 9203166530

tokenization_orion.py

@@ -3,7 +3,6 @@
 import os
 from shutil import copyfile
 from typing import Any, Dict, List, Optional, Tuple
-import re
 
 import sentencepiece as spm
 from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
@@ -71,7 +70,6 @@ class OrionTokenizer(PreTrainedTokenizer):
         self.add_eos_token = add_eos_token
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(vocab_file)
-        
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -120,8 +118,6 @@ class OrionTokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
-        zhPattern = re.compile(u'[\u4e00-\u9fa5]+')
-        need_convert_punctuation=(",",";","!","?",":","(",")")
         current_sub_tokens = []
         out_string = ""
         prev_is_special = False
@@ -133,22 +129,12 @@ class OrionTokenizer(PreTrainedTokenizer):
                 out_string += self.sp_model.decode(current_sub_tokens) + token
                 prev_is_special = True
                 current_sub_tokens = []
-            if any([True if punctuation in token else False for punctuation in need_convert_punctuation]):
-                out_string += self.sp_model.decode(current_sub_tokens)
-                token=self.sp_model.decode(token)
-                if zhPattern.search(out_string[-20:]):
-                    token = self.to_zh_punctuation(token)
-                out_string += token
-                current_sub_tokens = []
             else:
                 current_sub_tokens.append(token)
                 prev_is_special = False
         out_string += self.sp_model.decode(current_sub_tokens)
         return out_string
 
-    def to_zh_punctuation(self, token):
-        return token.replace(",","，").replace(";","；").replace("!","！").replace("?","？").replace(":","：").replace("(","（").replace(")","）")
-
     def save_vocabulary(
         self, save_directory, filename_prefix: Optional[str] = None
     ) -> Tuple[str]: