switch to fast tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52bd78402728bcc81b8fa7f036d8dffc6ec60d113361906bf2285a446e93d897
+size 17637528

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 65536,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "additional_special_tokens": ["<|system|>", "<|user|>", "<|assistant|>"],
+  "eos_token":                 "<|endoftext|>",
+  "add_generation_prompt": true,
+  "add_bos_token": false,
+  "padding_side": "left",
+  "add_eos_token": false,
+  "chat_template": "{% for m in messages %}{% if m.role == 'system' %}<|system|>{{ m.content | trim }}<|system|>{% elif m.role == 'user' %}<|user|>{{ m.content | trim }}<|user|>{% elif m.role == 'assistant' %}<|assistant|>{{ m.content | trim }}<|assistant|>{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1].role != 'assistant' %}<|assistant|>{% endif %}",
+  "unk_token": "<unk>"
+}