Upload tokenizer

special_tokens_map.json

@@ -1,6 +1,24 @@
 {
-  "bos_token": "<|startoftext|>",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<unk>"
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -9,7 +9,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -18,7 +18,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -27,7 +27,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -2132,17 +2132,17 @@
         },
         "content": " "
       },
-      {
-        "type": "ByteFallback"
-      },
-      {
-        "type": "Fuse"
-      },
       {
         "type": "Strip",
         "content": " ",
         "start": 1,
         "stop": 0
+      },
+      {
+        "type": "ByteFallback"
+      },
+      {
+        "type": "Fuse"
       }
     ]
   },

tokenizer_config.json

@@ -1,9 +1,10 @@
 {
+  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -11,7 +12,7 @@
     "1": {
       "content": "<|startoftext|>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -19,7 +20,7 @@
     "2": {
       "content": "<|endoftext|>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -1850,13 +1851,17 @@
     }
   },
   "bos_token": "<|startoftext|>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|im_end|>",
   "legacy": true,
   "model_max_length": 4096,
-  "pad_token": "<|endoftext|>",
+  "pad_token": "<|im_end|>",
+  "padding_side": "right",
   "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": true
+  "use_default_system_prompt": false
 }