Upload tokenizer

special_tokens_map.json

@@ -13,13 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<|placeholder6|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": true,
-    "single_word": false
-  },
+  "pad_token": "<unk>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {
@@ -155,6 +160,12 @@
           "id": "A",
           "type_id": 0
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|endoftext|>",
+          "type_id": 0
+        }
       }
     ],
     "pair": [
@@ -164,14 +175,36 @@
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<|endoftext|>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|endoftext|>",
+          "type_id": 1
+        }
       }
     ],
-    "special_tokens": {}
+    "special_tokens": {
+      "<|endoftext|>": {
+        "id": "<|endoftext|>",
+        "ids": [
+          32000
+        ],
+        "tokens": [
+          "<|endoftext|>"
+        ]
+      }
+    }
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "add_bos_token": false,
-  "add_eos_token": false,
+  "add_eos_token": true,
   "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
@@ -117,12 +117,12 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{% if 'role' in messages[0] %}{% for message in messages %}{% if message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% else %}{{'<|' + message['role'] + '|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% else %}{% for message in messages %}{% if message['about'] == 'human' %}{{'<|user|>\n' + message['news'] + '<|end|>\n'}}{% elif message['about'] == 'gpt' %}{{'<|assistant|>\n' + message['news'] + '<|end|>\n'}}{% else %}{{'<|' + message['about'] + '|>\n' + message['news'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endif %}",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
-  "model_max_length": 131072,
-  "pad_token": "<|placeholder6|>",
+  "model_max_length": 4096,
+  "pad_token": "<unk>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",