Upload tokenizer

added_tokens.json

@@ -1,14 +1,15 @@
 {
-  "<|assistant|>": 32001,
-  "<|continue|>": 32009,
+  "<|assistant|>": 32002,
+  "<|continue|>": 32010,
   "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|function_call|>": 32005,
-  "<|function_list|>": 32011,
-  "<|function_output|>": 32003,
-  "<|raw|>": 32008,
-  "<|step|>": 32002,
-  "<|system|>": 32006,
-  "<|tag|>": 32004,
-  "<|user|>": 32010
+  "<|end|>": 32008,
+  "<|function_call|>": 32006,
+  "<|function_list|>": 32012,
+  "<|function_output|>": 32004,
+  "<|padding|>": 32001,
+  "<|raw|>": 32009,
+  "<|step|>": 32003,
+  "<|system|>": 32007,
+  "<|tag|>": 32005,
+  "<|user|>": 32011
 }

special_tokens_map.json

@@ -86,6 +86,7 @@
     "single_word": false
   },
   "eos_token": "<|endoftext|>",
+  "pad_token": "<|padding|>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -41,6 +41,15 @@
     },
     {
       "id": 32001,
+      "content": "<|padding|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32002,
       "content": "<|assistant|>",
       "single_word": false,
       "lstrip": false,
@@ -49,7 +58,7 @@
       "special": true
     },
     {
-      "id": 32002,
+      "id": 32003,
       "content": "<|step|>",
       "single_word": false,
       "lstrip": false,
@@ -58,7 +67,7 @@
       "special": true
     },
     {
-      "id": 32003,
+      "id": 32004,
       "content": "<|function_output|>",
       "single_word": false,
       "lstrip": false,
@@ -67,7 +76,7 @@
       "special": true
     },
     {
-      "id": 32004,
+      "id": 32005,
       "content": "<|tag|>",
       "single_word": false,
       "lstrip": false,
@@ -76,7 +85,7 @@
       "special": true
     },
     {
-      "id": 32005,
+      "id": 32006,
       "content": "<|function_call|>",
       "single_word": false,
       "lstrip": false,
@@ -85,7 +94,7 @@
       "special": true
     },
     {
-      "id": 32006,
+      "id": 32007,
       "content": "<|system|>",
       "single_word": false,
       "lstrip": false,
@@ -94,7 +103,7 @@
       "special": true
     },
     {
-      "id": 32007,
+      "id": 32008,
       "content": "<|end|>",
       "single_word": false,
       "lstrip": false,
@@ -103,7 +112,7 @@
       "special": true
     },
     {
-      "id": 32008,
+      "id": 32009,
       "content": "<|raw|>",
       "single_word": false,
       "lstrip": false,
@@ -112,7 +121,7 @@
       "special": true
     },
     {
-      "id": 32009,
+      "id": 32010,
       "content": "<|continue|>",
       "single_word": false,
       "lstrip": false,
@@ -121,7 +130,7 @@
       "special": true
     },
     {
-      "id": 32010,
+      "id": 32011,
       "content": "<|user|>",
       "single_word": false,
       "lstrip": false,
@@ -130,7 +139,7 @@
       "special": true
     },
     {
-      "id": 32011,
+      "id": 32012,
       "content": "<|function_list|>",
       "single_word": false,
       "lstrip": false,

tokenizer_config.json

@@ -33,6 +33,14 @@
       "special": true
     },
     "32001": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
       "content": "<|assistant|>",
       "lstrip": false,
       "normalized": false,
@@ -40,7 +48,7 @@
       "single_word": false,
       "special": true
     },
-    "32002": {
+    "32003": {
       "content": "<|step|>",
       "lstrip": false,
       "normalized": false,
@@ -48,7 +56,7 @@
       "single_word": false,
       "special": true
     },
-    "32003": {
+    "32004": {
       "content": "<|function_output|>",
       "lstrip": false,
       "normalized": false,
@@ -56,7 +64,7 @@
       "single_word": false,
       "special": true
     },
-    "32004": {
+    "32005": {
       "content": "<|tag|>",
       "lstrip": false,
       "normalized": false,
@@ -64,7 +72,7 @@
       "single_word": false,
       "special": true
     },
-    "32005": {
+    "32006": {
       "content": "<|function_call|>",
       "lstrip": false,
       "normalized": false,
@@ -72,7 +80,7 @@
       "single_word": false,
       "special": true
     },
-    "32006": {
+    "32007": {
       "content": "<|system|>",
       "lstrip": false,
       "normalized": false,
@@ -80,7 +88,7 @@
       "single_word": false,
       "special": true
     },
-    "32007": {
+    "32008": {
       "content": "<|end|>",
       "lstrip": false,
       "normalized": false,
@@ -88,7 +96,7 @@
       "single_word": false,
       "special": true
     },
-    "32008": {
+    "32009": {
       "content": "<|raw|>",
       "lstrip": false,
       "normalized": false,
@@ -96,7 +104,7 @@
       "single_word": false,
       "special": true
     },
-    "32009": {
+    "32010": {
       "content": "<|continue|>",
       "lstrip": false,
       "normalized": false,
@@ -104,7 +112,7 @@
       "single_word": false,
       "special": true
     },
-    "32010": {
+    "32011": {
       "content": "<|user|>",
       "lstrip": false,
       "normalized": false,
@@ -112,7 +120,7 @@
       "single_word": false,
       "special": true
     },
-    "32011": {
+    "32012": {
       "content": "<|function_list|>",
       "lstrip": false,
       "normalized": false,
@@ -139,7 +147,7 @@
   "eos_token": "<|endoftext|>",
   "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_token": "<|padding|>",
   "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",