Upload tokenizer

added_tokens.json

@@ -4,6 +4,9 @@
   "<pad>": 8003,
   "<s>": 8000,
   "<unk>": 8002,
-  "<|assistant|>": 8006,
-  "<|endoftext|>": 8005
+  "<|assistant|>": 8008,
+  "<|endoftext|>": 8005,
+  "<|end|>": 8007,
+  "<|system|>": 8009,
+  "<|user|>": 8006
 }

merges.txt

@@ -7739,3 +7739,7 @@ bal anced
 Ġoverwhel med
 Ġlic ense
 Ġflood ing
+Ġcrow d
+Ġtow ns
+ĠEle phant
+Ġwond ers

tokenizer_config.json

@@ -1,22 +1,6 @@
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
-    "2": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
     "8000": {
       "content": "<s>",
       "lstrip": false,
@@ -66,12 +50,36 @@
       "special": true
     },
     "8006": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8008": {
       "content": "<|assistant|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "8009": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "bos_token": "<|user|>",