Upload tokenizer

added_tokens.json

@@ -1,6 +1,6 @@
 {
-  "<|a|>": 50259,
-  "<|c|>": 50257,
-  "<|f|>": 50260,
-  "<|u|>": 50258
+  "[assist]": 50259,
+  "[context]": 50257,
+  "[fin]": 50260,
+  "[user]": 50258
 }

special_tokens_map.json

@@ -1,28 +1,28 @@
 {
   "additional_special_tokens": [
     {
-      "content": "<|c|>",
+      "content": "[context]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "<|u|>",
+      "content": "[user]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "<|a|>",
+      "content": "[assist]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "<|f|>",
+      "content": "[fin]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,

tokenizer.json

@@ -14,7 +14,7 @@
     },
     {
       "id": 50257,
-      "content": "<|c|>",
+      "content": "[context]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 50258,
-      "content": "<|u|>",
+      "content": "[user]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 50259,
-      "content": "<|a|>",
+      "content": "[assist]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 50260,
-      "content": "<|f|>",
+      "content": "[fin]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,

tokenizer_config.json

@@ -10,7 +10,7 @@
       "special": true
     },
     "50257": {
-      "content": "<|c|>",
+      "content": "[context]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -18,7 +18,7 @@
       "special": true
     },
     "50258": {
-      "content": "<|u|>",
+      "content": "[user]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -26,7 +26,7 @@
       "special": true
     },
     "50259": {
-      "content": "<|a|>",
+      "content": "[assist]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -34,7 +34,7 @@
       "special": true
     },
     "50260": {
-      "content": "<|f|>",
+      "content": "[fin]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -43,10 +43,10 @@
     }
   },
   "additional_special_tokens": [
-    "<|c|>",
-    "<|u|>",
-    "<|a|>",
-    "<|f|>"
+    "[context]",
+    "[user]",
+    "[assist]",
+    "[fin]"
   ],
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": true,