Upload tokenizer

special_tokens_map.json

@@ -1,4 +1,11 @@
 {
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "cls_token": {
     "content": "[CLS]",
     "lstrip": false,
@@ -6,6 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "mask_token": {
     "content": "[MASK]",
     "lstrip": false,

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "[PAD]",
+      "content": "[CLS]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "[UNK]",
+      "content": "[PAD]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "[CLS]",
+      "content": "[SEP]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "[SEP]",
+      "content": "[UNK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -103,13 +103,13 @@
       {
         "Sequence": {
           "id": "B",
-          "type_id": 1
+          "type_id": 0
         }
       },
       {
         "SpecialToken": {
           "id": "[SEP]",
-          "type_id": 1
+          "type_id": 0
         }
       }
     ],
@@ -117,7 +117,7 @@
       "[CLS]": {
         "id": "[CLS]",
         "ids": [
-          2
+          0
         ],
         "tokens": [
           "[CLS]"
@@ -126,7 +126,7 @@
       "[SEP]": {
         "id": "[SEP]",
         "ids": [
-          3
+          2
         ],
         "tokens": [
           "[SEP]"
@@ -145,10 +145,10 @@
     "continuing_subword_prefix": "##",
     "max_input_chars_per_word": 100,
     "vocab": {
-      "[PAD]": 0,
-      "[UNK]": 1,
-      "[CLS]": 2,
-      "[SEP]": 3,
+      "[CLS]": 0,
+      "[PAD]": 1,
+      "[SEP]": 2,
+      "[UNK]": 3,
       "[MASK]": 4,
       "!": 5,
       "\"": 6,

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "[PAD]",
+      "content": "[CLS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "[UNK]",
+      "content": "[PAD]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,7 +17,7 @@
       "special": true
     },
     "2": {
-      "content": "[CLS]",
+      "content": "[SEP]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -25,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "[SEP]",
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -41,10 +41,12 @@
       "special": true
     }
   },
+  "bos_token": "[CLS]",
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "do_basic_tokenize": true,
   "do_lower_case": false,
+  "eos_token": "[SEP]",
   "mask_token": "[MASK]",
   "model_max_length": 512,
   "never_split": null,

vocab.txt

@@ -1,7 +1,7 @@
-[PAD]
-[UNK]
 [CLS]
+[PAD]
 [SEP]
+[UNK]
 [MASK]
 !
 "