Upload tokenizer

special_tokens_map.json

@@ -1,7 +1 @@
-{
-  "bos_token": "<|bos|>",
-  "eos_token": "<|endoftext|>",
-  "mask_token": "<|mask|>",
-  "pad_token": "<|pad|>",
-  "unk_token": "<|unk|>"
-}
+{}

tokenizer.json

@@ -5182,7 +5182,12 @@
       "Ġalongside": 5111,
       "ĠPass": 5112,
       "itled": 5113,
-      "ĠNetherlands": 5114
+      "ĠNetherlands": 5114,
+      "ĠDer": 5115,
+      "ĠFire": 5116,
+      "Ġtouch": 5117,
+      "astic": 5118,
+      "ĠSenate": 5119
     },
     "merges": [
       [
@@ -24776,6 +24781,26 @@
       [
         "ĠN",
         "etherlands"
+      ],
+      [
+        "ĠD",
+        "er"
+      ],
+      [
+        "ĠF",
+        "ire"
+      ],
+      [
+        "Ġto",
+        "uch"
+      ],
+      [
+        "ast",
+        "ic"
+      ],
+      [
+        "ĠSen",
+        "ate"
       ]
     ]
   }

tokenizer_config.json

@@ -41,12 +41,7 @@
       "special": true
     }
   },
-  "bos_token": "<|bos|>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "mask_token": "<|mask|>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|pad|>",
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "<|unk|>"
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }