Upload tokenizer

special_tokens_map.json

@@ -1,3 +1,3 @@
 {
-  "pad_token": "<s>"
+  "pad_token": "<pad>"
 }

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 81,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": {
     "strategy": {
       "Fixed": 81
@@ -14,12 +9,12 @@
     "pad_to_multiple_of": null,
     "pad_id": 0,
     "pad_type_id": 0,
-    "pad_token": "<s>"
+    "pad_token": "<pad>"
   },
   "added_tokens": [
     {
       "id": 0,
-      "content": "<s>",
+      "content": "<pad>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -28,7 +23,7 @@
     },
     {
       "id": 1,
-      "content": "<pad>",
+      "content": "<s>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -92,8 +87,8 @@
     "byte_fallback": false,
     "ignore_merges": false,
     "vocab": {
-      "<s>": 0,
-      "<pad>": 1,
+      "<pad>": 0,
+      "<s>": 1,
       "</s>": 2,
       "<unk>": 3,
       "<mask>": 4,

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "<s>",
+      "content": "<pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "<pad>",
+      "content": "<s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -42,9 +42,9 @@
     }
   },
   "clean_up_tokenization_spaces": true,
-  "max_length": 81,
-  "model_max_length": 81,
-  "pad_token": "<s>",
+  "max_length": 4096,
+  "model_max_length": 4096,
+  "pad_token": "<pad>",
   "stride": 0,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "right",