Upload tokenizer

tokenizer.json

@@ -2,13 +2,13 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 262,
+    "max_length": 268,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
-      "Fixed": 262
+      "Fixed": 268
     },
     "direction": "Right",
     "pad_to_multiple_of": null,
@@ -958,8 +958,8 @@
       {
         "type": "Metaspace",
         "replacement": "▁",
-        "add_prefix_space": true,
-        "prepend_scheme": "always"
+        "prepend_scheme": "always",
+        "split": true
       }
     ]
   },
@@ -1020,8 +1020,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
+    "prepend_scheme": "always",
+    "split": true
   },
   "model": {
     "type": "Unigram",

tokenizer_config.json

@@ -930,7 +930,7 @@
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,
-  "model_max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
   "tokenizer_class": "T5Tokenizer",
   "unk_token": "<unk>"