Upload tokenizer

tokenizer.json

@@ -265,7 +265,9 @@
       "special": true
     }
   ],
-  "normalizer": null,
+  "normalizer": {
+    "type": "NFKD"
+  },
   "pre_tokenizer": {
     "type": "Whitespace"
   },
@@ -344,7 +346,11 @@
       }
     }
   },
-  "decoder": null,
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
   "model": {
     "type": "WordPiece",
     "unk_token": "[UNK]",

tokenizer_config.json

@@ -1,5 +1,5 @@
 {
   "clean_up_tokenization_spaces": true,
-  "model_max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
   "tokenizer_class": "PreTrainedTokenizerFast"
 }