add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{}

tokenizer.json

@@ -0,0 +1,17 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {}
+  }
+}

tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "PreTrainedTokenizerFast"}