Add tokenizer files

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "mask_token": "_",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -66,32 +66,7 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": {
-    "type": "TemplateProcessing",
-    "single": [
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      }
-    ],
-    "pair": [
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "B",
-          "type_id": 1
-        }
-      }
-    ],
-    "special_tokens": {}
-  },
+  "post_processor": null,
   "decoder": null,
   "model": {
     "type": "WordLevel",

tokenizer_config.json

@@ -1,10 +1,53 @@
 {
-  "backend": "tokenizers",
+  "added_tokens_decoder": {
+    "0": {
+      "content": "_",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[BOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "[BOS]",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "[EOS]",
+  "extra_special_tokens": {},
   "mask_token": "_",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
-  "tokenizer_class": "TokenizersBackend",
+  "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "[UNK]"
 }