add tokenizer

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "unk_token": "<unk>",
+    "pad_token": "<pad>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+    "unk_token": "<unk>",
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "pad_token": "<pad>",
+    "do_lower_case": false,
+    "return_attention_mask": false,
+    "do_normalize": true
+}

vocab.json

@@ -0,0 +1,33 @@
+{
+    "<pad>": 0,
+    "<s>": 1,
+    "</s>": 2,
+    "<unk>": 3,
+    "E": 4,
+    "A": 5,
+    "O": 6,
+    "S": 7,
+    "R": 8,
+    "N": 9,
+    "I": 10,
+    "D": 11,
+    "L": 12,
+    "C": 13,
+    "T": 14,
+    "U": 15,
+    "M": 16,
+    "P": 17,
+    "B": 18,
+    "G": 19,
+    "V": 20,
+    "Y": 21,
+    "Q": 22,
+    "H": 23,
+    "F": 24,
+    "Z": 25,
+    "J": 26,
+    "Ñ": 27,
+    "X": 28,
+    "K": 29,
+    "W": 30
+}