add tokenizer

runs/May06_13-04-17_hydcorp-esxi09/1651822460.5175688/events.out.tfevents.1651822460.hydcorp-esxi09.1526819.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9e16b422881fba1343ec317f73333bf11428038eaf72572ead141a563cecd5b
+size 4789

runs/May06_13-04-17_hydcorp-esxi09/events.out.tfevents.1651822460.hydcorp-esxi09.1526819.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9140a82a5d10d3622db86f0983eb0bced097c55ad0c0111af26fff1a75804bf1
+size 4755

vocab.json

@@ -1 +1 @@
-{"m": 0, "f": 1, "n": 2, "x": 3, "t": 4, "g": 5, "0": 6, "c": 7, "2": 9, "j": 10, "p": 11, "d": 12, "4": 13, "e": 14, "v": 15, "r": 16, "h": 17, "8": 18, "a": 19, "u": 20, "z": 21, "q": 22, "3": 23, "6": 24, "o": 25, "k": 26, "9": 27, "s": 28, "l": 29, "i": 30, "y": 31, "5": 32, "b": 33, "w": 34, "1": 35, "'": 36, "|": 8, "[UNK]": 37, "[PAD]": 38}
+{"0": 0, "r": 1, "m": 2, "i": 3, "y": 4, "b": 5, "8": 6, "1": 7, "w": 8, "v": 9, "k": 10, "9": 11, "4": 12, "d": 13, "l": 14, "o": 15, "c": 16, "3": 17, "p": 18, "f": 19, "'": 20, "z": 21, "e": 22, "5": 23, "n": 24, "6": 25, "s": 26, "g": 27, "a": 28, "x": 29, "2": 31, "h": 32, "j": 33, "t": 34, "q": 35, "u": 36, "|": 30, "[UNK]": 37, "[PAD]": 38}