add tokenizer

runs/May06_13-05-28_hydcorp-esxi09/1651822531.5317035/events.out.tfevents.1651822531.hydcorp-esxi09.1527205.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:513e1b781b35abfe46108f235ab51515e45b3290df998f943c0956e7617a65b7
+size 4789

runs/May06_13-05-28_hydcorp-esxi09/events.out.tfevents.1651822531.hydcorp-esxi09.1527205.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2015090fec8f3ad27d7f207598f88a65e4b7c8e3f3aec679bc7ddcc044e4405b
+size 4755

vocab.json

@@ -1 +1 @@
-{"0": 0, "r": 1, "m": 2, "i": 3, "y": 4, "b": 5, "8": 6, "1": 7, "w": 8, "v": 9, "k": 10, "9": 11, "4": 12, "d": 13, "l": 14, "o": 15, "c": 16, "3": 17, "p": 18, "f": 19, "'": 20, "z": 21, "e": 22, "5": 23, "n": 24, "6": 25, "s": 26, "g": 27, "a": 28, "x": 29, "2": 31, "h": 32, "j": 33, "t": 34, "q": 35, "u": 36, "|": 30, "[UNK]": 37, "[PAD]": 38}
+{"c": 0, "e": 1, "o": 2, "6": 3, "h": 4, "p": 5, "0": 6, "a": 7, "n": 8, "q": 9, "w": 10, "2": 11, "f": 12, "'": 13, "8": 14, "1": 15, "t": 16, "j": 17, "s": 18, "5": 19, "l": 20, "z": 21, "k": 22, "x": 24, "9": 25, "i": 26, "3": 27, "b": 28, "d": 29, "u": 30, "y": 31, "v": 32, "g": 33, "4": 34, "m": 35, "r": 36, "|": 23, "[UNK]": 37, "[PAD]": 38}