add tokenizer

runs/Mar27_18-40-01_DESKTOP-C5B0HSF/1648386614.0137758/events.out.tfevents.1648386614.DESKTOP-C5B0HSF.8600.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4adb35da48e57d6531fce6aa5f0d31ced96bdb1e94670b5cbb8b3812ce321dca
+size 4757

runs/Mar27_18-40-01_DESKTOP-C5B0HSF/events.out.tfevents.1648386614.DESKTOP-C5B0HSF.8600.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1c21b069c4293bfaeb8cc2a37abbe1ebd803f54da9f500f15c703ce2a854f96
+size 5138

vocab.json

@@ -1 +1 @@
-{"t": 0, "x": 1, "l": 2, "z": 3, "'": 4, "f": 5, "q": 6, "c": 7, "g": 8, "e": 9, "w": 10, "s": 11, "y": 12, "a": 13, "p": 14, "h": 15, "n": 16, "r": 17, "m": 18, "o": 20, "j": 21, "b": 22, "d": 23, "i": 24, "u": 25, "k": 26, "v": 27, "|": 19, "[UNK]": 28, "[PAD]": 29}
+{"m": 0, "y": 1, "k": 2, "b": 3, "n": 4, "j": 5, "i": 6, "a": 7, "q": 9, "e": 10, "w": 11, "z": 12, "g": 13, "f": 14, "l": 15, "p": 16, "h": 17, "'": 18, "v": 19, "o": 20, "s": 21, "t": 22, "c": 23, "r": 24, "u": 25, "d": 26, "x": 27, "|": 8, "[UNK]": 28, "[PAD]": 29}