add tokenizer

runs/May10_19-16-59_DESKTOP-AGU7Q8N/1652206809.7839186/events.out.tfevents.1652206809.DESKTOP-AGU7Q8N.13420.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6309aee43d7fbe713c864ea29b895f297ca50712f0edc2de3ae3bea31379ea7a
+size 4547

runs/May10_19-16-59_DESKTOP-AGU7Q8N/events.out.tfevents.1652206809.DESKTOP-AGU7Q8N.13420.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:329aa1dc621914a8bb0bbc49001430c36b8717565e3f0884f2d6f2fdb6ba2ee9
+size 4463

vocab.json

@@ -1 +1 @@
-{"u": 0, "y": 1, "h": 2, "t": 3, "r": 4, "q": 5, "f": 6, "g": 7, "a": 8, "d": 9, "m": 10, "w": 11, "i": 12, "z": 13, "l": 14, "x": 15, "o": 16, "j": 17, "v": 18, "p": 20, "'": 21, "s": 22, "n": 23, "e": 24, "k": 25, "c": 26, "b": 27, "|": 19, "[UNK]": 28, "[PAD]": 29}
+{"x": 0, "w": 1, "g": 2, "s": 4, "u": 5, "'": 6, "d": 7, "o": 8, "i": 9, "v": 10, "j": 11, "c": 12, "m": 13, "e": 14, "t": 15, "z": 16, "q": 17, "h": 18, "a": 19, "f": 20, "n": 21, "r": 22, "l": 23, "y": 24, "p": 25, "k": 26, "b": 27, "|": 3, "[UNK]": 28, "[PAD]": 29}