add tokenizer

runs/Apr06_10-24-33_DESKTOP-5GECLH7/1649219150.819579/events.out.tfevents.1649219150.DESKTOP-5GECLH7.20404.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e90cff95c543ff1a60df3f6f5675a027c012554cf9df6e00947522193db1c9c
+size 4507

runs/Apr06_10-24-33_DESKTOP-5GECLH7/events.out.tfevents.1649219150.DESKTOP-5GECLH7.20404.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dd9d099c6b23f1464e78df989d1b358860a578c6d342a88d5cb8264193cf97f
+size 4437

vocab.json

@@ -1 +1 @@
-{"r": 0, "n": 1, "m": 2, "d": 3, "c": 4, "u": 5, "t": 6, "e": 7, "'": 8, "h": 9, "i": 10, "k": 11, "p": 12, "o": 13, "q": 14, "w": 15, "a": 16, "x": 18, "v": 19, "l": 20, "g": 21, "s": 22, "z": 23, "y": 24, "f": 25, "b": 26, "j": 27, "|": 17, "[UNK]": 28, "[PAD]": 29}
+{"h": 0, "w": 1, "a": 2, "d": 3, "c": 4, "x": 5, "r": 6, "i": 7, "f": 8, "b": 9, "q": 10, "s": 11, "t": 12, "l": 13, "z": 15, "y": 16, "m": 17, "u": 18, "k": 19, "v": 20, "e": 21, "p": 22, "o": 23, "j": 24, "n": 25, "g": 26, "'": 27, "|": 14, "[UNK]": 28, "[PAD]": 29}