add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Aug25_14-33-03_510311d14d2f/1661438023.9286213/events.out.tfevents.1661438023.510311d14d2f.70.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886854397d1da55632e6ee8c90c0c2e3c10f8476eb25312b375ae56844905bfb
+size 4762

runs/Aug25_14-33-03_510311d14d2f/events.out.tfevents.1661438023.510311d14d2f.70.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c76614005f19116c9d67c2175eb7e5eccfedb2347b8f4b1cd9d1dba0bb2f91c3
+size 4941

vocab.json

@@ -1 +1 @@
-{"s": 0, "R": 2, "T": 3, "t": 4, "o": 5, "w": 6, "N": 7, "j": 8, "S": 9, "\"": 10, "K": 11, "k": 12, "U": 13, "d": 14, "p": 15, "r": 16, "E": 17, "y": 18, "X": 19, "B": 20, "u": 21, ":": 22, "c": 23, "I": 24, "A": 25, "F": 26, ",": 27, "W": 28, "L": 29, "!": 30, "a": 31, "V": 32, "G": 33, "D": 34, "i": 35, "Y": 36, "n": 37, "q": 38, "Q": 39, "f": 40, "h": 41, "J": 42, "l": 43, "-": 44, "H": 45, "g": 46, "b": 47, "m": 48, "v": 49, "?": 50, "O": 51, "e": 52, "z": 53, "'": 54, ".": 55, ";": 56, "P": 57, "C": 58, "x": 59, "M": 60, "|": 1, "[UNK]": 61, "[PAD]": 62}
+{"s": 0, "c": 2, "t": 3, "o": 4, "a": 5, "w": 6, "i": 7, "j": 8, "n": 9, "q": 10, "f": 11, "h": 12, "k": 13, "l": 14, "g": 15, "b": 16, "d": 17, "p": 18, "r": 19, "m": 20, "v": 21, "y": 22, "e": 23, "z": 24, "'": 25, "u": 26, "x": 27, "|": 1, "[UNK]": 28, "[PAD]": 29}