add tokenizer

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7024d0755c377938e0fdc24d170ec11badbe927c3418cd420a6803b83d31bdd4
+oid sha256:491e1ef347ec8ad58e64b1a9b1b0f31909013e03622b94cc76402e3aa4b623e6
 size 1262052141

runs/Jan03_08-50-38_ip-172-31-16-234/events.out.tfevents.1672735927.ip-172-31-16-234.1044451.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7878b2a4fee29d4d889a1f6ecfe67734e3cb5170be78cb9cc5118826e2e87e4c
-size 4675
+oid sha256:925310a998bb409742c4f00fc8eb9af909a9a8365cd4de3792c01e1e6e31d1a8
+size 5150

vocab.json

@@ -1 +1 @@
-{"g": 0, "y": 1, "u": 2, "w": 3, "z": 4, "k": 5, "i": 6, "q": 7, "d": 8, "o": 9, "a": 10, "v": 11, "j": 12, "s": 13, "p": 14, "r": 15, "l": 16, "f": 17, "'": 18, "c": 19, "e": 20, "m": 21, "h": 22, "b": 23, "x": 25, "n": 26, "t": 27, "|": 24, "[unq]": 28, "[padded]": 29}
+{"y": 0, "c": 1, "j": 2, "i": 3, "r": 4, "n": 5, "h": 6, "v": 7, "s": 8, "x": 9, "p": 10, "u": 11, "b": 12, "t": 13, "d": 14, "w": 15, "o": 16, "z": 17, "l": 18, "e": 19, "a": 20, "'": 21, "g": 22, "m": 24, "f": 25, "k": 26, "q": 27, "|": 23, "[unq]": 28, "[padded]": 29}