add tokenizer

runs/Apr05_21-31-46_TUF/1649219571.1507864/events.out.tfevents.1649219571.TUF.9124.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c04856ae88b92d21ef08967f8951a67b11ec96f1e1aa54e62e1f8a65fe3cb7d
+size 4778

runs/Apr05_21-31-46_TUF/events.out.tfevents.1649219571.TUF.9124.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b51a241f68052cb04abc7110dc73ab6e9aa1bba3ceb541d12b2838a7e991051
+size 4994

vocab.json

@@ -1 +1 @@
-{"h": 0, "d": 1, "g": 2, "f": 3, "b": 4, "v": 5, "m": 6, "j": 7, "z": 8, "r": 9, "t": 10, "u": 11, "p": 12, "l": 13, "s": 14, "a": 15, "w": 16, "x": 17, "c": 18, "o": 19, "'": 20, "n": 21, "k": 22, "y": 23, "e": 25, "q": 26, "i": 27, "[UNK]": 28, "[PAD]": 29, "|": 24}
+{"n": 0, "y": 2, "w": 3, "v": 4, "e": 5, "k": 6, "m": 7, "p": 8, "o": 9, "f": 10, "j": 11, "c": 12, "r": 13, "x": 14, "l": 15, "h": 16, "q": 17, "b": 18, "d": 19, "g": 20, "i": 21, "'": 22, "u": 23, "t": 24, "z": 25, "a": 26, "s": 27, "[UNK]": 28, "[PAD]": 29, "|": 1}