add tokenizer
Browse files- vocab.json +1 -1
vocab.json
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
{"'":
|
|
|
|
| 1 |
+
{"'": 29, "a": 22, "b": 0, "c": 16, "d": 6, "e": 21, "f": 23, "g": 4, "h": 38, "i": 3, "j": 28, "k": 32, "l": 13, "m": 34, "n": 5, "o": 18, "p": 2, "q": 27, "r": 8, "s": 10, "t": 20, "u": 14, "v": 26, "w": 36, "x": 31, "y": 33, "z": 11, "|": 25, "[UNK]": 40, "[PAD]": 41}
|