duclee9x commited on
Commit
4a3119d
·
1 Parent(s): d9e4c81

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"'": 1, "a": 24, "b": 37, "c": 36, "d": 17, "e": 38, "f": 10, "g": 35, "h": 5, "i": 33, "j": 14, "k": 21, "l": 29, "m": 19, "n": 6, "o": 9, "p": 34, "q": 2, "r": 16, "s": 31, "t": 39, "u": 15, "v": 26, "w": 18, "x": 7, "y": 13, "z": 3, "|": 27, "[UNK]": 40, "[PAD]": 41}
 
1
+ {"'": 29, "a": 22, "b": 0, "c": 16, "d": 6, "e": 21, "f": 23, "g": 4, "h": 38, "i": 3, "j": 28, "k": 32, "l": 13, "m": 34, "n": 5, "o": 18, "p": 2, "q": 27, "r": 8, "s": 10, "t": 20, "u": 14, "v": 26, "w": 36, "x": 31, "y": 33, "z": 11, "|": 25, "[UNK]": 40, "[PAD]": 41}