modhp commited on
Commit
7b1d45d
·
1 Parent(s): e97d529

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"t": 0, "i": 1, "m": 2, "q": 3, " ": 4, "a": 5, "s": 6, "e": 7, "p": 8, "h": 9, "w": 10, "c": 11, "3": 12, "r": 13, "k": 14, "n": 15, "z": 16, "y": 17, "l": 18, "b": 19, "o": 20, "d": 21, "f": 22, "1": 23, "g": 24, "j": 25, "v": 26, "'": 27, "x": 28, "u": 29, "[UNK]": 30, "[PAD]": 31}
 
1
+ {"'": 0, "j": 1, "i": 2, "o": 3, "v": 4, "k": 5, "c": 6, "d": 7, "e": 8, "q": 9, "z": 10, "p": 11, " ": 12, "l": 13, "h": 14, "r": 15, "x": 16, "f": 17, "g": 18, "y": 19, "a": 20, "t": 21, "u": 22, "w": 23, "m": 24, "b": 25, "n": 26, "s": 27, "[UNK]": 28, "[PAD]": 29}