Shubham09 commited on
Commit
5aae044
·
1 Parent(s): 45bb495

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"l": 0, "o": 1, "j": 2, "i": 3, "n": 5, "w": 6, "g": 7, "k": 8, "m": 9, "d": 10, "r": 11, "t": 12, "s": 13, "u": 14, "e": 15, "p": 16, "/": 17, "f": 18, "z": 19, "(": 20, "b": 21, "x": 22, "c": 23, "y": 24, "h": 25, "a": 26, ")": 27, "q": 28, "v": 29, "|": 4, "[UNK]": 30, "[PAD]": 31}
 
1
+ {"p": 0, "c": 1, "n": 2, "g": 3, "q": 4, "b": 5, "m": 7, "d": 8, "a": 9, "z": 10, "h": 11, "e": 12, "t": 13, "j": 14, "w": 15, "k": 16, "i": 17, "r": 18, "(": 19, "f": 20, ")": 21, "y": 22, "o": 23, "u": 24, "x": 25, "s": 26, "v": 27, "/": 28, "l": 29, "|": 6, "[UNK]": 30, "[PAD]": 31}