add tokenizer

Browse files

Files changed (1) hide show

vocab.json +1 -1

vocab.json CHANGED Viewed

@@ -1 +1 @@

- {"ت": 0, "ذ": 1, "ق": 2, "ف": 3, "ئ": 4, "ٹ": 5, "ظ": 6, "ڈ": 7, "ک": 8, "ج": 9, "ا": 10, "ص": 11, "ھ": 12, "ہ": 13, "م": 14, "ر": 15, "ح": 16, "چ": 17, "ی": 18, "ء": 19, "گ": 20, "ے": 21, "آ": 22, "و": 23, "ڑ": 24, "ں": 25, "ض": 26, "پ": 27, "ل": 28, " ": 29, "ب": 30, "خ": 31, "س": 32, "ن": 33, "ؤ": 34, "ث": 35, "ط": 36, "ژ": 37, "غ": 38, "ش": 39, "ز": 40, "د": 41, "ع": 42, "[UNK]": 43, "[PAD]": 44}

+ {"ژ": 0, "ز": 1, "ظ": 2, "ش": 3, "ف": 4, "ا": 5, "ر": 6, "چ": 7, "د": 8, "ص": 9, "ب": 10, "و": 11, "ئ": 12, "ء": 13, "ھ": 14, "ض": 15, "ط": 16, "ں": 17, "ڈ": 18, "س": 19, "ہ": 20, "غ": 21, "ؤ": 22, "ڑ": 23, "ک": 24, "ن": 25, "خ": 26, "ج": 27, "ث": 28, "آ": 29, "ت": 30, "ی": 31, "ق": 32, "ح": 33, " ": 34, "ع": 35, "ذ": 36, "ے": 37, "پ": 38, "ل": 39, "گ": 40, "م": 41, "ٹ": 42, "[UNK]": 43, "[PAD]": 44}