add tokenizer

vocab.json

@@ -1 +1 @@
-{"ح": 0, "ظ": 1, "ے": 2, "خ": 3, "ؤ": 4, "س": 5, "و": 6, "ر": 7, "چ": 8, "‌": 9, "ی": 10, "ڈ": 11, "م": 12, "ں": 13, "ذ": 14, "ھ": 15, "ٹ": 16, "ئ": 17, "آ": 18, "ت": 19, "ً": 20, "ص": 21, "د": 22, "ع": 23, "ا": 24, "غ": 25, "ِ": 26, "ہ": 27, "ن": 29, "ء": 30, "ط": 31, "ّ": 32, "گ": 33, "ض": 34, "ک": 35, "ج": 36, "ٔ": 37, "ڑ": 38, "ق": 39, "ب": 40, "ث": 41, "ز": 42, "ل": 43, "پ": 44, "ُ": 45, "ف": 46, "ژ": 47, "َ": 48, "ش": 49, "|": 28, "[UNK]": 50, "[PAD]": 51}
+{"ں": 0, "ج": 1, "م": 2, "گ": 3, "ع": 4, "ن": 5, "چ": 6, "ُ": 7, "غ": 8, "ب": 9, "خ": 10, "ح": 11, "ل": 12, "ث": 13, "َ": 14, "د": 15, "ا": 16, "و": 17, "ً": 18, "ء": 19, "ر": 20, "ی": 21, "ف": 22, "ِ": 23, "ّ": 24, "ذ": 25, "ک": 26, "آ": 27, "ظ": 28, "‌": 29, "س": 30, "ڑ": 31, "ھ": 32, "ئ": 33, "ش": 34, "ت": 35, "ے": 36, "ہ": 37, "ڈ": 38, "ط": 39, "ص": 40, "ٔ": 41, "ز": 42, "ق": 43, "ؤ": 44, "ض": 45, "ژ": 47, "ٹ": 48, "پ": 49, "|": 46, "[UNK]": 50, "[PAD]": 51}