add tokenizer

vocab.json

@@ -1 +1 @@
-{"ফ": 0, "ড": 1, "উ": 2, "৩": 3, "ধ": 4, "র": 5, "৬": 6, "চ": 7, "ঋ": 8, "ভ": 9, "ঝ": 10, "া": 11, "ঃ": 12, "ী": 13, "ে": 14, "়": 15, "ঢ়": 16, "স": 17, "২": 18, "গ": 19, "এ": 20, "ক": 21, "য়": 22, "ং": 23, "ট": 24, "জ": 25, "ই": 26, "ঠ": 27, "ৗ": 28, "ঞ": 29, "ঘ": 30, "।": 31, "৫": 32, "দ": 33, "০": 34, "ঢ": 35, "ও": 36, "আ": 37, "ঈ": 38, "ঊ": 39, "ষ": 40, "ণ": 41, "ম": 42, "৯": 43, " ": 44, "ছ": 45, "ৌ": 46, "ূ": 47, "ড়": 48, "ব": 49, "৭": 50, "ৰ": 51, "শ": 52, "ঙ": 53, "ত": 54, "ল": 55, "ন": 56, "অ": 57, "ৃ": 58, "ৎ": 59, "১": 60, "ি": 61, "ৈ": 62, "ঁ": 63, "ঔ": 64, "য": 65, "প": 66, "খ": 67, "ো": 68, "৪": 69, "ঐ": 70, "হ": 71, "ু": 72, "্": 73, "থ": 74, "৮": 75, "[UNK]": 76, "[PAD]": 77}
+{"৮": 0, "্": 1, "ফ": 2, "জ": 3, "ল": 4, "থ": 5, "৬": 6, "য": 7, "খ": 8, "উ": 9, "ূ": 10, "ং": 11, "ই": 12, "ড": 13, "প": 14, "১": 15, "ঠ": 16, "ও": 17, "গ": 18, "ী": 19, "৩": 20, "ঈ": 21, "ঝ": 22, "ৎ": 23, "ু": 24, "ৃ": 25, "ড়": 26, "এ": 27, "৯": 28, "ণ": 29, "ট": 30, "৭": 31, "ষ": 32, "ৗ": 33, "শ": 34, "দ": 35, "ত": 36, "ছ": 37, "ধ": 38, "ক": 39, "ঁ": 40, "ঋ": 41, "য়": 42, "ৰ": 43, "ব": 44, "র": 45, "ঊ": 46, "ঐ": 47, "৪": 48, "ো": 49, "ঞ": 50, "া": 51, "২": 52, "ম": 53, "আ": 54, "ঙ": 55, "ন": 56, "হ": 57, "ঢ়": 58, "ঢ": 59, "ৌ": 60, "চ": 61, "ি": 62, "ৈ": 63, "ঔ": 64, "ঘ": 65, " ": 66, "।": 67, "০": 68, "৫": 69, "স": 70, "়": 71, "ঃ": 72, "ে": 73, "অ": 74, "ভ": 75, "[UNK]": 76, "[PAD]": 77}