add tokenizer

vocab.json

@@ -1 +1 @@
-{"ô": 0, "ổ": 1, "l": 2, "é": 3, "p": 4, "ứ": 5, "ờ": 6, "n": 7, "ỵ": 8, "ơ": 9, "e": 10, "ỡ": 11, "ă": 12, "â": 13, "ừ": 14, "ễ": 15, "ử": 16, "ồ": 17, "ỳ": 18, "ằ": 19, "ý": 20, "ầ": 21, "à": 22, "g": 23, "ế": 24, "ủ": 25, "ỉ": 26, "ỏ": 27, "a": 28, "ụ": 29, "è": 30, "b": 31, "k": 32, "r": 33, "o": 34, "v": 35, "ỗ": 36, "ỷ": 37, "q": 38, "ặ": 39, "ớ": 40, "ũ": 41, "á": 42, "ợ": 43, "ắ": 44, "ẫ": 45, "ó": 46, "ĩ": 47, "c": 48, "m": 49, "ể": 50, "ậ": 52, "ấ": 53, "ù": 54, "ê": 55, "x": 56, "ữ": 57, "ạ": 58, "ự": 59, "ẩ": 60, "ẹ": 61, "s": 62, "d": 63, "ọ": 64, "ề": 65, "í": 66, "ẳ": 67, "ì": 68, "ộ": 69, "ỹ": 70, "ẵ": 71, "h": 72, "u": 73, "ò": 74, "ệ": 75, "ú": 76, "i": 77, "ị": 78, "õ": 79, "t": 80, "ở": 81, "ã": 82, "4": 83, "ẽ": 84, "đ": 85, "y": 86, "ư": 87, "ẻ": 88, "ả": 89, "ố": 90, "|": 51, "[UNK]": 91, "[PAD]": 92}
+{"ọ": 0, "v": 1, "ẩ": 2, "a": 3, "ũ": 4, "e": 5, "u": 6, "ố": 7, "ù": 8, "4": 9, "ĩ": 10, "ẻ": 11, "ằ": 12, "ồ": 13, "m": 14, "d": 15, "í": 16, "ầ": 17, "ứ": 18, "ê": 19, "ợ": 20, "ỹ": 21, "é": 22, "ẽ": 23, "ý": 24, "ề": 25, "ử": 26, "ổ": 27, "l": 28, "ơ": 29, "ế": 30, "s": 31, "i": 32, "ò": 33, "â": 34, "ẫ": 35, "ệ": 36, "ú": 37, "y": 38, "õ": 39, "ộ": 40, "ẹ": 41, "ụ": 42, "ó": 43, "p": 44, "ủ": 45, "ấ": 46, "ỡ": 47, "đ": 48, "ậ": 49, "ặ": 50, "à": 51, "n": 52, "ỷ": 53, "ả": 54, "ỳ": 55, "ă": 56, "ờ": 57, "ữ": 59, "x": 60, "q": 61, "ể": 62, "ớ": 63, "á": 64, "ẵ": 65, "ị": 66, "è": 67, "ạ": 68, "c": 69, "ẳ": 70, "ắ": 71, "r": 72, "ỵ": 73, "ì": 74, "t": 75, "ở": 76, "ự": 77, "ỏ": 78, "ư": 79, "g": 80, "ễ": 81, "ỉ": 82, "ã": 83, "ỗ": 84, "ô": 85, "o": 86, "k": 87, "ừ": 88, "b": 89, "h": 90, "|": 58, "[UNK]": 91, "[PAD]": 92}