add tokenizer

vocab.json

@@ -1 +1 @@
-{"i": 0, "b": 1, "ị": 2, "u": 3, "ễ": 4, "v": 5, "ộ": 6, "r": 7, "â": 8, "ằ": 9, "o": 10, "í": 11, "h": 12, "ầ": 13, "ứ": 14, "s": 15, "õ": 16, "ổ": 17, "ố": 18, "ù": 19, "y": 20, "ĩ": 21, "ợ": 22, "ô": 23, "ờ": 24, "t": 25, "ì": 26, "ớ": 27, "è": 28, "à": 29, "m": 31, "f": 32, "ó": 33, "ũ": 34, "ỉ": 35, "ẽ": 36, "ư": 37, "ủ": 38, "é": 39, "ừ": 40, "ồ": 41, "q": 42, "ự": 43, "ế": 44, "'": 45, "ỡ": 46, "a": 47, "ẹ": 48, "g": 49, "ở": 50, "ă": 51, "ỹ": 52, "n": 53, "á": 54, "j": 55, "ể": 56, "d": 57, "e": 58, "ẫ": 59, "ỏ": 60, "ý": 61, "đ": 62, "ò": 63, "z": 64, "l": 65, "w": 66, "ơ": 67, "ả": 68, "ê": 69, "ỳ": 70, "ạ": 71, "ẻ": 72, "ậ": 73, "ụ": 74, "ẩ": 75, "p": 76, "ữ": 77, "ỵ": 78, "ỗ": 79, "ỷ": 80, "ặ": 81, "ấ": 82, "c": 83, "ề": 84, "x": 85, "ử": 86, "k": 87, "ẵ": 88, "ọ": 89, "ẳ": 90, "ú": 91, "ệ": 92, "ắ": 93, "ã": 94, "|": 30, "[UNK]": 95, "[PAD]": 96}
+{"c": 0, "ề": 1, "ữ": 2, "b": 3, "õ": 4, "ẽ": 5, "ĩ": 6, "w": 7, "ự": 8, "é": 9, "'": 10, "ổ": 11, "í": 12, "ớ": 13, "ắ": 14, "ó": 15, "ị": 16, "đ": 17, "z": 18, "y": 19, "p": 20, "ũ": 21, "a": 22, "m": 23, "ặ": 24, "ý": 25, "l": 26, "f": 27, "ỉ": 28, "h": 29, "ả": 30, "ằ": 31, "ô": 32, "ấ": 33, "ẩ": 34, "ử": 35, "ẻ": 36, "ứ": 37, "t": 38, "ì": 39, "ệ": 40, "ỏ": 41, "ụ": 42, "ờ": 43, "u": 44, "ể": 45, "o": 47, "ẫ": 48, "â": 49, "n": 50, "s": 51, "ố": 52, "ỡ": 53, "ầ": 54, "ế": 55, "à": 56, "ậ": 57, "ẵ": 58, "ú": 59, "ợ": 60, "v": 61, "ỵ": 62, "i": 63, "ẹ": 64, "x": 65, "è": 66, "ă": 67, "ã": 68, "ỗ": 69, "ù": 70, "ạ": 71, "ỷ": 72, "ồ": 73, "d": 74, "á": 75, "ở": 76, "ọ": 77, "ư": 78, "ê": 79, "ủ": 80, "ễ": 81, "ỹ": 82, "ơ": 83, "ò": 84, "ẳ": 85, "ừ": 86, "g": 87, "r": 88, "ỳ": 89, "q": 90, "e": 91, "ộ": 92, "k": 93, "j": 94, "|": 46, "[UNK]": 95, "[PAD]": 96}