add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"ẩ": 0, "ứ": 1, "ủ": 2, "n": 3, "ầ": 4, "ũ": 5, "ỗ": 6, "ở": 7, "ồ": 8, "ớ": 9, "ư": 10, "à": 11, "ỹ": 12, "ĩ": 13, "r": 14, "ệ": 15, "ó": 16, "d": 17, "ợ": 18, "ý": 19, "ờ": 20, "õ": 21, "u": 22, "ặ": 23, "ỳ": 24, "a": 25, "ễ": 26, "ỷ": 27, "ự": 28, "ữ": 29, "ằ": 30, "y": 31, "ẫ": 32, "ơ": 33, "t": 34, "è": 35, "ậ": 36, "đ": 37, "x": 38, "ổ": 39, "é": 40, "ố": 41, "ù": 43, "ử": 44, "ẳ": 45, "ả": 46, "p": 47, "ọ": 48, "â": 49, "ị": 50, "ụ": 51, "ì": 52, "c": 53, "q": 54, "ỡ": 55, "l": 56, "ề": 57, "ắ": 58, "ừ": 59, "4": 60, "ò": 61, "á": 62, "e": 63, "í": 64, "v": 65, "ú": 66, "ă": 67, "ê": 68, "ấ": 69, "ỏ": 70, "ẻ": 71, "m": 72, "h": 73, "b": 74, "ỵ": 75, "ỉ": 76, "ế": 77, "o": 78, "ẽ": 79, "s": 80, "g": 81, "ẵ": 82, "ẹ": 83, "ã": 84, "i": 85, "k": 86, "ể": 87, "ạ": 88, "ộ": 89, "ô": 90, "|": 42, "[UNK]": 91, "[PAD]": 92}
+{"ô": 0, "ổ": 1, "l": 2, "é": 3, "p": 4, "ứ": 5, "ờ": 6, "n": 7, "ỵ": 8, "ơ": 9, "e": 10, "ỡ": 11, "ă": 12, "â": 13, "ừ": 14, "ễ": 15, "ử": 16, "ồ": 17, "ỳ": 18, "ằ": 19, "ý": 20, "ầ": 21, "à": 22, "g": 23, "ế": 24, "ủ": 25, "ỉ": 26, "ỏ": 27, "a": 28, "ụ": 29, "è": 30, "b": 31, "k": 32, "r": 33, "o": 34, "v": 35, "ỗ": 36, "ỷ": 37, "q": 38, "ặ": 39, "ớ": 40, "ũ": 41, "á": 42, "ợ": 43, "ắ": 44, "ẫ": 45, "ó": 46, "ĩ": 47, "c": 48, "m": 49, "ể": 50, "ậ": 52, "ấ": 53, "ù": 54, "ê": 55, "x": 56, "ữ": 57, "ạ": 58, "ự": 59, "ẩ": 60, "ẹ": 61, "s": 62, "d": 63, "ọ": 64, "ề": 65, "í": 66, "ẳ": 67, "ì": 68, "ộ": 69, "ỹ": 70, "ẵ": 71, "h": 72, "u": 73, "ò": 74, "ệ": 75, "ú": 76, "i": 77, "ị": 78, "õ": 79, "t": 80, "ở": 81, "ã": 82, "4": 83, "ẽ": 84, "đ": 85, "y": 86, "ư": 87, "ẻ": 88, "ả": 89, "ố": 90, "|": 51, "[UNK]": 91, "[PAD]": 92}