add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 93, "</s>": 94}

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"ỹ": 0, "ờ": 1, "x": 2, "đ": 3, "i": 4, "ĩ": 5, "s": 6, "q": 8, "ẽ": 9, "v": 10, "õ": 11, "ũ": 12, "o": 13, "ỳ": 14, "ợ": 15, "ú": 16, "ế": 17, "ồ": 18, "m": 19, "ạ": 20, "ỡ": 21, "ổ": 22, "ấ": 23, "ớ": 24, "ễ": 25, "y": 26, "g": 27, "ẳ": 28, "r": 29, "ã": 30, "4": 31, "h": 32, "t": 33, "ủ": 34, "ỵ": 35, "d": 36, "ư": 37, "ẹ": 38, "ẩ": 39, "ệ": 40, "ừ": 41, "ử": 42, "ặ": 43, "ứ": 44, "ý": 45, "p": 46, "ề": 47, "u": 48, "ẻ": 49, "c": 50, "ể": 51, "ằ": 52, "ụ": 53, "í": 54, "k": 55, "ầ": 56, "ẫ": 57, "ỷ": 58, "b": 59, "ỏ": 60, "ù": 61, "ọ": 62, "ơ": 63, "ỉ": 64, "ả": 65, "ẵ": 66, "ê": 67, "ắ": 68, "è": 69, "á": 70, "ậ": 71, "é": 72, "ố": 73, "ộ": 74, "à": 75, "ữ": 76, "ó": 77, "n": 78, "ự": 79, "l": 80, "ỗ": 81, "â": 82, "ò": 83, "ô": 84, "e": 85, "ă": 86, "ì": 87, "ị": 88, "a": 89, "ở": 90, "|": 7, "[UNK]": 91, "[PAD]": 92}
+{"4": 1, "a": 2, "b": 3, "c": 4, "d": 5, "e": 6, "g": 7, "h": 8, "i": 9, "k": 10, "l": 11, "m": 12, "n": 13, "o": 14, "p": 15, "q": 16, "r": 17, "s": 18, "t": 19, "u": 20, "v": 21, "x": 22, "y": 23, "à": 24, "á": 25, "â": 26, "ã": 27, "è": 28, "é": 29, "ê": 30, "ì": 31, "í": 32, "ò": 33, "ó": 34, "ô": 35, "õ": 36, "ù": 37, "ú": 38, "ý": 39, "ă": 40, "đ": 41, "ĩ": 42, "ũ": 43, "ơ": 44, "ư": 45, "ạ": 46, "ả": 47, "ấ": 48, "ầ": 49, "ẩ": 50, "ẫ": 51, "ậ": 52, "ắ": 53, "ằ": 54, "ẳ": 55, "ẵ": 56, "ặ": 57, "ẹ": 58, "ẻ": 59, "ẽ": 60, "ế": 61, "ề": 62, "ể": 63, "ễ": 64, "ệ": 65, "ỉ": 66, "ị": 67, "ọ": 68, "ỏ": 69, "ố": 70, "ồ": 71, "ổ": 72, "ỗ": 73, "ộ": 74, "ớ": 75, "ờ": 76, "ở": 77, "ỡ": 78, "ợ": 79, "ụ": 80, "ủ": 81, "ứ": 82, "ừ": 83, "ử": 84, "ữ": 85, "ự": 86, "ỳ": 87, "ỵ": 88, "ỷ": 89, "ỹ": 90, "|": 0, "[UNK]": 91, "[PAD]": 92}