add tokenizer

added_tokens.json

@@ -1,4 +1 @@
-{
-  "</s>": 92,
-  "<s>": 91
-}
+{"<s>": 110, "</s>": 111}

special_tokens_map.json

@@ -1,22 +1 @@
-{
-  "additional_special_tokens": [
-    {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "[PAD]",
-  "unk_token": "[UNK]"
-}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1,12 +1 @@
-{
-  "bos_token": "<s>",
-  "do_lower_case": false,
-  "eos_token": "</s>",
-  "name_or_path": "./",
-  "pad_token": "[PAD]",
-  "replace_word_delimiter_char": " ",
-  "special_tokens_map_file": null,
-  "tokenizer_class": "Wav2Vec2CTCTokenizer",
-  "unk_token": "[UNK]",
-  "word_delimiter_token": "|"
-}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "C:\\Users\\stech-data/.cache\\huggingface\\transformers\\d60dd610960164f1b79222456db381022e8f93b2f92c21d276f4cd2cb7647737.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "nguyenvulebinh/wav2vec2-base-vietnamese-250h", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1,93 +1 @@
-{
-  "[PAD]": 90,
-  "[UNK]": 89,
-  "a": 1,
-  "b": 2,
-  "c": 3,
-  "d": 4,
-  "e": 5,
-  "g": 6,
-  "h": 7,
-  "i": 8,
-  "k": 9,
-  "l": 10,
-  "m": 11,
-  "n": 12,
-  "o": 13,
-  "p": 14,
-  "q": 15,
-  "r": 16,
-  "s": 17,
-  "t": 18,
-  "u": 19,
-  "v": 20,
-  "x": 21,
-  "y": 22,
-  "|": 0,
-  "à": 23,
-  "á": 24,
-  "â": 25,
-  "ã": 26,
-  "è": 27,
-  "é": 28,
-  "ê": 29,
-  "ì": 30,
-  "í": 31,
-  "ò": 32,
-  "ó": 33,
-  "ô": 34,
-  "õ": 35,
-  "ù": 36,
-  "ú": 37,
-  "ý": 38,
-  "ă": 39,
-  "đ": 40,
-  "ĩ": 41,
-  "ũ": 42,
-  "ơ": 43,
-  "ư": 44,
-  "ạ": 45,
-  "ả": 46,
-  "ấ": 47,
-  "ầ": 48,
-  "ẩ": 49,
-  "ẫ": 50,
-  "ậ": 51,
-  "ắ": 52,
-  "ằ": 53,
-  "ẳ": 54,
-  "ẵ": 55,
-  "ặ": 56,
-  "ẹ": 57,
-  "ẻ": 58,
-  "ẽ": 59,
-  "ế": 60,
-  "ề": 61,
-  "ể": 62,
-  "ễ": 63,
-  "ệ": 64,
-  "ỉ": 65,
-  "ị": 66,
-  "ọ": 67,
-  "ỏ": 68,
-  "ố": 69,
-  "ồ": 70,
-  "ổ": 71,
-  "ỗ": 72,
-  "ộ": 73,
-  "ớ": 74,
-  "ờ": 75,
-  "ở": 76,
-  "ỡ": 77,
-  "ợ": 78,
-  "ụ": 79,
-  "ủ": 80,
-  "ứ": 81,
-  "ừ": 82,
-  "ử": 83,
-  "ữ": 84,
-  "ự": 85,
-  "ỳ": 86,
-  "ỷ": 87,
-  "ỹ": 88
-}
+{"ẻ": 0, "6": 1, "ụ": 2, "í": 3, "3": 4, "ỹ": 5, "ý": 6, "ẩ": 7, "ở": 8, "ề": 9, "õ": 10, "7": 11, "ê": 12, "ứ": 13, "ỏ": 14, "v": 15, "ỷ": 16, "a": 17, "l": 18, "ự": 19, "q": 20, "ờ": 21, "j": 22, "ố": 23, "à": 24, "ỗ": 25, "n": 26, "é": 27, "ủ": 28, "у": 29, "ô": 30, "u": 31, "y": 32, "ằ": 33, "4": 34, "w": 35, "b": 36, "ệ": 37, "ễ": 38, "s": 39, "ì": 40, "ầ": 41, "ỵ": 42, "8": 43, "d": 44, "ể": 45, "r": 47, "ũ": 48, "c": 49, "ạ": 50, "9": 51, "ế": 52, "ù": 53, "ỡ": 54, "2": 55, "t": 56, "i": 57, "g": 58, "́": 59, "ử": 60, "̀": 61, "á": 62, "0": 63, "ậ": 64, "e": 65, "ộ": 66, "m": 67, "ẳ": 68, "ợ": 69, "ĩ": 70, "h": 71, "â": 72, "ú": 73, "ọ": 74, "ồ": 75, "ặ": 76, "f": 77, "ữ": 78, "ắ": 79, "ỳ": 80, "x": 81, "ó": 82, "ã": 83, "ổ": 84, "ị": 85, "̣": 86, "z": 87, "ả": 88, "đ": 89, "è": 90, "ừ": 91, "ò": 92, "ẵ": 93, "1": 94, "ơ": 95, "k": 96, "ẫ": 97, "p": 98, "ấ": 99, "ẽ": 100, "ỉ": 101, "ớ": 102, "ẹ": 103, "ă": 104, "o": 105, "ư": 106, "5": 107, "|": 46, "<unk>": 108, "<pad>": 109}