add tokenizer
Browse files- special_tokens_map.json +1 -0
- tokenizer_config.json +1 -0
- vocab.json +1 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
|
vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"ม": 0, "ง": 1, "ณ": 2, "ฏ": 3, "?": 4, "อ": 5, "ก": 6, "ป": 7, "ี": 8, "ศ": 9, "ษ": 10, "ธ": 11, "ห": 12, "ส": 13, "ล": 14, "เ": 15, "ฟ": 16, "จ": 17, "ำ": 18, "แ": 19, "บ": 20, "ผ": 21, "ย": 22, "ท": 23, "ด": 24, "้": 25, "ญ": 26, "ั": 27, "พ": 28, "!": 29, "ไ": 30, "ะ": 31, "ู": 32, "ื": 33, "ช": 34, "ุ": 36, "า": 37, "ใ": 38, "น": 39, "ึ": 40, "ว": 41, "ข": 42, "็": 43, "ค": 44, "ิ": 45, "ร": 46, "่": 47, "ต": 48, "ฉ": 49, "|": 35, "[UNK]": 50, "[PAD]": 51}
|