{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|bos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|eos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|TEXT_PROMPT_START|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "<|TEXT_PROMPT_END|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|SPEECH_GENERATION_START|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|SPEECH_GENERATION_END|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|audio_ref_slot|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|emotion_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|emotion_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|emotion_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|emotion_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|emotion_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|reserved_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "<|reserved_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "<|reserved_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "<|reserved_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "<|reserved_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "<|reserved_5|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 19, "content": "<|reserved_6|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 20, "content": "<|reserved_7|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "<|reserved_8|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "<|reserved_9|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 23, "content": "<|reserved_10|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 24, "content": "<|reserved_11|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 25, "content": "<|reserved_12|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 26, "content": "<|reserved_13|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 27, "content": "<|reserved_14|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 28, "content": "<|reserved_15|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 29, "content": "<|reserved_16|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 30, "content": "<|reserved_17|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 31, "content": "<|reserved_18|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 32, "content": "<|reserved_19|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 33, "content": "<|reserved_20|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 34, "content": "<|reserved_21|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 35, "content": "<|reserved_22|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 36, "content": "<|reserved_23|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 37, "content": "<|reserved_24|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 38, "content": "<|reserved_25|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 39, "content": "<|reserved_26|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 40, "content": "<|reserved_27|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 41, "content": "<|reserved_28|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 42, "content": "<|reserved_29|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 43, "content": "<|unk|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false } ] }, "post_processor": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "model": { "type": "BPE", "dropout": null, "unk_token": "<|unk|>", "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "<|pad|>": 0, "<|bos|>": 1, "<|eos|>": 2, "<|TEXT_PROMPT_START|>": 3, "<|TEXT_PROMPT_END|>": 4, "<|SPEECH_GENERATION_START|>": 5, "<|SPEECH_GENERATION_END|>": 6, "<|audio_ref_slot|>": 7, "<|emotion_0|>": 8, "<|emotion_1|>": 9, "<|emotion_2|>": 10, "<|emotion_3|>": 11, "<|emotion_4|>": 12, "<|reserved_0|>": 13, "<|reserved_1|>": 14, "<|reserved_2|>": 15, "<|reserved_3|>": 16, "<|reserved_4|>": 17, "<|reserved_5|>": 18, "<|reserved_6|>": 19, "<|reserved_7|>": 20, "<|reserved_8|>": 21, "<|reserved_9|>": 22, "<|reserved_10|>": 23, "<|reserved_11|>": 24, "<|reserved_12|>": 25, "<|reserved_13|>": 26, "<|reserved_14|>": 27, "<|reserved_15|>": 28, "<|reserved_16|>": 29, "<|reserved_17|>": 30, "<|reserved_18|>": 31, "<|reserved_19|>": 32, "<|reserved_20|>": 33, "<|reserved_21|>": 34, "<|reserved_22|>": 35, "<|reserved_23|>": 36, "<|reserved_24|>": 37, "<|reserved_25|>": 38, "<|reserved_26|>": 39, "<|reserved_27|>": 40, "<|reserved_28|>": 41, "<|reserved_29|>": 42, "<|unk|>": 43, "Ā": 44, "ā": 45, "Ă": 46, "ă": 47, "Ą": 48, "ą": 49, "Ć": 50, "ć": 51, "Ĉ": 52, "ĉ": 53, "Ċ": 54, "ċ": 55, "Č": 56, "č": 57, "Ď": 58, "ď": 59, "Đ": 60, "đ": 61, "Ē": 62, "ē": 63, "Ĕ": 64, "ĕ": 65, "Ė": 66, "ė": 67, "Ę": 68, "ę": 69, "Ě": 70, "ě": 71, "Ĝ": 72, "ĝ": 73, "Ğ": 74, "ğ": 75, "Ġ": 76, "!": 77, "\"": 78, "#": 79, "$": 80, "%": 81, "&": 82, "'": 83, "(": 84, ")": 85, "*": 86, "+": 87, ",": 88, "-": 89, ".": 90, "/": 91, "0": 92, "1": 93, "2": 94, "3": 95, "4": 96, "5": 97, "6": 98, "7": 99, "8": 100, "9": 101, ":": 102, ";": 103, "<": 104, "=": 105, ">": 106, "?": 107, "@": 108, "A": 109, "B": 110, "C": 111, "D": 112, "E": 113, "F": 114, "G": 115, "H": 116, "I": 117, "J": 118, "K": 119, "L": 120, "M": 121, "N": 122, "O": 123, "P": 124, "Q": 125, "R": 126, "S": 127, "T": 128, "U": 129, "V": 130, "W": 131, "X": 132, "Y": 133, "Z": 134, "[": 135, "\\": 136, "]": 137, "^": 138, "_": 139, "`": 140, "a": 141, "b": 142, "c": 143, "d": 144, "e": 145, "f": 146, "g": 147, "h": 148, "i": 149, "j": 150, "k": 151, "l": 152, "m": 153, "n": 154, "o": 155, "p": 156, "q": 157, "r": 158, "s": 159, "t": 160, "u": 161, "v": 162, "w": 163, "x": 164, "y": 165, "z": 166, "{": 167, "|": 168, "}": 169, "~": 170, "ġ": 171, "Ģ": 172, "ģ": 173, "Ĥ": 174, "ĥ": 175, "Ħ": 176, "ħ": 177, "Ĩ": 178, "ĩ": 179, "Ī": 180, "ī": 181, "Ĭ": 182, "ĭ": 183, "Į": 184, "į": 185, "İ": 186, "ı": 187, "IJ": 188, "ij": 189, "Ĵ": 190, "ĵ": 191, "Ķ": 192, "ķ": 193, "ĸ": 194, "Ĺ": 195, "ĺ": 196, "Ļ": 197, "ļ": 198, "Ľ": 199, "ľ": 200, "Ŀ": 201, "ŀ": 202, "Ł": 203, "ł": 204, "¡": 205, "¢": 206, "£": 207, "¤": 208, "¥": 209, "¦": 210, "§": 211, "¨": 212, "©": 213, "ª": 214, "«": 215, "¬": 216, "Ń": 217, "®": 218, "¯": 219, "°": 220, "±": 221, "²": 222, "³": 223, "´": 224, "µ": 225, "¶": 226, "·": 227, "¸": 228, "¹": 229, "º": 230, "»": 231, "¼": 232, "½": 233, "¾": 234, "¿": 235, "À": 236, "Á": 237, "Â": 238, "Ã": 239, "Ä": 240, "Å": 241, "Æ": 242, "Ç": 243, "È": 244, "É": 245, "Ê": 246, "Ë": 247, "Ì": 248, "Í": 249, "Î": 250, "Ï": 251, "Ð": 252, "Ñ": 253, "Ò": 254, "Ó": 255, "Ô": 256, "Õ": 257, "Ö": 258, "×": 259, "Ø": 260, "Ù": 261, "Ú": 262, "Û": 263, "Ü": 264, "Ý": 265, "Þ": 266, "ß": 267, "à": 268, "á": 269, "â": 270, "ã": 271, "ä": 272, "å": 273, "æ": 274, "ç": 275, "è": 276, "é": 277, "ê": 278, "ë": 279, "ì": 280, "í": 281, "î": 282, "ï": 283, "ð": 284, "ñ": 285, "ò": 286, "ó": 287, "ô": 288, "õ": 289, "ö": 290, "÷": 291, "ø": 292, "ù": 293, "ú": 294, "û": 295, "ü": 296, "ý": 297, "þ": 298, "ÿ": 299, "æ": 300, "ç": 301, "ð": 302, "Åĭ": 303, "ÉIJ": 304, "Éij": 305, "ÉĶ": 306, "ÉĹ": 307, "ÉĻ": 308, "Éļ": 309, "ÉĽ": 310, "Éľ": 311, "É¡": 312, "É£": 313, "ɪ": 314, "ɬ": 315, "ɲ": 316, "ɹ": 317, "ɾ": 318, "Êĥ": 319, "ÊĬ": 320, "ÊĮ": 321, "ÊĴ": 322, "ÊĶ": 323, "ʲ": 324, "ËĪ": 325, "ËĮ": 326, "ËIJ": 327, "Ìĥ": 328, "Ì©": 329, "̪": 330, "θ": 331, "áµ": 332, "áµ»": 333, "ĠĠ": 334, "Ġ-": 335, "Ġ1": 336, "Ġ2": 337, "Ġ4": 338, "Ġ5": 339, "Ġ6": 340, "Ġa": 341, "Ġb": 342, "Ġc": 343, "Ġd": 344, "Ġe": 345, "Ġf": 346, "Ġh": 347, "Ġi": 348, "Ġj": 349, "Ġk": 350, "Ġl": 351, "Ġm": 352, "Ġn": 353, "Ġo": 354, "Ġp": 355, "Ġr": 356, "Ġs": 357, "Ġt": 358, "Ġu": 359, "Ġv": 360, "Ġw": 361, "Ġx": 362, "Ġy": 363, "Ġz": 364, "ĠÃ": 365, "Ġæ": 366, "Ġç": 367, "Ġð": 368, "ĠÅ": 369, "ĠÅĭ": 370, "ĠÉ": 371, "ĠÉIJ": 372, "ĠÉij": 373, "ĠÉĶ": 374, "ĠÉĹ": 375, "ĠÉĻ": 376, "ĠÉļ": 377, "ĠÉĽ": 378, "ĠÉľ": 379, "ĠÉ¡": 380, "ĠÉ£": 381, "Ġɪ": 382, "Ġɬ": 383, "Ġɲ": 384, "Ġɹ": 385, "Ġɾ": 386, "ĠÊ": 387, "ĠÊĥ": 388, "ĠÊĬ": 389, "ĠÊĮ": 390, "ĠÊĴ": 391, "ĠÊĶ": 392, "Ġʲ": 393, "ĠË": 394, "ĠËĪ": 395, "ĠËĮ": 396, "ĠËIJ": 397, "ĠÌ": 398, "ĠÌĥ": 399, "ĠÌ©": 400, "Ġ̪": 401, "ĠÎ": 402, "Ġθ": 403, "Ġá": 404, "Ġáµ": 405, "Ġáµ»": 406, "Ġ.": 407, "Ġ?": 408, "Ġ!": 409, "Ġ,": 410, "Ġ;": 411, "Ġ:": 412, "Ġ(": 413, "Ġ)": 414, "Ġ[": 415, "Ġ]": 416, "Ġ{": 417, "Ġ}": 418 }, "merges": [ [ "Ã", "¦" ], [ "Ã", "§" ], [ "Ã", "°" ], [ "Å", "ĭ" ], [ "É", "IJ" ], [ "É", "ij" ], [ "É", "Ķ" ], [ "É", "Ĺ" ], [ "É", "Ļ" ], [ "É", "ļ" ], [ "É", "Ľ" ], [ "É", "ľ" ], [ "É", "¡" ], [ "É", "£" ], [ "É", "ª" ], [ "É", "¬" ], [ "É", "²" ], [ "É", "¹" ], [ "É", "¾" ], [ "Ê", "ĥ" ], [ "Ê", "Ĭ" ], [ "Ê", "Į" ], [ "Ê", "Ĵ" ], [ "Ê", "Ķ" ], [ "Ê", "²" ], [ "Ë", "Ī" ], [ "Ë", "Į" ], [ "Ë", "IJ" ], [ "Ì", "ĥ" ], [ "Ì", "©" ], [ "Ì", "ª" ], [ "Î", "¸" ], [ "á", "µ" ], [ "áµ", "»" ], [ "Ġ", "Ġ" ], [ "Ġ", "-" ], [ "Ġ", "1" ], [ "Ġ", "2" ], [ "Ġ", "4" ], [ "Ġ", "5" ], [ "Ġ", "6" ], [ "Ġ", "a" ], [ "Ġ", "b" ], [ "Ġ", "c" ], [ "Ġ", "d" ], [ "Ġ", "e" ], [ "Ġ", "f" ], [ "Ġ", "h" ], [ "Ġ", "i" ], [ "Ġ", "j" ], [ "Ġ", "k" ], [ "Ġ", "l" ], [ "Ġ", "m" ], [ "Ġ", "n" ], [ "Ġ", "o" ], [ "Ġ", "p" ], [ "Ġ", "r" ], [ "Ġ", "s" ], [ "Ġ", "t" ], [ "Ġ", "u" ], [ "Ġ", "v" ], [ "Ġ", "w" ], [ "Ġ", "x" ], [ "Ġ", "y" ], [ "Ġ", "z" ], [ "Ġ", "Ã" ], [ "ĠÃ", "¦" ], [ "ĠÃ", "§" ], [ "ĠÃ", "°" ], [ "Ġ", "Å" ], [ "ĠÅ", "ĭ" ], [ "Ġ", "É" ], [ "ĠÉ", "IJ" ], [ "ĠÉ", "ij" ], [ "ĠÉ", "Ķ" ], [ "ĠÉ", "Ĺ" ], [ "ĠÉ", "Ļ" ], [ "ĠÉ", "ļ" ], [ "ĠÉ", "Ľ" ], [ "ĠÉ", "ľ" ], [ "ĠÉ", "¡" ], [ "ĠÉ", "£" ], [ "ĠÉ", "ª" ], [ "ĠÉ", "¬" ], [ "ĠÉ", "²" ], [ "ĠÉ", "¹" ], [ "ĠÉ", "¾" ], [ "Ġ", "Ê" ], [ "ĠÊ", "ĥ" ], [ "ĠÊ", "Ĭ" ], [ "ĠÊ", "Į" ], [ "ĠÊ", "Ĵ" ], [ "ĠÊ", "Ķ" ], [ "ĠÊ", "²" ], [ "Ġ", "Ë" ], [ "ĠË", "Ī" ], [ "ĠË", "Į" ], [ "ĠË", "IJ" ], [ "Ġ", "Ì" ], [ "ĠÌ", "ĥ" ], [ "ĠÌ", "©" ], [ "ĠÌ", "ª" ], [ "Ġ", "Î" ], [ "ĠÎ", "¸" ], [ "Ġ", "á" ], [ "Ġá", "µ" ], [ "Ġáµ", "»" ], [ "Ġ", "." ], [ "Ġ", "?" ], [ "Ġ", "!" ], [ "Ġ", "," ], [ "Ġ", ";" ], [ "Ġ", ":" ], [ "Ġ", "(" ], [ "Ġ", ")" ], [ "Ġ", "[" ], [ "Ġ", "]" ], [ "Ġ", "{" ], [ "Ġ", "}" ] ] } }