qwen3-omni / tokenizer.json
yujiepan's picture
Duplicate from ShowMaker27/Qwen3-Omni-tiny-random
664a532
Raw
History Blame Contribute Delete
24.4 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|image_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<|video_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "<|audio_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6,
"content": "<|vision_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 7,
"content": "<|vision_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 8,
"content": "<|audio_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 9,
"content": "<|audio_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|endoftext|>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"<|im_start|>": 1,
"<|im_end|>": 2,
"<|image_pad|>": 3,
"<|video_pad|>": 4,
"<|audio_pad|>": 5,
"<|vision_start|>": 6,
"<|vision_end|>": 7,
"<|audio_start|>": 8,
"<|audio_end|>": 9,
"+": 10,
".": 11,
"0": 12,
"1": 13,
"2": 14,
"3": 15,
"4": 16,
"5": 17,
"6": 18,
"7": 19,
"8": 20,
"9": 21,
":": 22,
"<": 23,
"=": 24,
">": 25,
"?": 26,
"S": 27,
"T": 28,
"W": 29,
"_": 30,
"a": 31,
"b": 32,
"c": 33,
"d": 34,
"e": 35,
"f": 36,
"g": 37,
"h": 38,
"i": 39,
"j": 40,
"k": 41,
"l": 42,
"m": 43,
"n": 44,
"o": 45,
"p": 46,
"q": 47,
"r": 48,
"s": 49,
"t": 50,
"u": 51,
"v": 52,
"w": 53,
"x": 54,
"y": 55,
"z": 56,
"|": 57,
"Ċ": 58,
"Ġ": 59,
"Ġ1": 60,
"Ġ2": 61,
"Ġ4": 62,
"Ġ3": 63,
"Ġ5": 64,
"Ġ6": 65,
"Ġ7": 66,
"Ġ8": 67,
"Ġ9": 68,
"Ġ10": 69,
"Ġ11": 70,
"Ġ12": 71,
"Ġ13": 72,
"Ġ14": 73,
"Ġ15": 74,
"Ġ16": 75,
"Ġ17": 76,
"Ġ18": 77,
"Ġ19": 78,
"Ġ20": 79,
"Ġ21": 80,
"Ġ22": 81,
"Ġ23": 82,
"Ġ24": 83,
"Ġ25": 84,
"<|": 85,
"im": 86,
"|>": 87,
"er": 88,
"he": 89,
"is": 90,
"ta": 91,
"Ġf": 92,
"The": 93,
"en": 94,
"rt": 95,
"sta": 96,
"Ġ+": 97,
"Ġis": 98,
"Ġ42": 99,
"Ġfo": 100,
"end": 101,
"start": 102,
"?<|": 103,
"So": 104,
"Wh": 105,
"al": 106,
"an": 107,
"as": 108,
"at": 109,
"az": 110,
"br": 111,
"ck": 112,
"do": 113,
"in": 114,
"ick": 115,
"ju": 116,
"lv": 117,
"laz": 118,
"mp": 119,
"nt": 120,
"ov": 121,
"ow": 122,
"qu": 123,
"sw": 124,
"ser": 125,
"sis": 126,
"the": 127,
"user": 128,
"Ġ=": 129,
"Ġx": 130,
"ĠThe": 131,
"Ġan": 132,
"Ġbr": 133,
"Ġdo": 134,
"Ġju": 135,
"Ġlaz": 136,
"Ġov": 137,
"Ġqu": 138,
"Ġthe": 139,
"Ġ26": 140,
"Ġ27": 141,
"Ġ28": 142,
"Ġ29": 143,
"Ġ40": 144,
"Ġ41": 145,
"Ġ43": 146,
"Ġ44": 147,
"Ġ45": 148,
"Ġ46": 149,
"Ġ47": 150,
"Ġ48": 151,
"Ġ49": 152,
"Ġ30": 153,
"Ġ31": 154,
"Ġ32": 155,
"Ġ33": 156,
"Ġ34": 157,
"Ġ35": 158,
"Ġ36": 159,
"Ġ37": 160,
"Ġ38": 161,
"Ġ39": 162,
"Ġ50": 163,
"Ġ51": 164,
"Ġ52": 165,
"Ġ53": 166,
"Ġ54": 167,
"Ġ55": 168,
"Ġ56": 169,
"Ġ57": 170,
"Ġ58": 171,
"Ġ59": 172,
"Ġ60": 173,
"Ġ61": 174,
"Ġ62": 175,
"Ġ63": 176,
"Ġ64": 177,
"Ġ65": 178,
"Ġ66": 179,
"Ġ67": 180,
"Ġ68": 181,
"Ġ69": 182,
"Ġ70": 183,
"Ġ71": 184,
"Ġ72": 185,
"Ġ73": 186,
"Ġ74": 187,
"Ġ75": 188,
"Ġ76": 189,
"Ġ77": 190,
"Ġ78": 191,
"Ġ79": 192,
"Ġ80": 193,
"Ġ81": 194,
"Ġ82": 195,
"Ġ83": 196,
"Ġ84": 197,
"Ġ85": 198,
"Ġ86": 199,
"Ġ87": 200,
"Ġ88": 201,
"Ġ89": 202,
"Ġ90": 203,
"Ġ91": 204,
"Ġ92": 205,
"Ġ93": 206,
"Ġ94": 207,
"Ġ95": 208,
"Ġ96": 209,
"Ġ97": 210,
"Ġ98": 211,
"Ġ99": 212,
"Ġ100": 213,
"Ġ101": 214,
"Ġ102": 215,
"Ġ103": 216,
"Ġ104": 217,
"Ġ105": 218,
"Ġ106": 219,
"Ġ107": 220,
"Ġ108": 221,
"Ġ109": 222,
"Ġ110": 223,
"Ġ111": 224,
"Ġ112": 225,
"Ġ113": 226,
"Ġ114": 227,
"Ġ115": 228,
"Ġ116": 229,
"Ġ117": 230,
"Ġ118": 231,
"Ġ119": 232,
"Ġ120": 233,
"Ġ121": 234,
"Ġ122": 235,
"Ġ123": 236,
"Ġ124": 237,
"Ġ125": 238,
"Ġ126": 239,
"Ġ127": 240,
"Ġ128": 241,
"Ġ129": 242,
"Ġ130": 243,
"Ġ131": 244,
"Ġ132": 245,
"Ġ133": 246,
"Ġ134": 247,
"Ġ135": 248,
"Ġ136": 249,
"Ġ137": 250,
"Ġ138": 251,
"Ġ139": 252,
"Ġ140": 253,
"Ġ141": 254,
"Ġ142": 255,
"Ġ143": 256,
"Ġ144": 257,
"Ġ145": 258,
"Ġ146": 259,
"Ġ147": 260,
"Ġ148": 261,
"Ġ149": 262,
"Ġ150": 263,
"Ġ151": 264,
"Ġ152": 265,
"Ġ153": 266,
"Ġ154": 267,
"Ġ155": 268,
"Ġ156": 269,
"Ġ157": 270,
"Ġ158": 271,
"Ġ159": 272,
"Ġ160": 273,
"Ġ161": 274,
"Ġ162": 275,
"Ġ163": 276,
"Ġ164": 277,
"Ġ165": 278,
"Ġ166": 279,
"Ġ167": 280,
"Ġ168": 281,
"Ġ169": 282,
"Ġ170": 283,
"Ġ171": 284,
"Ġ172": 285,
"Ġ173": 286,
"Ġ174": 287,
"Ġ175": 288,
"Ġ176": 289,
"Ġ177": 290,
"Ġ178": 291,
"Ġ179": 292,
"Ġ180": 293,
"Ġ181": 294,
"Ġ182": 295,
"Ġ183": 296,
"Ġ184": 297,
"Ġ185": 298,
"Ġ186": 299,
"Ġ187": 300,
"Ġ188": 301,
"Ġ189": 302,
"Ġ190": 303,
"Ġ191": 304,
"Ġ192": 305,
"Ġ193": 306,
"Ġ194": 307,
"Ġ195": 308,
"Ġ196": 309,
"Ġ197": 310,
"Ġ198": 311,
"Ġ199": 312,
"Ġ200": 313,
"Ġ201": 314,
"Ġ202": 315,
"Ġ203": 316,
"Ġ204": 317,
"Ġ205": 318,
"Ġ206": 319,
"Ġ207": 320,
"Ġ208": 321,
"Ġ209": 322,
"Ġ210": 323,
"Ġ211": 324,
"Ġ212": 325,
"Ġ213": 326,
"Ġ214": 327,
"Ġ215": 328,
"Ġ216": 329,
"Ġ217": 330,
"Ġ218": 331,
"Ġ219": 332,
"Ġ220": 333,
"Ġ221": 334,
"Ġ222": 335,
"Ġ223": 336,
"Ġ224": 337,
"Ġ225": 338,
"Ġ226": 339,
"Ġ227": 340,
"Ġ228": 341,
"Ġ229": 342,
"Ġ230": 343,
"Ġ231": 344,
"Ġ232": 345,
"Ġ233": 346,
"Ġ234": 347,
"Ġ235": 348,
"Ġ236": 349,
"Ġ237": 350,
"Ġ238": 351,
"Ġ239": 352,
"Ġ240": 353,
"Ġ241": 354,
"Ġ242": 355,
"Ġ243": 356,
"Ġ244": 357,
"Ġ245": 358,
"Ġ246": 359,
"Ġ247": 360,
"Ġ248": 361,
"Ġ249": 362,
"Ġ250": 363,
"Ġ251": 364,
"Ġ252": 365,
"Ġ253": 366,
"Ġ254": 367,
"Ġ255": 368,
"tant": 369,
"Ġfin": 370,
"Ġfor": 371,
"Ġfox": 372,
"Solv": 373,
"What": 374,
"assis": 375,
"mps": 376,
"own": 377,
"swer": 378,
"Ġanswer": 379,
"Ġbrown": 380,
"Ġdog": 381,
"Ġjumps": 382,
"Ġlazy": 383,
"Ġover": 384,
"Ġquick": 385,
"Ġfinal": 386,
"Solve": 387,
"assistant": 388
},
"merges": [
[
"Ġ",
"1"
],
[
"Ġ",
"2"
],
[
"Ġ",
"4"
],
[
"Ġ",
"3"
],
[
"Ġ",
"5"
],
[
"Ġ",
"6"
],
[
"Ġ",
"7"
],
[
"Ġ",
"8"
],
[
"Ġ",
"9"
],
[
"Ġ1",
"0"
],
[
"Ġ1",
"1"
],
[
"Ġ1",
"2"
],
[
"Ġ1",
"3"
],
[
"Ġ1",
"4"
],
[
"Ġ1",
"5"
],
[
"Ġ1",
"6"
],
[
"Ġ1",
"7"
],
[
"Ġ1",
"8"
],
[
"Ġ1",
"9"
],
[
"Ġ2",
"0"
],
[
"Ġ2",
"1"
],
[
"Ġ2",
"2"
],
[
"Ġ2",
"3"
],
[
"Ġ2",
"4"
],
[
"Ġ2",
"5"
],
[
"<",
"|"
],
[
"i",
"m"
],
[
"|",
">"
],
[
"e",
"r"
],
[
"h",
"e"
],
[
"i",
"s"
],
[
"t",
"a"
],
[
"Ġ",
"f"
],
[
"T",
"he"
],
[
"e",
"n"
],
[
"r",
"t"
],
[
"s",
"ta"
],
[
"Ġ",
"+"
],
[
"Ġ",
"is"
],
[
"Ġ4",
"2"
],
[
"Ġf",
"o"
],
[
"en",
"d"
],
[
"sta",
"rt"
],
[
"?",
"<|"
],
[
"S",
"o"
],
[
"W",
"h"
],
[
"a",
"l"
],
[
"a",
"n"
],
[
"a",
"s"
],
[
"a",
"t"
],
[
"a",
"z"
],
[
"b",
"r"
],
[
"c",
"k"
],
[
"d",
"o"
],
[
"i",
"n"
],
[
"i",
"ck"
],
[
"j",
"u"
],
[
"l",
"v"
],
[
"l",
"az"
],
[
"m",
"p"
],
[
"n",
"t"
],
[
"o",
"v"
],
[
"o",
"w"
],
[
"q",
"u"
],
[
"s",
"w"
],
[
"s",
"er"
],
[
"s",
"is"
],
[
"t",
"he"
],
[
"u",
"ser"
],
[
"Ġ",
"="
],
[
"Ġ",
"x"
],
[
"Ġ",
"The"
],
[
"Ġ",
"an"
],
[
"Ġ",
"br"
],
[
"Ġ",
"do"
],
[
"Ġ",
"ju"
],
[
"Ġ",
"laz"
],
[
"Ġ",
"ov"
],
[
"Ġ",
"qu"
],
[
"Ġ",
"the"
],
[
"Ġ2",
"6"
],
[
"Ġ2",
"7"
],
[
"Ġ2",
"8"
],
[
"Ġ2",
"9"
],
[
"Ġ4",
"0"
],
[
"Ġ4",
"1"
],
[
"Ġ4",
"3"
],
[
"Ġ4",
"4"
],
[
"Ġ4",
"5"
],
[
"Ġ4",
"6"
],
[
"Ġ4",
"7"
],
[
"Ġ4",
"8"
],
[
"Ġ4",
"9"
],
[
"Ġ3",
"0"
],
[
"Ġ3",
"1"
],
[
"Ġ3",
"2"
],
[
"Ġ3",
"3"
],
[
"Ġ3",
"4"
],
[
"Ġ3",
"5"
],
[
"Ġ3",
"6"
],
[
"Ġ3",
"7"
],
[
"Ġ3",
"8"
],
[
"Ġ3",
"9"
],
[
"Ġ5",
"0"
],
[
"Ġ5",
"1"
],
[
"Ġ5",
"2"
],
[
"Ġ5",
"3"
],
[
"Ġ5",
"4"
],
[
"Ġ5",
"5"
],
[
"Ġ5",
"6"
],
[
"Ġ5",
"7"
],
[
"Ġ5",
"8"
],
[
"Ġ5",
"9"
],
[
"Ġ6",
"0"
],
[
"Ġ6",
"1"
],
[
"Ġ6",
"2"
],
[
"Ġ6",
"3"
],
[
"Ġ6",
"4"
],
[
"Ġ6",
"5"
],
[
"Ġ6",
"6"
],
[
"Ġ6",
"7"
],
[
"Ġ6",
"8"
],
[
"Ġ6",
"9"
],
[
"Ġ7",
"0"
],
[
"Ġ7",
"1"
],
[
"Ġ7",
"2"
],
[
"Ġ7",
"3"
],
[
"Ġ7",
"4"
],
[
"Ġ7",
"5"
],
[
"Ġ7",
"6"
],
[
"Ġ7",
"7"
],
[
"Ġ7",
"8"
],
[
"Ġ7",
"9"
],
[
"Ġ8",
"0"
],
[
"Ġ8",
"1"
],
[
"Ġ8",
"2"
],
[
"Ġ8",
"3"
],
[
"Ġ8",
"4"
],
[
"Ġ8",
"5"
],
[
"Ġ8",
"6"
],
[
"Ġ8",
"7"
],
[
"Ġ8",
"8"
],
[
"Ġ8",
"9"
],
[
"Ġ9",
"0"
],
[
"Ġ9",
"1"
],
[
"Ġ9",
"2"
],
[
"Ġ9",
"3"
],
[
"Ġ9",
"4"
],
[
"Ġ9",
"5"
],
[
"Ġ9",
"6"
],
[
"Ġ9",
"7"
],
[
"Ġ9",
"8"
],
[
"Ġ9",
"9"
],
[
"Ġ10",
"0"
],
[
"Ġ10",
"1"
],
[
"Ġ10",
"2"
],
[
"Ġ10",
"3"
],
[
"Ġ10",
"4"
],
[
"Ġ10",
"5"
],
[
"Ġ10",
"6"
],
[
"Ġ10",
"7"
],
[
"Ġ10",
"8"
],
[
"Ġ10",
"9"
],
[
"Ġ11",
"0"
],
[
"Ġ11",
"1"
],
[
"Ġ11",
"2"
],
[
"Ġ11",
"3"
],
[
"Ġ11",
"4"
],
[
"Ġ11",
"5"
],
[
"Ġ11",
"6"
],
[
"Ġ11",
"7"
],
[
"Ġ11",
"8"
],
[
"Ġ11",
"9"
],
[
"Ġ12",
"0"
],
[
"Ġ12",
"1"
],
[
"Ġ12",
"2"
],
[
"Ġ12",
"3"
],
[
"Ġ12",
"4"
],
[
"Ġ12",
"5"
],
[
"Ġ12",
"6"
],
[
"Ġ12",
"7"
],
[
"Ġ12",
"8"
],
[
"Ġ12",
"9"
],
[
"Ġ13",
"0"
],
[
"Ġ13",
"1"
],
[
"Ġ13",
"2"
],
[
"Ġ13",
"3"
],
[
"Ġ13",
"4"
],
[
"Ġ13",
"5"
],
[
"Ġ13",
"6"
],
[
"Ġ13",
"7"
],
[
"Ġ13",
"8"
],
[
"Ġ13",
"9"
],
[
"Ġ14",
"0"
],
[
"Ġ14",
"1"
],
[
"Ġ14",
"2"
],
[
"Ġ14",
"3"
],
[
"Ġ14",
"4"
],
[
"Ġ14",
"5"
],
[
"Ġ14",
"6"
],
[
"Ġ14",
"7"
],
[
"Ġ14",
"8"
],
[
"Ġ14",
"9"
],
[
"Ġ15",
"0"
],
[
"Ġ15",
"1"
],
[
"Ġ15",
"2"
],
[
"Ġ15",
"3"
],
[
"Ġ15",
"4"
],
[
"Ġ15",
"5"
],
[
"Ġ15",
"6"
],
[
"Ġ15",
"7"
],
[
"Ġ15",
"8"
],
[
"Ġ15",
"9"
],
[
"Ġ16",
"0"
],
[
"Ġ16",
"1"
],
[
"Ġ16",
"2"
],
[
"Ġ16",
"3"
],
[
"Ġ16",
"4"
],
[
"Ġ16",
"5"
],
[
"Ġ16",
"6"
],
[
"Ġ16",
"7"
],
[
"Ġ16",
"8"
],
[
"Ġ16",
"9"
],
[
"Ġ17",
"0"
],
[
"Ġ17",
"1"
],
[
"Ġ17",
"2"
],
[
"Ġ17",
"3"
],
[
"Ġ17",
"4"
],
[
"Ġ17",
"5"
],
[
"Ġ17",
"6"
],
[
"Ġ17",
"7"
],
[
"Ġ17",
"8"
],
[
"Ġ17",
"9"
],
[
"Ġ18",
"0"
],
[
"Ġ18",
"1"
],
[
"Ġ18",
"2"
],
[
"Ġ18",
"3"
],
[
"Ġ18",
"4"
],
[
"Ġ18",
"5"
],
[
"Ġ18",
"6"
],
[
"Ġ18",
"7"
],
[
"Ġ18",
"8"
],
[
"Ġ18",
"9"
],
[
"Ġ19",
"0"
],
[
"Ġ19",
"1"
],
[
"Ġ19",
"2"
],
[
"Ġ19",
"3"
],
[
"Ġ19",
"4"
],
[
"Ġ19",
"5"
],
[
"Ġ19",
"6"
],
[
"Ġ19",
"7"
],
[
"Ġ19",
"8"
],
[
"Ġ19",
"9"
],
[
"Ġ20",
"0"
],
[
"Ġ20",
"1"
],
[
"Ġ20",
"2"
],
[
"Ġ20",
"3"
],
[
"Ġ20",
"4"
],
[
"Ġ20",
"5"
],
[
"Ġ20",
"6"
],
[
"Ġ20",
"7"
],
[
"Ġ20",
"8"
],
[
"Ġ20",
"9"
],
[
"Ġ21",
"0"
],
[
"Ġ21",
"1"
],
[
"Ġ21",
"2"
],
[
"Ġ21",
"3"
],
[
"Ġ21",
"4"
],
[
"Ġ21",
"5"
],
[
"Ġ21",
"6"
],
[
"Ġ21",
"7"
],
[
"Ġ21",
"8"
],
[
"Ġ21",
"9"
],
[
"Ġ22",
"0"
],
[
"Ġ22",
"1"
],
[
"Ġ22",
"2"
],
[
"Ġ22",
"3"
],
[
"Ġ22",
"4"
],
[
"Ġ22",
"5"
],
[
"Ġ22",
"6"
],
[
"Ġ22",
"7"
],
[
"Ġ22",
"8"
],
[
"Ġ22",
"9"
],
[
"Ġ23",
"0"
],
[
"Ġ23",
"1"
],
[
"Ġ23",
"2"
],
[
"Ġ23",
"3"
],
[
"Ġ23",
"4"
],
[
"Ġ23",
"5"
],
[
"Ġ23",
"6"
],
[
"Ġ23",
"7"
],
[
"Ġ23",
"8"
],
[
"Ġ23",
"9"
],
[
"Ġ24",
"0"
],
[
"Ġ24",
"1"
],
[
"Ġ24",
"2"
],
[
"Ġ24",
"3"
],
[
"Ġ24",
"4"
],
[
"Ġ24",
"5"
],
[
"Ġ24",
"6"
],
[
"Ġ24",
"7"
],
[
"Ġ24",
"8"
],
[
"Ġ24",
"9"
],
[
"Ġ25",
"0"
],
[
"Ġ25",
"1"
],
[
"Ġ25",
"2"
],
[
"Ġ25",
"3"
],
[
"Ġ25",
"4"
],
[
"Ġ25",
"5"
],
[
"ta",
"nt"
],
[
"Ġf",
"in"
],
[
"Ġfo",
"r"
],
[
"Ġfo",
"x"
],
[
"So",
"lv"
],
[
"Wh",
"at"
],
[
"as",
"sis"
],
[
"mp",
"s"
],
[
"ow",
"n"
],
[
"sw",
"er"
],
[
"Ġan",
"swer"
],
[
"Ġbr",
"own"
],
[
"Ġdo",
"g"
],
[
"Ġju",
"mps"
],
[
"Ġlaz",
"y"
],
[
"Ġov",
"er"
],
[
"Ġqu",
"ick"
],
[
"Ġfin",
"al"
],
[
"Solv",
"e"
],
[
"assis",
"tant"
]
]
}
}