{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 500, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 501, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 502, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 503, "content": "<|AUDIO|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 504, "content": "<|audio_bos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 505, "content": "<|audio_eos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 506, "content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 507, "content": "<|quad_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 508, "content": "<|quad_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 509, "content": "<|vision_bos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 510, "content": "<|vision_eos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 511, "content": "<|vision_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 512, "content": "<|IMAGE|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 513, "content": "<|VIDEO|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 514, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 515, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 516, "content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 517, "content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 518, "content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 519, "content": "<|fim_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 520, "content": "<|repo_name|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 521, "content": "<|file_sep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false } ] }, "post_processor": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "ĠĠ": 256, "ĠĠĠĠ": 257, "in": 258, "Ġt": 259, "ĠĠĠĠĠĠĠĠ": 260, "er": 261, "ĠĠĠ": 262, "on": 263, "Ġa": 264, "re": 265, "at": 266, "st": 267, "en": 268, "or": 269, "Ġth": 270, "ĊĊ": 271, "Ġc": 272, "le": 273, "Ġs": 274, "it": 275, "an": 276, "ar": 277, "al": 278, "Ġthe": 279, ";Ċ": 280, "Ġp": 281, "Ġf": 282, "ou": 283, "Ġ=": 284, "is": 285, "ĠĠĠĠĠĠĠ": 286, "ing": 287, "es": 288, "Ġw": 289, "ion": 290, "ed": 291, "ic": 292, "Ġb": 293, "Ġd": 294, "et": 295, "Ġm": 296, "Ġo": 297, "ĉĉ": 298, "ro": 299, "as": 300, "el": 301, "ct": 302, "nd": 303, "Ġin": 304, "Ġh": 305, "ent": 306, "id": 307, "Ġn": 308, "am": 309, "ĠĠĠĠĠĠĠĠĠĠĠ": 310, "Ġto": 311, "Ġre": 312, "--": 313, "Ġ{": 314, "Ġof": 315, "om": 316, ");Ċ": 317, "im": 318, "čĊ": 319, "Ġ(": 320, "il": 321, "//": 322, "Ġand": 323, "ur": 324, "se": 325, "Ġl": 326, "ex": 327, "ĠS": 328, "ad": 329, "Ġ\"": 330, "ch": 331, "ut": 332, "if": 333, "**": 334, "Ġ}": 335, "em": 336, "ol": 337, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338, "th": 339, ")Ċ": 340, "Ġ{Ċ": 341, "Ġg": 342, "ig": 343, "iv": 344, ",Ċ": 345, "ce": 346, "od": 347, "Ġv": 348, "ate": 349, "ĠT": 350, "ag": 351, "ay": 352, "Ġ*": 353, "ot": 354, "us": 355, "ĠC": 356, "Ġst": 357, "ĠI": 358, "un": 359, "ul": 360, "ue": 361, "ĠA": 362, "ow": 363, "Ġ'": 364, "ew": 365, "Ġ<": 366, "ation": 367, "()": 368, "Ġfor": 369, "ab": 370, "ort": 371, "um": 372, "ame": 373, "Ġis": 374, "pe": 375, "tr": 376, "ck": 377, "âĢ": 378, "Ġy": 379, "ist": 380, "----": 381, ".ĊĊ": 382, "he": 383, "Ġe": 384, "lo": 385, "ĠM": 386, "Ġbe": 387, "ers": 388, "Ġon": 389, "Ġcon": 390, "ap": 391, "ub": 392, "ĠP": 393, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394, "ass": 395, "int": 396, ">Ċ": 397, "ly": 398, "urn": 399, "Ġ$": 400, ";ĊĊ": 401, "av": 402, "port": 403, "ir": 404, "->": 405, "nt": 406, "ction": 407, "end": 408, "Ġde": 409, "ith": 410, "out": 411, "turn": 412, "our": 413, "ĠĠĠĠĠ": 414, "lic": 415, "res": 416, "pt": 417, "==": 418, "Ġthis": 419, "Ġwh": 420, "Ġif": 421, "ĠD": 422, "ver": 423, "age": 424, "ĠB": 425, "ht": 426, "ext": 427, "=\"": 428, "Ġthat": 429, "****": 430, "ĠR": 431, "Ġit": 432, "ess": 433, "ĠF": 434, "Ġr": 435, "os": 436, "and": 437, "Ġas": 438, "ect": 439, "ke": 440, "rom": 441, "Ġ//": 442, "con": 443, "ĠL": 444, "(\"": 445, "qu": 446, "lass": 447, "Ġwith": 448, "iz": 449, "de": 450, "ĠN": 451, "Ġal": 452, "op": 453, "up": 454, "get": 455, "Ġ}Ċ": 456, "ile": 457, "Ġan": 458, "ata": 459, "ore": 460, "ri": 461, "Ġpro": 462, ";čĊ": 463, "ĉĉĉĉ": 464, "ter": 465, "ain": 466, "ĠW": 467, "ĠE": 468, "Ġcom": 469, "Ġreturn": 470, "art": 471, "ĠH": 472, "ack": 473, "import": 474, "ublic": 475, "Ġor": 476, "est": 477, "ment": 478, "ĠG": 479, "able": 480, "Ġ-": 481, "ine": 482, "ill": 483, "ind": 484, "ere": 485, "::": 486, "ity": 487, "Ġ+": 488, "Ġtr": 489, "elf": 490, "ight": 491, "('": 492, "orm": 493, "ult": 494, "str": 495, "..": 496, "\",": 497, "Ġyou": 498, "ype": 499 }, "merges": [ [ "Ġ", "Ġ" ], [ "ĠĠ", "ĠĠ" ], [ "i", "n" ], [ "Ġ", "t" ], [ "ĠĠĠĠ", "ĠĠĠĠ" ], [ "e", "r" ], [ "ĠĠ", "Ġ" ], [ "o", "n" ], [ "Ġ", "a" ], [ "r", "e" ], [ "a", "t" ], [ "s", "t" ], [ "e", "n" ], [ "o", "r" ], [ "Ġt", "h" ], [ "Ċ", "Ċ" ], [ "Ġ", "c" ], [ "l", "e" ], [ "Ġ", "s" ], [ "i", "t" ], [ "a", "n" ], [ "a", "r" ], [ "a", "l" ], [ "Ġth", "e" ], [ ";", "Ċ" ], [ "Ġ", "p" ], [ "Ġ", "f" ], [ "o", "u" ], [ "Ġ", "=" ], [ "i", "s" ], [ "ĠĠĠĠ", "ĠĠĠ" ], [ "in", "g" ], [ "e", "s" ], [ "Ġ", "w" ], [ "i", "on" ], [ "e", "d" ], [ "i", "c" ], [ "Ġ", "b" ], [ "Ġ", "d" ], [ "e", "t" ], [ "Ġ", "m" ], [ "Ġ", "o" ], [ "ĉ", "ĉ" ], [ "r", "o" ], [ "a", "s" ], [ "e", "l" ], [ "c", "t" ], [ "n", "d" ], [ "Ġ", "in" ], [ "Ġ", "h" ], [ "en", "t" ], [ "i", "d" ], [ "Ġ", "n" ], [ "a", "m" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠ" ], [ "Ġt", "o" ], [ "Ġ", "re" ], [ "-", "-" ], [ "Ġ", "{" ], [ "Ġo", "f" ], [ "o", "m" ], [ ")", ";Ċ" ], [ "i", "m" ], [ "č", "Ċ" ], [ "Ġ", "(" ], [ "i", "l" ], [ "/", "/" ], [ "Ġa", "nd" ], [ "u", "r" ], [ "s", "e" ], [ "Ġ", "l" ], [ "e", "x" ], [ "Ġ", "S" ], [ "a", "d" ], [ "Ġ", "\"" ], [ "c", "h" ], [ "u", "t" ], [ "i", "f" ], [ "*", "*" ], [ "Ġ", "}" ], [ "e", "m" ], [ "o", "l" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠ" ], [ "t", "h" ], [ ")", "Ċ" ], [ "Ġ{", "Ċ" ], [ "Ġ", "g" ], [ "i", "g" ], [ "i", "v" ], [ ",", "Ċ" ], [ "c", "e" ], [ "o", "d" ], [ "Ġ", "v" ], [ "at", "e" ], [ "Ġ", "T" ], [ "a", "g" ], [ "a", "y" ], [ "Ġ", "*" ], [ "o", "t" ], [ "u", "s" ], [ "Ġ", "C" ], [ "Ġ", "st" ], [ "Ġ", "I" ], [ "u", "n" ], [ "u", "l" ], [ "u", "e" ], [ "Ġ", "A" ], [ "o", "w" ], [ "Ġ", "'" ], [ "e", "w" ], [ "Ġ", "<" ], [ "at", "ion" ], [ "(", ")" ], [ "Ġf", "or" ], [ "a", "b" ], [ "or", "t" ], [ "u", "m" ], [ "am", "e" ], [ "Ġ", "is" ], [ "p", "e" ], [ "t", "r" ], [ "c", "k" ], [ "â", "Ģ" ], [ "Ġ", "y" ], [ "i", "st" ], [ "--", "--" ], [ ".", "ĊĊ" ], [ "h", "e" ], [ "Ġ", "e" ], [ "l", "o" ], [ "Ġ", "M" ], [ "Ġb", "e" ], [ "er", "s" ], [ "Ġ", "on" ], [ "Ġc", "on" ], [ "a", "p" ], [ "u", "b" ], [ "Ġ", "P" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠ" ], [ "as", "s" ], [ "in", "t" ], [ ">", "Ċ" ], [ "l", "y" ], [ "ur", "n" ], [ "Ġ", "$" ], [ ";", "ĊĊ" ], [ "a", "v" ], [ "p", "ort" ], [ "i", "r" ], [ "-", ">" ], [ "n", "t" ], [ "ct", "ion" ], [ "en", "d" ], [ "Ġd", "e" ], [ "it", "h" ], [ "ou", "t" ], [ "t", "urn" ], [ "ou", "r" ], [ "ĠĠĠĠ", "Ġ" ], [ "l", "ic" ], [ "re", "s" ], [ "p", "t" ], [ "=", "=" ], [ "Ġth", "is" ], [ "Ġw", "h" ], [ "Ġ", "if" ], [ "Ġ", "D" ], [ "v", "er" ], [ "ag", "e" ], [ "Ġ", "B" ], [ "h", "t" ], [ "ex", "t" ], [ "=", "\"" ], [ "Ġth", "at" ], [ "**", "**" ], [ "Ġ", "R" ], [ "Ġ", "it" ], [ "es", "s" ], [ "Ġ", "F" ], [ "Ġ", "r" ], [ "o", "s" ], [ "an", "d" ], [ "Ġa", "s" ], [ "e", "ct" ], [ "k", "e" ], [ "ro", "m" ], [ "Ġ", "//" ], [ "c", "on" ], [ "Ġ", "L" ], [ "(", "\"" ], [ "q", "u" ], [ "l", "ass" ], [ "Ġw", "ith" ], [ "i", "z" ], [ "d", "e" ], [ "Ġ", "N" ], [ "Ġa", "l" ], [ "o", "p" ], [ "u", "p" ], [ "g", "et" ], [ "Ġ}", "Ċ" ], [ "i", "le" ], [ "Ġa", "n" ], [ "at", "a" ], [ "o", "re" ], [ "r", "i" ], [ "Ġp", "ro" ], [ ";", "čĊ" ], [ "ĉĉ", "ĉĉ" ], [ "t", "er" ], [ "a", "in" ], [ "Ġ", "W" ], [ "Ġ", "E" ], [ "Ġc", "om" ], [ "Ġre", "turn" ], [ "ar", "t" ], [ "Ġ", "H" ], [ "a", "ck" ], [ "im", "port" ], [ "ub", "lic" ], [ "Ġ", "or" ], [ "e", "st" ], [ "m", "ent" ], [ "Ġ", "G" ], [ "ab", "le" ], [ "Ġ", "-" ], [ "in", "e" ], [ "il", "l" ], [ "in", "d" ], [ "er", "e" ], [ ":", ":" ], [ "it", "y" ], [ "Ġ", "+" ], [ "Ġt", "r" ], [ "el", "f" ], [ "ig", "ht" ], [ "(", "'" ], [ "or", "m" ], [ "ul", "t" ], [ "st", "r" ], [ ".", "." ], [ "\"", "," ], [ "Ġy", "ou" ], [ "y", "pe" ] ] } }