{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 500, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 501, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 502, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 503, "content": "<|object_ref_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 504, "content": "<|object_ref_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 505, "content": "<|box_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 506, "content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 507, "content": "<|quad_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 508, "content": "<|quad_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 509, "content": "<|vision_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 510, "content": "<|vision_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 511, "content": "<|vision_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 512, "content": "<|image_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 513, "content": "<|video_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 514, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 515, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 516, "content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 517, "content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 518, "content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 519, "content": "<|fim_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 520, "content": "<|repo_name|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 521, "content": "<|file_sep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 522, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 523, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 524, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 525, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false }, { "id": 526, "content": "<|audio_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 527, "content": "<|audio_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 528, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 529, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 530, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 531, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 532, "content": "<|audio_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 533, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 534, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 535, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 536, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 537, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 538, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 539, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 540, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 541, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 542, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 543, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 544, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 545, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 546, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 547, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 548, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 549, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false } ] }, "post_processor": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "ĠĠ": 256, "ĠĠĠĠ": 257, "in": 258, "Ġt": 259, "ĠĠĠĠĠĠĠĠ": 260, "er": 261, "ĠĠĠ": 262, "on": 263, "Ġa": 264, "re": 265, "at": 266, "st": 267, "en": 268, "or": 269, "Ġth": 270, "ĊĊ": 271, "Ġc": 272, "le": 273, "Ġs": 274, "it": 275, "an": 276, "ar": 277, "al": 278, "Ġthe": 279, "Ġp": 280, "Ġf": 281, "ou": 282, "Ġ=": 283, "is": 284, "ĠĠĠĠĠĠĠ": 285, "ing": 286, "es": 287, "Ġw": 288, "ion": 289, "ed": 290, "ic": 291, "Ġb": 292, "Ġd": 293, "et": 294, "Ġm": 295, "Ġo": 296, "ĉĉ": 297, "ro": 298, "as": 299, "el": 300, "ct": 301, "nd": 302, "Ġin": 303, "Ġh": 304, "ent": 305, "id": 306, "Ġn": 307, "am": 308, "ĠĠĠĠĠĠĠĠĠĠĠ": 309, "Ġto": 310, "Ġre": 311, "--": 312, "Ġ{": 313, "Ġof": 314, "om": 315, "im": 316, "čĊ": 317, "Ġ(": 318, "il": 319, "//": 320, "Ġand": 321, "ur": 322, "se": 323, "Ġl": 324, "ex": 325, "ĠS": 326, "ad": 327, "Ġ\"": 328, "ch": 329, "ut": 330, "if": 331, "**": 332, "Ġ}": 333, "em": 334, "ol": 335, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 336, "th": 337, "Ġg": 338, "ig": 339, "iv": 340, "ce": 341, "od": 342, "Ġv": 343, "ate": 344, "ĠT": 345, "ag": 346, "ay": 347, "Ġ*": 348, "ot": 349, "us": 350, "ĠC": 351, "Ġst": 352, "ĠI": 353, "un": 354, "ul": 355, "ue": 356, "ĠA": 357, "ow": 358, "Ġ'": 359, "ew": 360, "Ġ<": 361, "ation": 362, "()": 363, "Ġfor": 364, "ab": 365, "ort": 366, "um": 367, "ame": 368, "Ġis": 369, "pe": 370, "tr": 371, "ck": 372, "âĢ": 373, "Ġy": 374, "ist": 375, "----": 376, "he": 377, "Ġe": 378, "lo": 379, "ĠM": 380, "Ġbe": 381, "ers": 382, "Ġon": 383, "Ġcon": 384, "ap": 385, "ub": 386, "ĠP": 387, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 388, "ass": 389, "int": 390, "ly": 391, "urn": 392, "Ġ$": 393, "av": 394, "port": 395, "ir": 396, "->": 397, "nt": 398, "ction": 399, "end": 400, "Ġde": 401, "ith": 402, "out": 403, "turn": 404, "our": 405, "ĠĠĠĠĠ": 406, "lic": 407, "res": 408, "pt": 409, "==": 410, "Ġthis": 411, "Ġwh": 412, "Ġif": 413, "ĠD": 414, "ver": 415, "age": 416, "ĠB": 417, "ht": 418, "ext": 419, "=\"": 420, "Ġthat": 421, "****": 422, "ĠR": 423, "Ġit": 424, "ess": 425, "ĠF": 426, "Ġr": 427, "os": 428, "and": 429, "Ġas": 430, "ect": 431, "ke": 432, "rom": 433, "Ġ//": 434, "con": 435, "ĠL": 436, "(\"": 437, "qu": 438, "lass": 439, "Ġwith": 440, "iz": 441, "de": 442, "ĠN": 443, "Ġal": 444, "op": 445, "up": 446, "get": 447, "ile": 448, "Ġan": 449, "ata": 450, "ore": 451, "ri": 452, "Ġpro": 453, "ĉĉĉĉ": 454, "ter": 455, "ain": 456, "ĠW": 457, "ĠE": 458, "Ġcom": 459, "Ġreturn": 460, "art": 461, "ĠH": 462, "ack": 463, "import": 464, "ublic": 465, "Ġor": 466, "est": 467, "ment": 468, "ĠG": 469, "able": 470, "Ġ-": 471, "ine": 472, "ill": 473, "ind": 474, "ere": 475, "::": 476, "ity": 477, "Ġ+": 478, "Ġtr": 479, "elf": 480, "ight": 481, "('": 482, "orm": 483, "ult": 484, "str": 485, "..": 486, "\",": 487, "Ġyou": 488, "ype": 489, "pl": 490, "Ġnew": 491, "Ġj": 492, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 493, "Ġfrom": 494, "Ġex": 495, "ĠO": 496, "ld": 497, "Ġ[": 498, "oc": 499 }, "merges": [ [ "Ġ", "Ġ" ], [ "ĠĠ", "ĠĠ" ], [ "i", "n" ], [ "Ġ", "t" ], [ "ĠĠĠĠ", "ĠĠĠĠ" ], [ "e", "r" ], [ "ĠĠ", "Ġ" ], [ "o", "n" ], [ "Ġ", "a" ], [ "r", "e" ], [ "a", "t" ], [ "s", "t" ], [ "e", "n" ], [ "o", "r" ], [ "Ġt", "h" ], [ "Ċ", "Ċ" ], [ "Ġ", "c" ], [ "l", "e" ], [ "Ġ", "s" ], [ "i", "t" ], [ "a", "n" ], [ "a", "r" ], [ "a", "l" ], [ "Ġth", "e" ], [ "Ġ", "p" ], [ "Ġ", "f" ], [ "o", "u" ], [ "Ġ", "=" ], [ "i", "s" ], [ "ĠĠĠĠ", "ĠĠĠ" ], [ "in", "g" ], [ "e", "s" ], [ "Ġ", "w" ], [ "i", "on" ], [ "e", "d" ], [ "i", "c" ], [ "Ġ", "b" ], [ "Ġ", "d" ], [ "e", "t" ], [ "Ġ", "m" ], [ "Ġ", "o" ], [ "ĉ", "ĉ" ], [ "r", "o" ], [ "a", "s" ], [ "e", "l" ], [ "c", "t" ], [ "n", "d" ], [ "Ġ", "in" ], [ "Ġ", "h" ], [ "en", "t" ], [ "i", "d" ], [ "Ġ", "n" ], [ "a", "m" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠ" ], [ "Ġt", "o" ], [ "Ġ", "re" ], [ "-", "-" ], [ "Ġ", "{" ], [ "Ġo", "f" ], [ "o", "m" ], [ "i", "m" ], [ "č", "Ċ" ], [ "Ġ", "(" ], [ "i", "l" ], [ "/", "/" ], [ "Ġa", "nd" ], [ "u", "r" ], [ "s", "e" ], [ "Ġ", "l" ], [ "e", "x" ], [ "Ġ", "S" ], [ "a", "d" ], [ "Ġ", "\"" ], [ "c", "h" ], [ "u", "t" ], [ "i", "f" ], [ "*", "*" ], [ "Ġ", "}" ], [ "e", "m" ], [ "o", "l" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠ" ], [ "t", "h" ], [ "Ġ", "g" ], [ "i", "g" ], [ "i", "v" ], [ "c", "e" ], [ "o", "d" ], [ "Ġ", "v" ], [ "at", "e" ], [ "Ġ", "T" ], [ "a", "g" ], [ "a", "y" ], [ "Ġ", "*" ], [ "o", "t" ], [ "u", "s" ], [ "Ġ", "C" ], [ "Ġ", "st" ], [ "Ġ", "I" ], [ "u", "n" ], [ "u", "l" ], [ "u", "e" ], [ "Ġ", "A" ], [ "o", "w" ], [ "Ġ", "'" ], [ "e", "w" ], [ "Ġ", "<" ], [ "at", "ion" ], [ "(", ")" ], [ "Ġf", "or" ], [ "a", "b" ], [ "or", "t" ], [ "u", "m" ], [ "am", "e" ], [ "Ġ", "is" ], [ "p", "e" ], [ "t", "r" ], [ "c", "k" ], [ "â", "Ģ" ], [ "Ġ", "y" ], [ "i", "st" ], [ "--", "--" ], [ "h", "e" ], [ "Ġ", "e" ], [ "l", "o" ], [ "Ġ", "M" ], [ "Ġb", "e" ], [ "er", "s" ], [ "Ġ", "on" ], [ "Ġc", "on" ], [ "a", "p" ], [ "u", "b" ], [ "Ġ", "P" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠ" ], [ "as", "s" ], [ "in", "t" ], [ "l", "y" ], [ "ur", "n" ], [ "Ġ", "$" ], [ "a", "v" ], [ "p", "ort" ], [ "i", "r" ], [ "-", ">" ], [ "n", "t" ], [ "ct", "ion" ], [ "en", "d" ], [ "Ġd", "e" ], [ "it", "h" ], [ "ou", "t" ], [ "t", "urn" ], [ "ou", "r" ], [ "ĠĠĠĠ", "Ġ" ], [ "l", "ic" ], [ "re", "s" ], [ "p", "t" ], [ "=", "=" ], [ "Ġth", "is" ], [ "Ġw", "h" ], [ "Ġ", "if" ], [ "Ġ", "D" ], [ "v", "er" ], [ "ag", "e" ], [ "Ġ", "B" ], [ "h", "t" ], [ "ex", "t" ], [ "=", "\"" ], [ "Ġth", "at" ], [ "**", "**" ], [ "Ġ", "R" ], [ "Ġ", "it" ], [ "es", "s" ], [ "Ġ", "F" ], [ "Ġ", "r" ], [ "o", "s" ], [ "an", "d" ], [ "Ġa", "s" ], [ "e", "ct" ], [ "k", "e" ], [ "ro", "m" ], [ "Ġ", "//" ], [ "c", "on" ], [ "Ġ", "L" ], [ "(", "\"" ], [ "q", "u" ], [ "l", "ass" ], [ "Ġw", "ith" ], [ "i", "z" ], [ "d", "e" ], [ "Ġ", "N" ], [ "Ġa", "l" ], [ "o", "p" ], [ "u", "p" ], [ "g", "et" ], [ "i", "le" ], [ "Ġa", "n" ], [ "at", "a" ], [ "o", "re" ], [ "r", "i" ], [ "Ġp", "ro" ], [ "ĉĉ", "ĉĉ" ], [ "t", "er" ], [ "a", "in" ], [ "Ġ", "W" ], [ "Ġ", "E" ], [ "Ġc", "om" ], [ "Ġre", "turn" ], [ "ar", "t" ], [ "Ġ", "H" ], [ "a", "ck" ], [ "im", "port" ], [ "ub", "lic" ], [ "Ġ", "or" ], [ "e", "st" ], [ "m", "ent" ], [ "Ġ", "G" ], [ "ab", "le" ], [ "Ġ", "-" ], [ "in", "e" ], [ "il", "l" ], [ "in", "d" ], [ "er", "e" ], [ ":", ":" ], [ "it", "y" ], [ "Ġ", "+" ], [ "Ġt", "r" ], [ "el", "f" ], [ "ig", "ht" ], [ "(", "'" ], [ "or", "m" ], [ "ul", "t" ], [ "st", "r" ], [ ".", "." ], [ "\"", "," ], [ "Ġy", "ou" ], [ "y", "pe" ], [ "p", "l" ], [ "Ġn", "ew" ], [ "Ġ", "j" ], [ "ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠĠĠĠ" ], [ "Ġf", "rom" ], [ "Ġ", "ex" ], [ "Ġ", "O" ], [ "l", "d" ], [ "Ġ", "[" ], [ "o", "c" ] ] } }