Blame for tokenizer.json

{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 500, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 501, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 502, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 503, "content": "<|object_ref_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 504, "content": "<|object_ref_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 505, "content": "<|box_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 506, "content": "<|box_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 507, "content": "<|quad_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 508, "content": "<|quad_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 509, "content": "<|vision_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 510, "content": "<|vision_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 511, "content": "<|vision_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 512, "content": "<|image_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 513, "content": "<|video_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 514, "content": "<tool_call>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 515, "content": "</tool_call>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 516, "content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 517, "content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 518, "content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 519, "content": "<|fim_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 520, "content": "<|repo_name|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 521, "content": "<|file_sep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 522, "content": "<tool_response>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 523, "content": "</tool_response>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 524, "content": "<think>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 525, "content": "</think>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 526, "content": "<|audio_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 527, "content": "<|audio_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 528, "content": "<tts_pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 529, "content": "<tts_text_bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 530, "content": "<tts_text_eod>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 531, "content": "<tts_text_bos_single>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 532, "content": "<|audio_pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 533, "content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 534, "content": "<image>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 535, "content": "</image>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 536, "content": "<ref>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 537, "content": "</ref>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 538, "content": "<box>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 539, "content": "</box>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 540, "content": "<quad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 541, "content": "</quad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 542, "content": "<point>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 543, "content": "</point>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 544, "content": "<slice>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 545, "content": "</slice>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 546, "content": "<image_id>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 547, "content": "</image_id>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 548, "content": "<timestamp>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 549, "content": "</timestamp>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 550, "content": "<ecel>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 551, "content": "<fcel>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 552, "content": "<xcel>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 553, "content": "<lcel>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 554, "content": "<ucel>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 555, "content": "<nl>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}], "normalizer": {"type": "NFC"}, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": {"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "ĠĠ": 256, "ĠĠĠĠ": 257, "in": 258, "Ġt": 259, "ĠĠĠĠĠĠĠĠ": 260, "er": 261, "ĠĠĠ": 262, "on": 263, "Ġa": 264, "re": 265, "at": 266, "st": 267, "en": 268, "or": 269, "Ġth": 270, "ĊĊ": 271, "Ġc": 272, "le": 273, "Ġs": 274, "it": 275, "an": 276, "ar": 277, "al": 278, "Ġthe": 279, "Ġp": 280, "Ġf": 281, "ou": 282, "Ġ=": 283, "is": 284, "ĠĠĠĠĠĠĠ": 285, "ing": 286, "es": 287, "Ġw": 288, "ion": 289, "ed": 290, "ic": 291, "Ġb": 292, "Ġd": 293, "et": 294, "Ġm": 295, "Ġo": 296, "ĉĉ": 297, "ro": 298, "as": 299, "el": 300, "ct": 301, "nd": 302, "Ġin": 303, "Ġh": 304, "ent": 305, "id": 306, "Ġn": 307, "am": 308, "ĠĠĠĠĠĠĠĠĠĠĠ": 309, "Ġto": 310, "Ġre": 311, "--": 312, "Ġ{": 313, "Ġof": 314, "om": 315, "im": 316, "čĊ": 317, "Ġ(": 318, "il": 319, "//": 320, "Ġand": 321, "ur": 322, "se": 323, "Ġl": 324, "ex": 325, "ĠS": 326, "ad": 327, "Ġ\"": 328, "ch": 329, "ut": 330, "if": 331, "**": 332, "Ġ}": 333, "em": 334, "ol": 335, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 336, "th": 337, "Ġg": 338, "ig": 339, "iv": 340, "ce": 341, "od": 342, "Ġv": 343, "ate": 344, "ĠT": 345, "ag": 346, "ay": 347, "Ġ*": 348, "ot": 349, "us": 350, "ĠC": 351, "Ġst": 352, "ĠI": 353, "un": 354, "ul": 355, "ue": 356, "ĠA": 357, "ow": 358, "Ġ'": 359, "ew": 360, "Ġ<": 361, "ation": 362, "()": 363, "Ġfor": 364, "ab": 365, "ort": 366, "um": 367, "ame": 368, "Ġis": 369, "pe": 370, "tr": 371, "ck": 372, "âĢ": 373, "Ġy": 374, "ist": 375, "----": 376, "he": 377, "Ġe": 378, "lo": 379, "ĠM": 380, "Ġbe": 381, "ers": 382, "Ġon": 383, "Ġcon": 384, "ap": 385, "ub": 386, "ĠP": 387, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 388, "ass": 389, "int": 390, "ly": 391, "urn": 392, "Ġ$": 393, "av": 394, "port": 395, "ir": 396, "->": 397, "nt": 398, "ction": 399, "end": 400, "Ġde": 401, "ith": 402, "out": 403, "turn": 404, "our": 405, "ĠĠĠĠĠ": 406, "lic": 407, "res": 408, "pt": 409, "==": 410, "Ġthis": 411, "Ġwh": 412, "Ġif": 413, "ĠD": 414, "ver": 415, "age": 416, "ĠB": 417, "ht": 418, "ext": 419, "=\"": 420, "Ġthat": 421, "****": 422, "ĠR": 423, "Ġit": 424, "ess": 425, "ĠF": 426, "Ġr": 427, "os": 428, "and": 429, "Ġas": 430, "ect": 431, "ke": 432, "rom": 433, "Ġ//": 434, "con": 435, "ĠL": 436, "(\"": 437, "qu": 438, "lass": 439, "Ġwith": 440, "iz": 441, "de": 442, "ĠN": 443, "Ġal": 444, "op": 445, "up": 446, "get": 447, "ile": 448, "Ġan": 449, "ata": 450, "ore": 451, "ri": 452, "Ġpro": 453, "ĉĉĉĉ": 454, "ter": 455, "ain": 456, "ĠW": 457, "ĠE": 458, "Ġcom": 459, "Ġreturn": 460, "art": 461, "ĠH": 462, "ack": 463, "import": 464, "ublic": 465, "Ġor": 466, "est": 467, "ment": 468, "ĠG": 469, "able": 470, "Ġ-": 471, "ine": 472, "ill": 473, "ind": 474, "ere": 475, "::": 476, "ity": 477, "Ġ+": 478, "Ġtr": 479, "elf": 480, "ight": 481, "('": 482, "orm": 483, "ult": 484, "str": 485, "..": 486, "\",": 487, "Ġyou": 488, "ype": 489, "pl": 490, "Ġnew": 491, "Ġj": 492, "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 493, "Ġfrom": 494, "Ġex": 495, "ĠO": 496, "ld": 497, "Ġ[": 498, "oc": 499}, "merges": [["Ġ", "Ġ"], ["ĠĠ", "ĠĠ"], ["i", "n"], ["Ġ", "t"], ["ĠĠĠĠ", "ĠĠĠĠ"], ["e", "r"], ["ĠĠ", "Ġ"], ["o", "n"], ["Ġ", "a"], ["r", "e"], ["a", "t"], ["s", "t"], ["e", "n"], ["o", "r"], ["Ġt", "h"], ["Ċ", "Ċ"], ["Ġ", "c"], ["l", "e"], ["Ġ", "s"], ["i", "t"], ["a", "n"], ["a", "r"], ["a", "l"], ["Ġth", "e"], ["Ġ", "p"], ["Ġ", "f"], ["o", "u"], ["Ġ", "="], ["i", "s"], ["ĠĠĠĠ", "ĠĠĠ"], ["in", "g"], ["e", "s"], ["Ġ", "w"], ["i", "on"], ["e", "d"], ["i", "c"], ["Ġ", "b"], ["Ġ", "d"], ["e", "t"], ["Ġ", "m"], ["Ġ", "o"], ["ĉ", "ĉ"], ["r", "o"], ["a", "s"], ["e", "l"], ["c", "t"], ["n", "d"], ["Ġ", "in"], ["Ġ", "h"], ["en", "t"], ["i", "d"], ["Ġ", "n"], ["a", "m"], ["ĠĠĠĠĠĠĠĠ", "ĠĠĠ"], ["Ġt", "o"], ["Ġ", "re"], ["-", "-"], ["Ġ", "{"], ["Ġo", "f"], ["o", "m"], ["i", "m"], ["č", "Ċ"], ["Ġ", "("], ["i", "l"], ["/", "/"], ["Ġa", "nd"], ["u", "r"], ["s", "e"], ["Ġ", "l"], ["e", "x"], ["Ġ", "S"], ["a", "d"], ["Ġ", "\""], ["c", "h"], ["u", "t"], ["i", "f"], ["*", "*"], ["Ġ", "}"], ["e", "m"], ["o", "l"], ["ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠ"], ["t", "h"], ["Ġ", "g"], ["i", "g"], ["i", "v"], ["c", "e"], ["o", "d"], ["Ġ", "v"], ["at", "e"], ["Ġ", "T"], ["a", "g"], ["a", "y"], ["Ġ", "*"], ["o", "t"], ["u", "s"], ["Ġ", "C"], ["Ġ", "st"], ["Ġ", "I"], ["u", "n"], ["u", "l"], ["u", "e"], ["Ġ", "A"], ["o", "w"], ["Ġ", "'"], ["e", "w"], ["Ġ", "<"], ["at", "ion"], ["(", ")"], ["Ġf", "or"], ["a", "b"], ["or", "t"], ["u", "m"], ["am", "e"], ["Ġ", "is"], ["p", "e"], ["t", "r"], ["c", "k"], ["â", "Ģ"], ["Ġ", "y"], ["i", "st"], ["--", "--"], ["h", "e"], ["Ġ", "e"], ["l", "o"], ["Ġ", "M"], ["Ġb", "e"], ["er", "s"], ["Ġ", "on"], ["Ġc", "on"], ["a", "p"], ["u", "b"], ["Ġ", "P"], ["ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠ"], ["as", "s"], ["in", "t"], ["l", "y"], ["ur", "n"], ["Ġ", "$"], ["a", "v"], ["p", "ort"], ["i", "r"], ["-", ">"], ["n", "t"], ["ct", "ion"], ["en", "d"], ["Ġd", "e"], ["it", "h"], ["ou", "t"], ["t", "urn"], ["ou", "r"], ["ĠĠĠĠ", "Ġ"], ["l", "ic"], ["re", "s"], ["p", "t"], ["=", "="], ["Ġth", "is"], ["Ġw", "h"], ["Ġ", "if"], ["Ġ", "D"], ["v", "er"], ["ag", "e"], ["Ġ", "B"], ["h", "t"], ["ex", "t"], ["=", "\""], ["Ġth", "at"], ["**", "**"], ["Ġ", "R"], ["Ġ", "it"], ["es", "s"], ["Ġ", "F"], ["Ġ", "r"], ["o", "s"], ["an", "d"], ["Ġa", "s"], ["e", "ct"], ["k", "e"], ["ro", "m"], ["Ġ", "//"], ["c", "on"], ["Ġ", "L"], ["(", "\""], ["q", "u"], ["l", "ass"], ["Ġw", "ith"], ["i", "z"], ["d", "e"], ["Ġ", "N"], ["Ġa", "l"], ["o", "p"], ["u", "p"], ["g", "et"], ["i", "le"], ["Ġa", "n"], ["at", "a"], ["o", "re"], ["r", "i"], ["Ġp", "ro"], ["ĉĉ", "ĉĉ"], ["t", "er"], ["a", "in"], ["Ġ", "W"], ["Ġ", "E"], ["Ġc", "om"], ["Ġre", "turn"], ["ar", "t"], ["Ġ", "H"], ["a", "ck"], ["im", "port"], ["ub", "lic"], ["Ġ", "or"], ["e", "st"], ["m", "ent"], ["Ġ", "G"], ["ab", "le"], ["Ġ", "-"], ["in", "e"], ["il", "l"], ["in", "d"], ["er", "e"], [":", ":"], ["it", "y"], ["Ġ", "+"], ["Ġt", "r"], ["el", "f"], ["ig", "ht"], ["(", "'"], ["or", "m"], ["ul", "t"], ["st", "r"], [".", "."], ["\"", ","], ["Ġy", "ou"], ["y", "pe"], ["p", "l"], ["Ġn", "ew"], ["Ġ", "j"], ["ĠĠĠĠĠĠĠĠ", "ĠĠĠĠĠĠĠĠĠĠĠ"], ["Ġf", "rom"], ["Ġ", "ex"], ["Ġ", "O"], ["l", "d"], ["Ġ", "["], ["o", "c"]]}}