{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "<|endoftext|>": 0, "<|im_start|>": 1, "<|im_end|>": 2, "!": 3, "\"": 4, "#": 5, "$": 6, "%": 7, "&": 8, "'": 9, "(": 10, ")": 11, "*": 12, "+": 13, ",": 14, "-": 15, ".": 16, "/": 17, "0": 18, "1": 19, "2": 20, "3": 21, "4": 22, "5": 23, "6": 24, "7": 25, "8": 26, "9": 27, ":": 28, ";": 29, "<": 30, "=": 31, ">": 32, "?": 33, "@": 34, "A": 35, "B": 36, "C": 37, "D": 38, "E": 39, "F": 40, "G": 41, "H": 42, "I": 43, "J": 44, "K": 45, "L": 46, "M": 47, "N": 48, "O": 49, "P": 50, "Q": 51, "R": 52, "S": 53, "T": 54, "U": 55, "V": 56, "W": 57, "X": 58, "Y": 59, "Z": 60, "[": 61, "\\": 62, "]": 63, "^": 64, "_": 65, "`": 66, "a": 67, "b": 68, "c": 69, "d": 70, "e": 71, "f": 72, "g": 73, "h": 74, "i": 75, "j": 76, "k": 77, "l": 78, "m": 79, "n": 80, "o": 81, "p": 82, "q": 83, "r": 84, "s": 85, "t": 86, "u": 87, "v": 88, "w": 89, "x": 90, "y": 91, "z": 92, "{": 93, "|": 94, "}": 95, "~": 96, "Ċ": 97, "Ġ": 98, "Ġt": 99, "Ġa": 100, "in": 101, "on": 102, "he": 103, "ti": 104, "re": 105, "Ġthe": 106, "er": 107, "Ġs": 108, "Ġo": 109, "en": 110, "al": 111, "Ġc": 112, "or": 113, "at": 114, "tion": 115, "es": 116, "Ġof": 117, "is": 118, "Ġp": 119, "an": 120, "ed": 121, "Ġf": 122, "ar": 123, "it": 124, "ro": 125, "ing": 126, "Ġan": 127, "Ġin": 128, "Ġd": 129, "Ġw": 130, "Ġm": 131, "ic": 132, "le": 133, "Ġb": 134, "Ġand": 135, "ation": 136, "as": 137, "Ġe": 138, "el": 139, "Ġto": 140, "et": 141, "ent": 142, "Ġre": 143, "st": 144, "ct": 145, "Ġth": 146, "om": 147, "im": 148, "ul": 149, "ra": 150, "Ġn": 151, "ve": 152, "ly": 153, "od": 154, "ur": 155, "un": 156, "si": 157, "ce": 158, "Ġis": 159, "Ġl": 160, "Ġfor": 161, "us": 162, "ith": 163, "ĠT": 164, "ol": 165, "Ġg": 166, "Ġst": 167, "Ġpro": 168, "ch": 169, "ow": 170, "tic": 171, "Ġcon": 172, "qu": 173, "Ġh": 174, "per": 175, "Ġon": 176, "ig": 177, "am": 178, "res": 179, "Ġwith": 180, "Ġthat": 181, "ĠW": 182, "ver": 183, "um": 184, "Ġ$": 185, "il": 186, "Ġex": 187, "ut": 188, "se": 189, "ot": 190, "ate": 191, "ec": 192, "id": 193, "em": 194, "ity": 195, "Ġwe": 196, "pl": 197, "ĠWe": 198, "if": 199, "Ġ(": 200, "ab": 201, "ge": 202, "ĠThe": 203, "ri": 204, "ter": 205, "the": 206, "Ġal": 207, "Ġas": 208, "Ġbe": 209, "Ġare": 210, "ect": 211, "omp": 212, "ction": 213, "ations": 214, "ure": 215, "ĠI": 216, "Ġmod": 217, "Ġv": 218, "ir": 219, "os": 220, "Ġsu": 221, "Ġby": 222, "ts": 223, "Ġsp": 224, "pro": 225, "und": 226, "iz": 227, "ac": 228, "ical": 229, "for": 230, "ran": 231, "ant": 232, "Ġcomp": 233, "di": 234, "ain": 235, "Ġwh": 236, "ener": 237, "ren": 238, "rom": 239, "Ġmodel": 240, "Ġus": 241, "ĠA": 242, "Ġthis": 243, "ist": 244, "tive": 245, "Ġres": 246, "sion": 247, "ag": 248, "Ġde": 249, "du": 250, "Ġsh": 251, "ess": 252, "Ġat": 253, "ĠS": 254, "oc": 255 }, "merges": [ [ "Ġ", "t" ], [ "Ġ", "a" ], [ "i", "n" ], [ "o", "n" ], [ "h", "e" ], [ "t", "i" ], [ "r", "e" ], [ "Ġt", "he" ], [ "e", "r" ], [ "Ġ", "s" ], [ "Ġ", "o" ], [ "e", "n" ], [ "a", "l" ], [ "Ġ", "c" ], [ "o", "r" ], [ "a", "t" ], [ "ti", "on" ], [ "e", "s" ], [ "Ġo", "f" ], [ "i", "s" ], [ "Ġ", "p" ], [ "a", "n" ], [ "e", "d" ], [ "Ġ", "f" ], [ "a", "r" ], [ "i", "t" ], [ "r", "o" ], [ "in", "g" ], [ "Ġa", "n" ], [ "Ġ", "in" ], [ "Ġ", "d" ], [ "Ġ", "w" ], [ "Ġ", "m" ], [ "i", "c" ], [ "l", "e" ], [ "Ġ", "b" ], [ "Ġan", "d" ], [ "a", "tion" ], [ "a", "s" ], [ "Ġ", "e" ], [ "e", "l" ], [ "Ġt", "o" ], [ "e", "t" ], [ "en", "t" ], [ "Ġ", "re" ], [ "s", "t" ], [ "c", "t" ], [ "Ġt", "h" ], [ "o", "m" ], [ "i", "m" ], [ "u", "l" ], [ "r", "a" ], [ "Ġ", "n" ], [ "v", "e" ], [ "l", "y" ], [ "o", "d" ], [ "u", "r" ], [ "u", "n" ], [ "s", "i" ], [ "c", "e" ], [ "Ġ", "is" ], [ "Ġ", "l" ], [ "Ġf", "or" ], [ "u", "s" ], [ "it", "h" ], [ "Ġ", "T" ], [ "o", "l" ], [ "Ġ", "g" ], [ "Ġs", "t" ], [ "Ġp", "ro" ], [ "c", "h" ], [ "o", "w" ], [ "ti", "c" ], [ "Ġc", "on" ], [ "q", "u" ], [ "Ġ", "h" ], [ "p", "er" ], [ "Ġ", "on" ], [ "i", "g" ], [ "a", "m" ], [ "re", "s" ], [ "Ġw", "ith" ], [ "Ġth", "at" ], [ "Ġ", "W" ], [ "v", "er" ], [ "u", "m" ], [ "Ġ", "$" ], [ "i", "l" ], [ "Ġe", "x" ], [ "u", "t" ], [ "s", "e" ], [ "o", "t" ], [ "at", "e" ], [ "e", "c" ], [ "i", "d" ], [ "e", "m" ], [ "it", "y" ], [ "Ġw", "e" ], [ "p", "l" ], [ "ĠW", "e" ], [ "i", "f" ], [ "Ġ", "(" ], [ "a", "b" ], [ "g", "e" ], [ "ĠT", "he" ], [ "r", "i" ], [ "t", "er" ], [ "t", "he" ], [ "Ġa", "l" ], [ "Ġa", "s" ], [ "Ġb", "e" ], [ "Ġa", "re" ], [ "e", "ct" ], [ "om", "p" ], [ "c", "tion" ], [ "ation", "s" ], [ "u", "re" ], [ "Ġ", "I" ], [ "Ġm", "od" ], [ "Ġ", "v" ], [ "i", "r" ], [ "o", "s" ], [ "Ġs", "u" ], [ "Ġb", "y" ], [ "t", "s" ], [ "Ġs", "p" ], [ "p", "ro" ], [ "un", "d" ], [ "i", "z" ], [ "a", "c" ], [ "ic", "al" ], [ "f", "or" ], [ "r", "an" ], [ "an", "t" ], [ "Ġc", "omp" ], [ "d", "i" ], [ "a", "in" ], [ "Ġw", "h" ], [ "en", "er" ], [ "re", "n" ], [ "ro", "m" ], [ "Ġmod", "el" ], [ "Ġ", "us" ], [ "Ġ", "A" ], [ "Ġth", "is" ], [ "is", "t" ], [ "ti", "ve" ], [ "Ġre", "s" ], [ "si", "on" ], [ "a", "g" ], [ "Ġd", "e" ], [ "d", "u" ], [ "Ġs", "h" ], [ "es", "s" ], [ "Ġa", "t" ], [ "Ġ", "S" ], [ "o", "c" ] ] } }