{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 96, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 97, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 98, "content": "<|system|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 99, "content": "<|user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 100, "content": "<|assistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 101, "content": "<|tool|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 102, "content": "<|assistant_reasoning|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 103, "content": "<|assistant_reasoning_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "Prepend", "prepend": "▁" }, { "type": "Replace", "pattern": { "String": " " }, "content": "▁" } ] }, "pre_tokenizer": null, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "SpecialToken": { "id": "", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "", "type_id": 1 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": { "": { "id": "", "ids": [ 1 ], "tokens": [ "" ] } } }, "decoder": { "type": "Sequence", "decoders": [ { "type": "Replace", "pattern": { "String": "▁" }, "content": " " }, { "type": "ByteFallback" }, { "type": "Fuse" }, { "type": "Strip", "content": " ", "start": 1, "stop": 0 } ] }, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": true, "byte_fallback": true, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, "te": 4, "en": 5, "ar": 6, "▁=": 7, "▁[": 8, "▁": 9, "e": 10, "a": 11, "t": 12, "i": 13, "r": 14, "s": 15, "n": 16, "|": 17, "o": 18, ".": 19, "l": 20, "1": 21, "0": 22, "m": 23, "[": 24, "]": 25, "=": 26, "c": 27, "p": 28, "d": 29, "u": 30, "2": 31, "%": 32, "-": 33, "'": 34, "/": 35, "3": 36, "8": 37, "h": 38, "g": 39, "9": 40, "w": 41, ":": 42, "D": 43, "<": 44, ">": 45, "_": 46, "{": 47, "}": 48, "6": 49, "4": 50, "B": 51, "k": 52, "5": 53, ",": 54, "b": 55, "7": 56, "(": 57, ")": 58, "z": 59, "f": 60, "y": 61, "x": 62, "v": 63, "*": 64, "I": 65, "F": 66, "E": 67, "X": 68, "S": 69, "A": 70, "C": 71, "T": 72, "#": 73, "N": 74, "P": 75, "q": 76, "M": 77, "Z": 78, "H": 79, "Q": 80, "W": 81, "j": 82, "G": 83, "\"": 84, ";": 85, "L": 86, "U": 87, "R": 88, "K": 89, "J": 90, "V": 91, "?": 92, "O": 93, "&": 94, "+": 95 }, "merges": [ [ "t", "e" ], [ "e", "n" ], [ "a", "r" ], [ "▁", "=" ], [ "▁", "[" ] ] } }