{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 4, "content": "<|system|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 5, "content": "<|user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 6, "content": "<|assistant|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 7, "content": "<|end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 8, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 9, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 10, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 11, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 12, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 13, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 14, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 15, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 16, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 17, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 18, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 19, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 20, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 21, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 22, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 24, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 25, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 26, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 27, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 28, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 29, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 30, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 31, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 32, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFC" }, { "type": "Lowercase" } ] }, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": {}, "merges": [] } }