{ "tokenizer_class": "LlamaTokenizer", "bos_token": "", "eos_token": "<|end|>", "unk_token": "", "pad_token": "", "add_bos_token": true, "add_eos_token": false, "model_max_length": 1024, "chat_template": "{%- for message in messages -%}{{- '<|' + message['role'] + '|> ' + message['content'] + ' <|end|> ' -}}{%- endfor -%}{%- if add_generation_prompt -%}{{- '<|assistant|>' -}}{%- endif -%}", "added_tokens_decoder": { "0": { "content": "", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "1": { "content": "", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "2": { "content": "", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "3": { "content": "", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "4": { "content": "<|user|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "5": { "content": "<|assistant|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "6": { "content": "<|system|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "7": { "content": "<|end|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "8": { "content": "<|mem_l1|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "9": { "content": "<|mem_l2|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "10": { "content": "<|mem_l3|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "11": { "content": "<|sep|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false }, "12": { "content": "<|summary|>", "special": true, "lstrip": false, "rstrip": false, "normalized": false, "single_word": false } }, "additional_special_tokens": [ "<|user|>", "<|assistant|>", "<|system|>", "<|end|>", "<|mem_l1|>", "<|mem_l2|>", "<|mem_l3|>", "<|sep|>", "<|summary|>" ] }