Update tokenizer_config.json
Browse files- tokenizer_config.json +1 -0
tokenizer_config.json
CHANGED
|
@@ -1005,6 +1005,7 @@
|
|
| 1005 |
],
|
| 1006 |
"model_max_length": 1000000000000000019884624838656,
|
| 1007 |
"pad_token": "<pad>",
|
|
|
|
| 1008 |
"processor_class": "PixtralProcessor",
|
| 1009 |
"tokenizer_class": "TokenizersBackend"
|
| 1010 |
}
|
|
|
|
| 1005 |
],
|
| 1006 |
"model_max_length": 1000000000000000019884624838656,
|
| 1007 |
"pad_token": "<pad>",
|
| 1008 |
+
"chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ '[INST]' + message['content'] }}{% if loop.index0 == 1 and messages[0]['role'] == 'system' %}{{ '\n' + messages[0]['content'] }}{% endif %}{{ '[/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}",
|
| 1009 |
"processor_class": "PixtralProcessor",
|
| 1010 |
"tokenizer_class": "TokenizersBackend"
|
| 1011 |
}
|