Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
|
@@ -400,10 +400,10 @@
|
|
| 400 |
"bos_token": "<s>",
|
| 401 |
"chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
|
| 402 |
"clean_up_tokenization_spaces": false,
|
| 403 |
-
"eos_token": "<|
|
| 404 |
"legacy": false,
|
| 405 |
"model_max_length": 131072,
|
| 406 |
-
"pad_token": "<|
|
| 407 |
"padding_side": "right",
|
| 408 |
"processor_class": "Phi3VProcessor",
|
| 409 |
"sp_model_kwargs": {},
|
|
|
|
| 400 |
"bos_token": "<s>",
|
| 401 |
"chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
|
| 402 |
"clean_up_tokenization_spaces": false,
|
| 403 |
+
"eos_token": "<|endoftext|>",
|
| 404 |
"legacy": false,
|
| 405 |
"model_max_length": 131072,
|
| 406 |
+
"pad_token": "<|endoftext|>",
|
| 407 |
"padding_side": "right",
|
| 408 |
"processor_class": "Phi3VProcessor",
|
| 409 |
"sp_model_kwargs": {},
|