Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +1 -2
tokenizer_config.json
CHANGED
|
@@ -16,6 +16,5 @@
|
|
| 16 |
"model_max_length": 1048576,
|
| 17 |
"pad_token": "[PAD]",
|
| 18 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 19 |
-
"unk_token": "[UNK]"
|
| 20 |
-
"chat_template": "{%- for message in messages -%}{%- if loop.first and messages[0]['role'] != 'system' -%}{{'<|im_system|>system<|im_middle|>You are a helpful assistant<|im_end|>'}}{%- endif -%}{%- if message['role'] == 'system' -%}{{'<|im_system|>'}}{%- endif -%}{%- if message['role'] == 'user' -%}{{'<|im_user|>'}}{%- endif -%}{%- if message['role'] == 'assistant' -%}{{'<|im_assistant|>'}}{%- endif -%}{{- message['role'] -}}{{'<|im_middle|>'}}{%- if message['content'] is string -%}{{- message['content'] + '<|im_end|>' -}}{%- else -%}{%- for content in message['content'] -%}{%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}{{'<|media_start|>image<|media_content|><|media_pad|><|media_end|>'}}{%- else -%}{{content['text']}}{%- endif -%}{%- endfor -%}{{'<|im_end|>'}}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{'<|im_assistant|>assistant<|im_middle|>'}}{%- endif -%}"
|
| 21 |
}
|
|
|
|
| 16 |
"model_max_length": 1048576,
|
| 17 |
"pad_token": "[PAD]",
|
| 18 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 19 |
+
"unk_token": "[UNK]"
|
|
|
|
| 20 |
}
|