manaestras commited on
Commit
350e760
·
verified ·
1 Parent(s): 51c1a22

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -1
tokenizer_config.json CHANGED
@@ -6553,7 +6553,9 @@
6553
  "extra_sepcial_tokens": {
6554
  "image_token": "<|hy_place▁holder▁no▁102|>",
6555
  "image_start_token": "<|hy_place▁holder▁no▁100|>",
6556
- "image_end_token": "<|hy_place▁holder▁no▁101|>"
 
 
6557
  },
6558
  "tokenizer_class": "PreTrainedTokenizerFast",
6559
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% if messages[0]['content'] is string %}{% set system_message = messages[0]['content'] %}{% else %}{% set system_message = messages[0]['content']['text'] %}{% endif %}<|hy_begin▁of▁sentence|>{{ system_message }}<|hy_place▁holder▁no▁3|>{% else %}{% set loop_messages = messages %}<|hy_begin▁of▁sentence|>{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}<|hy_User|>{% elif message['role'] == 'assistant' %}{{ message['content'] }}<|hy_Assistant|>{% endif %}{% endfor %}"
 
6553
  "extra_sepcial_tokens": {
6554
  "image_token": "<|hy_place▁holder▁no▁102|>",
6555
  "image_start_token": "<|hy_place▁holder▁no▁100|>",
6556
+ "image_end_token": "<|hy_place▁holder▁no▁101|>",
6557
+ "video_start_token": "<|hy_place▁holder▁no▁104|>",
6558
+ "video_end_token": "<|hy_place▁holder▁no▁105|>"
6559
  },
6560
  "tokenizer_class": "PreTrainedTokenizerFast",
6561
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% if messages[0]['content'] is string %}{% set system_message = messages[0]['content'] %}{% else %}{% set system_message = messages[0]['content']['text'] %}{% endif %}<|hy_begin▁of▁sentence|>{{ system_message }}<|hy_place▁holder▁no▁3|>{% else %}{% set loop_messages = messages %}<|hy_begin▁of▁sentence|>{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}<|hy_User|>{% elif message['role'] == 'assistant' %}{{ message['content'] }}<|hy_Assistant|>{% endif %}{% endfor %}"