Xenova HF Staff commited on
Commit
7f9935c
·
verified ·
1 Parent(s): 514216a

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +3 -3
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b472c7c2d1a2ac607daa53c794575e59504c06b39579a0f2399bb95cea6a4949
3
- size 20323105
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726
3
+ size 33384567
tokenizer_config.json CHANGED
@@ -15,10 +15,10 @@
15
  "image_token": "<image_soft_token>"
16
  },
17
  "pad_token": "<pad>",
 
18
  "sp_model_kwargs": null,
19
  "spaces_between_special_tokens": false,
20
  "tokenizer_class": "GemmaTokenizer",
21
  "unk_token": "<unk>",
22
- "use_default_system_prompt": false,
23
- "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n {%- if messages[0]['content'] is string -%}\n {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n {%- else -%}\n {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n {%- set first_user_prefix = \"\" -%}\n {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif -%}\n {%- if (message['role'] == 'assistant') -%}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is iterable -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type\") }}\n {%- endif -%}\n {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{'<start_of_turn>model\n'}}\n{%- endif -%}\n"
24
- }
 
15
  "image_token": "<image_soft_token>"
16
  },
17
  "pad_token": "<pad>",
18
+ "padding_side": "left",
19
  "sp_model_kwargs": null,
20
  "spaces_between_special_tokens": false,
21
  "tokenizer_class": "GemmaTokenizer",
22
  "unk_token": "<unk>",
23
+ "use_default_system_prompt": false
24
+ }