mlxha commited on
Commit
bc111e1
·
verified ·
1 Parent(s): cee48e4

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -2053,7 +2053,7 @@
2053
  }
2054
  },
2055
  "bos_token": "<|begin▁of▁sentence|>",
2056
- "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{{'\n\nmessage['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{{'\n\n' + message['content']}}{%- endif %}{%- endfor %}{% if add_generation_prompt %}{{'\n\n'}}{%endif%}",
2057
  "clean_up_tokenization_spaces": false,
2058
  "eos_token": "<|end▁of▁sentence|>",
2059
  "extra_special_tokens": {},
 
2053
  }
2054
  },
2055
  "bos_token": "<|begin▁of▁sentence|>",
2056
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{{\"\n\n\" + message[\"content\"]}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{{\"\n\n\" + message[\"content\"]}}{%- endif %}{%- endfor %}{% if add_generation_prompt %}{{\"\n\n\"}}{%endif%}",
2057
  "clean_up_tokenization_spaces": false,
2058
  "eos_token": "<|end▁of▁sentence|>",
2059
  "extra_special_tokens": {},