Upload tokenizer

README.md

@@ -1,14 +1,13 @@
 ---
+library_name: transformers
+tags:
+- mergekit
+- merge
 base_model:
 - crestf411/gemma2-9B-sunfall-v0.5.2
 - Hastagaras/Gemma-Model1
 - IlyaGusev/gemma-2-9b-it-abliterated
 - Hastagaras/Gemmoy-9B-G2-MK.3
-library_name: transformers
-tags:
-- mergekit
-- merge
-
 ---
 i just realized it became 10b and I don't know why
 

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6baaffa9d0e48eb225a65139caaa8f6dbeea24f43defeb29b7ee35d4d011716d
-size 17518679
+oid sha256:c15eb04bc5ad609fb26533e8525302c5640a945e5f67f65b7c849900acda7d99
+size 17518497

tokenizer_config.json

@@ -1744,14 +1744,11 @@
     "<end_of_turn>"
   ],
   "bos_token": "<bos>",
-  "chat_template": "{% for message in messages -%}{%- if message['role'] == 'system' -%}<start_of_turn>user{{ '\n' }}{{ message['content'] | trim }}{%- if loop.nextitem is defined and loop.nextitem['role'] == 'user' %}{{ '\n\n' }}{{ loop.nextitem['content'] | trim }}<end_of_turn>{%- else -%}<end_of_turn>{%- endif -%}{%- elif message['role'] == 'user' and loop.previtem is defined and loop.previtem['role'] == 'system' -%}{# Skip this message as it was already included #}{%- else -%}<start_of_turn>{% if message['role'] == 'assistant' %}model{% else %}{{ message['role'] }}{% endif %}{{ '\n' }}{{ message['content'] | trim }}<end_of_turn>{%- endif -%}{%- if not loop.last and not (message['role'] == 'system' and loop.nextitem['role'] == 'user') -%}{{ '\n' }}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{ '\n' }}<start_of_turn>model{%- endif -%}",  "clean_up_tokenization_spaces": false,
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
-  "max_length": null,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "left",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "GemmaTokenizer",