tsor13 commited on
Commit
2b8d7ba
·
verified ·
1 Parent(s): 466634b

Initial upload of fine‑tuned Gemma + custom tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -7
  2. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -8,13 +8,7 @@
8
  "single_word": false
9
  },
10
  "eoi_token": "<end_of_image>",
11
- "eos_token": {
12
- "content": "<eos>",
13
- "lstrip": false,
14
- "normalized": false,
15
- "rstrip": false,
16
- "single_word": false
17
- },
18
  "image_token": "<image_soft_token>",
19
  "pad_token": {
20
  "content": "<pad>",
 
8
  "single_word": false
9
  },
10
  "eoi_token": "<end_of_image>",
11
+ "eos_token": "<end_of_turn>",
 
 
 
 
 
 
12
  "image_token": "<image_soft_token>",
13
  "pad_token": {
14
  "content": "<pad>",
tokenizer_config.json CHANGED
@@ -51328,7 +51328,7 @@
51328
  "chat_template": "{{ bos_token }}{%- for message in messages -%}\n {%- if message['role'] == 'description' -%}\n {{ '<start_of_turn>description\n' + message['content'] + '<end_of_turn>\n' }}\n {%- elif message['role'] == 'input' -%}\n {{ '<start_of_turn>input\n' + message['content'] + '<end_of_turn>\n' }}\n {%- elif message['role'] == 'output' -%}\n {{ '<start_of_turn>output\n' + message['content'] + '<end_of_turn>\n' }}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{ '<start_of_turn>output\n' }}\n{%- endif -%}",
51329
  "clean_up_tokenization_spaces": false,
51330
  "eoi_token": "<end_of_image>",
51331
- "eos_token": "<eos>",
51332
  "extra_special_tokens": {
51333
  "boi_token": "<start_of_image>",
51334
  "eoi_token": "<end_of_image>",
 
51328
  "chat_template": "{{ bos_token }}{%- for message in messages -%}\n {%- if message['role'] == 'description' -%}\n {{ '<start_of_turn>description\n' + message['content'] + '<end_of_turn>\n' }}\n {%- elif message['role'] == 'input' -%}\n {{ '<start_of_turn>input\n' + message['content'] + '<end_of_turn>\n' }}\n {%- elif message['role'] == 'output' -%}\n {{ '<start_of_turn>output\n' + message['content'] + '<end_of_turn>\n' }}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{ '<start_of_turn>output\n' }}\n{%- endif -%}",
51329
  "clean_up_tokenization_spaces": false,
51330
  "eoi_token": "<end_of_image>",
51331
+ "eos_token": "<end_of_turn>",
51332
  "extra_special_tokens": {
51333
  "boi_token": "<start_of_image>",
51334
  "eoi_token": "<end_of_image>",