Update tokenizer_config.json

#2
by iamPi - opened
Files changed (1) hide show
  1. tokenizer_config.json +3 -5
tokenizer_config.json CHANGED
@@ -31,16 +31,14 @@
31
  "<|im_start|>",
32
  "<|im_end|>"
33
  ],
34
- "bos_token": "<|im_start|>",
35
  "clean_up_tokenization_spaces": false,
36
  "eos_token": "<|im_end|>",
37
  "errors": "replace",
38
  "extra_special_tokens": {},
39
  "model_max_length": 262144,
40
  "pad_token": "<|endoftext|>",
41
- "vocab_size": 16,
42
  "split_special_tokens": false,
43
  "tokenizer_class": "Qwen2Tokenizer",
44
- "unk_token": null,
45
- "chat_template": "{%- for message in messages %}{{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}"
46
- }
 
31
  "<|im_start|>",
32
  "<|im_end|>"
33
  ],
34
+ "bos_token": null,
35
  "clean_up_tokenization_spaces": false,
36
  "eos_token": "<|im_end|>",
37
  "errors": "replace",
38
  "extra_special_tokens": {},
39
  "model_max_length": 262144,
40
  "pad_token": "<|endoftext|>",
 
41
  "split_special_tokens": false,
42
  "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null
44
+ }