luosuu commited on
Commit
b25926d
·
verified ·
1 Parent(s): 4e735b0

Fix eos_token in tokenizer_config.json to use im_end for Instruct Model

Browse files

eos_token for instruct model should be im_end instead of the original eos token in base model

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -82,7 +82,7 @@
82
  ],
83
  "bos_token": "[BOS]",
84
  "clean_up_tokenization_spaces": false,
85
- "eos_token": "[EOS]",
86
  "extra_special_tokens": {},
87
  "model_max_length": 1048576,
88
  "pad_token": "[PAD]",
 
82
  ],
83
  "bos_token": "[BOS]",
84
  "clean_up_tokenization_spaces": false,
85
+ "eos_token": "<|im_end|>",
86
  "extra_special_tokens": {},
87
  "model_max_length": 1048576,
88
  "pad_token": "[PAD]",