mazesmazes commited on
Commit
5506373
·
verified ·
1 Parent(s): b6009b5

Fix eos_token_id to use correct <|im_end|> and <|endoftext|> tokens

Browse files
Files changed (1) hide show
  1. generation_config.json +3 -6
generation_config.json CHANGED
@@ -1,15 +1,12 @@
1
  {
2
- "bos_token_id": 151643,
3
- "eos_token_id": [
4
- 151645,
5
- 151643
6
- ],
7
  "length_penalty": 1.0,
8
  "max_new_tokens": 128,
9
  "min_new_tokens": 0,
10
  "no_repeat_ngram_size": 0,
11
  "num_beams": 1,
12
- "pad_token_id": 151643,
13
  "repetition_penalty": 1.0,
14
  "transformers_version": "5.0.0.dev0"
15
  }
 
1
  {
2
+ "bos_token_id": 217181,
3
+ "eos_token_id": [217183, 217181],
 
 
 
4
  "length_penalty": 1.0,
5
  "max_new_tokens": 128,
6
  "min_new_tokens": 0,
7
  "no_repeat_ngram_size": 0,
8
  "num_beams": 1,
9
+ "pad_token_id": 217208,
10
  "repetition_penalty": 1.0,
11
  "transformers_version": "5.0.0.dev0"
12
  }