Ex0bit commited on
Commit
1ec183e
·
verified ·
1 Parent(s): c51ff0d

Fix: Set special=true for thinking tokens and add to additional_special_tokens

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -4
tokenizer_config.json CHANGED
@@ -174,7 +174,7 @@
174
  "normalized": false,
175
  "rstrip": false,
176
  "single_word": false,
177
- "special": false
178
  },
179
  "151351": {
180
  "content": "</think>",
@@ -182,7 +182,7 @@
182
  "normalized": false,
183
  "rstrip": false,
184
  "single_word": false,
185
- "special": false
186
  },
187
  "151352": {
188
  "content": "<tool_call>",
@@ -311,7 +311,8 @@
311
  "<|code_prefix|>",
312
  "<|code_middle|>",
313
  "<|code_suffix|>",
314
- "/nothink"
 
315
  ],
316
  "clean_up_tokenization_spaces": false,
317
  "do_lower_case": false,
@@ -322,4 +323,4 @@
322
  "padding_side": "left",
323
  "remove_space": false,
324
  "tokenizer_class": "PreTrainedTokenizerFast"
325
- }
 
174
  "normalized": false,
175
  "rstrip": false,
176
  "single_word": false,
177
+ "special": true
178
  },
179
  "151351": {
180
  "content": "</think>",
 
182
  "normalized": false,
183
  "rstrip": false,
184
  "single_word": false,
185
+ "special": true
186
  },
187
  "151352": {
188
  "content": "<tool_call>",
 
311
  "<|code_prefix|>",
312
  "<|code_middle|>",
313
  "<|code_suffix|>",
314
+ "<think>",
315
+ "</think>"
316
  ],
317
  "clean_up_tokenization_spaces": false,
318
  "do_lower_case": false,
 
323
  "padding_side": "left",
324
  "remove_space": false,
325
  "tokenizer_class": "PreTrainedTokenizerFast"
326
+ }