Fix: Set special=true for thinking tokens and add to additional_special_tokens
Browse files- tokenizer_config.json +5 -4
tokenizer_config.json
CHANGED
|
@@ -174,7 +174,7 @@
|
|
| 174 |
"normalized": false,
|
| 175 |
"rstrip": false,
|
| 176 |
"single_word": false,
|
| 177 |
-
"special":
|
| 178 |
},
|
| 179 |
"151351": {
|
| 180 |
"content": "</think>",
|
|
@@ -182,7 +182,7 @@
|
|
| 182 |
"normalized": false,
|
| 183 |
"rstrip": false,
|
| 184 |
"single_word": false,
|
| 185 |
-
"special":
|
| 186 |
},
|
| 187 |
"151352": {
|
| 188 |
"content": "<tool_call>",
|
|
@@ -311,7 +311,8 @@
|
|
| 311 |
"<|code_prefix|>",
|
| 312 |
"<|code_middle|>",
|
| 313 |
"<|code_suffix|>",
|
| 314 |
-
"
|
|
|
|
| 315 |
],
|
| 316 |
"clean_up_tokenization_spaces": false,
|
| 317 |
"do_lower_case": false,
|
|
@@ -322,4 +323,4 @@
|
|
| 322 |
"padding_side": "left",
|
| 323 |
"remove_space": false,
|
| 324 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
| 325 |
-
}
|
|
|
|
| 174 |
"normalized": false,
|
| 175 |
"rstrip": false,
|
| 176 |
"single_word": false,
|
| 177 |
+
"special": true
|
| 178 |
},
|
| 179 |
"151351": {
|
| 180 |
"content": "</think>",
|
|
|
|
| 182 |
"normalized": false,
|
| 183 |
"rstrip": false,
|
| 184 |
"single_word": false,
|
| 185 |
+
"special": true
|
| 186 |
},
|
| 187 |
"151352": {
|
| 188 |
"content": "<tool_call>",
|
|
|
|
| 311 |
"<|code_prefix|>",
|
| 312 |
"<|code_middle|>",
|
| 313 |
"<|code_suffix|>",
|
| 314 |
+
"<think>",
|
| 315 |
+
"</think>"
|
| 316 |
],
|
| 317 |
"clean_up_tokenization_spaces": false,
|
| 318 |
"do_lower_case": false,
|
|
|
|
| 323 |
"padding_side": "left",
|
| 324 |
"remove_space": false,
|
| 325 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
| 326 |
+
}
|