Update tokenizer_config.json
Browse files- tokenizer_config.json +10 -10
tokenizer_config.json
CHANGED
|
@@ -45,19 +45,19 @@
|
|
| 45 |
},
|
| 46 |
"151648": {
|
| 47 |
"content": "<think>",
|
| 48 |
-
"lstrip":
|
| 49 |
-
"normalized":
|
| 50 |
-
"rstrip":
|
| 51 |
-
"single_word":
|
| 52 |
-
"special":
|
| 53 |
},
|
| 54 |
"151649": {
|
| 55 |
"content": "</think>",
|
| 56 |
-
"lstrip":
|
| 57 |
-
"normalized":
|
| 58 |
-
"rstrip":
|
| 59 |
-
"single_word":
|
| 60 |
-
"special":
|
| 61 |
},
|
| 62 |
"151650": {
|
| 63 |
"content": "<|quad_start|>",
|
|
|
|
| 45 |
},
|
| 46 |
"151648": {
|
| 47 |
"content": "<think>",
|
| 48 |
+
"lstrip": true,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": true,
|
| 51 |
+
"single_word": true,
|
| 52 |
+
"special": true
|
| 53 |
},
|
| 54 |
"151649": {
|
| 55 |
"content": "</think>",
|
| 56 |
+
"lstrip": true,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": true,
|
| 59 |
+
"single_word": true,
|
| 60 |
+
"special": true
|
| 61 |
},
|
| 62 |
"151650": {
|
| 63 |
"content": "<|quad_start|>",
|