Update tokenizer_config.json
Browse files- tokenizer_config.json +6 -6
tokenizer_config.json
CHANGED
|
@@ -38,7 +38,7 @@
|
|
| 38 |
"normalized": false,
|
| 39 |
"rstrip": false,
|
| 40 |
"single_word": false,
|
| 41 |
-
"special":
|
| 42 |
},
|
| 43 |
"128005": {
|
| 44 |
"content": "<tool_response>",
|
|
@@ -46,7 +46,7 @@
|
|
| 46 |
"normalized": false,
|
| 47 |
"rstrip": false,
|
| 48 |
"single_word": false,
|
| 49 |
-
"special":
|
| 50 |
},
|
| 51 |
"128006": {
|
| 52 |
"content": "<|start_header_id|>",
|
|
@@ -70,7 +70,7 @@
|
|
| 70 |
"normalized": false,
|
| 71 |
"rstrip": false,
|
| 72 |
"single_word": false,
|
| 73 |
-
"special":
|
| 74 |
},
|
| 75 |
"128009": {
|
| 76 |
"content": "<|eot_id>",
|
|
@@ -86,7 +86,7 @@
|
|
| 86 |
"normalized": false,
|
| 87 |
"rstrip": false,
|
| 88 |
"single_word": false,
|
| 89 |
-
"special":
|
| 90 |
},
|
| 91 |
"128011": {
|
| 92 |
"content": "</tool_call>",
|
|
@@ -94,7 +94,7 @@
|
|
| 94 |
"normalized": false,
|
| 95 |
"rstrip": false,
|
| 96 |
"single_word": false,
|
| 97 |
-
"special":
|
| 98 |
},
|
| 99 |
"128012": {
|
| 100 |
"content": "</tool_response>",
|
|
@@ -102,7 +102,7 @@
|
|
| 102 |
"normalized": false,
|
| 103 |
"rstrip": false,
|
| 104 |
"single_word": false,
|
| 105 |
-
"special":
|
| 106 |
},
|
| 107 |
"128013": {
|
| 108 |
"content": "<|reserved_special_token_8|>",
|
|
|
|
| 38 |
"normalized": false,
|
| 39 |
"rstrip": false,
|
| 40 |
"single_word": false,
|
| 41 |
+
"special": false
|
| 42 |
},
|
| 43 |
"128005": {
|
| 44 |
"content": "<tool_response>",
|
|
|
|
| 46 |
"normalized": false,
|
| 47 |
"rstrip": false,
|
| 48 |
"single_word": false,
|
| 49 |
+
"special": false
|
| 50 |
},
|
| 51 |
"128006": {
|
| 52 |
"content": "<|start_header_id|>",
|
|
|
|
| 70 |
"normalized": false,
|
| 71 |
"rstrip": false,
|
| 72 |
"single_word": false,
|
| 73 |
+
"special": false
|
| 74 |
},
|
| 75 |
"128009": {
|
| 76 |
"content": "<|eot_id>",
|
|
|
|
| 86 |
"normalized": false,
|
| 87 |
"rstrip": false,
|
| 88 |
"single_word": false,
|
| 89 |
+
"special": false
|
| 90 |
},
|
| 91 |
"128011": {
|
| 92 |
"content": "</tool_call>",
|
|
|
|
| 94 |
"normalized": false,
|
| 95 |
"rstrip": false,
|
| 96 |
"single_word": false,
|
| 97 |
+
"special": false
|
| 98 |
},
|
| 99 |
"128012": {
|
| 100 |
"content": "</tool_response>",
|
|
|
|
| 102 |
"normalized": false,
|
| 103 |
"rstrip": false,
|
| 104 |
"single_word": false,
|
| 105 |
+
"special": false
|
| 106 |
},
|
| 107 |
"128013": {
|
| 108 |
"content": "<|reserved_special_token_8|>",
|