Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +4 -4
tokenizer.json
CHANGED
|
@@ -59,7 +59,7 @@
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 128006,
|
| 62 |
-
"content": "<|
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": false,
|
|
@@ -86,7 +86,7 @@
|
|
| 86 |
},
|
| 87 |
{
|
| 88 |
"id": 128009,
|
| 89 |
-
"content": "<|
|
| 90 |
"single_word": false,
|
| 91 |
"lstrip": false,
|
| 92 |
"rstrip": false,
|
|
@@ -167,7 +167,7 @@
|
|
| 167 |
},
|
| 168 |
{
|
| 169 |
"id": 128018,
|
| 170 |
-
"content": "<|
|
| 171 |
"single_word": false,
|
| 172 |
"lstrip": false,
|
| 173 |
"rstrip": false,
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
},
|
| 177 |
{
|
| 178 |
"id": 128019,
|
| 179 |
-
"content": "<|
|
| 180 |
"single_word": false,
|
| 181 |
"lstrip": false,
|
| 182 |
"rstrip": false,
|
|
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 128006,
|
| 62 |
+
"content": "<|start_header_id|>",
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": false,
|
|
|
|
| 86 |
},
|
| 87 |
{
|
| 88 |
"id": 128009,
|
| 89 |
+
"content": "<|eot_id|>",
|
| 90 |
"single_word": false,
|
| 91 |
"lstrip": false,
|
| 92 |
"rstrip": false,
|
|
|
|
| 167 |
},
|
| 168 |
{
|
| 169 |
"id": 128018,
|
| 170 |
+
"content": "<|im_start|>",
|
| 171 |
"single_word": false,
|
| 172 |
"lstrip": false,
|
| 173 |
"rstrip": false,
|
|
|
|
| 176 |
},
|
| 177 |
{
|
| 178 |
"id": 128019,
|
| 179 |
+
"content": "<|im_end|>",
|
| 180 |
"single_word": false,
|
| 181 |
"lstrip": false,
|
| 182 |
"rstrip": false,
|