Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +1 -10
tokenizer_config.json
CHANGED
|
@@ -225,14 +225,6 @@
|
|
| 225 |
"rstrip": false,
|
| 226 |
"single_word": false,
|
| 227 |
"special": true
|
| 228 |
-
},
|
| 229 |
-
"151671": {
|
| 230 |
-
"content": "<|end_of_audio|>",
|
| 231 |
-
"lstrip": false,
|
| 232 |
-
"normalized": false,
|
| 233 |
-
"rstrip": false,
|
| 234 |
-
"single_word": false,
|
| 235 |
-
"special": true
|
| 236 |
}
|
| 237 |
},
|
| 238 |
"additional_special_tokens": [
|
|
@@ -250,8 +242,7 @@
|
|
| 250 |
"<|image_pad|>",
|
| 251 |
"<|video_pad|>",
|
| 252 |
"<|AUDIO|>",
|
| 253 |
-
"<|start_of_audio|>"
|
| 254 |
-
"<|end_of_audio|>"
|
| 255 |
],
|
| 256 |
"bos_token": null,
|
| 257 |
"clean_up_tokenization_spaces": false,
|
|
|
|
| 225 |
"rstrip": false,
|
| 226 |
"single_word": false,
|
| 227 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
}
|
| 229 |
},
|
| 230 |
"additional_special_tokens": [
|
|
|
|
| 242 |
"<|image_pad|>",
|
| 243 |
"<|video_pad|>",
|
| 244 |
"<|AUDIO|>",
|
| 245 |
+
"<|start_of_audio|>"
|
|
|
|
| 246 |
],
|
| 247 |
"bos_token": null,
|
| 248 |
"clean_up_tokenization_spaces": false,
|