AlexWortega commited on
Commit
db16150
·
verified ·
1 Parent(s): 91edaac

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -10
tokenizer_config.json CHANGED
@@ -225,14 +225,6 @@
225
  "rstrip": false,
226
  "single_word": false,
227
  "special": true
228
- },
229
- "151671": {
230
- "content": "<|end_of_audio|>",
231
- "lstrip": false,
232
- "normalized": false,
233
- "rstrip": false,
234
- "single_word": false,
235
- "special": true
236
  }
237
  },
238
  "additional_special_tokens": [
@@ -250,8 +242,7 @@
250
  "<|image_pad|>",
251
  "<|video_pad|>",
252
  "<|AUDIO|>",
253
- "<|start_of_audio|>",
254
- "<|end_of_audio|>"
255
  ],
256
  "bos_token": null,
257
  "clean_up_tokenization_spaces": false,
 
225
  "rstrip": false,
226
  "single_word": false,
227
  "special": true
 
 
 
 
 
 
 
 
228
  }
229
  },
230
  "additional_special_tokens": [
 
242
  "<|image_pad|>",
243
  "<|video_pad|>",
244
  "<|AUDIO|>",
245
+ "<|start_of_audio|>"
 
246
  ],
247
  "bos_token": null,
248
  "clean_up_tokenization_spaces": false,