Stardust-minus commited on
Commit
cf387f0
·
verified ·
1 Parent(s): 0141e51

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -37213,7 +37213,7 @@
37213
  "model_max_length": 131072,
37214
  "pad_token": "<|endoftext|>",
37215
  "split_special_tokens": false,
37216
- "tokenizer_class": "PreTrainedTokenizerFast",
37217
  "unk_token": null,
37218
  "video_token": "<|video_pad|>",
37219
  "vision_bos_token": "<|vision_start|>",
 
37213
  "model_max_length": 131072,
37214
  "pad_token": "<|endoftext|>",
37215
  "split_special_tokens": false,
37216
+ "tokenizer_class": "Qwen2Tokenizer",
37217
  "unk_token": null,
37218
  "video_token": "<|video_pad|>",
37219
  "vision_bos_token": "<|vision_start|>",