SeaWolf-AI commited on
Commit
a6fc8a9
·
verified ·
1 Parent(s): 31c167d

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -304,9 +304,9 @@
304
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
305
  "processor_class": "Qwen3VLProcessor",
306
  "split_special_tokens": false,
307
- "tokenizer_class": "Qwen2Tokenizer",
308
  "unk_token": null,
309
  "video_token": "<|video_pad|>",
310
  "vision_bos_token": "<|vision_start|>",
311
  "vision_eos_token": "<|vision_end|>"
312
- }
 
304
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
305
  "processor_class": "Qwen3VLProcessor",
306
  "split_special_tokens": false,
307
+ "tokenizer_class": "Qwen2TokenizerFast",
308
  "unk_token": null,
309
  "video_token": "<|video_pad|>",
310
  "vision_bos_token": "<|vision_start|>",
311
  "vision_eos_token": "<|vision_end|>"
312
+ }