--- tokenizer_config.json +++ tokenizer_config.json @@ -24,7 +24,7 @@ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", "processor_class": "Qwen3VLProcessor", "split_special_tokens": false, - "tokenizer_class": "TokenizersBackend", + "tokenizer_class": "Qwen2Tokenizer", "unk_token": null, "video_token": "<|video_pad|>", "vision_bos_token": "<|vision_start|>",