alphaXiv commited on
Commit
0be270c
·
verified ·
1 Parent(s): a5a8818

fix: change tokenizer_class from TokenizersBackend to Qwen2Tokenizer for vLLM compat

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -23,9 +23,9 @@
23
  "pad_token": "<|endoftext|>",
24
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
25
  "split_special_tokens": false,
26
- "tokenizer_class": "TokenizersBackend",
27
  "unk_token": null,
28
  "video_token": "<|video_pad|>",
29
  "vision_bos_token": "<|vision_start|>",
30
  "vision_eos_token": "<|vision_end|>"
31
- }
 
23
  "pad_token": "<|endoftext|>",
24
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
25
  "split_special_tokens": false,
26
+ "tokenizer_class": "Qwen2Tokenizer",
27
  "unk_token": null,
28
  "video_token": "<|video_pad|>",
29
  "vision_bos_token": "<|vision_start|>",
30
  "vision_eos_token": "<|vision_end|>"
31
+ }