mediusware-ai commited on
Commit
0008130
·
verified ·
1 Parent(s): f6694e7

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -23,11 +23,11 @@
23
  "pad_token": "<|vision_pad|>",
24
  "padding_side": "right",
25
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
- "processor_class": "Qwen3VLProcessor",
27
  "split_special_tokens": false,
28
  "tokenizer_class": "TokenizersBackend",
29
  "unk_token": null,
30
  "video_token": "<|video_pad|>",
31
  "vision_bos_token": "<|vision_start|>",
32
  "vision_eos_token": "<|vision_end|>"
33
- }
 
23
  "pad_token": "<|vision_pad|>",
24
  "padding_side": "right",
25
  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
26
+ "processor_class": "IntellixProcessor",
27
  "split_special_tokens": false,
28
  "tokenizer_class": "TokenizersBackend",
29
  "unk_token": null,
30
  "video_token": "<|video_pad|>",
31
  "vision_bos_token": "<|vision_start|>",
32
  "vision_eos_token": "<|vision_end|>"
33
+ }