nieila commited on
Commit
39d7793
·
verified ·
1 Parent(s): 78efccd

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -3
tokenizer_config.json CHANGED
@@ -20,11 +20,10 @@
20
  "<|image_pad|>",
21
  "<|video_pad|>"
22
  ],
23
- "is_local": true,
24
- "local_files_only": true,
25
  "model_max_length": 131072,
26
  "pad_token": "<|endoftext|>",
27
- "padding_side": "left",
28
  "split_special_tokens": false,
29
  "tokenizer_class": "Qwen2Tokenizer",
30
  "unk_token": null
 
20
  "<|image_pad|>",
21
  "<|video_pad|>"
22
  ],
23
+ "is_local": false,
24
+ "local_files_only": false,
25
  "model_max_length": 131072,
26
  "pad_token": "<|endoftext|>",
 
27
  "split_special_tokens": false,
28
  "tokenizer_class": "Qwen2Tokenizer",
29
  "unk_token": null