PolPc13 commited on
Commit
3c45d6e
·
verified ·
1 Parent(s): 3d71ffe

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -0
tokenizer_config.json CHANGED
@@ -51334,13 +51334,19 @@
51334
  "image_token": "<image_soft_token>"
51335
  },
51336
  "image_token": "<image_soft_token>",
 
51337
  "model_max_length": 32768,
 
51338
  "pad_token": "<pad>",
 
51339
  "padding_side": "left",
51340
  "processor_class": "Gemma3Processor",
51341
  "sp_model_kwargs": null,
51342
  "spaces_between_special_tokens": false,
 
51343
  "tokenizer_class": "GemmaTokenizer",
 
 
51344
  "unk_token": "<unk>",
51345
  "use_default_system_prompt": false
51346
  }
 
51334
  "image_token": "<image_soft_token>"
51335
  },
51336
  "image_token": "<image_soft_token>",
51337
+ "max_length": 512,
51338
  "model_max_length": 32768,
51339
+ "pad_to_multiple_of": null,
51340
  "pad_token": "<pad>",
51341
+ "pad_token_type_id": 0,
51342
  "padding_side": "left",
51343
  "processor_class": "Gemma3Processor",
51344
  "sp_model_kwargs": null,
51345
  "spaces_between_special_tokens": false,
51346
+ "stride": 0,
51347
  "tokenizer_class": "GemmaTokenizer",
51348
+ "truncation_side": "right",
51349
+ "truncation_strategy": "longest_first",
51350
  "unk_token": "<unk>",
51351
  "use_default_system_prompt": false
51352
  }