scilons
/

sciroberta-tokenizer-v1

Model card Files Files and versions

lfoppiano commited on Jun 13, 2025

Commit

df4a4bf

·

verified ·

1 Parent(s): d31c7d5

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -46,7 +46,7 @@
   "eos_token": "</s>",
   "extra_special_tokens": {},
   "mask_token": "<mask>",
-  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
   "tokenizer_class": "PreTrainedTokenizer",
   "unk_token": "<unk>"

   "eos_token": "</s>",
   "extra_special_tokens": {},
   "mask_token": "<mask>",
+  "model_max_length": 512,
   "pad_token": "<pad>",
   "tokenizer_class": "PreTrainedTokenizer",
   "unk_token": "<unk>"