sdadas commited on
Commit
f3b336d
·
verified ·
1 Parent(s): 0af012d

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -49,14 +49,14 @@
49
  "errors": "replace",
50
  "mask_token": "<mask>",
51
  "max_length": 512,
52
- "model_max_length": 1000000000000000019884624838656,
53
  "pad_to_multiple_of": null,
54
  "pad_token": "<pad>",
55
  "pad_token_type_id": 0,
56
  "padding_side": "right",
57
  "sep_token": "</s>",
58
  "stride": 0,
59
- "tokenizer_class": "RobertaTokenizer",
60
  "trim_offsets": true,
61
  "truncation_side": "right",
62
  "truncation_strategy": "longest_first",
 
49
  "errors": "replace",
50
  "mask_token": "<mask>",
51
  "max_length": 512,
52
+ "model_max_length": 512,
53
  "pad_to_multiple_of": null,
54
  "pad_token": "<pad>",
55
  "pad_token_type_id": 0,
56
  "padding_side": "right",
57
  "sep_token": "</s>",
58
  "stride": 0,
59
+ "tokenizer_class": "XLMRobertaTokenizer",
60
  "trim_offsets": true,
61
  "truncation_side": "right",
62
  "truncation_strategy": "longest_first",