guillermoruiz commited on
Commit
073e47e
·
verified ·
1 Parent(s): 356f6a9

Update tokenizer_config.json

Browse files

Changed tokenizer_class to PreTrainedTokenizerFast

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -107,7 +107,7 @@
107
  "model_max_length": 1000000000000000019884624838656,
108
  "pad_token": "[PAD]",
109
  "sep_token": "[SEP]",
110
- "tokenizer_class": "RobertaTokenizer",
111
  "trim_offsets": true,
112
  "unk_token": "[UNK]"
113
  }
 
107
  "model_max_length": 1000000000000000019884624838656,
108
  "pad_token": "[PAD]",
109
  "sep_token": "[SEP]",
110
+ "tokenizer_class": "PreTrainedTokenizerFast",
111
  "trim_offsets": true,
112
  "unk_token": "[UNK]"
113
  }