almanach
/

camembertv2-base

Model card Files Files and versions

Metrics Training metrics Community

Update tokenizer_config.json

#3

by itazap HF Staff - opened 12 days ago

base: refs/heads/main

←

from: refs/pr/3

Discussion Files changed

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -51,7 +51,7 @@
   "model_max_length": 1024,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "tokenizer_class": "RobertaTokenizer",
   "trim_offsets": true,
   "unk_token": "[UNK]"
 }

   "model_max_length": 1024,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
   "trim_offsets": true,
   "unk_token": "[UNK]"
 }