hellosindh commited on
Commit
4097a30
·
verified ·
1 Parent(s): b9a1b96

Fix mask token mapping for HF pipeline

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -48,6 +48,6 @@
48
  "mask_token": "<mask>",
49
  "model_max_length": 1000000000000000019884624838656,
50
  "pad_token": "<pad>",
51
- "tokenizer_class": "SindhiTokenizer",
52
  "unk_token": "<unk>"
53
- }
 
48
  "mask_token": "<mask>",
49
  "model_max_length": 1000000000000000019884624838656,
50
  "pad_token": "<pad>",
51
+ "tokenizer_class": "XLMRobertaTokenizer",
52
  "unk_token": "<unk>"
53
+ }