hyoo14 commited on
Commit
8290dbe
·
verified ·
1 Parent(s): cd10673

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -43,6 +43,7 @@
43
  },
44
  "clean_up_tokenization_spaces": false,
45
  "cls_token": "[CLS]",
 
46
  "mask_token": "[MASK]",
47
  "model_max_length": 1000000000000000019884624838656,
48
  "pad_token": "[PAD]",
 
43
  },
44
  "clean_up_tokenization_spaces": false,
45
  "cls_token": "[CLS]",
46
+ "extra_special_tokens": {},
47
  "mask_token": "[MASK]",
48
  "model_max_length": 1000000000000000019884624838656,
49
  "pad_token": "[PAD]",