Haaaaeun commited on
Commit
c97a88c
·
1 Parent(s): cf930c1

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +14 -3
tokenizer_config.json CHANGED
@@ -1,10 +1,21 @@
1
- {
 
 
2
  "bos_token": "[CLS]",
3
  "eos_token": "[SEP]",
4
  "unk_token": "[UNK]",
5
  "sep_token": "[SEP]",
6
  "pad_token": "[PAD]",
7
  "cls_token": "[CLS]",
8
- "mask_token": "[MASK]",
9
- "special_tokens_map_file": "~/.cache/huggingface/transformers/Haaaaeun/kobert-hatespeech/"
 
 
 
 
 
 
 
 
 
10
  }
 
1
+ {"do_lower_case": false,
2
+ "remove_space": true,
3
+ "keep_accents": false,
4
  "bos_token": "[CLS]",
5
  "eos_token": "[SEP]",
6
  "unk_token": "[UNK]",
7
  "sep_token": "[SEP]",
8
  "pad_token": "[PAD]",
9
  "cls_token": "[CLS]",
10
+ "mask_token": {
11
+ "content": "[MASK]",
12
+ "single_word": false,
13
+ "lstrip": true,
14
+ "rstrip": false,
15
+ "normalized": true,
16
+ "__type": "AddedToken"
17
+ },
18
+ "additional_special_tokens": null,
19
+ "sp_model_kwargs": {},
20
+ "tokenizer_class": "XLNetTokenizer"
21
  }