Bingsu
/

ko_BBPE_tokenizer_bert

Model card Files Files and versions

Bingsu commited on Aug 22, 2022

Commit

a33d392

·

1 Parent(s): e3b51e1

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -6,7 +6,7 @@
   "name_or_path": "ko_BBPE_tokenizer_bert",
   "pad_token": "<pad>",
   "sep_token": "</s>",
-  "special_tokens_map_file": "ko_BBPE_tokenizer_bert\\special_tokens_map.json",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",

   "name_or_path": "ko_BBPE_tokenizer_bert",
   "pad_token": "<pad>",
   "sep_token": "</s>",
+  "special_tokens_map_file": null,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",