bart_r3f_sample / tokenizer_config.json
seoyoung's picture
add tokenizer
d420f03
raw
history blame contribute delete
957 Bytes
{
"additional_special_tokens": [
"#@μ£Όμ†Œ#",
"#@이λͺ¨ν‹°μ½˜#",
"#@이름#",
"#@URL#",
"#@μ†Œμ†#",
"#@기타#",
"#@μ „λ²ˆ#",
"#@계정#",
"#@url#",
"#@번호#",
"#@금육#",
"#@신원#",
"#@μž₯μ†Œ#",
"#@μ‹œμŠ€ν…œ#사진#",
"#@μ‹œμŠ€ν…œ#λ™μ˜μƒ#",
"#@μ‹œμŠ€ν…œ#기타#",
"#@μ‹œμŠ€ν…œ#검색#",
"#@μ‹œμŠ€ν…œ#지도#",
"#@μ‹œμŠ€ν…œ#μ‚­μ œ#",
"#@μ‹œμŠ€ν…œ#파일#",
"#@μ‹œμŠ€ν…œ#μ†‘κΈˆ#",
"#@μ‹œμŠ€ν…œ#"
],
"bos_token": "[BOS]",
"cls_token": "[BOS]",
"eos_token": "[EOS]",
"mask_token": "[MASK]",
"name_or_path": "alaggung/bart-r3f",
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"special_tokens_map_file": "/root/.cache/huggingface/transformers/24d71f9503b120c3ec8f1a015af8aa5131c5bb79a824fdd4a43c937f5ef6bb25.fa860d1b6246633ccda064e867611d04b4fa718efef676d6b5268c3b1def2231",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "[UNK]"
}