BART_BaseModel2 / tokenizer_config.json
seoyoung's picture
add tokenizer
80ad5ba
raw
history blame contribute delete
964 Bytes
{
"additional_special_tokens": [
"#@μ£Όμ†Œ#",
"#@이λͺ¨ν‹°μ½˜#",
"#@이름#",
"#@URL#",
"#@μ†Œμ†#",
"#@기타#",
"#@μ „λ²ˆ#",
"#@계정#",
"#@url#",
"#@번호#",
"#@금육#",
"#@신원#",
"#@μž₯μ†Œ#",
"#@μ‹œμŠ€ν…œ#사진#",
"#@μ‹œμŠ€ν…œ#λ™μ˜μƒ#",
"#@μ‹œμŠ€ν…œ#기타#",
"#@μ‹œμŠ€ν…œ#검색#",
"#@μ‹œμŠ€ν…œ#지도#",
"#@μ‹œμŠ€ν…œ#μ‚­μ œ#",
"#@μ‹œμŠ€ν…œ#파일#",
"#@μ‹œμŠ€ν…œ#μ†‘κΈˆ#",
"#@μ‹œμŠ€ν…œ#"
],
"bos_token": "[BOS]",
"cls_token": "[BOS]",
"eos_token": "[EOS]",
"mask_token": "[MASK]",
"name_or_path": "alaggung/bart-pretrained",
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"special_tokens_map_file": "/root/.cache/huggingface/transformers/846fe2996aec3b0232d18049d01f3b998ebfaead13ce84937a0541c086d50103.fa860d1b6246633ccda064e867611d04b4fa718efef676d6b5268c3b1def2231",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "[UNK]"
}