deskull commited on
Commit
55c078d
·
verified ·
1 Parent(s): 2129260

Fix tokenizer: add <unk> to BPE vocab

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -32776,5 +32776,5 @@
32776
  "pad_token": "[PAD]",
32777
  "sep_token": "[SEP]",
32778
  "tokenizer_class": "PreTrainedTokenizerFast",
32779
- "unk_token": "[UNK]"
32780
- }
 
32776
  "pad_token": "[PAD]",
32777
  "sep_token": "[SEP]",
32778
  "tokenizer_class": "PreTrainedTokenizerFast",
32779
+ "unk_token": "<unk>"
32780
+ }