Fix tokenizer: add <unk> to BPE vocab
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
|
@@ -32776,5 +32776,5 @@
|
|
| 32776 |
"pad_token": "[PAD]",
|
| 32777 |
"sep_token": "[SEP]",
|
| 32778 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 32779 |
-
"unk_token": "
|
| 32780 |
-
}
|
|
|
|
| 32776 |
"pad_token": "[PAD]",
|
| 32777 |
"sep_token": "[SEP]",
|
| 32778 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 32779 |
+
"unk_token": "<unk>"
|
| 32780 |
+
}
|