Fix tokenizer: add <unk> to BPE vocab
Browse files- tokenizer.json +4 -2
tokenizer.json
CHANGED
|
@@ -36875,13 +36875,15 @@
|
|
| 36875 |
"model": {
|
| 36876 |
"type": "BPE",
|
| 36877 |
"dropout": null,
|
| 36878 |
-
"unk_token": "
|
| 36879 |
"continuing_subword_prefix": null,
|
| 36880 |
"end_of_word_suffix": null,
|
| 36881 |
"fuse_unk": false,
|
| 36882 |
"byte_fallback": false,
|
| 36883 |
"ignore_merges": false,
|
| 36884 |
-
"vocab": {
|
|
|
|
|
|
|
| 36885 |
"merges": []
|
| 36886 |
}
|
| 36887 |
}
|
|
|
|
| 36875 |
"model": {
|
| 36876 |
"type": "BPE",
|
| 36877 |
"dropout": null,
|
| 36878 |
+
"unk_token": "<unk>",
|
| 36879 |
"continuing_subword_prefix": null,
|
| 36880 |
"end_of_word_suffix": null,
|
| 36881 |
"fuse_unk": false,
|
| 36882 |
"byte_fallback": false,
|
| 36883 |
"ignore_merges": false,
|
| 36884 |
+
"vocab": {
|
| 36885 |
+
"<unk>": 0
|
| 36886 |
+
},
|
| 36887 |
"merges": []
|
| 36888 |
}
|
| 36889 |
}
|