deskull commited on
Commit
2129260
·
verified ·
1 Parent(s): 7116a9c

Fix tokenizer: add <unk> to BPE vocab

Browse files
Files changed (1) hide show
  1. tokenizer.json +4 -2
tokenizer.json CHANGED
@@ -36875,13 +36875,15 @@
36875
  "model": {
36876
  "type": "BPE",
36877
  "dropout": null,
36878
- "unk_token": "[UNK]",
36879
  "continuing_subword_prefix": null,
36880
  "end_of_word_suffix": null,
36881
  "fuse_unk": false,
36882
  "byte_fallback": false,
36883
  "ignore_merges": false,
36884
- "vocab": {},
 
 
36885
  "merges": []
36886
  }
36887
  }
 
36875
  "model": {
36876
  "type": "BPE",
36877
  "dropout": null,
36878
+ "unk_token": "<unk>",
36879
  "continuing_subword_prefix": null,
36880
  "end_of_word_suffix": null,
36881
  "fuse_unk": false,
36882
  "byte_fallback": false,
36883
  "ignore_merges": false,
36884
+ "vocab": {
36885
+ "<unk>": 0
36886
+ },
36887
  "merges": []
36888
  }
36889
  }