Hugging Face's logo

Lakoc
/
libri_1000

Transformers
Model card Files
xet
 Community
Lakoc commited on
Commit
6fa4c03
·
verified ·
1 Parent(s): 32da7e9

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -128,7 +128,7 @@
128
  "model": {
129
  "type": "BPE",
130
  "dropout": null,
131
- "unk_token": null,
132
  "continuing_subword_prefix": null,
133
  "end_of_word_suffix": null,
134
  "fuse_unk": false,
 
128
  "model": {
129
  "type": "BPE",
130
  "dropout": null,
131
+ "unk_token": "([unk])",
132
  "continuing_subword_prefix": null,
133
  "end_of_word_suffix": null,
134
  "fuse_unk": false,