Lakoc
/

libri_1000

Lakoc commited on Oct 11, 2024

Commit

6fa4c03

verified ·

1 Parent(s): 32da7e9

Upload tokenizer

Files changed (1) hide show

tokenizer.json CHANGED Viewed

@@ -128,7 +128,7 @@
   "model": {
     "type": "BPE",
     "dropout": null,
-    "unk_token": null,
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
     "fuse_unk": false,

   "model": {
     "type": "BPE",
     "dropout": null,
+    "unk_token": "([unk])",
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
     "fuse_unk": false,