add decoder to tokenizer config
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
|
@@ -19,7 +19,7 @@
|
|
| 19 |
"special": true,
|
| 20 |
"content": "<mask>",
|
| 21 |
"single_word": false,
|
| 22 |
-
"lstrip":
|
| 23 |
"rstrip": false,
|
| 24 |
"normalized": false
|
| 25 |
},
|
|
@@ -81,7 +81,7 @@
|
|
| 81 |
"trim_offsets": true,
|
| 82 |
"add_prefix_space": true
|
| 83 |
},
|
| 84 |
-
"decoder":
|
| 85 |
"model": {
|
| 86 |
"type": "BPE",
|
| 87 |
"dropout": null,
|
|
|
|
| 19 |
"special": true,
|
| 20 |
"content": "<mask>",
|
| 21 |
"single_word": false,
|
| 22 |
+
"lstrip": true,
|
| 23 |
"rstrip": false,
|
| 24 |
"normalized": false
|
| 25 |
},
|
|
|
|
| 81 |
"trim_offsets": true,
|
| 82 |
"add_prefix_space": true
|
| 83 |
},
|
| 84 |
+
"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},
|
| 85 |
"model": {
|
| 86 |
"type": "BPE",
|
| 87 |
"dropout": null,
|