{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-7.208079548493886
],
[
"",
-10.123786659852527
],
[
"",
-11.946748313207657
],
[
"
",
-9.580141754762195
],
[
"",
-9.393839569703603
],
[
"",
-11.946748313207657
],
[
"
",
-8.665046425477529
],
[
"",
-8.863847466126943
],
[
"",
-11.946748313207657
],
[
"
",
-8.472898216859365
],
[
"",
-9.03262397827294
],
[
"",
-17.935744019166492
],
[
"
",
-8.448865624085927
],
[
"",
-10.258083077088855
],
[
"",
-11.946748313207657
],
[
"
",
-9.6387297827163
],
[
"
",
-8.327250214234853
],
[
"
",
-9.369502754823232
],
[
"",
-12.174393387452628
],
[
"",
-8.69839260162499
],
[
"",
-11.946748313207657
],
[
"",
-8.5253789144912
],
[
"