{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-8.756058981392087
],
[
"",
-10.4357646012041
],
[
"",
-12.1905626605856
],
[
"
",
-9.97888689276904
],
[
"",
-10.44176702735156
],
[
"",
-12.1905626605856
],
[
"
",
-9.942392637703122
],
[
"",
-10.293550779792966
],
[
"",
-12.1905626605856
],
[
"
",
-9.57562761542529
],
[
"",
-10.426496794197265
],
[
"",
-12.1905626605856
],
[
"
",
-9.70542841193408
],
[
"",
-11.222253134224118
],
[
"",
-12.1905626605856
],
[
"
",
-10.267948439094724
],
[
"
",
-8.977778723213376
],
[
"
",
-10.55605154273242
],
[
"",
-12.1122782730791
],
[
"",
-9.66551904914111
],
[
"",
-12.1905626605856
],
[
"",
-9.59677248236865
],
[
"