{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-9.041355068460174
],
[
"",
-12.18913834787912
],
[
"",
-11.555872735219891
],
[
"
",
-11.625665600076385
],
[
"",
-12.097417766824432
],
[
"",
-11.555872735219891
],
[
"
",
-11.521586353555389
],
[
"",
-11.637447292581268
],
[
"",
-19.28327553965158
],
[
"
",
-11.270300800576871
],
[
"",
-12.504688198342986
],
[
"",
-11.555872735219891
],
[
"
",
-12.20918172098703
],
[
"",
-14.15751927591867
],
[
"",
-11.555872735219891
],
[
"
",
-13.489652569070524
],
[
"
",
-8.470621998086639
],
[
"
",
-8.708232815042205
],
[
"",
-11.370367939248748
],
[
"",
-9.715926105752654
],
[
"",
-11.555872735219891
],
[
"",
-9.69740670420236
],
[
"