{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-9.538965301548956
],
[
"",
-15.972170906102178
],
[
"",
-12.266834612492444
],
[
"
",
-15.972142295872686
],
[
"",
-13.40733249667835
],
[
"",
-12.266834612492444
],
[
"
",
-13.403175430333135
],
[
"",
-11.467962341343878
],
[
"",
-12.266834612492444
],
[
"
",
-11.467965202366829
],
[
"",
-16.74943168643665
],
[
"",
-12.266834612492444
],
[
"
",
-16.74941070560169
],
[
"",
-19.467750625645635
],
[
"",
-12.266834612492444
],
[
"
",
-19.467273788487432
],
[
"
",
-12.742645340001104
],
[
"
",
-15.072524147068975
],
[
"",
-18.467260437047003
],
[
"",
-14.707968788182257
],
[
"",
-12.266834612492444
],
[
"",
-14.70806606296253
],
[
"