{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-13.094395083161189
],
[
"",
-15.612391871186093
],
[
"",
-12.393932314270192
],
[
"
",
-15.57239000198443
],
[
"",
-15.150848787995171
],
[
"",
-12.393932314270192
],
[
"
",
-15.10008947250445
],
[
"",
-15.536234301300835
],
[
"",
-12.393932314270192
],
[
"
",
-15.46515409347613
],
[
"",
-16.836208742829157
],
[
"",
-12.393932314270192
],
[
"
",
-16.83394662735064
],
[
"",
-12.393932314270192
],
[
"",
-12.393932314270192
],
[
"
",
-12.393932314270192
],
[
"
",
-12.149604243012265
],
[
"
",
-14.259485644074276
],
[
"",
-17.34361497757037
],
[
"",
-17.595093172760798
],
[
"",
-12.393932314270192
],
[
"",
-17.594067019196345
],
[
"