{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-8.778226178140951
],
[
"",
-10.460364621134117
],
[
"",
-12.074152116693512
],
[
"
",
-10.003508847208334
],
[
"",
-10.466417592020344
],
[
"",
-12.074152116693512
],
[
"
",
-9.96704510972054
],
[
"",
-10.318198483438804
],
[
"",
-12.074152116693512
],
[
"
",
-9.600275319071129
],
[
"",
-10.451139729471516
],
[
"",
-12.074152116693512
],
[
"
",
-9.730077069254232
],
[
"",
-11.246878903360678
],
[
"",
-12.074152116693512
],
[
"
",
-10.29258565232308
],
[
"
",
-9.00225476548226
],
[
"
",
-10.58068494126351
],
[
"",
-12.132314007730796
],
[
"",
-9.690123837442709
],
[
"",
-12.074152116693512
],
[
"",
-9.621513646097494
],
[
"