{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-12.069286666327574
],
[
"",
-13.816990218573668
],
[
"",
-12.14144171147333
],
[
"
",
-13.672939620429135
],
[
"",
-14.085961661749938
],
[
"",
-12.14144171147333
],
[
"
",
-14.09457238715277
],
[
"",
-14.698506675177672
],
[
"",
-12.14144171147333
],
[
"
",
-14.666484198981385
],
[
"",
-16.214637122565367
],
[
"",
-12.14144171147333
],
[
"
",
-15.36706765692816
],
[
"",
-15.955322585517028
],
[
"",
-12.14144171147333
],
[
"
",
-15.749436698371031
],
[
"
",
-11.376566253119568
],
[
"
",
-12.426416716986754
],
[
"",
-16.56578763526068
],
[
"",
-14.08574613135443
],
[
"",
-12.14144171147333
],
[
"",
-13.938525519782164
],
[
"