{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-7.221322486106599
],
[
"",
-10.110804030601228
],
[
"",
-12.054902082883592
],
[
"
",
-9.560461470786777
],
[
"",
-9.375757643882478
],
[
"",
-12.054902082883592
],
[
"
",
-8.644626090232576
],
[
"",
-8.844606826011384
],
[
"",
-12.054902082883592
],
[
"
",
-8.452477881614412
],
[
"",
-9.013242194358552
],
[
"",
-17.92461533278629
],
[
"
",
-8.428427169028962
],
[
"",
-10.238794753257478
],
[
"",
-12.054902082883592
],
[
"
",
-9.618479201499666
],
[
"
",
-8.328210303489412
],
[
"
",
-9.34941143721744
],
[
"",
-12.38912148207828
],
[
"",
-8.693252989951814
],
[
"",
-12.054902082883592
],
[
"",
-8.501059004966462
],
[
"