{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-9.212673746147017
],
[
"",
-12.405258737602097
],
[
"",
-12.261438550247393
],
[
"
",
-12.062103830375532
],
[
"",
-12.541362367668013
],
[
"",
-12.261438550247393
],
[
"
",
-12.048115335502484
],
[
"",
-12.763678155937056
],
[
"",
-12.261438550247393
],
[
"
",
-12.333918176689007
],
[
"",
-13.549840532340864
],
[
"",
-12.261438550247393
],
[
"
",
-13.09902819255815
],
[
"",
-14.67851694683061
],
[
"",
-12.261438550247393
],
[
"
",
-13.857587419547896
],
[
"
",
-11.640755258598189
],
[
"
",
-12.679698549308638
],
[
"",
-15.58734377483354
],
[
"",
-11.68580397228227
],
[
"",
-12.261438550247393
],
[
"",
-11.651303850211958
],
[
"