{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-10.665724525597954
],
[
"",
-12.707528839257622
],
[
"",
-12.20149708827138
],
[
"
",
-12.384113083032036
],
[
"",
-12.163093338158989
],
[
"",
-12.20149708827138
],
[
"
",
-11.934233436730766
],
[
"",
-12.857015381005668
],
[
"",
-12.20149708827138
],
[
"
",
-12.538424263146782
],
[
"",
-12.393817672875786
],
[
"",
-12.20149708827138
],
[
"
",
-12.197772751000786
],
[
"",
-14.295806656030082
],
[
"",
-12.20149708827138
],
[
"
",
-13.465474853661918
],
[
"
",
-12.468868026879692
],
[
"
",
-11.763583908227348
],
[
"",
-14.65403057113018
],
[
"",
-11.66285968795147
],
[
"",
-12.20149708827138
],
[
"",
-11.64148117080059
],
[
"