{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-10.654902477968603
],
[
"",
-12.68785669397202
],
[
"",
-12.325575858659416
],
[
"
",
-12.364229222048191
],
[
"",
-12.14335443567124
],
[
"",
-12.325575858659416
],
[
"
",
-11.914340992678076
],
[
"",
-12.837265034426125
],
[
"",
-12.325575858659416
],
[
"
",
-12.518536587465674
],
[
"",
-12.373956700075537
],
[
"",
-12.325575858659416
],
[
"
",
-12.17787935327378
],
[
"",
-14.276015301454931
],
[
"",
-12.325575858659416
],
[
"
",
-13.445588131655128
],
[
"
",
-12.45167734216538
],
[
"
",
-11.743744869936377
],
[
"",
-14.639000912416844
],
[
"",
-11.643003483522802
],
[
"",
-12.325575858659416
],
[
"",
-11.621583004702002
],
[
"