{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-9.223671164715157
],
[
"",
-12.421696868145332
],
[
"",
-12.140532115196567
],
[
"
",
-12.07884427567707
],
[
"",
-12.557939734661446
],
[
"",
-12.140532115196567
],
[
"
",
-12.0649473335384
],
[
"",
-12.78031083604084
],
[
"",
-12.140532115196567
],
[
"
",
-12.350752082073557
],
[
"",
-13.56620236893879
],
[
"",
-12.140532115196567
],
[
"
",
-13.115757193767893
],
[
"",
-14.693405356609688
],
[
"",
-12.140532115196567
],
[
"
",
-13.873846259319649
],
[
"
",
-11.643085684978828
],
[
"
",
-12.691888060772284
],
[
"",
-15.205886092388496
],
[
"",
-11.701565947735176
],
[
"",
-12.140532115196567
],
[
"",
-11.6677248146175
],
[
"