{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
" ",
-12.026919882773452
],
[
"",
-13.792365592001968
],
[
"",
-12.262909819649614
],
[
"
",
-13.648134749411636
],
[
"",
-14.061679404257829
],
[
"",
-12.262909819649614
],
[
"
",
-14.069762747763686
],
[
"",
-14.673770468710952
],
[
"",
-12.262909819649614
],
[
"
",
-14.641680281638198
],
[
"",
-16.190063994406753
],
[
"",
-12.262909819649614
],
[
"
",
-15.342749159812026
],
[
"",
-15.93043188381105
],
[
"",
-12.262909819649614
],
[
"
",
-15.724700491904311
],
[
"
",
-11.351758521079116
],
[
"
",
-12.401619475363784
],
[
"",
-16.54108099269777
],
[
"",
-14.060882132529311
],
[
"",
-12.262909819649614
],
[
"",
-13.913686316489272
],
[
"