Hugging Face's logo

yijiang-archive
/
tutorial-model4

Safetensors
collie
Model card Files
xet
 Community
tutorial-model4 / tokenizer.json
yijiang-ts's picture
yijiang-ts
Upload folder using huggingface_hub
dfecbfd verified
Raw History Blame Contribute Delete
3.42 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[BOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"decoder": {
"type": "Fuse"
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"[PAD]": 0,
"[BOS]": 1,
" ": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"0 ": 34,
"0 0 ": 35,
"0!": 36,
"/?": 37,
"0 0 0 0 ": 38,
"0\"": 39,
"/>": 40,
"0 /?": 41,
"0 0!": 42,
"0 0 0 0 0 0 0 0 ": 43,
"0 0 /?": 44,
"0%": 45,
"0 0 0!": 46,
"/;": 47,
"0#": 48,
"/=": 49,
"/3": 50,
"0-": 51,
"0 /?0!": 52,
"0 0 0 ": 53,
"0 0 0 0 0 0 0 0 0 0 0 ": 54,
"/?0!": 55,
"0 /": 56,
"0$": 57,
",,": 58,
"34": 59,
"0 0 /?0!": 60,
"0 0 0 0 0 0 0 0 0 0 0 0 ": 61,
"0 0 0!0!": 62,
"0 />": 63,
"0 0 0!/?": 64,
"0 0!0!": 65
},
"merges": [
[
"0",
" "
],
[
"0 ",
"0 "
],
[
"0",
"!"
],
[
"/",
"?"
],
[
"0 0 ",
"0 0 "
],
[
"0",
"\""
],
[
"/",
">"
],
[
"0 ",
"/?"
],
[
"0 ",
"0!"
],
[
"0 0 0 0 ",
"0 0 0 0 "
],
[
"0 0 ",
"/?"
],
[
"0",
"%"
],
[
"0 0 ",
"0!"
],
[
"/",
";"
],
[
"0",
"#"
],
[
"/",
"="
],
[
"/",
"3"
],
[
"0",
"-"
],
[
"0 /?",
"0!"
],
[
"0 0 ",
"0 "
],
[
"0 0 0 0 0 0 0 0 ",
"0 0 0 "
],
[
"/?",
"0!"
],
[
"0 ",
"/"
],
[
"0",
"$"
],
[
",",
","
],
[
"3",
"4"
],
[
"0 0 /?",
"0!"
],
[
"0 0 0 0 0 0 0 0 ",
"0 0 0 0 "
],
[
"0 0 0!",
"0!"
],
[
"0 ",
"/>"
],
[
"0 0 0!",
"/?"
],
[
"0 0!",
"0!"
]
]
}
}