Hugging Face's logo

Zaid
/
ashaar_tokenizer

Model card Files
xet
Community
ashaar_tokenizer / tokenizer.json
Zaid's picture
Zaid
Upload tokenizer
f11724f
raw
history blame contribute delete
8.32 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|psep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</|psep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|vsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<|bsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "</|bsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 7,
"content": "<|meter_0|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 8,
"content": "<|meter_1|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 9,
"content": "<|meter_2|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 10,
"content": "<|meter_3|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 11,
"content": "<|meter_4|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 12,
"content": "<|meter_5|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 13,
"content": "<|meter_6|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 14,
"content": "<|meter_7|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 15,
"content": "<|meter_8|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 16,
"content": "<|meter_9|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 17,
"content": "<|meter_10|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 18,
"content": "<|meter_11|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 19,
"content": "<|meter_12|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 20,
"content": "<|meter_13|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 21,
"content": "<|meter_14|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 22,
"content": "<|meter_15|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 23,
"content": "<|res_0|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 24,
"content": "<|res_1|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 25,
"content": "<|res_2|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 26,
"content": "<|res_3|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 27,
"content": "<|res_4|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 28,
"content": "<|res_5|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 29,
"content": "<|res_6|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 30,
"content": "<|res_7|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 31,
"content": "<|res_8|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 32,
"content": "<|res_9|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "UnicodeScripts"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"vocab": {
"<|endoftext|>": 0,
"<|psep|>": 1,
"</|psep|>": 2,
"<|vsep|>": 3,
"<|bsep|>": 4,
"</|bsep|>": 5,
"<|pad|>": 6,
"<|meter_0|>": 7,
"<|meter_1|>": 8,
"<|meter_2|>": 9,
"<|meter_3|>": 10,
"<|meter_4|>": 11,
"<|meter_5|>": 12,
"<|meter_6|>": 13,
"<|meter_7|>": 14,
"<|meter_8|>": 15,
"<|meter_9|>": 16,
"<|meter_10|>": 17,
"<|meter_11|>": 18,
"<|meter_12|>": 19,
"<|meter_13|>": 20,
"<|meter_14|>": 21,
"<|meter_15|>": 22,
"<|res_0|>": 23,
"<|res_1|>": 24,
"<|res_2|>": 25,
"<|res_3|>": 26,
"<|res_4|>": 27,
"<|res_5|>": 28,
"<|res_6|>": 29,
"<|res_7|>": 30,
"<|res_8|>": 31,
"<|res_9|>": 32,
" ": 33,
"/": 34,
"0": 35,
"1": 36,
"2": 37,
"3": 38,
"4": 39,
"5": 40,
"6": 41,
"7": 42,
"8": 43,
"9": 44,
"<": 45,
">": 46,
"_": 47,
"b": 48,
"e": 49,
"m": 50,
"p": 51,
"r": 52,
"s": 53,
"t": 54,
"v": 55,
"|": 56,
"~": 57,
"ء": 58,
"أ": 59,
"ؤ": 60,
"ئ": 61,
"ا": 62,
"ب": 63,
"ة": 64,
"ت": 65,
"ث": 66,
"ج": 67,
"ح": 68,
"خ": 69,
"د": 70,
"ذ": 71,
"ر": 72,
"ز": 73,
"س": 74,
"ش": 75,
"ص": 76,
"ض": 77,
"ط": 78,
"ظ": 79,
"ع": 80,
"غ": 81,
"ف": 82,
"ق": 83,
"ك": 84,
"ل": 85,
"م": 86,
"ن": 87,
"ه": 88,
"و": 89,
"ى": 90,
"ي": 91,
"ً": 92,
"ٌ": 93,
"ٍ": 94,
"َ": 95,
"ُ": 96,
"ِ": 97,
"ّ": 98,
"ْ": 99
},
"merges": []
}
}