Hugging Face's logo

abbassix
/
pn6_800

Text Classification
Transformers
TensorBoard
Safetensors
bert
Generated from Trainer
text-embeddings-inference
Model card Files
xet
 Metrics Community
pn6_800 / tokenizer_config.json
abbassix's picture
abbassix
Upload tokenizer
da0fd88
raw
history blame contribute delete
11.5 kB
{
 "added_tokens_decoder": {
 "0": {
 "content": "[PAD]",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "100": {
 "content": "[UNK]",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "101": {
 "content": "[CLS]",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "102": {
 "content": "[SEP]",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "103": {
 "content": "[MASK]",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "30522": {
 "content": "1𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30523": {
 "content": "1π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30524": {
 "content": "1𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30525": {
 "content": "1π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30526": {
 "content": "1𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30527": {
 "content": "1π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30528": {
 "content": "2𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30529": {
 "content": "2π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30530": {
 "content": "2𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30531": {
 "content": "2π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30532": {
 "content": "2𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30533": {
 "content": "2π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30534": {
 "content": "3𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30535": {
 "content": "3π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30536": {
 "content": "3𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30537": {
 "content": "3π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30538": {
 "content": "3𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30539": {
 "content": "3π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30540": {
 "content": "4𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30541": {
 "content": "4π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30542": {
 "content": "4𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30543": {
 "content": "4π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30544": {
 "content": "4𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30545": {
 "content": "4π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30546": {
 "content": "5𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30547": {
 "content": "5π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30548": {
 "content": "5𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30549": {
 "content": "5π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30550": {
 "content": "5𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30551": {
 "content": "5π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30552": {
 "content": "6𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30553": {
 "content": "6π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30554": {
 "content": "6𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30555": {
 "content": "6π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30556": {
 "content": "6𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30557": {
 "content": "6π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30558": {
 "content": "7𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30559": {
 "content": "7π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30560": {
 "content": "7𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30561": {
 "content": "7π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30562": {
 "content": "7𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30563": {
 "content": "7π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30564": {
 "content": "8𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30565": {
 "content": "8π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30566": {
 "content": "8𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30567": {
 "content": "8π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30568": {
 "content": "8𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30569": {
 "content": "8π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30570": {
 "content": "9𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30571": {
 "content": "9π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30572": {
 "content": "9𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30573": {
 "content": "9π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30574": {
 "content": "9𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30575": {
 "content": "9π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30576": {
 "content": "0𐄇",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30577": {
 "content": "0π„ˆ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30578": {
 "content": "0𐄉",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30579": {
 "content": "0π„Š",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30580": {
 "content": "0𐄋",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 },
 "30581": {
 "content": "0π„Œ",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": false
 }
 },
 "clean_up_tokenization_spaces": true,
 "cls_token": "[CLS]",
 "do_lower_case": true,
 "mask_token": "[MASK]",
 "model_max_length": 512,
 "pad_token": "[PAD]",
 "sep_token": "[SEP]",
 "strip_accents": null,
 "tokenize_chinese_chars": true,
 "tokenizer_class": "BertTokenizer",
 "unk_token": "[UNK]"
}