File size: 2,558 Bytes
67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f 56be64d 67d3b9f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | {
"add_bos_token": false,
"add_eos_token": false,
"added_tokens_decoder": {
"75": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"76": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"77": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"78": {
"content": "<|mdm_mask|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"79": {
"content": "<role>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"80": {
"content": "</role>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"81": {
"content": "<|arithmetic_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"82": {
"content": "<|arithmetic_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"83": {
"content": "<|number_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"84": {
"content": "<|number_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|mdm_mask|>",
"<role>",
"</role>",
"<|arithmetic_start|>",
"<|arithmetic_end|>",
"<|number_start|>",
"<|number_end|>"
],
"bos_token": "<|startoftext|>",
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"fast_tokenizer": true,
"gmask_token": "[gMASK]",
"mask_token": "<|mdm_mask|>",
"merges_file": null,
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"padding_side": "right",
"tokenizer_class": "PreTrainedTokenizerFast",
"trust_remote_code": true
}
|