File size: 1,866 Bytes
552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 18c6ab3 9df32ff 18c6ab3 9df32ff 552ed2f 9df32ff 552ed2f 9df32ff 552ed2f 18c6ab3 552ed2f 9df32ff 552ed2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | {
"add_prefix_space": false,
"added_tokens_decoder": {
"0": {
"content": "aha",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "wait",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "BoS",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "EoS",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "UNK",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "PAD",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"7": {
"content": "EoT",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"8": {
"content": "BoT",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"EoT",
"BoT",
"aha",
"wait"
],
"bos_token": "BoS",
"clean_up_tokenization_spaces": false,
"eos_token": "EoS",
"extra_special_tokens": {},
"model_max_length": 1024,
"pad_token": "PAD",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "UNK"
}
|