| { | |
| "additional_special_tokens": [ | |
| "<GN>", | |
| "<FN>", | |
| "<JN>", | |
| "<IN>", | |
| "<KN>", | |
| "<HN>", | |
| "<LN>", | |
| "<SN>", | |
| "<MN>", | |
| "<TN>", | |
| "<NN>", | |
| "<UN>", | |
| "<ON>", | |
| "<VN>", | |
| "<PN>", | |
| "<WN>", | |
| "<QN>", | |
| "<XN>", | |
| "<RN>", | |
| "<YN>" | |
| ], | |
| "bos_token": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false | |
| }, | |
| "eos_token": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false | |
| }, | |
| "pad_token": { | |
| "content": "<|pad|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false | |
| }, | |
| "unk_token": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false | |
| } | |
| } | |