| { | |
| "tokenizer_class": "SrnaTokenizer", | |
| "auto_map": { | |
| "AutoTokenizer": ["tokenization_srna.SrnaTokenizer", null] | |
| }, | |
| "add_prefix_space": false, | |
| "added_tokens_decoder": { | |
| "248044": { | |
| "content": "[PAD]", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "248047": { | |
| "content": "<csta>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "248048": { | |
| "content": "<cend>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "248049": { | |
| "content": "<capi>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "248050": { | |
| "content": "<uppe>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "clean_up_tokenization_spaces": false, | |
| "errors": "replace", | |
| "model_max_length": 50001, | |
| "pad_token": "[PAD]", | |
| "split_special_tokens": false, | |
| "unk_token": null, | |
| "add_bos_token": false, | |
| "boc_token": "<csta>", | |
| "eoc_token": "<cend>", | |
| "cap_token": "<capi>", | |
| "up_token": "<uppe>", | |
| "case_compression": true, | |
| "script_compression": true, | |
| "omit_tags": false, | |
| "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" | |
| } |