| { |
| "version": "1.0", |
| "truncation": { |
| "direction": "Right", |
| "max_length": 128, |
| "strategy": "LongestFirst", |
| "stride": 0 |
| }, |
| "padding": { |
| "strategy": "BatchLongest", |
| "direction": "Right", |
| "pad_to_multiple_of": null, |
| "pad_id": 0, |
| "pad_type_id": 0, |
| "pad_token": "<|endoftext|>" |
| }, |
| "added_tokens": [ |
| { |
| "id": 0, |
| "content": "<|endoftext|>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 1, |
| "content": "<pad>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 2, |
| "content": "0", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 3, |
| "content": "1", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 4, |
| "content": "2", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 5, |
| "content": "3", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 6, |
| "content": "4", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 7, |
| "content": "5", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 8, |
| "content": "6", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 9, |
| "content": "7", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 10, |
| "content": "8", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 11, |
| "content": "9", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 12, |
| "content": "a", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 13, |
| "content": "b", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 14, |
| "content": "c", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 15, |
| "content": "d", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 16, |
| "content": "e", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 17, |
| "content": "f", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 18, |
| "content": "g", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 19, |
| "content": "h", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 20, |
| "content": "i", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 21, |
| "content": "j", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 22, |
| "content": "k", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 23, |
| "content": "l", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 24, |
| "content": "m", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 25, |
| "content": "n", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 26, |
| "content": "o", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 27, |
| "content": "p", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 28, |
| "content": "q", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 29, |
| "content": "r", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 30, |
| "content": "s", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 31, |
| "content": "t", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 32, |
| "content": "u", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 33, |
| "content": "v", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 34, |
| "content": "w", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 35, |
| "content": "x", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 36, |
| "content": "y", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 37, |
| "content": "z", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 38, |
| "content": "A", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 39, |
| "content": "B", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 40, |
| "content": "C", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 41, |
| "content": "D", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 42, |
| "content": "E", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 43, |
| "content": "F", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 44, |
| "content": "G", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 45, |
| "content": "H", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 46, |
| "content": "I", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 47, |
| "content": "J", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 48, |
| "content": "K", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 49, |
| "content": "L", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 50, |
| "content": "M", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 51, |
| "content": "N", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 52, |
| "content": "O", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 53, |
| "content": "P", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 54, |
| "content": "Q", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 55, |
| "content": "R", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 56, |
| "content": "S", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 57, |
| "content": "T", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 58, |
| "content": "U", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 59, |
| "content": "V", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 60, |
| "content": "W", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 61, |
| "content": "X", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 62, |
| "content": "Y", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 63, |
| "content": "Z", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 64, |
| "content": "!", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 65, |
| "content": "\"", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 66, |
| "content": "#", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 67, |
| "content": "$", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 68, |
| "content": "%", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 69, |
| "content": "&", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 70, |
| "content": "'", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 71, |
| "content": "(", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 72, |
| "content": ")", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 73, |
| "content": "*", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 74, |
| "content": "+", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 75, |
| "content": ",", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 76, |
| "content": "-", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 77, |
| "content": ".", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 78, |
| "content": "/", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 79, |
| "content": ":", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 80, |
| "content": ";", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 81, |
| "content": "<", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 82, |
| "content": "=", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 83, |
| "content": ">", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 84, |
| "content": "?", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 85, |
| "content": "@", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 86, |
| "content": "[", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 87, |
| "content": "\\", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 88, |
| "content": "]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 89, |
| "content": "^", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 90, |
| "content": "_", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 91, |
| "content": "`", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 92, |
| "content": "{", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 93, |
| "content": "|", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 94, |
| "content": "}", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 95, |
| "content": "~", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 96, |
| "content": " ", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 97, |
| "content": "\t", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 98, |
| "content": "\n", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 99, |
| "content": "\r", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 100, |
| "content": "\u000b", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 101, |
| "content": "\f", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 102, |
| "content": "ö", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 103, |
| "content": "Ö", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 104, |
| "content": "ä", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 105, |
| "content": "Ä", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 106, |
| "content": "ü", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 107, |
| "content": "Ü", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 108, |
| "content": "ß", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| }, |
| { |
| "id": 109, |
| "content": "ẞ", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": true, |
| "special": false |
| } |
| ], |
| "normalizer": null, |
| "pre_tokenizer": null, |
| "post_processor": null, |
| "decoder": null, |
| "model": { |
| "type": "BPE", |
| "dropout": null, |
| "unk_token": null, |
| "continuing_subword_prefix": null, |
| "end_of_word_suffix": null, |
| "fuse_unk": false, |
| "byte_fallback": false, |
| "ignore_merges": false, |
| "vocab": {}, |
| "merges": [] |
| } |
| } |
