{ "version": "1.0", "truncation": null, "padding": { "strategy": "BatchLongest", "direction": "Right", "pad_to_multiple_of": null, "pad_id": 0, "pad_type_id": 0, "pad_token": "" }, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 19, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 20, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 23, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 24, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 25, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 26, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 27, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 28, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 29, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 30, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 31, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 32, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 33, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 34, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 35, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 36, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 37, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 38, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 39, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 40, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 41, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 42, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 43, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 44, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 45, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 46, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 47, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 48, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 49, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 50, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 51, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 52, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 53, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 54, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 55, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 56, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 57, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 58, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 59, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 60, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 61, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 62, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 63, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 64, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 65, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 66, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 67, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 68, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 69, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 70, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 71, "content": "d__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 72, "content": "p__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 73, "content": "c__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 74, "content": "o__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 75, "content": "f__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 76, "content": "g__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 77, "content": "s__", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 78, "content": ";", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 79, "content": "|", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 80, "content": "_", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 81, "content": "-", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 82, "content": "A", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 83, "content": "U", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 84, "content": "G", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 85, "content": "C", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 86, "content": "5", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 87, "content": "3", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 88, "content": "a", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 89, "content": "b", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 90, "content": "c", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 91, "content": "d", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 92, "content": "e", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 93, "content": "f", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 94, "content": "g", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 95, "content": "h", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 96, "content": "i", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 97, "content": "j", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 98, "content": "k", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 99, "content": "l", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 100, "content": "m", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 101, "content": "n", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 102, "content": "o", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 103, "content": "p", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 104, "content": "q", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 105, "content": "r", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 106, "content": "s", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 107, "content": "t", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 108, "content": "u", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 109, "content": "v", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 110, "content": "w", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 111, "content": "x", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 112, "content": "y", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 113, "content": "z", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, "": 4, "": 5, "": 6, "": 7, "": 8, "": 9, "": 10, "": 11, "": 12, "": 13, "": 14, "": 15, "": 16, "": 17, "": 18, "": 19, "": 20, "": 21, "": 22, "": 23, "": 24, "": 25, "": 26, "": 27, "": 28, "": 29, "": 30, "": 31, "": 32, "": 33, "": 34, "": 35, "": 36, "": 37, "": 38, "": 39, "": 40, "": 41, "": 42, "": 43, "": 44, "": 45, "": 46, "": 47, "": 48, "": 49, "": 50, "": 51, "": 52, "": 53, "": 54, "": 55, "": 56, "": 57, "": 58, "": 59, "": 60, "": 61, "": 62, "": 63, "": 64, "": 65, "": 66, "": 67, "": 68, "": 69, "": 70, "d__": 71, "p__": 72, "c__": 73, "o__": 74, "f__": 75, "g__": 76, "s__": 77, ";": 78, "|": 79, "_": 80, "-": 81, "A": 82, "U": 83, "G": 84, "C": 85, "5": 86, "3": 87, "a": 88, "b": 89, "c": 90, "d": 91, "e": 92, "f": 93, "g": 94, "h": 95, "i": 96, "j": 97, "k": 98, "l": 99, "m": 100, "n": 101, "o": 102, "p": 103, "q": 104, "r": 105, "s": 106, "t": 107, "u": 108, "v": 109, "w": 110, "x": 111, "y": 112, "z": 113 }, "merges": [] } }