{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, ",": 1, "-": 2, ".": 3, "A": 4, "B": 5, "C": 6, "D": 7, "H": 8, "L": 9, "M": 10, "N": 11, "O": 12, "R": 13, "T": 14, "U": 15, "a": 16, "b": 17, "c": 18, "d": 19, "e": 20, "f": 21, "g": 22, "h": 23, "i": 24, "j": 25, "k": 26, "l": 27, "m": 28, "n": 29, "o": 30, "p": 31, "q": 32, "r": 33, "s": 34, "t": 35, "u": 36, "v": 37, "w": 38, "x": 39, "y": 40, "z": 41, "in": 42, "ti": 43, "on": 44, "al": 45, "er": 46, "ing": 47, "an": 48, "es": 49, "ar": 50, "en": 51, "he": 52, "or": 53, "tion": 54, "at": 55, "ne": 56, "ur": 57, "it": 58, "le": 59, "de": 60, "ro": 61, "and": 62, "ch": 63 }, "merges": [ [ "i", "n" ], [ "t", "i" ], [ "o", "n" ], [ "a", "l" ], [ "e", "r" ], [ "in", "g" ], [ "a", "n" ], [ "e", "s" ], [ "a", "r" ], [ "e", "n" ], [ "h", "e" ], [ "o", "r" ], [ "ti", "on" ], [ "a", "t" ], [ "n", "e" ], [ "u", "r" ], [ "i", "t" ], [ "l", "e" ], [ "d", "e" ], [ "r", "o" ], [ "an", "d" ], [ "c", "h" ] ] } }