{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": "[PAD]", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "[PAD]": 0, "[BOS]": 1, "[EOS]": 2, "A": 3, "B": 4, "C": 5, "D": 6, "E": 7, "F": 8, "G": 9, "H": 10, "I": 11, "K": 12, "L": 13, "M": 14, "N": 15, "O": 16, "P": 17, "Q": 18, "R": 19, "S": 20, "T": 21, "U": 22, "V": 23, "W": 24, "X": 25, "Y": 26, "[": 27, "]": 28, "Ġ": 29, "OS": 30, "][": 31, "AC": 32, "AD": 33, "BOS": 34, "DE": 35, "EOS": 36, "FG": 37, "HI": 38, "KL": 39, "MN": 40, "PQ": 41, "PAD": 42, "RS": 43, "TU": 44, "VW": 45, "XY": 46, "Ġ[": 47, "ACDE": 48, "FGHI": 49, "KLMN": 50, "PQRS": 51, "TUVW": 52, "ACDEFGHI": 53, "KLMNPQRS": 54, "TUVWXY": 55, "ACDEFGHIKLMNPQRS": 56, "ACDEFGHIKLMNPQRSTUVWXY": 57 }, "merges": [ [ "O", "S" ], [ "]", "[" ], [ "A", "C" ], [ "A", "D" ], [ "B", "OS" ], [ "D", "E" ], [ "E", "OS" ], [ "F", "G" ], [ "H", "I" ], [ "K", "L" ], [ "M", "N" ], [ "P", "Q" ], [ "P", "AD" ], [ "R", "S" ], [ "T", "U" ], [ "V", "W" ], [ "X", "Y" ], [ "Ġ", "[" ], [ "AC", "DE" ], [ "FG", "HI" ], [ "KL", "MN" ], [ "PQ", "RS" ], [ "TU", "VW" ], [ "ACDE", "FGHI" ], [ "KLMN", "PQRS" ], [ "TUVW", "XY" ], [ "ACDEFGHI", "KLMNPQRS" ], [ "ACDEFGHIKLMNPQRS", "TUVWXY" ] ] } }