{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": null, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "[UNK]", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "[PAD]": 3, "[MASK]": 4, "(": 5, ")": 6, "*": 7, "+": 8, "-": 9, "0": 10, "1": 11, "2": 12, "3": 13, "4": 14, "5": 15, "6": 16, "7": 17, "8": 18, "9": 19, ":": 20, "<": 21, "=": 22, ">": 23, "F": 24, "I": 25, "R": 26, "S": 27, "a": 28, "d": 29, "e": 30, "i": 31, "l": 32, "n": 33, "p": 34, "s": 35, "t": 36, "u": 37, "Ċ": 38, "Ġ": 39, "Ġ-": 40, "(-": 41, "Ġ+": 42, "Ġ*": 43, "Ġ1": 44 }, "merges": [ [ "Ġ", "-" ], [ "(", "-" ], [ "Ġ", "+" ], [ "Ġ", "*" ], [ "Ġ", "1" ] ] } }