{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 128, "strategy": "LongestFirst", "stride": 0 }, "padding": { "strategy": { "Fixed": 128 }, "direction": "Right", "pad_to_multiple_of": null, "pad_id": 22, "pad_type_id": 0, "pad_token": "" }, "added_tokens": [ { "id": 20, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Split", "pattern": { "String": "" }, "behavior": "Removed", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": null, "model": { "type": "WordLevel", "vocab": { "(": 0, ")": 1, "*": 2, "+": 3, "-": 4, ".": 5, "/": 6, "0": 7, "1": 8, "2": 9, "3": 10, "4": 11, "5": 12, "6": 13, "7": 14, "8": 15, "9": 16, ":": 17, ";": 18, "<": 19, "": 20, "": 21, "": 22, "=": 23, ">": 24, "a": 25, "b": 26, "c": 27, "d": 28, "e": 29, "n": 30, "o": 31, "p": 32, "s": 33, "t": 34, "x": 35, "|": 36, "²": 37 }, "unk_token": "" } }