{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 100, "strategy": "LongestFirst", "stride": 0 }, "padding": { "strategy": { "Fixed": 100 }, "direction": "Right", "pad_to_multiple_of": null, "pad_id": 0, "pad_type_id": 0, "pad_token": "[PAD]" }, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": "[UNK]", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "[PAD]": 0, "[BOS]": 1, "[EOS]": 2, "[UNK]": 3, "!": 4, "(": 5, ")": 6, "+": 7, ",": 8, "-": 9, ".": 10, "/": 11, "0": 12, "1": 13, "2": 14, "3": 15, "4": 16, "5": 17, "6": 18, "7": 19, "8": 20, "9": 21, "<": 22, "=": 23, ">": 24, "A": 25, "B": 26, "C": 27, "D": 28, "E": 29, "F": 30, "G": 31, "H": 32, "I": 33, "L": 34, "M": 35, "N": 36, "P": 37, "R": 38, "S": 39, "T": 40, "V": 41, "X": 42, "Y": 43, "[": 44, "\\": 45, "]": 46, "^": 47, "_": 48, "a": 49, "b": 50, "c": 51, "d": 52, "e": 53, "f": 54, "g": 55, "h": 56, "i": 57, "j": 58, "k": 59, "l": 60, "m": 61, "n": 62, "o": 63, "p": 64, "q": 65, "r": 66, "s": 67, "t": 68, "u": 69, "v": 70, "w": 71, "x": 72, "y": 73, "z": 74, "{": 75, "|": 76, "}": 77, "ra": 78, "fra": 79, "frac": 80, "im": 81, "qr": 82, "sqr": 83, "sqrt": 84, "in": 85, "ta": 86, "lim": 87, "ts": 88, "its": 89, "limits": 90, "eta": 91, "sin": 92, "ime": 93, "su": 94, "sum": 95, "int": 96, "time": 97, "times": 98, "co": 99, "cos": 100, "heta": 101, "theta": 102, "pi": 103, "eq": 104, "ph": 105, "al": 106, "pha": 107, "alpha": 108, "ft": 109, "inft": 110, "infty": 111, "ig": 112, "do": 113, "hta": 114, "ow": 115, "rr": 116, "rig": 117, "htarr": 118, "rightarr": 119, "rightarrow": 120, "beta": 121, "lo": 122, "log": 123, "tan": 124, "dots": 125, "leq": 126, "pm": 127, "geq": 128, "ma": 129, "di": 130, "div": 131, "ldots": 132, "am": 133, "cdots": 134, "cdo": 135, "neq": 136, "cdot": 137, "gam": 138, "gamma": 139, "phi": 140, "pr": 141, "prime": 142, "\\{": 143, "\\}": 144, "sig": 145, "sigma": 146, "mu": 147, "De": 148, "lta": 149, "Delta": 150, "bd": 151, "lam": 152, "bda": 153, "lambda": 154, "fo": 155, "ll": 156, "rall": 157, "forall": 158, "ex": 159, "is": 160, "exis": 161, "exists": 162 }, "merges": [ [ "r", "a" ], [ "f", "ra" ], [ "fra", "c" ], [ "i", "m" ], [ "q", "r" ], [ "s", "qr" ], [ "sqr", "t" ], [ "i", "n" ], [ "t", "a" ], [ "l", "im" ], [ "t", "s" ], [ "i", "ts" ], [ "lim", "its" ], [ "e", "ta" ], [ "s", "in" ], [ "im", "e" ], [ "s", "u" ], [ "su", "m" ], [ "in", "t" ], [ "t", "ime" ], [ "time", "s" ], [ "c", "o" ], [ "co", "s" ], [ "h", "eta" ], [ "t", "heta" ], [ "p", "i" ], [ "e", "q" ], [ "p", "h" ], [ "a", "l" ], [ "ph", "a" ], [ "al", "pha" ], [ "f", "t" ], [ "in", "ft" ], [ "inft", "y" ], [ "i", "g" ], [ "d", "o" ], [ "h", "ta" ], [ "o", "w" ], [ "r", "r" ], [ "r", "ig" ], [ "hta", "rr" ], [ "rig", "htarr" ], [ "rightarr", "ow" ], [ "b", "eta" ], [ "l", "o" ], [ "lo", "g" ], [ "ta", "n" ], [ "do", "ts" ], [ "l", "eq" ], [ "p", "m" ], [ "g", "eq" ], [ "m", "a" ], [ "d", "i" ], [ "di", "v" ], [ "l", "dots" ], [ "a", "m" ], [ "c", "dots" ], [ "c", "do" ], [ "n", "eq" ], [ "cdo", "t" ], [ "g", "am" ], [ "gam", "ma" ], [ "ph", "i" ], [ "p", "r" ], [ "pr", "ime" ], [ "\\", "{" ], [ "\\", "}" ], [ "s", "ig" ], [ "sig", "ma" ], [ "m", "u" ], [ "D", "e" ], [ "l", "ta" ], [ "De", "lta" ], [ "b", "d" ], [ "l", "am" ], [ "bd", "a" ], [ "lam", "bda" ], [ "f", "o" ], [ "l", "l" ], [ "ra", "ll" ], [ "fo", "rall" ], [ "e", "x" ], [ "i", "s" ], [ "ex", "is" ], [ "exis", "ts" ] ] } }