{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, "": 4, "'": 5, "*": 6, ";": 7, "=": 8, "C": 9, "E": 10, "F": 11, "H": 12, "L": 13, "M": 14, "O": 15, "R": 16, "S": 17, "T": 18, "W": 19, "a": 20, "c": 21, "e": 22, "i": 23, "r": 24, "s": 25, "t": 26, "u": 27, "v": 28, "Ċ": 29, "Ġ": 30, "us": 31, "';": 32, "CT": 33, "EL": 34, "ER": 35, "ECT": 36, "FR": 37, "HER": 38, "OM": 39, "SEL": 40, "WHER": 41, "ac": 42, "at": 43, "er": 44, "iv": 45, "st": 46, "tiv": 47, "Ġ'": 48, "Ġ*": 49, "Ġ=": 50, "Ġus": 51, "ĠFR": 52, "ĠWHER": 53, "Ġst": 54, "SELECT": 55, "activ": 56, "atus": 57, "ers": 58, "Ġusers": 59, "ĠFROM": 60, "ĠWHERE": 61, "Ġstatus": 62, "active": 63 }, "merges": [ [ "u", "s" ], [ "'", ";" ], [ "C", "T" ], [ "E", "L" ], [ "E", "R" ], [ "E", "CT" ], [ "F", "R" ], [ "H", "ER" ], [ "O", "M" ], [ "S", "EL" ], [ "W", "HER" ], [ "a", "c" ], [ "a", "t" ], [ "e", "r" ], [ "i", "v" ], [ "s", "t" ], [ "t", "iv" ], [ "Ġ", "'" ], [ "Ġ", "*" ], [ "Ġ", "=" ], [ "Ġ", "us" ], [ "Ġ", "FR" ], [ "Ġ", "WHER" ], [ "Ġ", "st" ], [ "SEL", "ECT" ], [ "ac", "tiv" ], [ "at", "us" ], [ "er", "s" ], [ "Ġus", "ers" ], [ "ĠFR", "OM" ], [ "ĠWHER", "E" ], [ "Ġst", "atus" ], [ "activ", "e" ] ] } }