{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "[PAD]": 1, "<|endoftext|>": 2, "AA": 3, "TT": 4, "TG": 5, "CA": 6, "CC": 7, "TA": 8, "GG": 9, "TC": 10, "GA": 11, "AAA": 12, "GC": 13, "TAA": 14, "TTTT": 15, "TCA": 16, "TGA": 17, "TTA": 18, "GAA": 19, "TCC": 20, "CAA": 21, "CTG": 22, "CTT": 23, "GTG": 24, "GTT": 25, "GCA": 26, "A": 27, "T": 28, "G": 29, "C": 30, "▁": 31 }, "merges": [ [ "A", "A" ], [ "T", "T" ], [ "T", "G" ], [ "C", "A" ], [ "C", "C" ], [ "T", "A" ], [ "G", "G" ], [ "T", "C" ], [ "G", "A" ], [ "A", "AA" ], [ "AA", "A" ], [ "G", "C" ], [ "T", "AA" ], [ "TA", "A" ], [ "TT", "TT" ], [ "T", "CA" ], [ "TC", "A" ], [ "T", "GA" ], [ "TG", "A" ], [ "T", "TA" ], [ "TT", "A" ], [ "G", "AA" ], [ "GA", "A" ], [ "T", "CC" ], [ "TC", "C" ], [ "C", "AA" ], [ "CA", "A" ], [ "C", "TG" ], [ "C", "TT" ], [ "G", "TG" ], [ "G", "TT" ], [ "G", "CA" ], [ "GC", "A" ] ] } }