{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "[BOS]": 0, "[EOS]": 1, "[PAD]": 2, "[UNK]": 3, " ": 4, ".": 5, "T": 6, "a": 7, "b": 8, "e": 9, "f": 10, "h": 11, "i": 12, "l": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "v": 19, "y": 20, " a": 21, " b": 22, " f": 23, " i": 24, "Th": 25, "ai": 26, "e.": 27, "e f": 28, "iv": 29, "lo": 30, "liv": 31, "n i": 32, "pp": 33, "rai": 34, "s a": 35, "y b": 36, "The f": 37, "lopp": 38, "live.": 39, "n is a": 40, "rain is a": 41, "y brain is a": 42, "The flopp": 43, "y brain is alive.": 44, "The floppy brain is alive.": 45 }, "merges": [ [ " ", "a" ], [ " ", "b" ], [ " ", "f" ], [ " ", "i" ], [ "T", "h" ], [ "a", "i" ], [ "e", "." ], [ "e", " f" ], [ "i", "v" ], [ "l", "o" ], [ "l", "iv" ], [ "n", " i" ], [ "p", "p" ], [ "r", "ai" ], [ "s", " a" ], [ "y", " b" ], [ "Th", "e f" ], [ "lo", "pp" ], [ "liv", "e." ], [ "n i", "s a" ], [ "rai", "n is a" ], [ "y b", "rain is a" ], [ "The f", "lopp" ], [ "y brain is a", "live." ], [ "The flopp", "y brain is alive." ] ] } }