{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 24, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "r": 0, "u": 1, "f": 2, "l": 3, "d": 4, "b": 5, "R": 6, "RR": 7, "RRR": 8, "U": 9, "UU": 10, "UUU": 11, "F": 12, "FF": 13, "FFF": 14, "L": 15, "LL": 16, "LLL": 17, "D": 18, "DD": 19, "DDD": 20, "B": 21, "BB": 22, "BBB": 23, "<|endoftext|>": 24 }, "unk_token": "<|endoftext|>" } }