{ "version": "1.0", "model": { "type": "BPE", "vocab": { "<|endoftext|>": 0, "hello": 1, "world": 2 }, "merges": [] }, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true } ], "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "use_regex": true }, "decoder": { "type": "ByteLevel" }, "normalizer": null, "post_processor": null }