Download tokenizer.json from uckenkare/bb-probe-model: direct link, hf CLI and curl.
- Browser
- Download file 563 Bytes
-
https://huggingface.co/uckenkare/bb-probe-model/resolve/main/tokenizer.json
- Command line
-
hf download hf://uckenkare/bb-probe-model/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/uckenkare/bb-probe-model/resolve/main/tokenizer.json
563 Bytes
| { | |
| "version": "1.0", | |
| "model": { | |
| "type": "BPE", | |
| "vocab": { | |
| "<|endoftext|>": 0, | |
| "hello": 1, | |
| "world": 2 | |
| }, | |
| "merges": [] | |
| }, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<|endoftext|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": true | |
| } | |
| ], | |
| "pre_tokenizer": { | |
| "type": "ByteLevel", | |
| "add_prefix_space": false, | |
| "use_regex": true | |
| }, | |
| "decoder": { | |
| "type": "ByteLevel" | |
| }, | |
| "normalizer": null, | |
| "post_processor": null | |
| } |