| { |
| "manifest_sha256": "dd8c22806f385226e5c06b486b72e7742f969d8c3a6226f295cd799c16cf8671", |
| "comparison_note": "Uniform-code bits/byte isolates segmentation efficiency; final learned-model BPB is measured after training and remains the release gate.", |
| "candidates": { |
| "8192": { |
| "path": "artifacts/phase3/tokenizer-8192/tokenizer.json", |
| "train_sample": { |
| "tokenizer_sha256": "60d9c718c1c67653f925c7818e56d9ce5b7c9fb460e53b605877f134af8f94c6", |
| "vocab_size": 8192, |
| "documents": 500, |
| "characters": 6428387, |
| "bytes": 19037413, |
| "tokens": 4225138, |
| "tokens_per_character": 0.6572625450210138, |
| "tokens_per_byte": 0.22193866361989414, |
| "uniform_code_bits_per_byte": 2.885202627058624, |
| "roundtrip_passed": true |
| }, |
| "fixed_historical_test": { |
| "tokenizer_sha256": "60d9c718c1c67653f925c7818e56d9ce5b7c9fb460e53b605877f134af8f94c6", |
| "vocab_size": 8192, |
| "documents": 24, |
| "characters": 401323, |
| "bytes": 1190148, |
| "tokens": 261977, |
| "tokens_per_character": 0.6527834188421795, |
| "tokens_per_byte": 0.22012136305736765, |
| "uniform_code_bits_per_byte": 2.8615777197457795, |
| "roundtrip_passed": true |
| }, |
| "proxy_evaluation": { |
| "path": "artifacts/phase3/tokenizer-proxy-8192/evaluation.json", |
| "sha256": "c746301a89a9c2ea61ea1c007fa56c0d0a49b48bf993a570d918d5a2cb8920a8", |
| "historical_test_bpb": 2.3338388985944745 |
| } |
| }, |
| "16384": { |
| "path": "artifacts/phase3/tokenizer/tokenizer.json", |
| "train_sample": { |
| "tokenizer_sha256": "2c014c2f018dd70ab48d66b59e20c9afc12c5a0dd7b5bbdd48d7b849c4bf1d42", |
| "vocab_size": 16384, |
| "documents": 500, |
| "characters": 6428387, |
| "bytes": 19037413, |
| "tokens": 3709383, |
| "tokens_per_character": 0.577031687731308, |
| "tokens_per_byte": 0.1948470099377473, |
| "uniform_code_bits_per_byte": 2.7278581391284624, |
| "roundtrip_passed": true |
| }, |
| "fixed_historical_test": { |
| "tokenizer_sha256": "2c014c2f018dd70ab48d66b59e20c9afc12c5a0dd7b5bbdd48d7b849c4bf1d42", |
| "vocab_size": 16384, |
| "documents": 24, |
| "characters": 401323, |
| "bytes": 1190148, |
| "tokens": 227953, |
| "tokens_per_character": 0.5680038273410695, |
| "tokens_per_byte": 0.1915333219061831, |
| "uniform_code_bits_per_byte": 2.6814665066865633, |
| "roundtrip_passed": true |
| }, |
| "proxy_evaluation": { |
| "path": "artifacts/phase3/tokenizer-proxy-16384/evaluation.json", |
| "sha256": "cd46a1371e33ac31acf40c8fabcbbf463a417ee02c57c0fb96bd5afe83ea6931", |
| "historical_test_bpb": 2.287155362650661 |
| } |
| } |
| }, |
| "selected_vocab_size": 16384, |
| "decision_rule": "lowest matched 200-step proxy-model historical test BPB", |
| "version": "phase3" |
| } |
|
|
