File size: 2,909 Bytes
4f43093 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | {
"manifest_sha256": "dd8c22806f385226e5c06b486b72e7742f969d8c3a6226f295cd799c16cf8671",
"comparison_note": "Uniform-code bits/byte isolates segmentation efficiency; final learned-model BPB is measured after training and remains the release gate.",
"candidates": {
"8192": {
"path": "artifacts/phase3/tokenizer-8192/tokenizer.json",
"train_sample": {
"tokenizer_sha256": "60d9c718c1c67653f925c7818e56d9ce5b7c9fb460e53b605877f134af8f94c6",
"vocab_size": 8192,
"documents": 500,
"characters": 6428387,
"bytes": 19037413,
"tokens": 4225138,
"tokens_per_character": 0.6572625450210138,
"tokens_per_byte": 0.22193866361989414,
"uniform_code_bits_per_byte": 2.885202627058624,
"roundtrip_passed": true
},
"fixed_historical_test": {
"tokenizer_sha256": "60d9c718c1c67653f925c7818e56d9ce5b7c9fb460e53b605877f134af8f94c6",
"vocab_size": 8192,
"documents": 24,
"characters": 401323,
"bytes": 1190148,
"tokens": 261977,
"tokens_per_character": 0.6527834188421795,
"tokens_per_byte": 0.22012136305736765,
"uniform_code_bits_per_byte": 2.8615777197457795,
"roundtrip_passed": true
},
"proxy_evaluation": {
"path": "artifacts/phase3/tokenizer-proxy-8192/evaluation.json",
"sha256": "c746301a89a9c2ea61ea1c007fa56c0d0a49b48bf993a570d918d5a2cb8920a8",
"historical_test_bpb": 2.3338388985944745
}
},
"16384": {
"path": "artifacts/phase3/tokenizer/tokenizer.json",
"train_sample": {
"tokenizer_sha256": "2c014c2f018dd70ab48d66b59e20c9afc12c5a0dd7b5bbdd48d7b849c4bf1d42",
"vocab_size": 16384,
"documents": 500,
"characters": 6428387,
"bytes": 19037413,
"tokens": 3709383,
"tokens_per_character": 0.577031687731308,
"tokens_per_byte": 0.1948470099377473,
"uniform_code_bits_per_byte": 2.7278581391284624,
"roundtrip_passed": true
},
"fixed_historical_test": {
"tokenizer_sha256": "2c014c2f018dd70ab48d66b59e20c9afc12c5a0dd7b5bbdd48d7b849c4bf1d42",
"vocab_size": 16384,
"documents": 24,
"characters": 401323,
"bytes": 1190148,
"tokens": 227953,
"tokens_per_character": 0.5680038273410695,
"tokens_per_byte": 0.1915333219061831,
"uniform_code_bits_per_byte": 2.6814665066865633,
"roundtrip_passed": true
},
"proxy_evaluation": {
"path": "artifacts/phase3/tokenizer-proxy-16384/evaluation.json",
"sha256": "cd46a1371e33ac31acf40c8fabcbbf463a417ee02c57c0fb96bd5afe83ea6931",
"historical_test_bpb": 2.287155362650661
}
}
},
"selected_vocab_size": 16384,
"decision_rule": "lowest matched 200-step proxy-model historical test BPB",
"version": "phase3"
}
|