lexicon byte-state gauges: A 31997, B 29523, overlap 10801, union 50719, irreducibles 1048 (atoms: 10 digits + letters + punct; zero word-starts), 95/256 single bytes present
9918010 verified | { | |
| "A_sentencepiece": 31997, | |
| "B_wordpiece": 29523, | |
| "overlap": 10801, | |
| "union_required": 50719, | |
| "overlap_census": { | |
| "word-start": 8738, | |
| "continuation": 1701, | |
| "digit-cont": 76, | |
| "digit-start": 202, | |
| "punct/mixed": 84 | |
| }, | |
| "folded_overlap": 13071, | |
| "irreducible_required": 1048, | |
| "irreducible_census": { | |
| "punct/mixed": 970, | |
| "continuation": 68, | |
| "digit-cont": 10 | |
| }, | |
| "irreducible_from": { | |
| "overlap": 91, | |
| "only_A": 51, | |
| "only_B": 906 | |
| }, | |
| "composite": 49671, | |
| "single_bytes_in_union": 95 | |
| } |