lexicon byte-state gauges: A 31997, B 29523, overlap 10801, union 50719, irreducibles 1048 (atoms: 10 digits + letters + punct; zero word-starts), 95/256 single bytes present

lexicon_states_ledger.json

@@ -0,0 +1,27 @@
+{
+ "A_sentencepiece": 31997,
+ "B_wordpiece": 29523,
+ "overlap": 10801,
+ "union_required": 50719,
+ "overlap_census": {
+  "word-start": 8738,
+  "continuation": 1701,
+  "digit-cont": 76,
+  "digit-start": 202,
+  "punct/mixed": 84
+ },
+ "folded_overlap": 13071,
+ "irreducible_required": 1048,
+ "irreducible_census": {
+  "punct/mixed": 970,
+  "continuation": 68,
+  "digit-cont": 10
+ },
+ "irreducible_from": {
+  "overlap": 91,
+  "only_A": 51,
+  "only_B": 906
+ },
+ "composite": 49671,
+ "single_bytes_in_union": 95
+}