mapwise state representations: byte language C built from codex, both vocabs projected through it (full row contract); joint content-state map + words-of-C table

mapwise_summary.json

@@ -0,0 +1,38 @@
+{
+ "lexicon": {
+  "n_bytes": 256016,
+  "contexts": {
+   "1": 56,
+   "2": 432,
+   "3": 2049,
+   "4": 4733
+  },
+  "word_grams": 999
+ },
+ "projections": {
+  "flan-t5-small": {
+   "tokens": 31997,
+   "word_frac": 0.0182,
+   "divides_frac": 0.0143,
+   "cohesive_frac": 0.9857,
+   "mean_internal_hmax_bits": 0.3549,
+   "split_h_bits": 3.0
+  },
+  "tinybert-4l-312d": {
+   "tokens": 29523,
+   "word_frac": 0.0187,
+   "divides_frac": 0.0479,
+   "cohesive_frac": 0.9521,
+   "mean_internal_hmax_bits": 0.5213,
+   "split_h_bits": 3.0
+  }
+ },
+ "joint": {
+  "content_states": 45162,
+  "both": 11407,
+  "only_A": 18348,
+  "only_B": 15407,
+  "as_words_in_C": 491,
+  "divides_hmax3": 0
+ }
+}