mapwise state representations: byte language C built from codex, both vocabs projected through it (full row contract); joint content-state map + words-of-C table
8e42b95 verified | { | |
| "lexicon": { | |
| "n_bytes": 256016, | |
| "contexts": { | |
| "1": 56, | |
| "2": 432, | |
| "3": 2049, | |
| "4": 4733 | |
| }, | |
| "word_grams": 999 | |
| }, | |
| "projections": { | |
| "flan-t5-small": { | |
| "tokens": 31997, | |
| "word_frac": 0.0182, | |
| "divides_frac": 0.0143, | |
| "cohesive_frac": 0.9857, | |
| "mean_internal_hmax_bits": 0.3549, | |
| "split_h_bits": 3.0 | |
| }, | |
| "tinybert-4l-312d": { | |
| "tokens": 29523, | |
| "word_frac": 0.0187, | |
| "divides_frac": 0.0479, | |
| "cohesive_frac": 0.9521, | |
| "mean_internal_hmax_bits": 0.5213, | |
| "split_h_bits": 3.0 | |
| } | |
| }, | |
| "joint": { | |
| "content_states": 45162, | |
| "both": 11407, | |
| "only_A": 18348, | |
| "only_B": 15407, | |
| "as_words_in_C": 491, | |
| "divides_hmax3": 0 | |
| } | |
| } |