matrix_summary.json

{
 "split_H_bits": 3.0,
 "corpus_bytes": 8192000,
 "rows": [
  {
   "tokenizer": "bert-wordpiece",
   "tokens": 30517,
   "word_frac": 0.469,
   "divides_frac": 0.27,
   "cohesive_frac": 0.73,
   "mean_internal_Hmax_bits": 2.587,
   "secs": 1.3
  },
  {
   "tokenizer": "byt5-control",
   "tokens": 256,
   "word_frac": 0.301,
   "divides_frac": 0.0,
   "cohesive_frac": 0.0,
   "mean_internal_Hmax_bits": 0.0,
   "secs": 0.0
  },
  {
   "tokenizer": "cl100k_base",
   "tokens": 100256,
   "word_frac": 0.32,
   "divides_frac": 0.287,
   "cohesive_frac": 0.713,
   "mean_internal_Hmax_bits": 2.544,
   "secs": 3.0
  },
  {
   "tokenizer": "deepseek-v3",
   "tokens": 127997,
   "word_frac": 0.216,
   "divides_frac": 0.317,
   "cohesive_frac": 0.683,
   "mean_internal_Hmax_bits": 2.606,
   "secs": 2.7
  },
  {
   "tokenizer": "gpt2",
   "tokens": 50256,
   "word_frac": 0.479,
   "divides_frac": 0.343,
   "cohesive_frac": 0.657,
   "mean_internal_Hmax_bits": 2.67,
   "secs": 0.9
  },
  {
   "tokenizer": "llama3",
   "tokens": 128000,
   "word_frac": 0.251,
   "divides_frac": 0.283,
   "cohesive_frac": 0.717,
   "mean_internal_Hmax_bits": 2.528,
   "secs": 2.6
  },
  {
   "tokenizer": "mistral-v03",
   "tokens": 31997,
   "word_frac": 0.413,
   "divides_frac": 0.292,
   "cohesive_frac": 0.708,
   "mean_internal_Hmax_bits": 2.577,
   "secs": 0.6
  },
  {
   "tokenizer": "o200k_base",
   "tokens": 199998,
   "word_frac": 0.167,
   "divides_frac": 0.288,
   "cohesive_frac": 0.712,
   "mean_internal_Hmax_bits": 2.522,
   "secs": 4.0
  },
  {
   "tokenizer": "qwen3-newest",
   "tokens": 248044,
   "word_frac": 0.128,
   "divides_frac": 0.299,
   "cohesive_frac": 0.701,
   "mean_internal_Hmax_bits": 2.557,
   "secs": 4.7
  },
  {
   "tokenizer": "smollm2",
   "tokens": 49135,
   "word_frac": 0.502,
   "divides_frac": 0.356,
   "cohesive_frac": 0.644,
   "mean_internal_Hmax_bits": 2.703,
   "secs": 0.8
  },
  {
   "tokenizer": "t5-unigram",
   "tokens": 31997,
   "word_frac": 0.482,
   "divides_frac": 0.348,
   "cohesive_frac": 0.652,
   "mean_internal_Hmax_bits": 2.695,
   "secs": 0.6
  },
  {
   "tokenizer": "xlmr",
   "tokens": 249997,
   "word_frac": 0.047,
   "divides_frac": 0.282,
   "cohesive_frac": 0.718,
   "mean_internal_Hmax_bits": 2.489,
   "secs": 4.8
  }
 ]
}