{
  "custom_bpe": {
    "tokenizer_name": "custom_bpe",
    "total_words_evaluated": 158106,
    "total_tokens_produced": 589509,
    "compression_ratio": 1.7228,
    "avg_tokens_per_word": 3.7286,
    "coverage_percentage": 97.72,
    "unknown_word_percentage": 0.0
  },
  "phase2_subword_discovery": {
    "tokenizer_name": "phase2_subword_discovery",
    "total_words_evaluated": 158106,
    "total_tokens_produced": 158110,
    "compression_ratio": 6.4233,
    "avg_tokens_per_word": 1.0,
    "coverage_percentage": 99.97,
    "unknown_word_percentage": 0.0
  },
  "sentencepiece_bpe": {
    "tokenizer_name": "sentencepiece_bpe",
    "total_words_evaluated": 158106,
    "total_tokens_produced": 724187,
    "compression_ratio": 1.4024,
    "avg_tokens_per_word": 4.5804,
    "coverage_percentage": 89.39,
    "unknown_word_percentage": 0.0
  },
  "sentencepiece_unigram": {
    "tokenizer_name": "sentencepiece_unigram",
    "total_words_evaluated": 158106,
    "total_tokens_produced": 751698,
    "compression_ratio": 1.351,
    "avg_tokens_per_word": 4.7544,
    "coverage_percentage": 86.42,
    "unknown_word_percentage": 0.0
  }
}