{
  "tokenizer_config": {
    "name": "sudanizer-72k",
    "version": "1.0.0",
    "vocab_size": 72000,
    "min_frequency": 3,
    "limit_alphabet": 5000,
    "pre_tokenizer": "whitespace",
    "dialect_normalization": false,
    "hamza_normalization": true
  },
  "training_metrics": {
    "total_samples": 500,
    "total_characters": 86872,
    "total_tokens": 19153,
    "total_words": 16772,
    "avg_chars_per_token": 4.536,
    "compression_ratio": 0.2205,
    "unk_count": 0,
    "unk_rate_percent": 0.0,
    "avg_subwords_per_word": 1.159,
    "vocab_size": 72000,
    "max_tokens_in_sample": 868,
    "min_tokens_in_sample": 2,
    "avg_tokens_per_sample": 38.31
  },
  "comprehensive_evaluation": {
    "by_category": {
      "pure_dialect": {
        "total_samples": 10,
        "total_characters": 311,
        "total_tokens": 66,
        "total_words": 62,
        "avg_chars_per_token": 4.712,
        "compression_ratio": 0.2122,
        "unk_count": 0,
        "unk_rate_percent": 0.0,
        "avg_subwords_per_word": 1.064,
        "vocab_size": 72000,
        "max_tokens_in_sample": 8,
        "min_tokens_in_sample": 5,
        "avg_tokens_per_sample": 6.6
      },
      "code_switching": {
        "total_samples": 8,
        "total_characters": 349,
        "total_tokens": 156,
        "total_words": 53,
        "avg_chars_per_token": 2.237,
        "compression_ratio": 0.447,
        "unk_count": 0,
        "unk_rate_percent": 0.0,
        "avg_subwords_per_word": 2.984,
        "vocab_size": 72000,
        "max_tokens_in_sample": 27,
        "min_tokens_in_sample": 13,
        "avg_tokens_per_sample": 19.5
      },
      "social_media": {
        "total_samples": 10,
        "total_characters": 378,
        "total_tokens": 129,
        "total_words": 72,
        "avg_chars_per_token": 2.93,
        "compression_ratio": 0.3413,
        "unk_count": 29,
        "unk_rate_percent": 22.4806,
        "avg_subwords_per_word": 1.853,
        "vocab_size": 72000,
        "max_tokens_in_sample": 25,
        "min_tokens_in_sample": 10,
        "avg_tokens_per_sample": 12.9
      },
      "formal": {
        "total_samples": 5,
        "total_characters": 356,
        "total_tokens": 68,
        "total_words": 51,
        "avg_chars_per_token": 5.235,
        "compression_ratio": 0.191,
        "unk_count": 1,
        "unk_rate_percent": 1.4706,
        "avg_subwords_per_word": 1.315,
        "vocab_size": 72000,
        "max_tokens_in_sample": 19,
        "min_tokens_in_sample": 9,
        "avg_tokens_per_sample": 13.6
      },
      "edge_case": {
        "total_samples": 8,
        "total_characters": 355,
        "total_tokens": 76,
        "total_words": 50,
        "avg_chars_per_token": 4.671,
        "compression_ratio": 0.2141,
        "unk_count": 1,
        "unk_rate_percent": 1.3158,
        "avg_subwords_per_word": 1.525,
        "vocab_size": 72000,
        "max_tokens_in_sample": 13,
        "min_tokens_in_sample": 8,
        "avg_tokens_per_sample": 9.5
      },
      "morphology": {
        "total_samples": 8,
        "total_characters": 216,
        "total_tokens": 48,
        "total_words": 40,
        "avg_chars_per_token": 4.5,
        "compression_ratio": 0.2222,
        "unk_count": 0,
        "unk_rate_percent": 0.0,
        "avg_subwords_per_word": 1.205,
        "vocab_size": 72000,
        "max_tokens_in_sample": 8,
        "min_tokens_in_sample": 4,
        "avg_tokens_per_sample": 6.0
      },
      "cultural": {
        "total_samples": 8,
        "total_characters": 267,
        "total_tokens": 59,
        "total_words": 49,
        "avg_chars_per_token": 4.525,
        "compression_ratio": 0.221,
        "unk_count": 0,
        "unk_rate_percent": 0.0,
        "avg_subwords_per_word": 1.187,
        "vocab_size": 72000,
        "max_tokens_in_sample": 12,
        "min_tokens_in_sample": 5,
        "avg_tokens_per_sample": 7.38
      },
      "mixed_content": {
        "total_samples": 8,
        "total_characters": 224,
        "total_tokens": 81,
        "total_words": 33,
        "avg_chars_per_token": 2.765,
        "compression_ratio": 0.3616,
        "unk_count": 1,
        "unk_rate_percent": 1.2346,
        "avg_subwords_per_word": 3.017,
        "vocab_size": 72000,
        "max_tokens_in_sample": 18,
        "min_tokens_in_sample": 5,
        "avg_tokens_per_sample": 10.12
      }
    },
    "by_difficulty": {
      "easy": {
        "total_samples": 4,
        "total_characters": 157,
        "total_tokens": 28,
        "total_words": 28,
        "avg_chars_per_token": 5.607,
        "compression_ratio": 0.1783,
        "unk_count": 0,
        "unk_rate_percent": 0.0,
        "avg_subwords_per_word": 1.0,
        "vocab_size": 72000,
        "max_tokens_in_sample": 9,
        "min_tokens_in_sample": 5,
        "avg_tokens_per_sample": 7.0
      },
      "medium": {
        "total_samples": 22,
        "total_characters": 721,
        "total_tokens": 167,
        "total_words": 129,
        "avg_chars_per_token": 4.317,
        "compression_ratio": 0.2316,
        "unk_count": 4,
        "unk_rate_percent": 2.3952,
        "avg_subwords_per_word": 1.349,
        "vocab_size": 72000,
        "max_tokens_in_sample": 16,
        "min_tokens_in_sample": 4,
        "avg_tokens_per_sample": 7.59
      },
      "hard": {
        "total_samples": 27,
        "total_characters": 1073,
        "total_tokens": 315,
        "total_words": 180,
        "avg_chars_per_token": 3.406,
        "compression_ratio": 0.2936,
        "unk_count": 21,
        "unk_rate_percent": 6.6667,
        "avg_subwords_per_word": 1.794,
        "vocab_size": 72000,
        "max_tokens_in_sample": 27,
        "min_tokens_in_sample": 5,
        "avg_tokens_per_sample": 11.67
      },
      "extreme": {
        "total_samples": 12,
        "total_characters": 505,
        "total_tokens": 173,
        "total_words": 73,
        "avg_chars_per_token": 2.919,
        "compression_ratio": 0.3426,
        "unk_count": 7,
        "unk_rate_percent": 4.0462,
        "avg_subwords_per_word": 2.748,
        "vocab_size": 72000,
        "max_tokens_in_sample": 27,
        "min_tokens_in_sample": 6,
        "avg_tokens_per_sample": 14.42
      }
    },
    "overall": {
      "total_samples": 65,
      "total_characters": 2456,
      "total_tokens": 683,
      "total_words": 410,
      "avg_chars_per_token": 3.596,
      "compression_ratio": 0.2781,
      "unk_count": 32,
      "unk_rate_percent": 4.6852,
      "avg_subwords_per_word": 1.771,
      "vocab_size": 72000,
      "max_tokens_in_sample": 27,
      "min_tokens_in_sample": 4,
      "avg_tokens_per_sample": 10.51
    },
    "problematic_samples_count": 18
  },
  "benchmark_results": {
    "Sudanizer-72K-v2": {
      "avg_chars_per_token": 3.596,
      "unk_rate_percent": 4.6852,
      "avg_subwords_per_word": 1.771,
      "total_tokens": 683
    }
  }
}