training_config.json

{
  "tokenizer_name": "sudanizer-72k",
  "tokenizer_version": "1.0.0",
  "corpus_domain": "social+stories+youtube+qa",
  "vocab_size": 72000,
  "min_frequency": 3,
  "pre_tokenizer_mode": "whitespace",
  "dialect_normalization": false,
  "special_tokens": [
    "<pad>",
    "<unk>",
    "<s>",
    "</s>",
    "<mask>",
    "<sep>"
  ],
  "additional_special_tokens": [
    "<user>",
    "<assistant>",
    "<system>",
    "<sudan>",
    "<url>",
    "<mention>",
    "<hashtag>",
    "<elongation>",
    "<emoji_purple_heart>",
    "<emoji_sleep>",
    "<emoji_money>",
    "<emoji_confetti>",
    "<emoji_trophy>",
    "<emoji_heart>",
    "<emoji_broken_heart>",
    "<emoji_angry>",
    "<emoji_soccer>",
    "<emoji_camera>",
    "<emoji_arrow_right>",
    "<emoji_hearts>",
    "<emoji_tea>",
    "<emoji_green_circle>",
    "<emoji_ok>",
    "<emoji_sparkles>",
    "<emoji_party>",
    "<emoji_star>",
    "<emoji_green_heart>",
    "<emoji_notes>",
    "<emoji_joy>",
    "<emoji_sad>",
    "<emoji_wow>",
    "<emoji_rofl>",
    "<emoji_sweat>",
    "<emoji_check>",
    "<emoji_100>",
    "<emoji_love>",
    "<emoji_mask>",
    "<emoji_thumbsdown>",
    "<emoji_bell>",
    "<emoji_blue_heart>",
    "<emoji_rage>",
    "<emoji_grin>",
    "<emoji_black_heart>",
    "<emoji_sun>",
    "<emoji_white_heart>",
    "<emoji_cross>",
    "<emoji_rainbow>",
    "<emoji_red_circle>",
    "<emoji_arrow_left>",
    "<emoji_shock>",
    "<emoji_music>",
    "<emoji_phone>",
    "<emoji_star2>",
    "<emoji_neutral>",
    "<emoji_coffee>",
    "<emoji_fire>",
    "<emoji_clapper>",
    "<emoji_moon>",
    "<emoji_clap>",
    "<emoji_rose>",
    "<emoji_gift_heart>",
    "<emoji_mic>",
    "<emoji_pray>",
    "<emoji_yellow_heart>",
    "<emoji_computer>",
    "<emoji_think>",
    "<emoji_cry>",
    "<emoji_thumbsup>",
    "<emoji_smile>"
  ],
  "corpus_stats": {
    "total_lines": 609024,
    "total_characters": 126490282,
    "total_words": 24414457,
    "files_processed": 2,
    "total_files": 2,
    "avg_line_length": 207.69342751681378,
    "avg_words_per_line": 40.087840544871796,
    "min_line_length": 10,
    "max_line_length": 26332
  },
  "training_stats": {
    "training_time_seconds": 24.370618104934692,
    "final_vocab_size": 72000,
    "training_files": [
      "./output/sudanizer/training_corpus.txt"
    ]
  },
  "evaluation_metrics": {
    "total_samples": 500,
    "total_characters": 86872,
    "total_tokens": 19153,
    "total_words": 16772,
    "avg_chars_per_token": 4.536,
    "compression_ratio": 0.2205,
    "unk_count": 0,
    "unk_rate_percent": 0.0,
    "avg_subwords_per_word": 1.159,
    "vocab_size": 72000,
    "max_tokens_in_sample": 868,
    "min_tokens_in_sample": 2,
    "avg_tokens_per_sample": 38.31
  }
}