| { |
| "tokenizer_config": { |
| "name": "sudanizer-72k", |
| "version": "1.0.0", |
| "vocab_size": 72000, |
| "min_frequency": 3, |
| "limit_alphabet": 5000, |
| "pre_tokenizer": "whitespace", |
| "dialect_normalization": false, |
| "hamza_normalization": true |
| }, |
| "training_metrics": { |
| "total_samples": 500, |
| "total_characters": 86872, |
| "total_tokens": 19153, |
| "total_words": 16772, |
| "avg_chars_per_token": 4.536, |
| "compression_ratio": 0.2205, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 1.159, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 868, |
| "min_tokens_in_sample": 2, |
| "avg_tokens_per_sample": 38.31 |
| }, |
| "comprehensive_evaluation": { |
| "by_category": { |
| "pure_dialect": { |
| "total_samples": 10, |
| "total_characters": 311, |
| "total_tokens": 66, |
| "total_words": 62, |
| "avg_chars_per_token": 4.712, |
| "compression_ratio": 0.2122, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 1.064, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 8, |
| "min_tokens_in_sample": 5, |
| "avg_tokens_per_sample": 6.6 |
| }, |
| "code_switching": { |
| "total_samples": 8, |
| "total_characters": 349, |
| "total_tokens": 156, |
| "total_words": 53, |
| "avg_chars_per_token": 2.237, |
| "compression_ratio": 0.447, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 2.984, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 27, |
| "min_tokens_in_sample": 13, |
| "avg_tokens_per_sample": 19.5 |
| }, |
| "social_media": { |
| "total_samples": 10, |
| "total_characters": 378, |
| "total_tokens": 129, |
| "total_words": 72, |
| "avg_chars_per_token": 2.93, |
| "compression_ratio": 0.3413, |
| "unk_count": 29, |
| "unk_rate_percent": 22.4806, |
| "avg_subwords_per_word": 1.853, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 25, |
| "min_tokens_in_sample": 10, |
| "avg_tokens_per_sample": 12.9 |
| }, |
| "formal": { |
| "total_samples": 5, |
| "total_characters": 356, |
| "total_tokens": 68, |
| "total_words": 51, |
| "avg_chars_per_token": 5.235, |
| "compression_ratio": 0.191, |
| "unk_count": 1, |
| "unk_rate_percent": 1.4706, |
| "avg_subwords_per_word": 1.315, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 19, |
| "min_tokens_in_sample": 9, |
| "avg_tokens_per_sample": 13.6 |
| }, |
| "edge_case": { |
| "total_samples": 8, |
| "total_characters": 355, |
| "total_tokens": 76, |
| "total_words": 50, |
| "avg_chars_per_token": 4.671, |
| "compression_ratio": 0.2141, |
| "unk_count": 1, |
| "unk_rate_percent": 1.3158, |
| "avg_subwords_per_word": 1.525, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 13, |
| "min_tokens_in_sample": 8, |
| "avg_tokens_per_sample": 9.5 |
| }, |
| "morphology": { |
| "total_samples": 8, |
| "total_characters": 216, |
| "total_tokens": 48, |
| "total_words": 40, |
| "avg_chars_per_token": 4.5, |
| "compression_ratio": 0.2222, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 1.205, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 8, |
| "min_tokens_in_sample": 4, |
| "avg_tokens_per_sample": 6.0 |
| }, |
| "cultural": { |
| "total_samples": 8, |
| "total_characters": 267, |
| "total_tokens": 59, |
| "total_words": 49, |
| "avg_chars_per_token": 4.525, |
| "compression_ratio": 0.221, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 1.187, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 12, |
| "min_tokens_in_sample": 5, |
| "avg_tokens_per_sample": 7.38 |
| }, |
| "mixed_content": { |
| "total_samples": 8, |
| "total_characters": 224, |
| "total_tokens": 81, |
| "total_words": 33, |
| "avg_chars_per_token": 2.765, |
| "compression_ratio": 0.3616, |
| "unk_count": 1, |
| "unk_rate_percent": 1.2346, |
| "avg_subwords_per_word": 3.017, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 18, |
| "min_tokens_in_sample": 5, |
| "avg_tokens_per_sample": 10.12 |
| } |
| }, |
| "by_difficulty": { |
| "easy": { |
| "total_samples": 4, |
| "total_characters": 157, |
| "total_tokens": 28, |
| "total_words": 28, |
| "avg_chars_per_token": 5.607, |
| "compression_ratio": 0.1783, |
| "unk_count": 0, |
| "unk_rate_percent": 0.0, |
| "avg_subwords_per_word": 1.0, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 9, |
| "min_tokens_in_sample": 5, |
| "avg_tokens_per_sample": 7.0 |
| }, |
| "medium": { |
| "total_samples": 22, |
| "total_characters": 721, |
| "total_tokens": 167, |
| "total_words": 129, |
| "avg_chars_per_token": 4.317, |
| "compression_ratio": 0.2316, |
| "unk_count": 4, |
| "unk_rate_percent": 2.3952, |
| "avg_subwords_per_word": 1.349, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 16, |
| "min_tokens_in_sample": 4, |
| "avg_tokens_per_sample": 7.59 |
| }, |
| "hard": { |
| "total_samples": 27, |
| "total_characters": 1073, |
| "total_tokens": 315, |
| "total_words": 180, |
| "avg_chars_per_token": 3.406, |
| "compression_ratio": 0.2936, |
| "unk_count": 21, |
| "unk_rate_percent": 6.6667, |
| "avg_subwords_per_word": 1.794, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 27, |
| "min_tokens_in_sample": 5, |
| "avg_tokens_per_sample": 11.67 |
| }, |
| "extreme": { |
| "total_samples": 12, |
| "total_characters": 505, |
| "total_tokens": 173, |
| "total_words": 73, |
| "avg_chars_per_token": 2.919, |
| "compression_ratio": 0.3426, |
| "unk_count": 7, |
| "unk_rate_percent": 4.0462, |
| "avg_subwords_per_word": 2.748, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 27, |
| "min_tokens_in_sample": 6, |
| "avg_tokens_per_sample": 14.42 |
| } |
| }, |
| "overall": { |
| "total_samples": 65, |
| "total_characters": 2456, |
| "total_tokens": 683, |
| "total_words": 410, |
| "avg_chars_per_token": 3.596, |
| "compression_ratio": 0.2781, |
| "unk_count": 32, |
| "unk_rate_percent": 4.6852, |
| "avg_subwords_per_word": 1.771, |
| "vocab_size": 72000, |
| "max_tokens_in_sample": 27, |
| "min_tokens_in_sample": 4, |
| "avg_tokens_per_sample": 10.51 |
| }, |
| "problematic_samples_count": 18 |
| }, |
| "benchmark_results": { |
| "Sudanizer-72K-v2": { |
| "avg_chars_per_token": 3.596, |
| "unk_rate_percent": 4.6852, |
| "avg_subwords_per_word": 1.771, |
| "total_tokens": 683 |
| } |
| } |
| } |
