| { | |
| "custom_bpe": { | |
| "tokenizer_name": "custom_bpe", | |
| "total_words_evaluated": 158106, | |
| "total_tokens_produced": 589509, | |
| "compression_ratio": 1.7228, | |
| "avg_tokens_per_word": 3.7286, | |
| "coverage_percentage": 97.72, | |
| "unknown_word_percentage": 0.0 | |
| }, | |
| "phase2_subword_discovery": { | |
| "tokenizer_name": "phase2_subword_discovery", | |
| "total_words_evaluated": 158106, | |
| "total_tokens_produced": 158110, | |
| "compression_ratio": 6.4233, | |
| "avg_tokens_per_word": 1.0, | |
| "coverage_percentage": 99.97, | |
| "unknown_word_percentage": 0.0 | |
| }, | |
| "sentencepiece_bpe": { | |
| "tokenizer_name": "sentencepiece_bpe", | |
| "total_words_evaluated": 158106, | |
| "total_tokens_produced": 724187, | |
| "compression_ratio": 1.4024, | |
| "avg_tokens_per_word": 4.5804, | |
| "coverage_percentage": 89.39, | |
| "unknown_word_percentage": 0.0 | |
| }, | |
| "sentencepiece_unigram": { | |
| "tokenizer_name": "sentencepiece_unigram", | |
| "total_words_evaluated": 158106, | |
| "total_tokens_produced": 751698, | |
| "compression_ratio": 1.351, | |
| "avg_tokens_per_word": 4.7544, | |
| "coverage_percentage": 86.42, | |
| "unknown_word_percentage": 0.0 | |
| } | |
| } |