| { |
| "mt_en": { |
| "mean_tokens": 18.8248, |
| "p95_tokens": 50.0, |
| "max_tokens": 112, |
| "tokens_per_char": 0.2717304496837542, |
| "tokens_per_whitespace_word": 1.5909059147501896, |
| "unk_tokens": 0, |
| "byte_fallback_tokens": 71 |
| }, |
| "mt_vi": { |
| "mean_tokens": 22.5454, |
| "p95_tokens": 60.0, |
| "max_tokens": 120, |
| "tokens_per_char": 0.30452669132286336, |
| "tokens_per_whitespace_word": 1.3601911156958297, |
| "unk_tokens": 0, |
| "byte_fallback_tokens": 105 |
| }, |
| "mt_ja": { |
| "mean_tokens": 17.9854, |
| "p95_tokens": 47.0, |
| "max_tokens": 110, |
| "tokens_per_char": 0.48894536698039154, |
| "tokens_per_whitespace_word": 10.451642071573078, |
| "unk_tokens": 0, |
| "byte_fallback_tokens": 1100 |
| }, |
| "lm_tokens_per_split": { |
| "train": 40141738, |
| "val": 2217353, |
| "test": 2248878 |
| } |
| } |
