| { |
| "SARFTokenizer-v6-unigram": { |
| "vocab": 65000, |
| "ar_tokens": 130289, |
| "en_tokens": 19681, |
| "ar_cpt": 3.6826439684086916, |
| "en_cpt": 3.5215690259641277, |
| "parity": 1.0457395386138897, |
| "ar_tpw": 1.570428137505424, |
| "en_tpw": 2.0072412034676184 |
| }, |
| "SARFTokenizer-v4": { |
| "vocab": 62912, |
| "ar_tokens": 147068, |
| "en_tokens": 25401, |
| "ar_cpt": 3.262490820572796, |
| "en_cpt": 2.7285539939372465, |
| "parity": 1.195684904100098, |
| "ar_tpw": 1.7726724844510873, |
| "en_tpw": 2.590617032126466 |
| }, |
| "tiktoken/o200k_base": { |
| "vocab": 200019, |
| "ar_tokens": 155409, |
| "en_tokens": 20332, |
| "ar_cpt": 3.0873887612686524, |
| "en_cpt": 3.4088136927011607, |
| "parity": 0.9057076858964945, |
| "ar_tpw": 1.8732100670170195, |
| "en_tpw": 2.0736359000509945 |
| }, |
| "tiktoken/cl100k_base": { |
| "vocab": 100277, |
| "ar_tokens": 335815, |
| "en_tokens": 22607, |
| "ar_cpt": 1.428786683143993, |
| "en_cpt": 3.065776087052683, |
| "parity": 0.4660440431961137, |
| "ar_tpw": 4.047719492792054, |
| "en_tpw": 2.3056603773584907 |
| }, |
| "Qwen/Qwen2.5-0.5B": { |
| "vocab": 151665, |
| "ar_tokens": 185741, |
| "en_tokens": 23714, |
| "ar_cpt": 2.5832099536451296, |
| "en_cpt": 2.922661718815889, |
| "parity": 0.8838552669351389, |
| "ar_tpw": 2.2388144255339664, |
| "en_tpw": 2.4185619581845996 |
| }, |
| "tiiuae/falcon-7b": { |
| "vocab": 65024, |
| "ar_tokens": 484375, |
| "en_tokens": 25481, |
| "ar_cpt": 0.9905713548387097, |
| "en_cpt": 2.71998744162317, |
| "parity": 0.3641823266094125, |
| "ar_tpw": 5.838375681018273, |
| "en_tpw": 2.5987761346251914 |
| }, |
| "google/gemma-2-2b": { |
| "vocab": 256000, |
| "ar_tokens": 172625, |
| "en_tokens": 22237, |
| "ar_cpt": 2.7794815351194786, |
| "en_cpt": 3.1167873364212797, |
| "parity": 0.8917777297924028, |
| "ar_tpw": 2.0807217588351574, |
| "en_tpw": 2.267924528301887 |
| }, |
| "google/gemma-3-1b-pt": { |
| "vocab": 262145, |
| "ar_tokens": 169379, |
| "en_tokens": 22580, |
| "ar_cpt": 2.8327478613051205, |
| "en_cpt": 3.0694419840566876, |
| "parity": 0.9228869208211118, |
| "ar_tpw": 2.041596355045562, |
| "en_tpw": 2.302906680265171 |
| }, |
| "ALLaM-AI/ALLaM-7B-Instruct-preview": { |
| "vocab": 64000, |
| "ar_tokens": 168111, |
| "en_tokens": 27522, |
| "ar_cpt": 2.854114245944644, |
| "en_cpt": 2.5182762880604606, |
| "parity": 1.1333602510083756, |
| "ar_tpw": 2.0263126175208526, |
| "en_tpw": 2.8069352371239162 |
| }, |
| "QCRI/Fanar-1-9B-Instruct": { |
| "vocab": 128256, |
| "ar_tokens": 172741, |
| "en_tokens": 22743, |
| "ar_cpt": 2.777615042172964, |
| "en_cpt": 3.047443169326826, |
| "parity": 0.911457535986318, |
| "ar_tpw": 2.0821199556434116, |
| "en_tpw": 2.319530851606323 |
| }, |
| "hammh0a/Hala-350M": { |
| "vocab": 64400, |
| "ar_tokens": 216262, |
| "en_tokens": 21525, |
| "ar_cpt": 2.218642202513618, |
| "en_cpt": 3.219883855981417, |
| "parity": 0.689044171078456, |
| "ar_tpw": 2.6066968805747073, |
| "en_tpw": 2.195308516063233 |
| } |
| } |
