| { |
| "eval": "/tmp/v03_eval/eval.parquet", |
| "results": [ |
| { |
| "name": "SARFTokenizer v0.3.1", |
| "vocab": 100000, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 119828, |
| "cpt": 4.004139266281671, |
| "tpw": 1.4443373029265705 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 18568, |
| "cpt": 3.73265833692374, |
| "tpw": 1.8937276899541051 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 122248, |
| "cpt": 4.242744257574766, |
| "tpw": 1.3433698530785376 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 142865, |
| "cpt": 4.1997690127043015, |
| "tpw": 1.237215626163692 |
| } |
| } |
| }, |
| { |
| "name": "SARFTokenizer v0.3", |
| "vocab": 80000, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 150332, |
| "cpt": 3.191655801825293, |
| "tpw": 1.8120148498143773 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 19088, |
| "cpt": 3.6309723386420787, |
| "tpw": 1.9467618561958184 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 121769, |
| "cpt": 4.259433846052772, |
| "tpw": 1.3381061746574214 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 142051, |
| "cpt": 4.223835101477638, |
| "tpw": 1.2301663592354923 |
| } |
| } |
| }, |
| { |
| "name": "SARFTokenizer v0.2", |
| "vocab": 65000, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 130289, |
| "cpt": 3.6826439684086916, |
| "tpw": 1.570428137505424 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 19681, |
| "cpt": 3.5215690259641277, |
| "tpw": 2.0072412034676184 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 132253, |
| "cpt": 3.921778712014094, |
| "tpw": 1.4533136998494522 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 153321, |
| "cpt": 3.913358248380848, |
| "tpw": 1.3277649320620404 |
| } |
| } |
| }, |
| { |
| "name": "Qwen3.6-35B-A3B", |
| "vocab": 248077, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 153331, |
| "cpt": 3.129230227416504, |
| "tpw": 1.8481630586760522 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 23217, |
| "cpt": 2.985226342766077, |
| "tpw": 2.3678735339112698 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 160453, |
| "cpt": 3.2325166871295643, |
| "tpw": 1.7632004043911604 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 174818, |
| "cpt": 3.4321408550606916, |
| "tpw": 1.5139296632113135 |
| } |
| } |
| }, |
| { |
| "name": "GPT-4o (o200k_base)", |
| "vocab": 200019, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 155409, |
| "cpt": 3.0873887612686524, |
| "tpw": 1.8732100670170195 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 20332, |
| "cpt": 3.4088136927011607, |
| "tpw": 2.0736359000509945 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 147972, |
| "cpt": 3.505169897007542, |
| "tpw": 1.6260480654058747 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 165658, |
| "cpt": 3.621919858986587, |
| "tpw": 1.434603760186364 |
| } |
| } |
| }, |
| { |
| "name": "ALLaM-7B-Instruct", |
| "vocab": 64000, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 168111, |
| "cpt": 2.854114245944644, |
| "tpw": 2.0263126175208526 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 27522, |
| "cpt": 2.5182762880604606, |
| "tpw": 2.8069352371239162 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 172904, |
| "cpt": 2.9997397399713135, |
| "tpw": 1.9000230766694872 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 184611, |
| "cpt": 3.2500771893332465, |
| "tpw": 1.5987373671767426 |
| } |
| } |
| }, |
| { |
| "name": "google/gemma-4-31B-it", |
| "vocab": 262144, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 169379, |
| "cpt": 2.8327478613051205, |
| "tpw": 2.041596355045562 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 22580, |
| "cpt": 3.0694419840566876, |
| "tpw": 2.302906680265171 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 160000, |
| "cpt": 3.24166875, |
| "tpw": 1.7582224371160757 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 177333, |
| "cpt": 3.38346500651317, |
| "tpw": 1.5357096464108493 |
| } |
| } |
| }, |
| { |
| "name": "google/gemma-3-1b-pt", |
| "vocab": 262145, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 169379, |
| "cpt": 2.8327478613051205, |
| "tpw": 2.041596355045562 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 22580, |
| "cpt": 3.0694419840566876, |
| "tpw": 2.302906680265171 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 160000, |
| "cpt": 3.24166875, |
| "tpw": 1.7582224371160757 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 177301, |
| "cpt": 3.384075667931935, |
| "tpw": 1.535432525352247 |
| } |
| } |
| }, |
| { |
| "name": "google/gemma-2-2b", |
| "vocab": 256000, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 172625, |
| "cpt": 2.7794815351194786, |
| "tpw": 2.0807217588351574 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 22237, |
| "cpt": 3.1167873364212797, |
| "tpw": 2.267924528301887 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 158685, |
| "cpt": 3.268531997353247, |
| "tpw": 1.7437720464610278 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 177356, |
| "cpt": 3.3830262297300346, |
| "tpw": 1.5359088271717198 |
| } |
| } |
| }, |
| { |
| "name": "QCRI/Fanar-1-9B", |
| "vocab": 128256, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 172741, |
| "cpt": 2.777615042172964, |
| "tpw": 2.0821199556434116 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 22743, |
| "cpt": 3.047443169326826, |
| "tpw": 2.319530851606323 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 161011, |
| "cpt": 3.2213140717093864, |
| "tpw": 1.7693322051406029 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 179306, |
| "cpt": 3.3462349280001784, |
| "tpw": 1.5527958916803062 |
| } |
| } |
| }, |
| { |
| "name": "Qwen2.5-0.5B", |
| "vocab": 151665, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 185741, |
| "cpt": 2.5832099536451296, |
| "tpw": 2.2388144255339664 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 23714, |
| "cpt": 2.922661718815889, |
| "tpw": 2.4185619581845996 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 157198, |
| "cpt": 3.299450374686701, |
| "tpw": 1.7274315666860804 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 170867, |
| "cpt": 3.5115030989014846, |
| "tpw": 1.479713872506993 |
| } |
| } |
| }, |
| { |
| "name": "Hala-350M", |
| "vocab": 64400, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 216262, |
| "cpt": 2.218642202513618, |
| "tpw": 2.6066968805747073 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 21525, |
| "cpt": 3.219883855981417, |
| "tpw": 2.195308516063233 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 154027, |
| "cpt": 3.367377148162335, |
| "tpw": 1.6925857957604862 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 172555, |
| "cpt": 3.477152212338095, |
| "tpw": 1.4943320083482718 |
| } |
| } |
| }, |
| { |
| "name": "Kimi-K2.6", |
| "vocab": 163840, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 231367, |
| "cpt": 2.0737961766371176, |
| "tpw": 2.788763801166771 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 21401, |
| "cpt": 3.238540255128265, |
| "tpw": 2.182661907190209 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 147361, |
| "cpt": 3.5197033136311506, |
| "tpw": 1.6193338534741377 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 165268, |
| "cpt": 3.630466878040516, |
| "tpw": 1.4312263472846467 |
| } |
| } |
| }, |
| { |
| "name": "GPT-4 (cl100k_base)", |
| "vocab": 100277, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 335815, |
| "cpt": 1.428786683143993, |
| "tpw": 4.047719492792054 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 22607, |
| "cpt": 3.065776087052683, |
| "tpw": 2.3056603773584907 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 149086, |
| "cpt": 3.478978576123848, |
| "tpw": 1.6382896891242953 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 166345, |
| "cpt": 3.606961435570651, |
| "tpw": 1.440553202913235 |
| } |
| } |
| }, |
| { |
| "name": "Falcon-7B", |
| "vocab": 65024, |
| "by_label": { |
| "arabic": { |
| "chars": 479808, |
| "words": 82964, |
| "tokens": 484375, |
| "cpt": 0.9905713548387097, |
| "tpw": 5.838375681018273 |
| }, |
| "english": { |
| "chars": 69308, |
| "words": 9805, |
| "tokens": 25481, |
| "cpt": 2.71998744162317, |
| "tpw": 2.5987761346251914 |
| }, |
| "math": { |
| "chars": 518667, |
| "words": 91001, |
| "tokens": 166874, |
| "cpt": 3.1081354794635474, |
| "tpw": 1.833760068570675 |
| }, |
| "code": { |
| "chars": 600000, |
| "words": 115473, |
| "tokens": 186896, |
| "cpt": 3.2103415803441484, |
| "tpw": 1.6185255427675733 |
| } |
| } |
| } |
| ] |
| } |
