{
  "SARFTokenizer-v6-unigram": {
    "vocab": 65000,
    "ar_tokens": 130289,
    "en_tokens": 19681,
    "ar_cpt": 3.6826439684086916,
    "en_cpt": 3.5215690259641277,
    "parity": 1.0457395386138897,
    "ar_tpw": 1.570428137505424,
    "en_tpw": 2.0072412034676184
  },
  "SARFTokenizer-v4": {
    "vocab": 62912,
    "ar_tokens": 147068,
    "en_tokens": 25401,
    "ar_cpt": 3.262490820572796,
    "en_cpt": 2.7285539939372465,
    "parity": 1.195684904100098,
    "ar_tpw": 1.7726724844510873,
    "en_tpw": 2.590617032126466
  },
  "tiktoken/o200k_base": {
    "vocab": 200019,
    "ar_tokens": 155409,
    "en_tokens": 20332,
    "ar_cpt": 3.0873887612686524,
    "en_cpt": 3.4088136927011607,
    "parity": 0.9057076858964945,
    "ar_tpw": 1.8732100670170195,
    "en_tpw": 2.0736359000509945
  },
  "tiktoken/cl100k_base": {
    "vocab": 100277,
    "ar_tokens": 335815,
    "en_tokens": 22607,
    "ar_cpt": 1.428786683143993,
    "en_cpt": 3.065776087052683,
    "parity": 0.4660440431961137,
    "ar_tpw": 4.047719492792054,
    "en_tpw": 2.3056603773584907
  },
  "Qwen/Qwen2.5-0.5B": {
    "vocab": 151665,
    "ar_tokens": 185741,
    "en_tokens": 23714,
    "ar_cpt": 2.5832099536451296,
    "en_cpt": 2.922661718815889,
    "parity": 0.8838552669351389,
    "ar_tpw": 2.2388144255339664,
    "en_tpw": 2.4185619581845996
  },
  "tiiuae/falcon-7b": {
    "vocab": 65024,
    "ar_tokens": 484375,
    "en_tokens": 25481,
    "ar_cpt": 0.9905713548387097,
    "en_cpt": 2.71998744162317,
    "parity": 0.3641823266094125,
    "ar_tpw": 5.838375681018273,
    "en_tpw": 2.5987761346251914
  },
  "google/gemma-2-2b": {
    "vocab": 256000,
    "ar_tokens": 172625,
    "en_tokens": 22237,
    "ar_cpt": 2.7794815351194786,
    "en_cpt": 3.1167873364212797,
    "parity": 0.8917777297924028,
    "ar_tpw": 2.0807217588351574,
    "en_tpw": 2.267924528301887
  },
  "google/gemma-3-1b-pt": {
    "vocab": 262145,
    "ar_tokens": 169379,
    "en_tokens": 22580,
    "ar_cpt": 2.8327478613051205,
    "en_cpt": 3.0694419840566876,
    "parity": 0.9228869208211118,
    "ar_tpw": 2.041596355045562,
    "en_tpw": 2.302906680265171
  },
  "ALLaM-AI/ALLaM-7B-Instruct-preview": {
    "vocab": 64000,
    "ar_tokens": 168111,
    "en_tokens": 27522,
    "ar_cpt": 2.854114245944644,
    "en_cpt": 2.5182762880604606,
    "parity": 1.1333602510083756,
    "ar_tpw": 2.0263126175208526,
    "en_tpw": 2.8069352371239162
  },
  "QCRI/Fanar-1-9B-Instruct": {
    "vocab": 128256,
    "ar_tokens": 172741,
    "en_tokens": 22743,
    "ar_cpt": 2.777615042172964,
    "en_cpt": 3.047443169326826,
    "parity": 0.911457535986318,
    "ar_tpw": 2.0821199556434116,
    "en_tpw": 2.319530851606323
  },
  "hammh0a/Hala-350M": {
    "vocab": 64400,
    "ar_tokens": 216262,
    "en_tokens": 21525,
    "ar_cpt": 2.218642202513618,
    "en_cpt": 3.219883855981417,
    "parity": 0.689044171078456,
    "ar_tpw": 2.6066968805747073,
    "en_tpw": 2.195308516063233
  }
}