[
  {
    "name": "shared_bpe_8000",
    "tokenizer_type": "shared",
    "algorithm": "bpe",
    "vocab_size": 8000,
    "fertility_overall": 1.5744684296670828,
    "fertility_ar": 1.4755656463463651,
    "fertility_az": 1.7595859759130557,
    "cpt_overall": 3.5313313486070697,
    "cpt_ar": 3.812953050264278,
    "cpt_az": 3.0042165735724513,
    "fertility_disparity": 0.16141315823985886,
    "cpt_disparity": 0.21210239571026782,
    "oov_rate": 0.0,
    "vocab_gini": 0.7419718947527658,
    "shannon_entropy": 10.725417379440934,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_bpe_8000",
    "tokenizer_type": "concatenated",
    "algorithm": "bpe",
    "vocab_size": 8000,
    "fertility_overall": 1.625089409755206,
    "fertility_ar": 1.677255929872786,
    "fertility_az": 1.5274486975717831,
    "cpt_overall": 3.4121822358457132,
    "cpt_ar": 3.3745329363977343,
    "cpt_az": 3.4826508889784487,
    "fertility_disparity": 0.0893168595399542,
    "cpt_disparity": 0.031044728865266227,
    "oov_rate": 0.0,
    "vocab_gini": 0.7155357357913302,
    "shannon_entropy": 10.758976129080413,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  },
  {
    "name": "shared_unigram_8000",
    "tokenizer_type": "shared",
    "algorithm": "unigram",
    "vocab_size": 8000,
    "fertility_overall": 1.5860234519252308,
    "fertility_ar": 1.5098217608254827,
    "fertility_az": 1.7286510868870875,
    "cpt_overall": 3.513922137310213,
    "cpt_ar": 3.7495598617218993,
    "cpt_az": 3.072876133532698,
    "fertility_disparity": 0.12658964421540223,
    "cpt_disparity": 0.18047017600579116,
    "oov_rate": 0.0,
    "vocab_gini": 0.8011276471980863,
    "shannon_entropy": 10.278366552533573,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_unigram_8000",
    "tokenizer_type": "concatenated",
    "algorithm": "unigram",
    "vocab_size": 8000,
    "fertility_overall": 1.630557495524919,
    "fertility_ar": 1.6828221737842166,
    "fertility_az": 1.5327330595500248,
    "cpt_overall": 3.4256065103693167,
    "cpt_ar": 3.3890978047599254,
    "cpt_az": 3.4939403000980764,
    "fertility_disparity": 0.08918893307465849,
    "cpt_disparity": 0.030006950987459077,
    "oov_rate": 0.0,
    "vocab_gini": 0.7843741193591066,
    "shannon_entropy": 10.258501048163726,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  },
  {
    "name": "shared_wordpiece_8000",
    "tokenizer_type": "shared",
    "algorithm": "wordpiece",
    "vocab_size": 8000,
    "fertility_overall": 1.5720141231575526,
    "fertility_ar": 1.4715587661386824,
    "fertility_az": 1.7600376405371791,
    "cpt_overall": 3.5349251084819273,
    "cpt_ar": 3.819823570325556,
    "cpt_az": 3.001677181426908,
    "fertility_disparity": 0.16390494598197936,
    "cpt_disparity": 0.21418433962616734,
    "oov_rate": 0.0,
    "vocab_gini": 0.7254354187229622,
    "shannon_entropy": 10.734803471006908,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "concat_wordpiece_8000",
    "tokenizer_type": "concatenated",
    "algorithm": "wordpiece",
    "vocab_size": 8000,
    "fertility_overall": 1.6305196797375283,
    "fertility_ar": 1.679880351462672,
    "fertility_az": 1.5381307082241507,
    "cpt_overall": 3.397665609111008,
    "cpt_ar": 3.366673383613362,
    "cpt_az": 3.4556741357085015,
    "fertility_disparity": 0.08438079718897826,
    "cpt_disparity": 0.02575496085567456,
    "oov_rate": 0.0,
    "vocab_gini": 0.6940886180639433,
    "shannon_entropy": 10.748321790799139,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "shared_bbpe_8000",
    "tokenizer_type": "shared",
    "algorithm": "bbpe",
    "vocab_size": 8000,
    "fertility_overall": 1.7155041180862605,
    "fertility_ar": 1.5432842580927282,
    "fertility_az": 2.037850131276259,
    "cpt_overall": 3.2675384671739276,
    "cpt_ar": 3.6389791874263038,
    "cpt_az": 2.5723083391604016,
    "fertility_disparity": 0.24269001218151182,
    "cpt_disparity": 0.2931236463103581,
    "oov_rate": 0.0,
    "vocab_gini": 0.7852085662214892,
    "shannon_entropy": 10.17036760209476,
    "exact_match_ar": 0.9994106980961015,
    "exact_match_az": 0.9994909214322077
  },
  {
    "name": "concat_bbpe_8000",
    "tokenizer_type": "concatenated",
    "algorithm": "bbpe",
    "vocab_size": 8000,
    "fertility_overall": 1.7695328842122233,
    "fertility_ar": 1.733122664722859,
    "fertility_az": 1.8376823362685084,
    "cpt_overall": 3.130408968636395,
    "cpt_ar": 3.268439481034545,
    "cpt_az": 2.872055574482044,
    "fertility_disparity": 0.056897576627940066,
    "cpt_disparity": 0.12127619582756817,
    "oov_rate": 0.0,
    "vocab_gini": 0.7652784949485903,
    "shannon_entropy": 10.232825855115562,
    "exact_match_ar": 0.999909338168631,
    "exact_match_az": 0.9997454607161038
  },
  {
    "name": "shared_bpe_16000",
    "tokenizer_type": "shared",
    "algorithm": "bpe",
    "vocab_size": 16000,
    "fertility_overall": 1.406466326610023,
    "fertility_ar": 1.3322457895485136,
    "fertility_az": 1.545385811386613,
    "cpt_overall": 3.9381818445330614,
    "cpt_ar": 4.211736441809671,
    "cpt_az": 3.426166367193676,
    "fertility_disparity": 0.13792026577936378,
    "cpt_disparity": 0.18651928615895444,
    "oov_rate": 0.0,
    "vocab_gini": 0.7705612275154623,
    "shannon_entropy": 11.22051844286477,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_bpe_16000",
    "tokenizer_type": "concatenated",
    "algorithm": "bpe",
    "vocab_size": 16000,
    "fertility_overall": 1.4427306790554728,
    "fertility_ar": 1.4863282848922221,
    "fertility_az": 1.3611285082800872,
    "cpt_overall": 3.843167233342444,
    "cpt_ar": 3.806560946253488,
    "cpt_az": 3.9116836675170874,
    "fertility_disparity": 0.08423426902705648,
    "cpt_disparity": 0.02687403435419539,
    "oov_rate": 0.0,
    "vocab_gini": 0.7502784093339115,
    "shannon_entropy": 11.289933167496804,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  },
  {
    "name": "shared_unigram_16000",
    "tokenizer_type": "shared",
    "algorithm": "unigram",
    "vocab_size": 16000,
    "fertility_overall": 1.43915848471923,
    "fertility_ar": 1.383798509647741,
    "fertility_az": 1.542776425502961,
    "cpt_overall": 3.856083364492832,
    "cpt_ar": 4.074326867754011,
    "cpt_az": 3.4475943366681565,
    "fertility_disparity": 0.10304663282846806,
    "cpt_disparity": 0.1538248038089647,
    "oov_rate": 0.0,
    "vocab_gini": 0.8202410337846489,
    "shannon_entropy": 10.661726018051654,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_unigram_16000",
    "tokenizer_type": "concatenated",
    "algorithm": "unigram",
    "vocab_size": 16000,
    "fertility_overall": 1.4690109341605053,
    "fertility_ar": 1.5133852390561142,
    "fertility_az": 1.3859550062802124,
    "cpt_overall": 3.7990700186667543,
    "cpt_ar": 3.7636569223952008,
    "cpt_az": 3.865353143030446,
    "fertility_disparity": 0.08420211158883709,
    "cpt_disparity": 0.026309684231209756,
    "oov_rate": 0.0,
    "vocab_gini": 0.8131049651034313,
    "shannon_entropy": 10.680674921480614,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  },
  {
    "name": "shared_wordpiece_16000",
    "tokenizer_type": "shared",
    "algorithm": "wordpiece",
    "vocab_size": 16000,
    "fertility_overall": 1.4017223613822598,
    "fertility_ar": 1.3278822099406427,
    "fertility_az": 1.5399298737530456,
    "cpt_overall": 3.949283003677344,
    "cpt_ar": 4.223103615213092,
    "cpt_az": 3.43676962420351,
    "fertility_disparity": 0.1376995585491242,
    "cpt_disparity": 0.18619812883040157,
    "oov_rate": 0.0,
    "vocab_gini": 0.7570477677642601,
    "shannon_entropy": 11.225647699841687,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "concat_wordpiece_16000",
    "tokenizer_type": "concatenated",
    "algorithm": "wordpiece",
    "vocab_size": 16000,
    "fertility_overall": 1.441782460775622,
    "fertility_ar": 1.4835885687430344,
    "fertility_az": 1.3635334584201908,
    "cpt_overall": 3.8430675831500807,
    "cpt_ar": 3.81082194270718,
    "cpt_az": 3.9034221417934196,
    "fertility_disparity": 0.08092210526032824,
    "cpt_disparity": 0.0237228246708911,
    "oov_rate": 0.0,
    "vocab_gini": 0.7327455145352297,
    "shannon_entropy": 11.280193007623335,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "shared_bbpe_16000",
    "tokenizer_type": "shared",
    "algorithm": "bbpe",
    "vocab_size": 16000,
    "fertility_overall": 1.563251758755881,
    "fertility_ar": 1.407745203379972,
    "fertility_az": 1.8543152757754426,
    "cpt_overall": 3.5796704536935944,
    "cpt_ar": 3.9802348152214964,
    "cpt_az": 2.8299292509695566,
    "fertility_disparity": 0.24082747860054313,
    "cpt_disparity": 0.2890044476403401,
    "oov_rate": 0.0,
    "vocab_gini": 0.8152607254871892,
    "shannon_entropy": 10.485685853996797,
    "exact_match_ar": 0.9994106980961015,
    "exact_match_az": 0.9994909214322077
  },
  {
    "name": "concat_bbpe_16000",
    "tokenizer_type": "concatenated",
    "algorithm": "bbpe",
    "vocab_size": 16000,
    "fertility_overall": 1.6044565681716603,
    "fertility_ar": 1.5508741409303313,
    "fertility_az": 1.7047474509939684,
    "cpt_overall": 3.4578261987966146,
    "cpt_ar": 3.653251881231358,
    "cpt_az": 3.0920455646111016,
    "fertility_disparity": 0.09026164548532975,
    "cpt_disparity": 0.15361829265139457,
    "oov_rate": 0.0,
    "vocab_gini": 0.8039903620272112,
    "shannon_entropy": 10.550324007405273,
    "exact_match_ar": 0.999909338168631,
    "exact_match_az": 0.9997454607161038
  },
  {
    "name": "shared_bpe_32000",
    "tokenizer_type": "shared",
    "algorithm": "bpe",
    "vocab_size": 32000,
    "fertility_overall": 1.2796960189731112,
    "fertility_ar": 1.2326026481342502,
    "fertility_az": 1.3678412557544855,
    "cpt_overall": 4.304399960106754,
    "cpt_ar": 4.537455476649384,
    "cpt_az": 3.8681871063030555,
    "fertility_disparity": 0.09887010429849855,
    "cpt_disparity": 0.1474986088107117,
    "oov_rate": 0.0,
    "vocab_gini": 0.7839455718558345,
    "shannon_entropy": 11.593993601001019,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_bpe_32000",
    "tokenizer_type": "concatenated",
    "algorithm": "bpe",
    "vocab_size": 32000,
    "fertility_overall": 1.2695706222898995,
    "fertility_ar": 1.3077466165859795,
    "fertility_az": 1.1981161479838143,
    "cpt_overall": 4.4229147884371125,
    "cpt_ar": 4.388664556275568,
    "cpt_az": 4.487021365858093,
    "fertility_disparity": 0.08383158267185423,
    "cpt_disparity": 0.021920290001497422,
    "oov_rate": 0.0,
    "vocab_gini": 0.7084770707714895,
    "shannon_entropy": 12.560099600380696,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "shared_unigram_32000",
    "tokenizer_type": "shared",
    "algorithm": "unigram",
    "vocab_size": 32000,
    "fertility_overall": 1.3340159750699838,
    "fertility_ar": 1.3027084505800648,
    "fertility_az": 1.3926146506382522,
    "cpt_overall": 4.136639454094978,
    "cpt_ar": 4.308287618942264,
    "cpt_az": 3.8153634896854167,
    "fertility_disparity": 0.06455928064305677,
    "cpt_disparity": 0.11441300415729108,
    "oov_rate": 0.0,
    "vocab_gini": 0.8250969225299254,
    "shannon_entropy": 10.965056609467814,
    "exact_match_ar": 0.9990480507706255,
    "exact_match_az": 0.9992363821483116
  },
  {
    "name": "concat_unigram_32000",
    "tokenizer_type": "concatenated",
    "algorithm": "unigram",
    "vocab_size": 32000,
    "fertility_overall": 1.3498732692189985,
    "fertility_ar": 1.392269196943319,
    "fertility_az": 1.2705202940282205,
    "cpt_overall": 4.117905456086874,
    "cpt_ar": 4.071549633983907,
    "cpt_az": 4.204670213900507,
    "fertility_disparity": 0.08744638119007023,
    "cpt_disparity": 0.03166017146279571,
    "oov_rate": 0.0,
    "vocab_gini": 0.8281405210808994,
    "shannon_entropy": 11.002763195930038,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  },
  {
    "name": "shared_wordpiece_32000",
    "tokenizer_type": "shared",
    "algorithm": "wordpiece",
    "vocab_size": 32000,
    "fertility_overall": 1.2742505256521945,
    "fertility_ar": 1.2274699882356,
    "fertility_az": 1.3618102283002589,
    "cpt_overall": 4.319881028657028,
    "cpt_ar": 4.553239284995183,
    "cpt_az": 3.883101533084343,
    "fertility_disparity": 0.0986482824646834,
    "cpt_disparity": 0.1471782416793298,
    "oov_rate": 0.0,
    "vocab_gini": 0.7747440551637088,
    "shannon_entropy": 11.589595542112136,
    "exact_match_ar": 0.0,
    "exact_match_az": 0.0
  },
  {
    "name": "concat_wordpiece_32000",
    "tokenizer_type": "concatenated",
    "algorithm": "wordpiece",
    "vocab_size": 32000,
    "fertility_overall": 1.303008786024938,
    "fertility_ar": 1.3416327230816638,
    "fertility_az": 1.2307158917884393,
    "cpt_overall": 4.240397119793184,
    "cpt_ar": 4.201360994624003,
    "cpt_az": 4.313461511548837,
    "fertility_disparity": 0.08267302174804893,
    "cpt_disparity": 0.025988528383688383,
    "oov_rate": 0.0,
    "vocab_gini": 0.7687350341879174,
    "shannon_entropy": 11.65382526739417,
    "exact_match_ar": 0.001042611060743427,
    "exact_match_az": 0.00033938571186153065
  },
  {
    "name": "shared_bbpe_32000",
    "tokenizer_type": "shared",
    "algorithm": "bbpe",
    "vocab_size": 32000,
    "fertility_overall": 1.4537480484685892,
    "fertility_ar": 1.317581832517785,
    "fertility_az": 1.7086120162163183,
    "cpt_overall": 3.834237272964042,
    "cpt_ar": 4.240919303648946,
    "cpt_az": 3.0730455542376744,
    "fertility_disparity": 0.22885838328847793,
    "cpt_disparity": 0.2753822145133525,
    "oov_rate": 0.0,
    "vocab_gini": 0.8293407455122993,
    "shannon_entropy": 10.685544129991571,
    "exact_match_ar": 0.9994106980961015,
    "exact_match_az": 0.9994909214322077
  },
  {
    "name": "concat_bbpe_32000",
    "tokenizer_type": "concatenated",
    "algorithm": "bbpe",
    "vocab_size": 32000,
    "fertility_overall": 1.306778190060823,
    "fertility_ar": 1.3463628334026025,
    "fertility_az": 1.2326871301490916,
    "cpt_overall": 4.231787559188005,
    "cpt_ar": 4.190301018162676,
    "cpt_az": 4.3094384241989285,
    "fertility_disparity": 0.08443170030639024,
    "cpt_disparity": 0.02764569169088408,
    "oov_rate": 0.0,
    "vocab_gini": 0.7818011980546499,
    "shannon_entropy": 11.6730644846771,
    "exact_match_ar": 0.9989120580235721,
    "exact_match_az": 0.9964364500254539
  }
]