| [ |
| { |
| "name": "shared_bpe_8000", |
| "tokenizer_type": "shared", |
| "algorithm": "bpe", |
| "vocab_size": 8000, |
| "fertility_overall": 1.5744684296670828, |
| "fertility_ar": 1.4755656463463651, |
| "fertility_az": 1.7595859759130557, |
| "cpt_overall": 3.5313313486070697, |
| "cpt_ar": 3.812953050264278, |
| "cpt_az": 3.0042165735724513, |
| "fertility_disparity": 0.16141315823985886, |
| "cpt_disparity": 0.21210239571026782, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7419718947527658, |
| "shannon_entropy": 10.725417379440934, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_bpe_8000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bpe", |
| "vocab_size": 8000, |
| "fertility_overall": 1.625089409755206, |
| "fertility_ar": 1.677255929872786, |
| "fertility_az": 1.5274486975717831, |
| "cpt_overall": 3.4121822358457132, |
| "cpt_ar": 3.3745329363977343, |
| "cpt_az": 3.4826508889784487, |
| "fertility_disparity": 0.0893168595399542, |
| "cpt_disparity": 0.031044728865266227, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7155357357913302, |
| "shannon_entropy": 10.758976129080413, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| }, |
| { |
| "name": "shared_unigram_8000", |
| "tokenizer_type": "shared", |
| "algorithm": "unigram", |
| "vocab_size": 8000, |
| "fertility_overall": 1.5860234519252308, |
| "fertility_ar": 1.5098217608254827, |
| "fertility_az": 1.7286510868870875, |
| "cpt_overall": 3.513922137310213, |
| "cpt_ar": 3.7495598617218993, |
| "cpt_az": 3.072876133532698, |
| "fertility_disparity": 0.12658964421540223, |
| "cpt_disparity": 0.18047017600579116, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8011276471980863, |
| "shannon_entropy": 10.278366552533573, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_unigram_8000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "unigram", |
| "vocab_size": 8000, |
| "fertility_overall": 1.630557495524919, |
| "fertility_ar": 1.6828221737842166, |
| "fertility_az": 1.5327330595500248, |
| "cpt_overall": 3.4256065103693167, |
| "cpt_ar": 3.3890978047599254, |
| "cpt_az": 3.4939403000980764, |
| "fertility_disparity": 0.08918893307465849, |
| "cpt_disparity": 0.030006950987459077, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7843741193591066, |
| "shannon_entropy": 10.258501048163726, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| }, |
| { |
| "name": "shared_wordpiece_8000", |
| "tokenizer_type": "shared", |
| "algorithm": "wordpiece", |
| "vocab_size": 8000, |
| "fertility_overall": 1.5720141231575526, |
| "fertility_ar": 1.4715587661386824, |
| "fertility_az": 1.7600376405371791, |
| "cpt_overall": 3.5349251084819273, |
| "cpt_ar": 3.819823570325556, |
| "cpt_az": 3.001677181426908, |
| "fertility_disparity": 0.16390494598197936, |
| "cpt_disparity": 0.21418433962616734, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7254354187229622, |
| "shannon_entropy": 10.734803471006908, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "concat_wordpiece_8000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "wordpiece", |
| "vocab_size": 8000, |
| "fertility_overall": 1.6305196797375283, |
| "fertility_ar": 1.679880351462672, |
| "fertility_az": 1.5381307082241507, |
| "cpt_overall": 3.397665609111008, |
| "cpt_ar": 3.366673383613362, |
| "cpt_az": 3.4556741357085015, |
| "fertility_disparity": 0.08438079718897826, |
| "cpt_disparity": 0.02575496085567456, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.6940886180639433, |
| "shannon_entropy": 10.748321790799139, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "shared_bbpe_8000", |
| "tokenizer_type": "shared", |
| "algorithm": "bbpe", |
| "vocab_size": 8000, |
| "fertility_overall": 1.7155041180862605, |
| "fertility_ar": 1.5432842580927282, |
| "fertility_az": 2.037850131276259, |
| "cpt_overall": 3.2675384671739276, |
| "cpt_ar": 3.6389791874263038, |
| "cpt_az": 2.5723083391604016, |
| "fertility_disparity": 0.24269001218151182, |
| "cpt_disparity": 0.2931236463103581, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7852085662214892, |
| "shannon_entropy": 10.17036760209476, |
| "exact_match_ar": 0.9994106980961015, |
| "exact_match_az": 0.9994909214322077 |
| }, |
| { |
| "name": "concat_bbpe_8000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bbpe", |
| "vocab_size": 8000, |
| "fertility_overall": 1.7695328842122233, |
| "fertility_ar": 1.733122664722859, |
| "fertility_az": 1.8376823362685084, |
| "cpt_overall": 3.130408968636395, |
| "cpt_ar": 3.268439481034545, |
| "cpt_az": 2.872055574482044, |
| "fertility_disparity": 0.056897576627940066, |
| "cpt_disparity": 0.12127619582756817, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7652784949485903, |
| "shannon_entropy": 10.232825855115562, |
| "exact_match_ar": 0.999909338168631, |
| "exact_match_az": 0.9997454607161038 |
| }, |
| { |
| "name": "shared_bpe_16000", |
| "tokenizer_type": "shared", |
| "algorithm": "bpe", |
| "vocab_size": 16000, |
| "fertility_overall": 1.406466326610023, |
| "fertility_ar": 1.3322457895485136, |
| "fertility_az": 1.545385811386613, |
| "cpt_overall": 3.9381818445330614, |
| "cpt_ar": 4.211736441809671, |
| "cpt_az": 3.426166367193676, |
| "fertility_disparity": 0.13792026577936378, |
| "cpt_disparity": 0.18651928615895444, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7705612275154623, |
| "shannon_entropy": 11.22051844286477, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_bpe_16000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bpe", |
| "vocab_size": 16000, |
| "fertility_overall": 1.4427306790554728, |
| "fertility_ar": 1.4863282848922221, |
| "fertility_az": 1.3611285082800872, |
| "cpt_overall": 3.843167233342444, |
| "cpt_ar": 3.806560946253488, |
| "cpt_az": 3.9116836675170874, |
| "fertility_disparity": 0.08423426902705648, |
| "cpt_disparity": 0.02687403435419539, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7502784093339115, |
| "shannon_entropy": 11.289933167496804, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| }, |
| { |
| "name": "shared_unigram_16000", |
| "tokenizer_type": "shared", |
| "algorithm": "unigram", |
| "vocab_size": 16000, |
| "fertility_overall": 1.43915848471923, |
| "fertility_ar": 1.383798509647741, |
| "fertility_az": 1.542776425502961, |
| "cpt_overall": 3.856083364492832, |
| "cpt_ar": 4.074326867754011, |
| "cpt_az": 3.4475943366681565, |
| "fertility_disparity": 0.10304663282846806, |
| "cpt_disparity": 0.1538248038089647, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8202410337846489, |
| "shannon_entropy": 10.661726018051654, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_unigram_16000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "unigram", |
| "vocab_size": 16000, |
| "fertility_overall": 1.4690109341605053, |
| "fertility_ar": 1.5133852390561142, |
| "fertility_az": 1.3859550062802124, |
| "cpt_overall": 3.7990700186667543, |
| "cpt_ar": 3.7636569223952008, |
| "cpt_az": 3.865353143030446, |
| "fertility_disparity": 0.08420211158883709, |
| "cpt_disparity": 0.026309684231209756, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8131049651034313, |
| "shannon_entropy": 10.680674921480614, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| }, |
| { |
| "name": "shared_wordpiece_16000", |
| "tokenizer_type": "shared", |
| "algorithm": "wordpiece", |
| "vocab_size": 16000, |
| "fertility_overall": 1.4017223613822598, |
| "fertility_ar": 1.3278822099406427, |
| "fertility_az": 1.5399298737530456, |
| "cpt_overall": 3.949283003677344, |
| "cpt_ar": 4.223103615213092, |
| "cpt_az": 3.43676962420351, |
| "fertility_disparity": 0.1376995585491242, |
| "cpt_disparity": 0.18619812883040157, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7570477677642601, |
| "shannon_entropy": 11.225647699841687, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "concat_wordpiece_16000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "wordpiece", |
| "vocab_size": 16000, |
| "fertility_overall": 1.441782460775622, |
| "fertility_ar": 1.4835885687430344, |
| "fertility_az": 1.3635334584201908, |
| "cpt_overall": 3.8430675831500807, |
| "cpt_ar": 3.81082194270718, |
| "cpt_az": 3.9034221417934196, |
| "fertility_disparity": 0.08092210526032824, |
| "cpt_disparity": 0.0237228246708911, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7327455145352297, |
| "shannon_entropy": 11.280193007623335, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "shared_bbpe_16000", |
| "tokenizer_type": "shared", |
| "algorithm": "bbpe", |
| "vocab_size": 16000, |
| "fertility_overall": 1.563251758755881, |
| "fertility_ar": 1.407745203379972, |
| "fertility_az": 1.8543152757754426, |
| "cpt_overall": 3.5796704536935944, |
| "cpt_ar": 3.9802348152214964, |
| "cpt_az": 2.8299292509695566, |
| "fertility_disparity": 0.24082747860054313, |
| "cpt_disparity": 0.2890044476403401, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8152607254871892, |
| "shannon_entropy": 10.485685853996797, |
| "exact_match_ar": 0.9994106980961015, |
| "exact_match_az": 0.9994909214322077 |
| }, |
| { |
| "name": "concat_bbpe_16000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bbpe", |
| "vocab_size": 16000, |
| "fertility_overall": 1.6044565681716603, |
| "fertility_ar": 1.5508741409303313, |
| "fertility_az": 1.7047474509939684, |
| "cpt_overall": 3.4578261987966146, |
| "cpt_ar": 3.653251881231358, |
| "cpt_az": 3.0920455646111016, |
| "fertility_disparity": 0.09026164548532975, |
| "cpt_disparity": 0.15361829265139457, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8039903620272112, |
| "shannon_entropy": 10.550324007405273, |
| "exact_match_ar": 0.999909338168631, |
| "exact_match_az": 0.9997454607161038 |
| }, |
| { |
| "name": "shared_bpe_32000", |
| "tokenizer_type": "shared", |
| "algorithm": "bpe", |
| "vocab_size": 32000, |
| "fertility_overall": 1.2796960189731112, |
| "fertility_ar": 1.2326026481342502, |
| "fertility_az": 1.3678412557544855, |
| "cpt_overall": 4.304399960106754, |
| "cpt_ar": 4.537455476649384, |
| "cpt_az": 3.8681871063030555, |
| "fertility_disparity": 0.09887010429849855, |
| "cpt_disparity": 0.1474986088107117, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7839455718558345, |
| "shannon_entropy": 11.593993601001019, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_bpe_32000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bpe", |
| "vocab_size": 32000, |
| "fertility_overall": 1.2695706222898995, |
| "fertility_ar": 1.3077466165859795, |
| "fertility_az": 1.1981161479838143, |
| "cpt_overall": 4.4229147884371125, |
| "cpt_ar": 4.388664556275568, |
| "cpt_az": 4.487021365858093, |
| "fertility_disparity": 0.08383158267185423, |
| "cpt_disparity": 0.021920290001497422, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7084770707714895, |
| "shannon_entropy": 12.560099600380696, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "shared_unigram_32000", |
| "tokenizer_type": "shared", |
| "algorithm": "unigram", |
| "vocab_size": 32000, |
| "fertility_overall": 1.3340159750699838, |
| "fertility_ar": 1.3027084505800648, |
| "fertility_az": 1.3926146506382522, |
| "cpt_overall": 4.136639454094978, |
| "cpt_ar": 4.308287618942264, |
| "cpt_az": 3.8153634896854167, |
| "fertility_disparity": 0.06455928064305677, |
| "cpt_disparity": 0.11441300415729108, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8250969225299254, |
| "shannon_entropy": 10.965056609467814, |
| "exact_match_ar": 0.9990480507706255, |
| "exact_match_az": 0.9992363821483116 |
| }, |
| { |
| "name": "concat_unigram_32000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "unigram", |
| "vocab_size": 32000, |
| "fertility_overall": 1.3498732692189985, |
| "fertility_ar": 1.392269196943319, |
| "fertility_az": 1.2705202940282205, |
| "cpt_overall": 4.117905456086874, |
| "cpt_ar": 4.071549633983907, |
| "cpt_az": 4.204670213900507, |
| "fertility_disparity": 0.08744638119007023, |
| "cpt_disparity": 0.03166017146279571, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8281405210808994, |
| "shannon_entropy": 11.002763195930038, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| }, |
| { |
| "name": "shared_wordpiece_32000", |
| "tokenizer_type": "shared", |
| "algorithm": "wordpiece", |
| "vocab_size": 32000, |
| "fertility_overall": 1.2742505256521945, |
| "fertility_ar": 1.2274699882356, |
| "fertility_az": 1.3618102283002589, |
| "cpt_overall": 4.319881028657028, |
| "cpt_ar": 4.553239284995183, |
| "cpt_az": 3.883101533084343, |
| "fertility_disparity": 0.0986482824646834, |
| "cpt_disparity": 0.1471782416793298, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7747440551637088, |
| "shannon_entropy": 11.589595542112136, |
| "exact_match_ar": 0.0, |
| "exact_match_az": 0.0 |
| }, |
| { |
| "name": "concat_wordpiece_32000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "wordpiece", |
| "vocab_size": 32000, |
| "fertility_overall": 1.303008786024938, |
| "fertility_ar": 1.3416327230816638, |
| "fertility_az": 1.2307158917884393, |
| "cpt_overall": 4.240397119793184, |
| "cpt_ar": 4.201360994624003, |
| "cpt_az": 4.313461511548837, |
| "fertility_disparity": 0.08267302174804893, |
| "cpt_disparity": 0.025988528383688383, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7687350341879174, |
| "shannon_entropy": 11.65382526739417, |
| "exact_match_ar": 0.001042611060743427, |
| "exact_match_az": 0.00033938571186153065 |
| }, |
| { |
| "name": "shared_bbpe_32000", |
| "tokenizer_type": "shared", |
| "algorithm": "bbpe", |
| "vocab_size": 32000, |
| "fertility_overall": 1.4537480484685892, |
| "fertility_ar": 1.317581832517785, |
| "fertility_az": 1.7086120162163183, |
| "cpt_overall": 3.834237272964042, |
| "cpt_ar": 4.240919303648946, |
| "cpt_az": 3.0730455542376744, |
| "fertility_disparity": 0.22885838328847793, |
| "cpt_disparity": 0.2753822145133525, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.8293407455122993, |
| "shannon_entropy": 10.685544129991571, |
| "exact_match_ar": 0.9994106980961015, |
| "exact_match_az": 0.9994909214322077 |
| }, |
| { |
| "name": "concat_bbpe_32000", |
| "tokenizer_type": "concatenated", |
| "algorithm": "bbpe", |
| "vocab_size": 32000, |
| "fertility_overall": 1.306778190060823, |
| "fertility_ar": 1.3463628334026025, |
| "fertility_az": 1.2326871301490916, |
| "cpt_overall": 4.231787559188005, |
| "cpt_ar": 4.190301018162676, |
| "cpt_az": 4.3094384241989285, |
| "fertility_disparity": 0.08443170030639024, |
| "cpt_disparity": 0.02764569169088408, |
| "oov_rate": 0.0, |
| "vocab_gini": 0.7818011980546499, |
| "shannon_entropy": 11.6730644846771, |
| "exact_match_ar": 0.9989120580235721, |
| "exact_match_az": 0.9964364500254539 |
| } |
| ] |
