| [ |
| { |
| "name":"shared_bpe_8000", |
| "tokenizer_type":"shared", |
| "algorithm":"BPE", |
| "vocab_size":8000, |
| "fertility_overall":1.7787163674, |
| "cpt_overall":3.0049111942, |
| "fertility_disparity":0.2198677878, |
| "cpt_disparity":0.645823621, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7709496335, |
| "shannon_entropy":10.5561176463, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1755236174, |
| "morph_consistency_precision":0.1706382107, |
| "morph_consistency_recall":0.1556559587, |
| "morph_consistency_f1":0.161694338, |
| "ar_fertility":1.656426646, |
| "ar_cpt":3.3838216268, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":17.6595461798, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.8762944338, |
| "az_cpt":2.7379980058, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":25.9109200496, |
| "az_median_seq_len":26.0 |
| }, |
| { |
| "name":"concat_bpe_8000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BPE", |
| "vocab_size":8000, |
| "fertility_overall":1.6889873506, |
| "cpt_overall":3.1645498835, |
| "fertility_disparity":0.1648715075, |
| "cpt_disparity":0.0316881659, |
| "oov_disparity":0.0, |
| "vocab_gini":0.6849702921, |
| "shannon_entropy":10.9244878509, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.2157843708, |
| "morph_consistency_precision":0.1569809449, |
| "morph_consistency_recall":0.1856069998, |
| "morph_consistency_f1":0.1679847615, |
| "ar_fertility":1.7806883265, |
| "ar_cpt":3.1476885788, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":19.1354369793, |
| "ar_median_seq_len":19.0, |
| "az_fertility":1.615816819, |
| "az_cpt":3.1793767446, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.036429711, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"shared_unigram_8000", |
| "tokenizer_type":"shared", |
| "algorithm":"Unigram", |
| "vocab_size":8000, |
| "fertility_overall":1.7790411363, |
| "cpt_overall":3.0043626393, |
| "fertility_disparity":0.1474708507, |
| "cpt_disparity":0.5176722057, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8189437499, |
| "shannon_entropy":10.165482266, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1823029664, |
| "morph_consistency_precision":0.1498923892, |
| "morph_consistency_recall":0.2196576099, |
| "morph_consistency_f1":0.1770370715, |
| "ar_fertility":1.6970183469, |
| "ar_cpt":3.3028825634, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":18.1438574721, |
| "ar_median_seq_len":18.0, |
| "az_fertility":1.8444891977, |
| "az_cpt":2.7852103577, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":25.4391065414, |
| "az_median_seq_len":25.0 |
| }, |
| { |
| "name":"concat_unigram_8000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"Unigram", |
| "vocab_size":8000, |
| "fertility_overall":1.6947867958, |
| "cpt_overall":3.1537210089, |
| "fertility_disparity":0.1669302238, |
| "cpt_disparity":0.0343315623, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7623152124, |
| "shannon_entropy":10.4233732784, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.2111496914, |
| "morph_consistency_precision":0.1450557842, |
| "morph_consistency_recall":0.2404900444, |
| "morph_consistency_f1":0.1800784056, |
| "ar_fertility":1.7876328228, |
| "ar_cpt":3.1354606137, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":19.2194646339, |
| "ar_median_seq_len":19.0, |
| "az_fertility":1.620702599, |
| "az_cpt":3.169792176, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.1099982273, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"shared_wordpiece_8000", |
| "tokenizer_type":"shared", |
| "algorithm":"WordPiece", |
| "vocab_size":8000, |
| "fertility_overall":1.7761712394, |
| "cpt_overall":3.0092170198, |
| "fertility_disparity":0.226757188, |
| "cpt_disparity":0.6596486988, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7564900789, |
| "shannon_entropy":10.5708706321, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1765983213, |
| "morph_consistency_precision":0.163672929, |
| "morph_consistency_recall":0.1493781435, |
| "morph_consistency_f1":0.1551907022, |
| "ar_fertility":1.6500496569, |
| "ar_cpt":3.3968991687, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":17.5850912959, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.8768068449, |
| "az_cpt":2.7372504699, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":25.9190746322, |
| "az_median_seq_len":25.0 |
| }, |
| { |
| "name":"concat_wordpiece_8000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"WordPiece", |
| "vocab_size":8000, |
| "fertility_overall":1.6962184303, |
| "cpt_overall":3.1510592198, |
| "fertility_disparity":0.1560458869, |
| "cpt_disparity":0.0140028965, |
| "oov_disparity":0.0, |
| "vocab_gini":0.6600277207, |
| "shannon_entropy":10.9155665053, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.219108083, |
| "morph_consistency_precision":0.1522420473, |
| "morph_consistency_recall":0.175816502, |
| "morph_consistency_f1":0.1618608459, |
| "ar_fertility":1.7830106258, |
| "ar_cpt":3.1435888416, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":19.163623471, |
| "ar_median_seq_len":19.0, |
| "az_fertility":1.626964739, |
| "az_cpt":3.1575917381, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.2024463748, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"shared_bbpe_8000", |
| "tokenizer_type":"shared", |
| "algorithm":"BBPE", |
| "vocab_size":8000, |
| "fertility_overall":1.9554271209, |
| "cpt_overall":2.7333592066, |
| "fertility_disparity":0.4718091571, |
| "cpt_disparity":0.9376231111, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8121558079, |
| "shannon_entropy":9.9528287353, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1921228413, |
| "morph_consistency_precision":0.1550132007, |
| "morph_consistency_recall":0.2827125976, |
| "morph_consistency_f1":0.1995081577, |
| "ar_fertility":1.6930084603, |
| "ar_cpt":3.310705433, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":18.0962595285, |
| "ar_median_seq_len":18.0, |
| "az_fertility":2.1648176174, |
| "az_cpt":2.3730823219, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":30.2043963836, |
| "az_median_seq_len":30.0 |
| }, |
| { |
| "name":"concat_bbpe_8000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BBPE", |
| "vocab_size":8000, |
| "fertility_overall":1.8827749848, |
| "cpt_overall":2.8388335126, |
| "fertility_disparity":0.116792759, |
| "cpt_disparity":0.4279320309, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7491107045, |
| "shannon_entropy":10.2263454675, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.2316542111, |
| "morph_consistency_precision":0.1588581248, |
| "morph_consistency_recall":0.3282934754, |
| "morph_consistency_f1":0.2135590301, |
| "ar_fertility":1.8178152465, |
| "ar_cpt":3.0834004272, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":19.5777344442, |
| "ar_median_seq_len":19.0, |
| "az_fertility":1.9346080055, |
| "az_cpt":2.6554683963, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":26.7797376352, |
| "az_median_seq_len":26.0 |
| }, |
| { |
| "name":"shared_bpe_16000", |
| "tokenizer_type":"shared", |
| "algorithm":"BPE", |
| "vocab_size":16000, |
| "fertility_overall":1.5971771615, |
| "cpt_overall":3.3464570197, |
| "fertility_disparity":0.1510951681, |
| "cpt_disparity":0.6173754613, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7920106017, |
| "shannon_entropy":11.090783206, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1164562127, |
| "morph_consistency_precision":0.1645828627, |
| "morph_consistency_recall":0.111253923, |
| "morph_consistency_f1":0.1323611335, |
| "ar_fertility":1.513138539, |
| "ar_cpt":3.7042558651, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.9586952668, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.6642337071, |
| "az_cpt":3.0868804038, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.756248892, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"concat_bpe_16000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BPE", |
| "vocab_size":16000, |
| "fertility_overall":1.5165781285, |
| "cpt_overall":3.5243055556, |
| "fertility_disparity":0.1426774709, |
| "cpt_disparity":0.0229367001, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7224875042, |
| "shannon_entropy":11.453646644, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1531217, |
| "morph_consistency_precision":0.1543768971, |
| "morph_consistency_recall":0.1249030296, |
| "morph_consistency_f1":0.1376491323, |
| "ar_fertility":1.5959348561, |
| "ar_cpt":3.5120808886, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":16.9423861018, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.4532573853, |
| "az_cpt":3.5350175887, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":19.6181528098, |
| "az_median_seq_len":19.0 |
| }, |
| { |
| "name":"shared_unigram_16000", |
| "tokenizer_type":"shared", |
| "algorithm":"Unigram", |
| "vocab_size":16000, |
| "fertility_overall":1.6212862838, |
| "cpt_overall":3.296693975, |
| "fertility_disparity":0.0885280481, |
| "cpt_disparity":0.4717666206, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8334856469, |
| "shannon_entropy":10.5776002094, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1315210046, |
| "morph_consistency_precision":0.1557524276, |
| "morph_consistency_recall":0.1706022629, |
| "morph_consistency_f1":0.1622742892, |
| "ar_fertility":1.5720472823, |
| "ar_cpt":3.5654476623, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":16.6604325474, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.6605753304, |
| "az_cpt":3.0936810417, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.7031554689, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"concat_unigram_16000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"Unigram", |
| "vocab_size":16000, |
| "fertility_overall":1.541731151, |
| "cpt_overall":3.4668072447, |
| "fertility_disparity":0.1464437495, |
| "cpt_disparity":0.0256827057, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7918225207, |
| "shannon_entropy":10.8508668291, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1582156543, |
| "morph_consistency_precision":0.1464142753, |
| "morph_consistency_recall":0.1741476553, |
| "morph_consistency_f1":0.1574918159, |
| "ar_fertility":1.6231826701, |
| "ar_cpt":3.4531247844, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":17.2674171246, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.4767389206, |
| "az_cpt":3.4788074901, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":19.9683566743, |
| "az_median_seq_len":20.0 |
| }, |
| { |
| "name":"shared_wordpiece_16000", |
| "tokenizer_type":"shared", |
| "algorithm":"WordPiece", |
| "vocab_size":16000, |
| "fertility_overall":1.5914307397, |
| "cpt_overall":3.3585405825, |
| "fertility_disparity":0.1511682691, |
| "cpt_disparity":0.6209619713, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7797479401, |
| "shannon_entropy":11.1061302427, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1158134243, |
| "morph_consistency_precision":0.1653640001, |
| "morph_consistency_recall":0.1209851544, |
| "morph_consistency_f1":0.1385333865, |
| "ar_fertility":1.5073514587, |
| "ar_cpt":3.7184773832, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.8912426875, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.6585197278, |
| "az_cpt":3.0975154119, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":22.6717780535, |
| "az_median_seq_len":22.0 |
| }, |
| { |
| "name":"concat_wordpiece_16000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"WordPiece", |
| "vocab_size":16000, |
| "fertility_overall":1.5161705103, |
| "cpt_overall":3.5252530551, |
| "fertility_disparity":0.1360666065, |
| "cpt_disparity":0.0077904651, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7024364505, |
| "shannon_entropy":11.4443519139, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1506488459, |
| "morph_consistency_precision":0.1740477945, |
| "morph_consistency_recall":0.1384490501, |
| "morph_consistency_f1":0.1533202072, |
| "ar_fertility":1.5918502976, |
| "ar_cpt":3.5210926029, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":16.8945222478, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.455783691, |
| "az_cpt":3.528883068, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":19.6559120723, |
| "az_median_seq_len":19.0 |
| }, |
| { |
| "name":"shared_bbpe_16000", |
| "tokenizer_type":"shared", |
| "algorithm":"BBPE", |
| "vocab_size":16000, |
| "fertility_overall":1.7936656802, |
| "cpt_overall":2.9798667514, |
| "fertility_disparity":0.4264991353, |
| "cpt_disparity":1.0104466245, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8352512331, |
| "shannon_entropy":10.2895041219, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1365421097, |
| "morph_consistency_precision":0.154622604, |
| "morph_consistency_recall":0.2260888365, |
| "morph_consistency_f1":0.1823875169, |
| "ar_fertility":1.5564483008, |
| "ar_cpt":3.6011811666, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":16.4752703421, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.9829474362, |
| "az_cpt":2.590734542, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":27.498847722, |
| "az_median_seq_len":27.0 |
| }, |
| { |
| "name":"concat_bbpe_16000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BBPE", |
| "vocab_size":16000, |
| "fertility_overall":1.7340440692, |
| "cpt_overall":3.0823234649, |
| "fertility_disparity":0.1722144964, |
| "cpt_disparity":0.5838072031, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7908187337, |
| "shannon_entropy":10.5217570863, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1719709073, |
| "morph_consistency_precision":0.1620182352, |
| "morph_consistency_recall":0.2599866609, |
| "morph_consistency_f1":0.1986209129, |
| "ar_fertility":1.6382589476, |
| "ar_cpt":3.4213469831, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":17.4463747562, |
| "ar_median_seq_len":17.0, |
| "az_fertility":1.810473444, |
| "az_cpt":2.83753978, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":24.9330792413, |
| "az_median_seq_len":24.0 |
| }, |
| { |
| "name":"shared_bpe_32000", |
| "tokenizer_type":"shared", |
| "algorithm":"BPE", |
| "vocab_size":32000, |
| "fertility_overall":1.4562440141, |
| "cpt_overall":3.670322193, |
| "fertility_disparity":0.0726954505, |
| "cpt_disparity":0.507593217, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7969114377, |
| "shannon_entropy":11.521256996, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.0705763007, |
| "morph_consistency_precision":0.1610095066, |
| "morph_consistency_recall":0.0899012584, |
| "morph_consistency_f1":0.1149172148, |
| "ar_fertility":1.41581105, |
| "ar_cpt":3.9588985464, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":14.8034036518, |
| "ar_median_seq_len":14.0, |
| "az_fertility":1.4885065005, |
| "az_cpt":3.4513053294, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":20.1420847367, |
| "az_median_seq_len":20.0 |
| }, |
| { |
| "name":"concat_bpe_32000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BPE", |
| "vocab_size":32000, |
| "fertility_overall":1.3911576687, |
| "cpt_overall":3.8420409447, |
| "fertility_disparity":0.134007069, |
| "cpt_disparity":0.0335692487, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7567616479, |
| "shannon_entropy":11.8182442917, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.0970004442, |
| "morph_consistency_precision":0.148728366, |
| "morph_consistency_recall":0.0942436549, |
| "morph_consistency_f1":0.1146174635, |
| "ar_fertility":1.4656919481, |
| "ar_cpt":3.8241680422, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.3963836199, |
| "ar_median_seq_len":15.0, |
| "az_fertility":1.3316848791, |
| "az_cpt":3.8577372909, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":17.8096082255, |
| "az_median_seq_len":17.0 |
| }, |
| { |
| "name":"shared_unigram_32000", |
| "tokenizer_type":"shared", |
| "algorithm":"Unigram", |
| "vocab_size":32000, |
| "fertility_overall":1.5033984749, |
| "cpt_overall":3.55520164, |
| "fertility_disparity":0.019300727, |
| "cpt_disparity":0.3573083439, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8324888295, |
| "shannon_entropy":10.9257151273, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.0979264521, |
| "morph_consistency_precision":0.1631020109, |
| "morph_consistency_recall":0.1383463815, |
| "morph_consistency_f1":0.1482409685, |
| "ar_fertility":1.4926634757, |
| "ar_cpt":3.7550676352, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.7181350824, |
| "ar_median_seq_len":15.0, |
| "az_fertility":1.5119642027, |
| "az_cpt":3.3977592913, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":20.4923772381, |
| "az_median_seq_len":20.0 |
| }, |
| { |
| "name":"concat_unigram_32000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"Unigram", |
| "vocab_size":32000, |
| "fertility_overall":1.4304050001, |
| "cpt_overall":3.7366233501, |
| "fertility_disparity":0.1449358921, |
| "cpt_disparity":0.0511471793, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8069023024, |
| "shannon_entropy":11.1617375452, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1111640248, |
| "morph_consistency_precision":0.1486025565, |
| "morph_consistency_recall":0.1393090166, |
| "morph_consistency_f1":0.1420302247, |
| "ar_fertility":1.5110178541, |
| "ar_cpt":3.709454718, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.9360042546, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.3660819619, |
| "az_cpt":3.7606018973, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":18.3221946463, |
| "az_median_seq_len":18.0 |
| }, |
| { |
| "name":"shared_wordpiece_32000", |
| "tokenizer_type":"shared", |
| "algorithm":"WordPiece", |
| "vocab_size":32000, |
| "fertility_overall":1.4501065441, |
| "cpt_overall":3.6858565637, |
| "fertility_disparity":0.0723742616, |
| "cpt_disparity":0.5097029774, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7885550388, |
| "shannon_entropy":11.5290013722, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.0695593122, |
| "morph_consistency_precision":0.1813894046, |
| "morph_consistency_recall":0.0999073414, |
| "morph_consistency_f1":0.1273030756, |
| "ar_fertility":1.4098522241, |
| "ar_cpt":3.975631071, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":14.7339124269, |
| "ar_median_seq_len":14.0, |
| "az_fertility":1.4822264857, |
| "az_cpt":3.4659280936, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":20.0491934054, |
| "az_median_seq_len":20.0 |
| }, |
| { |
| "name":"concat_wordpiece_32000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"WordPiece", |
| "vocab_size":32000, |
| "fertility_overall":1.3885462613, |
| "cpt_overall":3.8492665836, |
| "fertility_disparity":0.1308885617, |
| "cpt_disparity":0.0257554455, |
| "oov_disparity":0.0, |
| "vocab_gini":0.7418783914, |
| "shannon_entropy":11.7951971085, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.097541522, |
| "morph_consistency_precision":0.1635636627, |
| "morph_consistency_recall":0.112578089, |
| "morph_consistency_f1":0.1328802305, |
| "ar_fertility":1.4613460375, |
| "ar_cpt":3.8355407712, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.3454174792, |
| "ar_median_seq_len":15.0, |
| "az_fertility":1.3304574758, |
| "az_cpt":3.8612962167, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":17.7915263251, |
| "az_median_seq_len":17.0 |
| }, |
| { |
| "name":"shared_bbpe_32000", |
| "tokenizer_type":"shared", |
| "algorithm":"BBPE", |
| "vocab_size":32000, |
| "fertility_overall":1.673696699, |
| "cpt_overall":3.1934607548, |
| "fertility_disparity":0.3736950757, |
| "cpt_disparity":1.0310609307, |
| "oov_disparity":0.0, |
| "vocab_gini":0.843090999, |
| "shannon_entropy":10.5235434613, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.0973786272, |
| "morph_consistency_precision":0.157092523, |
| "morph_consistency_recall":0.1976111484, |
| "morph_consistency_f1":0.1737146088, |
| "ar_fertility":1.4658487593, |
| "ar_cpt":3.8237589465, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.3998404538, |
| "ar_median_seq_len":15.0, |
| "az_fertility":1.839543835, |
| "az_cpt":2.7926980158, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":25.3655380252, |
| "az_median_seq_len":25.0 |
| }, |
| { |
| "name":"concat_bbpe_32000", |
| "tokenizer_type":"concatenated", |
| "algorithm":"BBPE", |
| "vocab_size":32000, |
| "fertility_overall":1.6299390561, |
| "cpt_overall":3.2791929879, |
| "fertility_disparity":0.2066556668, |
| "cpt_disparity":0.7157815641, |
| "oov_disparity":0.0, |
| "vocab_gini":0.8217315637, |
| "shannon_entropy":10.6841474757, |
| "exact_match_rate":0.99, |
| "morph_edit_distance_ar":4.1215903942, |
| "morph_consistency_precision":0.1574322292, |
| "morph_consistency_recall":0.2143887572, |
| "morph_consistency_f1":0.1797621337, |
| "ar_fertility":1.5149978718, |
| "ar_cpt":3.6997096906, |
| "ar_oov_rate":0.0, |
| "ar_mean_seq_len":15.9832476511, |
| "ar_median_seq_len":16.0, |
| "az_fertility":1.7216535386, |
| "az_cpt":2.9839281265, |
| "az_oov_rate":0.0, |
| "az_mean_seq_len":23.6117709626, |
| "az_median_seq_len":23.0 |
| } |
| ] |
