| name,tokenizer_type,algorithm,vocab_size,fertility_overall,cpt_overall,fertility_disparity,cpt_disparity,oov_disparity,vocab_gini,shannon_entropy,exact_match_rate,morph_edit_distance_ar,morph_consistency_precision,morph_consistency_recall,morph_consistency_f1,ar_fertility,ar_cpt,ar_oov_rate,ar_mean_seq_len,ar_median_seq_len,az_fertility,az_cpt,az_oov_rate,az_mean_seq_len,az_median_seq_len | |
| shared_bpe_8000,shared,BPE,8000,1.7787163673600594,3.0049111942064304,0.2198677878281334,0.6458236210091002,0.0,0.7709496335029786,10.55611764627618,0.99,4.175523617379729,0.1706382106949654,0.1556559587129042,0.1616943380250088,1.6564266459576311,3.3838216267631984,0.0,17.659546179755363,17.0,1.8762944337857643,2.7379980057540982,0.0,25.91092004963659,26.0 | |
| concat_bpe_8000,concatenated,BPE,8000,1.688987350581436,3.164549883549132,0.1648715075260096,0.0316881658708809,0.0,0.6849702920748841,10.92448785086656,0.99,4.215784370758192,0.1569809449375984,0.185606999819998,0.1679847615020598,1.7806883265257356,3.1476885787610915,0.0,19.135436979259,19.0,1.615816818999726,3.1793767446319725,0.0,22.036429711044143,22.0 | |
| shared_unigram_8000,shared,Unigram,8000,1.7790411362935912,3.004362639266361,0.1474708507483997,0.517672205714006,0.0,0.8189437498677725,10.16548226597516,0.99,4.182302966402401,0.1498923892103122,0.2196576099244517,0.1770370714512047,1.6970183469111926,3.3028825633737124,0.0,18.14385747207941,18.0,1.8444891976595923,2.7852103576597065,0.0,25.43910654139337,25.0 | |
| concat_unigram_8000,concatenated,Unigram,8000,1.6947867958230738,3.153721008902889,0.1669302238291101,0.0343315622881306,0.0,0.7623152123667158,10.423373278367707,0.99,4.211149691358025,0.1450557842465286,0.2404900444290288,0.1800784055672141,1.787632822825738,3.135460613706046,0.0,19.219464633930155,19.0,1.6207025989966275,3.1697921759941767,0.0,22.10999822726467,22.0 | |
| shared_wordpiece_8000,shared,WordPiece,8000,1.7761712393911575,3.009217019768082,0.2267571879764651,0.6596486988235135,0.0,0.7564900788547064,10.570870632114326,0.99,4.176598321292855,0.1636729290438088,0.1493781435194006,0.1551907022401313,1.6500496568821452,3.396899168676716,0.0,17.58509129586953,17.0,1.8768068448586104,2.7372504698532025,0.0,25.91907463215741,25.0 | |
| concat_wordpiece_8000,concatenated,WordPiece,8000,1.696218430305581,3.151059219793724,0.1560458868577927,0.0140028965147487,0.0,0.6600277207146565,10.915566505300117,0.99,4.219108082997832,0.1522420472868043,0.1758165020159975,0.161860845924899,1.783010625826059,3.143588841564794,0.0,19.16362347101578,19.0,1.6269647389682662,3.157591738079543,0.0,22.20244637475625,22.0 | |
| shared_bbpe_8000,shared,BBPE,8000,1.9554271208571248,2.7333592065837817,0.4718091570697111,0.937623111133436,0.0,0.8121558079192106,9.952828735258391,0.99,4.192122841316389,0.1550132006575307,0.2827125975809275,0.1995081577247993,1.6930084603379656,3.3107054329896,0.0,18.096259528452403,18.0,2.1648176174076768,2.373082321856164,0.0,30.20439638361993,30.0 | |
| concat_bbpe_8000,concatenated,BBPE,8000,1.882774984838593,2.8388335125754005,0.1167927589996542,0.4279320309094849,0.0,0.7491107045054002,10.226345467462991,0.99,4.231654211116816,0.1588581247865784,0.3282934753752605,0.2135590301482825,1.817815246529619,3.08340042720999,0.0,19.577734444247472,19.0,1.934608005529273,2.655468396300505,0.0,26.77973763517107,26.0 | |
| shared_bpe_16000,shared,BPE,16000,1.5971771614532415,3.346457019667893,0.151095168147779,0.6173754612936979,0.0,0.7920106016655506,11.09078320597655,0.99,4.116456212736052,0.1645828627466545,0.1112539229561153,0.13236113351836,1.5131385389675849,3.704255865138819,0.0,15.958695266796669,16.0,1.664233707115364,3.086880403845121,0.0,22.75624889204041,22.0 | |
| concat_bpe_16000,concatenated,BPE,16000,1.516578128469311,3.524305555555556,0.1426774708708416,0.0229367001288101,0.0,0.7224875042122751,11.453646643959052,0.99,4.153121699963362,0.1543768970896132,0.1249030296230426,0.1376491322945311,1.5959348561443858,3.512080888614394,0.0,16.94238610175501,17.0,1.4532573852735442,3.5350175887432043,0.0,19.6181528097855,19.0 | |
| shared_unigram_16000,shared,Unigram,16000,1.6212862838149082,3.2966939749973427,0.0885280480735342,0.471766620605258,0.0,0.8334856469131753,10.577600209435063,0.99,4.131521004556255,0.155752427565577,0.1706022628755263,0.1622742892278042,1.5720472823124425,3.5654476622950977,0.0,16.66043254742067,16.0,1.6605753303859767,3.09368104168984,0.0,22.703155468888497,22.0 | |
| concat_unigram_16000,concatenated,Unigram,16000,1.5417311509744724,3.4668072447132765,0.1464437495213746,0.0256827057459552,0.0,0.7918225207382892,10.850866829106874,0.99,4.158215654309487,0.146414275274185,0.174147655324929,0.1574918159115514,1.623182670121491,3.453124784358827,0.0,17.267417124623293,17.0,1.4767389206001167,3.4788074901047823,0.0,19.96835667434852,20.0 | |
| shared_wordpiece_16000,shared,WordPiece,16000,1.5914307397109555,3.3585405825258894,0.1511682691087217,0.6209619712571679,0.0,0.7797479401440701,11.10613024268468,0.99,4.115813424345848,0.1653640001478121,0.120985154439956,0.1385333865232417,1.507351458717583,3.7184773831757183,0.0,15.89124268746676,16.0,1.6585197278263046,3.0975154119185504,0.0,22.67177805353661,22.0 | |
| concat_wordpiece_16000,concatenated,WordPiece,16000,1.5161705103180416,3.525253055131156,0.1360666065398934,0.0077904650518956,0.0,0.7024364504781917,11.444351913907624,0.99,4.1506488458964,0.1740477945332265,0.1384490501465639,0.1533202071555139,1.591850297567933,3.521092602929932,0.0,16.894522247828398,17.0,1.4557836910280395,3.528883067981828,0.0,19.6559120723276,19.0 | |
| shared_bbpe_16000,shared,BBPE,16000,1.7936656802086477,2.9798667514097152,0.4264991353185887,1.0104466245363932,0.0,0.8352512330927735,10.28950412191232,0.99,4.136542109674858,0.1546226040137439,0.2260888365142326,0.1823875169336714,1.5564483008385666,3.601181166581909,0.0,16.475270342137918,16.0,1.9829474361571555,2.590734542045516,0.0,27.4988477220351,27.0 | |
| concat_bbpe_16000,concatenated,BBPE,16000,1.734044069155899,3.0823234649395226,0.1722144964254499,0.5838072030895862,0.0,0.7908187337120249,10.521757086263657,0.99,4.1719709072563695,0.1620182352343743,0.2599866608795151,0.1986209129423522,1.6382589475727865,3.4213469830533194,0.0,17.446374756248893,17.0,1.8104734439982364,2.837539779963733,0.0,24.93307924126928,24.0 | |
| shared_bpe_32000,shared,BPE,32000,1.4562440141440185,3.6703221930427423,0.0726954505076658,0.5075932170496387,0.0,0.7969114377091979,11.521256996016453,0.99,4.070576300661804,0.1610095065982786,0.0899012583680101,0.1149172147859238,1.4158110499630374,3.958898546444168,0.0,14.80340365183478,14.0,1.4885065004707032,3.451305329394529,0.0,20.142084736748803,20.0 | |
| concat_bpe_32000,concatenated,BPE,32000,1.306777890060823,4.231787559188005,0.08443170030690308,0.02764569169088408,0.0,0.781806,11.673012,0.99,4.097000444205457,0.1487283660267555,0.0942436549240109,0.1146174634727862,1.3463628334026025,4.190301018162676,0.0,15.396383619925546,15.0,1.2326871301490916,4.309438424198928,0.0,17.809608225491935,17.0 | |
| shared_unigram_32000,shared,Unigram,32000,1.5033984749115998,3.555201640012785,0.0193007270410581,0.3573083438742164,0.0,0.8324888295232007,10.925715127299506,0.99,4.097926452056364,0.163102010903185,0.1383463815290126,0.1482409685423633,1.492663475683062,3.7550676351702887,0.0,15.718135082432193,15.0,1.5119642027241202,3.3977592912960723,0.0,20.492377238078355,20.0 | |
| concat_unigram_32000,concatenated,Unigram,32000,1.430405000115989,3.736623350145611,0.1449358921373891,0.051147179315421,0.0,0.8069023023999202,11.161737545172748,0.99,4.111164024763589,0.1486025564558298,0.1393090165709961,0.1420302246786408,1.5110178540759711,3.709454717969499,0.0,15.936004254564793,16.0,1.3660819619385822,3.76060189728492,0.0,18.3221946463393,18.0 | |
| shared_wordpiece_32000,shared,WordPiece,32000,1.4501065440940106,3.685856563689799,0.0723742615885816,0.5097029774009858,0.0,0.7885550388247331,11.529001372228322,0.99,4.069559312168029,0.1813894046486048,0.0999073413879291,0.1273030755908827,1.4098522241056162,3.975631071046471,0.0,14.733912426874667,14.0,1.4822264856941978,3.465928093645485,0.0,20.04919340542457,20.0 | |
| concat_wordpiece_32000,concatenated,WordPiece,32000,1.3885462613462003,3.8492665836113775,0.1308885616909267,0.0257554455085005,0.0,0.7418783914457496,11.795197108498098,0.99,4.097541522004317,0.1635636626759207,0.1125780890343642,0.1328802305480902,1.4613460375301488,3.835540771172497,0.0,15.34541747917036,15.0,1.330457475839222,3.861296216680997,0.0,17.79152632511966,17.0 | |
| shared_bbpe_32000,shared,BBPE,32000,1.6736966989557684,3.193460754826817,0.3736950756551351,1.03106093074395,0.0,0.8430909990084379,10.523543461335844,0.99,4.097378627166955,0.1570925230003272,0.1976111484132842,0.1737146087871132,1.4658487593246663,3.823758946537276,0.0,15.399840453820245,15.0,1.8395438349798008,2.7926980157933263,0.0,25.36553802517284,25.0 | |
| concat_bbpe_32000,concatenated,BBPE,32000,1.629939056115432,3.2791929879493087,0.2066556667676287,0.715781564145126,0.0,0.821731563653521,10.68414747568906,0.99,4.121590394200272,0.1574322292342262,0.2143887571653681,0.17976213372643,1.514997871847908,3.699709690615959,0.0,15.983247651125687,16.0,1.7216535386155367,2.9839281264708326,0.0,23.611770962595283,23.0 | |