| name,tokenizer_type,algorithm,vocab_size,fertility_overall,fertility_ar,fertility_az,cpt_overall,cpt_ar,cpt_az,fertility_disparity,cpt_disparity,oov_rate,vocab_gini,shannon_entropy,exact_match_ar,exact_match_az | |
| shared_bpe_8000,shared,bpe,8000,1.5744684296670828,1.4755656463463651,1.7595859759130557,3.5313313486070697,3.812953050264278,3.0042165735724513,0.16141315823985886,0.21210239571026782,0.0,0.7419718947527658,10.725417379440934,0.9990480507706255,0.9992363821483116 | |
| concat_bpe_8000,concatenated,bpe,8000,1.625089409755206,1.677255929872786,1.5274486975717831,3.4121822358457132,3.3745329363977343,3.4826508889784487,0.0893168595399542,0.031044728865266227,0.0,0.7155357357913302,10.758976129080413,0.9989120580235721,0.9964364500254539 | |
| shared_unigram_8000,shared,unigram,8000,1.5860234519252308,1.5098217608254827,1.7286510868870875,3.513922137310213,3.7495598617218993,3.072876133532698,0.12658964421540223,0.18047017600579116,0.0,0.8011276471980863,10.278366552533573,0.9990480507706255,0.9992363821483116 | |
| concat_unigram_8000,concatenated,unigram,8000,1.630557495524919,1.6828221737842166,1.5327330595500248,3.4256065103693167,3.3890978047599254,3.4939403000980764,0.08918893307465849,0.030006950987459077,0.0,0.7843741193591066,10.258501048163726,0.9989120580235721,0.9964364500254539 | |
| shared_wordpiece_8000,shared,wordpiece,8000,1.5720141231575526,1.4715587661386824,1.7600376405371791,3.5349251084819273,3.819823570325556,3.001677181426908,0.16390494598197936,0.21418433962616734,0.0,0.7254354187229622,10.734803471006908,0.9990480507706255,0.9992363821483116 | |
| concat_wordpiece_8000,concatenated,wordpiece,8000,1.6305196797375283,1.679880351462672,1.5381307082241507,3.397665609111008,3.366673383613362,3.4556741357085015,0.08438079718897826,0.02575496085567456,0.0,0.6940886180639433,10.748321790799139,0.9990480507706255,0.9992363821483116 | |
| shared_bbpe_8000,shared,bbpe,8000,1.7155041180862605,1.5432842580927282,2.037850131276259,3.2675384671739276,3.6389791874263038,2.5723083391604016,0.24269001218151182,0.2931236463103581,0.0,0.7852085662214892,10.17036760209476,0.9994106980961015,0.9994909214322077 | |
| concat_bbpe_8000,concatenated,bbpe,8000,1.7695328842122233,1.733122664722859,1.8376823362685084,3.130408968636395,3.268439481034545,2.872055574482044,0.056897576627940066,0.12127619582756817,0.0,0.7652784949485903,10.232825855115562,0.999909338168631,0.9997454607161038 | |
| shared_bpe_16000,shared,bpe,16000,1.406466326610023,1.3322457895485136,1.545385811386613,3.9381818445330614,4.211736441809671,3.426166367193676,0.13792026577936378,0.18651928615895444,0.0,0.7705612275154623,11.22051844286477,0.9990480507706255,0.9992363821483116 | |
| concat_bpe_16000,concatenated,bpe,16000,1.4427306790554728,1.4863282848922221,1.3611285082800872,3.843167233342444,3.806560946253488,3.9116836675170874,0.08423426902705648,0.02687403435419539,0.0,0.7502784093339115,11.289933167496804,0.9989120580235721,0.9964364500254539 | |
| shared_unigram_16000,shared,unigram,16000,1.43915848471923,1.383798509647741,1.542776425502961,3.856083364492832,4.074326867754011,3.4475943366681565,0.10304663282846806,0.1538248038089647,0.0,0.8202410337846489,10.661726018051654,0.9990480507706255,0.9992363821483116 | |
| concat_unigram_16000,concatenated,unigram,16000,1.4690109341605053,1.5133852390561142,1.3859550062802124,3.7990700186667543,3.7636569223952008,3.865353143030446,0.08420211158883709,0.026309684231209756,0.0,0.8131049651034313,10.680674921480614,0.9989120580235721,0.9964364500254539 | |
| shared_wordpiece_16000,shared,wordpiece,16000,1.4017223613822598,1.3278822099406427,1.5399298737530456,3.949283003677344,4.223103615213092,3.43676962420351,0.1376995585491242,0.18619812883040157,0.0,0.7570477677642601,11.225647699841687,0.9990480507706255,0.9992363821483116 | |
| concat_wordpiece_16000,concatenated,wordpiece,16000,1.441782460775622,1.4835885687430344,1.3635334584201908,3.8430675831500807,3.81082194270718,3.9034221417934196,0.08092210526032824,0.0237228246708911,0.0,0.7327455145352297,11.280193007623335,0.9990480507706255,0.9992363821483116 | |
| shared_bbpe_16000,shared,bbpe,16000,1.563251758755881,1.407745203379972,1.8543152757754426,3.5796704536935944,3.9802348152214964,2.8299292509695566,0.24082747860054313,0.2890044476403401,0.0,0.8152607254871892,10.485685853996797,0.9994106980961015,0.9994909214322077 | |
| concat_bbpe_16000,concatenated,bbpe,16000,1.6044565681716603,1.5508741409303313,1.7047474509939684,3.4578261987966146,3.653251881231358,3.0920455646111016,0.09026164548532975,0.15361829265139457,0.0,0.8039903620272112,10.550324007405273,0.999909338168631,0.9997454607161038 | |
| shared_bpe_32000,shared,bpe,32000,1.2796960189731112,1.2326026481342502,1.3678412557544855,4.304399960106754,4.537455476649384,3.8681871063030555,0.09887010429849855,0.1474986088107117,0.0,0.7839455718558345,11.593993601001019,0.9990480507706255,0.9992363821483116 | |
| concat_bpe_32000,concatenated,bpe,32000,1.306777890060823,1.3463628334026025,1.2326871301490916,4.231787559188005,4.190301018162676,4.309438424198928,0.08443170030690308,0.02764569169088408,0.0,0.781806,11.673012,0.9989120580235721,0.9964364500254539 | |
| shared_unigram_32000,shared,unigram,32000,1.3340159750699838,1.3027084505800648,1.3926146506382522,4.136639454094978,4.308287618942264,3.8153634896854167,0.06455928064305677,0.11441300415729108,0.0,0.8250969225299254,10.965056609467814,0.9990480507706255,0.9992363821483116 | |
| concat_unigram_32000,concatenated,unigram,32000,1.3498732692189985,1.392269196943319,1.2705202940282205,4.117905456086874,4.071549633983907,4.204670213900507,0.08744638119007023,0.03166017146279571,0.0,0.8281405210808994,11.002763195930038,0.9989120580235721,0.9964364500254539 | |
| shared_wordpiece_32000,shared,wordpiece,32000,1.2742505256521945,1.2274699882356,1.3618102283002589,4.319881028657028,4.553239284995183,3.883101533084343,0.0986482824646834,0.1471782416793298,0.0,0.7747440551637088,11.589595542112136,0.9990480507706255,0.9992363821483116 | |
| concat_wordpiece_32000,concatenated,wordpiece,32000,1.303008786024938,1.3416327230816638,1.2307158917884393,4.240397119793184,4.201360994624003,4.313461511548837,0.08267302174804893,0.025988528383688383,0.0,0.7687350341879174,11.65382526739417,0.9990480507706255,0.9992363821483116 | |
| shared_bbpe_32000,shared,bbpe,32000,1.4537480484685892,1.317581832517785,1.7086120162163183,3.834237272964042,4.240919303648946,3.0730455542376744,0.22885838328847793,0.2753822145133525,0.0,0.8293407455122993,10.685544129991571,0.9994106980961015,0.9994909214322077 | |
| concat_bbpe_32000,concatenated,bbpe,32000,1.306778190060823,1.3463628334026025,1.2326871301490916,4.231787559188005,4.190301018162676,4.3094384241989285,0.08443170030639024,0.02764569169088408,0.0,0.7818011980546499,11.6730644846771,0.9989120580235721,0.9964364500254539 | |
| concat_bbpe_110000,concatenated,bbpe,110000,1.369840129142714,1.291780298027959,1.5159456674416696,4.043612215909571,4.3488864907186695,3.472226631123494,0.14787163829691546,0.20158260314821513,0.0,0.844799841338989,10.833579674164659,0.999909338168631,0.9997454607161038 | |
| shared_bpe_80000,shared,bpe,80000,1.177242372794901,1.1548666992636545,1.2191231939470568,4.655727755357667,4.826059096802318,4.336916505377266,0.05270713821411621,0.10135445538765812,0.0,0.7887947170139161,11.836876829668867,0.9990480507706255,0.9992363821483116 | |
| concat_bpe_80000,concatenated,bpe,80000,1.1868700228302995,1.225374017834228,1.1148016255974247,4.638864843976511,4.579226090671073,4.75049142618574,0.09023562653322222,0.03605213022185787,0.0,0.7947575553850832,11.9499512347738,0.9989120580235721,0.9964364500254539 | |
| shared_unigram_80000,shared,unigram,80000,1.2534560090741458,1.2470098171873627,1.265521425078085,4.37138176247215,4.477721430004725,4.172344509310044,0.014627652700214197,0.06819917796770109,0.0,0.8174151163558648,11.268598596008202,0.9990480507706255,0.9992363821483116 | |
| concat_unigram_80000,concatenated,unigram,80000,1.2476267949156266,1.2883186022081112,1.1714634427289465,4.422981152713373,4.361745756956336,4.537596190079757,0.09070361887104712,0.038754094846048935,0.0,0.8298065088537141,11.290097476176376,0.9989120580235721,0.9964364500254539 | |
| shared_wordpiece_80000,shared,wordpiece,80000,1.1709878085719998,1.150260806985456,1.209782790329946,4.676308894292271,4.841961646933312,4.36625461605866,0.04920055386823321,0.09824675731910108,0.0,0.7836904531915302,11.81965352183478,0.9990480507706255,0.9992363821483116 | |
| concat_wordpiece_80000,concatenated,wordpiece,80000,1.1828299158213738,1.2210699643670833,1.1112555503947363,4.651191183804086,4.591976174635943,4.762024638941472,0.08993294174528893,0.03570927855243706,0.0,0.790169825684203,11.915695743938125,0.9989120580235721,0.9964364500254539 | |
| shared_bbpe_80000,shared,bbpe,80000,1.3918874956781102,1.2680445723448455,1.6236858062781299,3.993843528050164,4.399361273164519,3.23483101683154,0.21903328375364556,0.2647043932119076,0.0,0.8356430315299456,10.747799658790331,0.9994106980961015,0.9994909214322077 | |
| concat_bbpe_80000,concatenated,bbpe,80000,1.3876971544644245,1.3106268890712012,1.5319505105287814,3.9922755286919944,4.28966310338533,3.435651576737558,0.1444717828261869,0.19908592028446243,0.0,0.8436032909102977,10.837904356803623,0.999909338168631,0.9997454607161038 | |
| shared_bpe_110000,shared,bpe,110000,1.177242372794901,1.1548666992636545,1.2191231939470568,4.655727755357667,4.826059096802318,4.336916505377266,0.05270713821411621,0.10135445538765812,0.0,0.7887947170139161,11.836876829668867,0.9990480507706255,0.9992363821483116 | |
| concat_bpe_110000,concatenated,bpe,110000,1.1586982113474131,1.1977431197805206,1.0856173798495037,4.744883406249378,4.676879065575831,4.872167960403322,0.09361417993497914,0.040082545678766936,0.0,0.79473268581714,11.994554793395306,0.9989120580235721,0.9964364500254539 | |
| shared_unigram_110000,shared,unigram,110000,1.2534560090741458,1.2470098171873627,1.265521425078085,4.37138176247215,4.477721430004725,4.172344509310044,0.014627652700214197,0.06819917796770109,0.0,0.8174151163558648,11.268598596008202,0.9990480507706255,0.9992363821483116 | |
| concat_unigram_110000,concatenated,unigram,110000,1.2244484890106007,1.2644141439392171,1.1496442850630972,4.493675111251978,4.428505794184029,4.615653317133443,0.09076919886277159,0.04054626942078096,0.0,0.8266388154322091,11.374113019138779,0.9989120580235721,0.9964364500254539 | |
| shared_wordpiece_110000,shared,wordpiece,110000,1.1709601158762446,1.1502558111859764,1.2097126155765094,4.676409948729624,4.842007732100854,4.366458557148992,0.04914952826394876,0.09821322089164262,0.0,0.7836380631024655,11.81985559664599,0.9990480507706255,0.9992363821483116 | |
| concat_wordpiece_110000,concatenated,wordpiece,110000,1.1546447596001155,1.1934305662132043,1.0820488921400155,4.758025910123524,4.690435896236248,4.884534963776444,0.09332899393268149,0.03973747121878104,0.0,0.7918619340248275,11.958933472578055,0.9989120580235721,0.9964364500254539 | |
| shared_bbpe_110000,shared,bbpe,110000,1.3918874956781102,1.2680445723448455,1.6236858062781299,3.993843528050164,4.399361273164519,3.23483101683154,0.21903328375364556,0.2647043932119076,0.0,0.8356430315299456,10.747799658790331,0.9994106980961015,0.9994909214322077 | |