tokenizer_test3 / evaluation /language_fertility.csv
ljcamargo's picture
Upload evaluation/language_fertility.csv with huggingface_hub
0702974 verified
Raw
History Blame Contribute Delete
18.4 kB
code,autonym,spanish_name,inali_name,english_name,samples,chars,utf8_bytes,tokens,chars_per_token,tokens_per_char,bytes_per_token,tokens_per_byte,round_trip
trc,Xnaꞌánj nu̱ꞌ,Triqui de Copala,triqui de San Juan Copala,Copala Triqui,3,1511,1791,218,6.931192660550459,0.1442753143613501,8.215596330275229,0.12171970965940815,True
sei,Cmiique Iitom,Seri,seri,Seri,3,1511,1531,248,6.092741935483871,0.16412971542025148,6.173387096774194,0.1619856303069889,True
cya,Chaꞌtñan,Chatino de Nopala,chatino oriental bajo,Nopala Chatino,2,1007,1019,166,6.066265060240964,0.16484607745779542,6.13855421686747,0.16290480863591755,True
tzo,Bats'i k'op,Tsotsil,tsotsil del centro,Tsotsil,4,2015,2199,355,5.676056338028169,0.1761786600496278,6.194366197183099,0.16143701682582992,True
amu,"Ñomndaa, Ñomndaa Suljaaꞌ",Amuzgo de Guerrero,amuzgo del norte,Guerrero Amuzgo,4,2015,2521,396,5.088383838383838,0.1965260545905707,6.366161616161616,0.15708052360174535,True
tzh,Bats'il k'op,Tzeltal,tseltal del occidente,Tzeltal,2,1007,1102,207,4.8647342995169085,0.20556107249255212,5.323671497584541,0.18784029038112524,True
mat,bot’una,Matlatzinca de San Francisco,matlatzinca,"Matlatzinca, San Francisco",3,1511,1619,319,4.736677115987461,0.21111846459298478,5.075235109717869,0.19703520691785054,True
maz,Jñatjo,Mazahua Central,mazahua de oriente,Central Mazahua,3,1511,1702,328,4.6067073170731705,0.21707478491065518,5.189024390243903,0.1927144535840188,True
ctu,"Ch'ol, Lakty’añ",Ch'ol,ch’ol del noroeste,Chol,3,1511,1753,338,4.470414201183432,0.22369291859695567,5.186390532544379,0.19281232173416998,True
nhw,Nahuatl,Náhuatl Huasteco Occidental,náhuatl de la Huasteca potosina,Western Huasteca Nahuatl,4,2015,2015,453,4.448123620309051,0.2248138957816377,4.448123620309051,0.2248138957816377,True
ocu,pjiekak'joo,Tlahuica,tlahuica,Tlahuica,3,1511,1644,347,4.354466858789626,0.22964923891462607,4.737752161383285,0.2110705596107056,True
cux,Cuicateco yeⁿꞌe Tepeuxila,Cuicateco de Tepeuxila,cuicateco del centro,Tepeuxila Cuicatec,3,1511,1828,349,4.329512893982808,0.23097286565188616,5.237822349570201,0.19091903719912473,True
zai,Diidxazá,Zapoteco del Istmo,zapoteco de la planicie costera,Isthmus Zapotec,4,2015,2234,472,4.2690677966101696,0.23424317617866006,4.733050847457627,0.21128021486123547,True
ntp,Oodamicʌdʌ,Tepehuano del Norte,tepehuano del norte,Northern Tepehuan,3,1511,1638,361,4.185595567867036,0.23891462607544672,4.537396121883656,0.22039072039072038,True
pei,,Chichimeco Jonaz,chichimeco jonaz,Chichimeco-Jonaz,3,1511,1725,361,4.185595567867036,0.23891462607544672,4.778393351800554,0.20927536231884059,True
toj,Tojol-ab'al,Tojolabal,tojolabal,Tojolabal,4,2015,2196,483,4.171842650103519,0.23970223325062034,4.546583850931677,0.21994535519125682,True
yua,Maaya t'aan,Maya de Yucatán,maya,Yucatec Maya,5,2519,2945,605,4.163636363636364,0.24017467248908297,4.867768595041323,0.20543293718166383,True
hus,Teenek,Huasteco,huasteco del occidente,Huastec,3,1450,1585,351,4.131054131054131,0.24206896551724139,4.515669515669516,0.22145110410094637,True
huv,Ombeayiüts,Huave de San Mateo del Mar,huave del oeste,San Mateo del Mar Huave,3,1511,1600,366,4.128415300546448,0.24222369291859697,4.371584699453552,0.22875,True
top,"Lichiwin Tutunaku, Tutunakú",Totonaco de Papantla,totonaco de la costa,Papantla Totonac,5,2519,2582,620,4.062903225806451,0.24612941643509328,4.164516129032258,0.24012393493415957,True
lac,Jach-t'aan,Lacandón,lacandón,Lacandon,5,2519,2820,624,4.0368589743589745,0.24771734815402938,4.519230769230769,0.22127659574468084,True
zoq,,Zoque de Tabasco,zoque del este,"Zoque, Tabasco",3,1511,1605,375,4.029333333333334,0.24818001323626737,4.28,0.2336448598130841,True
yaq,Yo'em noki,Yaqui,yaqui,Yaqui,4,2015,2110,504,3.998015873015873,0.2501240694789082,4.186507936507937,0.23886255924170616,True
nhe,Mexkatl,Náhuatl Huasteco Oriental,náhuatl de la Huasteca veracruzana,Eastern Huasteca Nahuatl,4,2015,2083,508,3.966535433070866,0.2521091811414392,4.100393700787402,0.24387902064330294,True
tsz,"P’urhe, P’urhépecha",Purépecha,purépecha,Purepecha,3,1511,1590,385,3.9246753246753245,0.25479814692256786,4.12987012987013,0.24213836477987422,True
pia,Oob No'ok,Pima Bajo,pima del sur,Pima Bajo,4,2015,2189,514,3.9202334630350193,0.25508684863523573,4.2587548638132295,0.23481041571493833,True
tpp,Lhichiwin,Tepehua de Pisaflores,tepehua del norte,Pisaflores Tepehua,4,2015,2099,522,3.8601532567049808,0.25905707196029776,4.021072796934866,0.2486898523106241,True
poi,Nuntajɨ̱yi,Popoluca de la Sierra,popoluca de la Sierra,Highland Popoluca,3,1511,1764,399,3.7869674185463658,0.2640635340833885,4.421052631578948,0.2261904761904762,True
pmq,Xi'iuy,Pame del Norte,pame del norte,Northern Pame,3,1511,1738,430,3.513953488372093,0.28457974851091994,4.0418604651162795,0.2474108170310702,True
thh,ralámuli raicha,Tarahumara del Norte,tarahumara del norte,"Tarahumara, Northern",3,1511,1676,430,3.513953488372093,0.28457974851091994,3.8976744186046512,0.256563245823389,True
tee,Lhima'alh'ama',Tepehua de Huehuetla,tepehua del sur,Huehuetla Tepehua,4,2015,2052,583,3.456260720411664,0.2893300248138958,3.5197255574614066,0.28411306042884993,True
mio,Tuhun Ñuun Ñucaan,Mixteco de Pinotepa Nacional,mixteco del oeste de la costa,Pinotepa Nacional Mixtec,4,2015,2186,584,3.4503424657534247,0.28982630272952853,3.743150684931507,0.26715462031107046,True
maj,"Kinꞌekꞌatiya, Ntaxjo̱",Mazateco de Jalapa de Díaz,mazateco del este bajo,Jalapa de Díaz Mazatec,1,503,507,146,3.4452054794520546,0.29025844930417494,3.4726027397260273,0.2879684418145957,True
ncx,,Náhuatl de Puebla Central,náhuatl del centro de Puebla,"Nahuatl, Central Puebla",5,2519,2554,735,3.4272108843537414,0.29178245335450576,3.4748299319727893,0.28778386844166015,True
meh,Tu̱ꞌun káꞌánꞌ kájí,Mixteco del Suroeste de Tlaxiaco,mixteco del suroeste,Southwestern Tlaxiaco Mixtec,3,1511,1552,444,3.4031531531531534,0.2938451356717406,3.4954954954954953,0.2860824742268041,True
azn,Meshikan de San Agustin Buenaventura y de Santa Cruz,Náhuatl de Durango Occidental,mexicano del noroeste,Western Durango Nahuatl,4,2007,2026,604,3.3228476821192054,0.3009466865969108,3.3543046357615895,0.2981243830207305,True
nhm,"Ika mejikano, Ika nauatl",Náhuatl de Morelos,mexicano central bajo,Morelos Nahuatl,3,1511,1511,455,3.320879120879121,0.3011250827266711,3.320879120879121,0.3011250827266711,True
mie,Tūhun sāsávī,Mixteco de Ocotepec,mixteco de Sierra sur noroeste,Ocotepec Mixtec,4,2015,2238,612,3.292483660130719,0.3037220843672457,3.656862745098039,0.2734584450402145,True
mfy,Yorem Nokpo,Mayo,mayo,Mayo,3,1511,1555,465,3.249462365591398,0.30774321641297153,3.3440860215053765,0.2990353697749196,True
ncj,Mexicatl,Náhuatl del Norte de Puebla,náhuatl alto del norte de Puebla,Northern Puebla Nahuatl,4,2015,2035,625,3.224,0.31017369727047145,3.256,0.3071253071253071,True
trs,Nânj Nï̀nꞌïn,Triqui de Chicahuaxtla,triqui de la alta,Chicahuaxtla Triqui,5,2519,2774,783,3.2171136653895274,0.3108376339817388,3.5427841634738186,0.2822638788752704,True
zaq,Tisa,Zapoteco de Aloápam,zapoteco de San Miguel Aloápam,Aloápam Zapotec,3,1511,1675,471,3.208067940552017,0.3117140966247518,3.5562632696390657,0.28119402985074626,True
ngu,Nauatlajtojli,Náhuatl de Guerrero,mexicano de Guerrero,Guerrero Nahuatl,4,2015,2020,631,3.1933438985736924,0.31315136476426797,3.2012678288431062,0.3123762376237624,True
quc,"K'iche', Qach'abel",Kiché,K’iche’ (oriental),K’iche’,3,1511,1558,475,3.1810526315789476,0.314361350099272,3.28,0.3048780487804878,True
nhy,Nāhuatlahtōlli,Náhuatl del Norte de Oaxaca,náhuatl de Oaxaca,Northern Oaxaca Nahuatl,5,2519,2587,797,3.1606022584692597,0.3163953949980151,3.245922208281054,0.3080788558175493,True
nch,Nāhuatlahtōlli,Náhuatl Huasteco Central,mexicano de la Huasteca Hidalguense,Central Huasteca Nahuatl,3,1511,1511,481,3.141372141372141,0.3183322303110523,3.141372141372141,0.3183322303110523,True
mam,"Qyol Mam, Yol Mam",Mam,Mam de la frontera,Mam,4,2015,2083,642,3.1386292834890965,0.3186104218362283,3.2445482866043616,0.3082093134901584,True
mix,"Sa'an Savi Snuviko, Sa'an savi",Mixteco de Mixtepec,mixteco de Ñumi,Mixtepec Mixtec,3,1511,1669,482,3.1348547717842323,0.3189940436796823,3.462655601659751,0.28879568603954464,True
tqt,Xinulajgsípij Totonaco,Totonaco del Cerro Xinolatépetl,totonaco del cerro Xinolatépetl,Totonaco del cerro Xinolatépetl,4,2015,2057,643,3.1337480559875583,0.31910669975186107,3.1990668740279937,0.31259115216334465,True
pls,Ngigua,Popoloca de San Marcos Tlacoyalco,popoloca del norte,San Marcos Tlacoyalco Popoloca,1,503,513,162,3.1049382716049383,0.3220675944333996,3.1666666666666665,0.3157894736842105,True
mau,Én‑ná,Mazateco de Huautla,mazateco del centro,Huautla Mazatec,5,2519,2838,818,3.0794621026894866,0.32473203652242955,3.469437652811736,0.288231148696265,True
mhc,Mocho',Motozintleco,mocho’,Mocho,4,2015,2021,655,3.0763358778625953,0.3250620347394541,3.0854961832061067,0.32409698169223156,True
tos,Totonaco,Totonaco de la Sierra,totonaco central del sur,Highland Totonac,4,2015,2088,658,3.0623100303951367,0.32655086848635234,3.1732522796352582,0.31513409961685823,True
xtn,Saꞌan savi,Mixteco de Tlaxiaco del Norte,mixteco del norte bajo,Northern Tlaxiaco Mixtec,4,2015,2174,664,3.0346385542168677,0.32952853598014886,3.2740963855421685,0.30542778288868444,True
nsu,"Nahuatl, Nahuatlajtoli",Náhuatl de la Sierra Negra,"náhuatl de la Sierra negra, sur",Sierra Negra Nahuatl,3,1511,1548,500,3.022,0.3309066843150232,3.096,0.32299741602067183,True
mzi,En ningotsie,Mazateco de Ixcatlán,mazateco de la presa bajo,Ixcatlán Mazatec,3,1511,1686,502,3.00996015936255,0.33223031105228323,3.358565737051793,0.2977461447212337,True
zaa,Titsaꞌ,Zapoteco de la Sierra de Juárez,"zapoteco serrano, bajo",Sierra de Juárez Zapotec,3,1511,1620,504,2.998015873015873,0.33355393778954334,3.2142857142857144,0.3111111111111111,True
dih,Kumiai,Kumiai,kumiai,Kumiai,2,1007,1048,337,2.9881305637982196,0.3346573982125124,3.1097922848664687,0.32156488549618323,True
mza,Tu'un va'a,Mixteco de Santa María Zacatepec,mixteco de Sierra sur oeste,Santa María Zacatepec Mixtec,3,1511,1780,508,2.9744094488188977,0.33620119126406356,3.5039370078740157,0.2853932584269663,True
mxt,Sa'an Xiñi Savi,Mixteco de Jamiltepec,mixteco de Oaxaca de costa central baja,Jamiltepec Mixtec,3,1511,1672,510,2.962745098039216,0.3375248180013236,3.2784313725490195,0.3050239234449761,True
cac,Koti',Chuj,Chuj,Chuj,3,1445,1614,489,2.955010224948875,0.3384083044982699,3.3006134969325154,0.30297397769516726,True
hch,"Wixárika, Wixárika Niukiyari",Huichol,huichol del norte,Huichol,5,2519,2752,856,2.942757009345794,0.33981738785232235,3.2149532710280373,0.311046511627907,True
vmp,En naxijen,Mazateco de Soyaltepec,mazateco del noreste,Soyaltepec Mazatec,3,1511,1661,514,2.9396887159533076,0.34017207147584383,3.2315175097276265,0.3094521372667068,True
ctp,Chaꞌtñan,Chatino de la Zona Alta Occidental,chatino occidental alto,Western Highland Chatino,2,1007,1167,345,2.918840579710145,0.3426017874875869,3.382608695652174,0.29562982005141386,True
pbs,Xi'oi,Pame Central,pame del centro,Central Pame,3,1511,1671,518,2.916988416988417,0.342819324950364,3.2258687258687258,0.3099940155595452,True
nhx,Melaꞌtájto̱l,Náhuatl del Istmo-Mecayapan,náhuatl del Istmo,Isthmus-Mecayapan Nahuatl,1,503,503,173,2.907514450867052,0.34393638170974156,2.907514450867052,0.34393638170974156,True
knj,K'anjob'al,Kaanjobal,Q’anjob’al,Akateko,3,1511,1635,526,2.8726235741444865,0.34811383189940437,3.1083650190114067,0.3217125382262997,True
maq,"Hne naja, Nne nangui ngaxni",Mazateco de Chiquihuitlán,mazateco del sur,Chiquihuitlán Mazatec,2,1007,1099,351,2.868945868945869,0.3485600794438927,3.131054131054131,0.31938125568698816,True
miy,Tu̱n savi̱ꞌ Tu̱tyoo̱ꞌ,Mixteco de Ayutla,mixteco de Ayutla,Ayutla Mixtec,4,2015,2417,704,2.862215909090909,0.34937965260545906,3.4332386363636362,0.29127016963177493,True
xta,Tu̱'un sâvi,Mixteco de Alcozauca,mixteco de Guerrero del noreste central,Alcozauca Mixtec,2,1007,1075,353,2.8526912181303117,0.35054617676266137,3.045325779036827,0.3283720930232558,True
ood,"Oʼodham ha-ñeʼokĭ, Oʼodham ñiok",Pápago-Pima,pápago,Tohono O’odham,3,1511,1539,530,2.8509433962264152,0.35076108537392453,2.9037735849056605,0.3443794671864847,True
zpr,Diste’,Zapoteco de Santiago Xanica,"zapoteco de la Sierra sur, del sureste bajo",Santiago Xanica Zapotec,3,1511,1589,530,2.8509433962264152,0.35076108537392453,2.9981132075471697,0.33354310887350536,True
mks,Tu̱hun ndáhví,Mixteco de Silacayoapan,mixteco del oeste alto,Silacayoapan Mixtec,1,503,549,177,2.84180790960452,0.3518886679920477,3.1016949152542375,0.3224043715846995,True
chf,Yocot'an,Chontal de Tabasco,chontal de Tabasco central,Tabasco Chontal,4,2015,2165,712,2.830056179775281,0.3533498759305211,3.0407303370786516,0.3288683602771363,True
zoc,Tzunitzame,Zoque de Copainalá,zoque del centro,Copainalá Zoque,4,2015,2235,717,2.810320781032078,0.3558312655086849,3.1171548117154813,0.3208053691275168,True
mmc,Jñatjo,Mazahua de Michoacán,mazahua de occidente,Michoacán Mazahua,2,1007,1069,359,2.8050139275766015,0.35650446871896724,2.977715877437326,0.33582787652011226,True
var,Warijio,Guarijío,guarijío del norte,Huarijío,3,1511,1648,539,2.803339517625232,0.356717405691595,3.057513914656772,0.3270631067961165,True
tar,Ralámuli raꞌíchala,Tarahumara Central,tarahumara del centro,Central Tarahumara,3,1511,1585,541,2.7929759704251387,0.35804103242885504,2.9297597042513863,0.34132492113564666,True
zos,Otetzame,Zoque de Francisco León,zoque del noroeste,Francisco León Zoque,3,1511,1590,541,2.7929759704251387,0.35804103242885504,2.9390018484288354,0.340251572327044,True
coz,,Chocholteco,chocholteco del oeste,Chocholtec,3,1511,1591,546,2.7673992673992673,0.3613500992720053,2.913919413919414,0.3431803896920176,True
kek,Q'eqchi',Q'eqchi',Q’eqchi’,Q’eqchi’,3,1511,1736,546,2.7673992673992673,0.3613500992720053,3.1794871794871793,0.31451612903225806,True
tla,,Tepehuano del Suroeste,tepehuano del sur alto,"Tepehuan, Southwestern",2,1007,1216,366,2.751366120218579,0.3634558093346574,3.3224043715846996,0.30098684210526316,True
klb,Kiliwa,Kiliwa,kiliwa,Kiliwa,1,503,528,185,2.718918918918919,0.36779324055666,2.854054054054054,0.3503787878787879,True
mto,Ayöök,Mixe de Totontepec,mixe alto del norte,Totontepec Mixe,4,2015,2331,744,2.7083333333333335,0.36923076923076925,3.1330645161290325,0.31917631917631917,True
otq,Ñhöñhö,Otomí de Querétaro,otomí del noroeste,Querétaro Otomi,3,1511,1834,558,2.707885304659498,0.3692918596955658,3.2867383512544803,0.3042529989094875,True
jac,"Abxubal, Abꞌxubꞌal Poptiꞌ",Jakalteko,Jakalteko,Jakalteko,3,1511,1689,563,2.6838365896980463,0.37260092653871607,3.0,0.3333333333333333,True
vmy,Enre naxinanda nguifi,Mazateco de Ayautla,mazateco del sureste,Ayautla Mazatec,1,503,556,188,2.675531914893617,0.37375745526838966,2.9574468085106385,0.3381294964028777,True
zor,Otetzame,Zoque de Rayón,zoque del norte bajo,Rayón Zoque,3,1511,1712,568,2.660211267605634,0.3759099933818663,3.0140845070422535,0.3317757009345794,True
ppi,Paipai,Paipai,paipai,Paipai,3,1511,1554,571,2.6462346760070052,0.37789543348775645,2.7215411558669,0.36743886743886744,True
crn,Naáyeri Nyuuca,Cora de El Nayar,cora meseño,El Nayar Cora,3,1511,1664,585,2.582905982905983,0.3871608206485771,2.8444444444444446,0.3515625,True
otm,Nyuhu,Otomí de la Sierra Oriental,otomí de la Sierra,Eastern Highland Otomi,3,1511,1745,590,2.5610169491525423,0.39046988749172734,2.957627118644068,0.33810888252148996,True
zoh,Tzunitzame,Zoque de Chimalapa,zoque del oeste,Chimalapa Zoque,4,2015,2225,788,2.5571065989847717,0.3910669975186104,2.8236040609137056,0.3541573033707865,True
cok,Cuéimarusaꞌna,Cora de Santa Teresa,cora tereseño,Santa Teresa Cora,1,503,503,198,2.54040404040404,0.39363817097415504,2.54040404040404,0.39363817097415504,True
plo,,Popoluca de Oluta,oluteco,"Popoluca, Oluta",1,499,540,199,2.507537688442211,0.39879759519038077,2.71356783919598,0.3685185185185185,True
mxp,Ayuujk,Mixe de Tlahuitoltepec,mixe alto del centro,Tlahuitoltepec Mixe,2,1007,1141,405,2.4864197530864196,0.40218470705064546,2.8172839506172838,0.3549517966695881,True
stp,Oꞌdam,Tepehuano del Sureste,tepehuano del sur bajo,Southeastern Tepehuan,3,1511,1570,610,2.4770491803278687,0.40370615486432826,2.5737704918032787,0.3885350318471338,True
cco,Jmíiˊ,Chinanteco de Comaltepec,chinanteco de la Sierra,Comaltepec Chinantec,1,503,599,204,2.465686274509804,0.40556660039761433,2.9362745098039214,0.34056761268781305,True
azg,Jnꞌoon,Amuzgo de San Pedro Amuzgos,amuzgo alto del este,San Pedro Amuzgos Amuzgo,1,503,608,206,2.441747572815534,0.4095427435387674,2.9514563106796117,0.33881578947368424,True
mxq,Ayuk,Mixe de Juquila,mixe medio del este,Juquila Mixe,4,2015,2509,830,2.427710843373494,0.4119106699751861,3.02289156626506,0.33080908728577124,True
toc,Tutunakú,Totonaco de Coyutla,totonaco central del norte,Coyutla Totonac,1,503,505,209,2.4066985645933014,0.415506958250497,2.416267942583732,0.41386138613861384,True
tpx,Meꞌpa̱a̱ Wíꞌi̱i̱n,Tlapaneco de Acatepec,tlapaneco del suroeste,Acatepec Me’phaa,3,1511,1695,630,2.3984126984126983,0.4169424222369292,2.6904761904761907,0.37168141592920356,True
tpl,Meꞌpa̱a̱ Wíꞌi̱i̱n,Tlapaneco de Tacoapa,tlapaneco del centro,Tlacoapa Me’phaa,3,1511,1948,659,2.2928679817905917,0.43613500992720056,2.9559939301972684,0.33829568788501024,True
tcf,Mè'phàà Mañuwìín,Tlapaneco a de Malinaltepec,tlapaneco central bajo,Malinaltepec Me’phaa,2,1007,1248,449,2.242761692650334,0.4458788480635551,2.779510022271715,0.359775641025641,True
mxv,"Tuꞌun Savi de Metlatónoc y Cochoapa el Grande, Tuꞌun savi",Mixteco de Metlatónoc,mixteco de Guerrero del este medio,Metlatónoc Mixtec,1,503,568,235,2.1404255319148935,0.4671968190854871,2.4170212765957446,0.4137323943661972,True
chj,"Jmiih kia’ dzä ‘vï ï, Jujmi",Chinanteco de Ojitlán,chinanteco del norte,Ojitlán Chinantec,3,1511,1740,708,2.134180790960452,0.4685638649900728,2.457627118644068,0.4068965517241379,True
cuc,Jau² jm⁴³,Chinanteco de Usila,chinanteco del oeste central alto,Usila Chinantec,4,2015,2521,953,2.114375655823715,0.47295285359801487,2.645330535152151,0.37802459341531136,True