dataset,tokenizer,source,vocab_size,fertility_ar,fertility_az,fertility_overall,cpt_ar,cpt_az,cpt_overall,gain_pct,n_texts
DODa,DarijaBERT-ar,external,80000,0.0,2.648,2.648,0.0,2.194,2.194,0.0,10000
DODa,DarijaBERT-az,external,110000,0.0,1.246,1.246,0.0,4.677,4.677,0.0,10000
DODa,DarijaBERT-mix,external,160000,0.0,1.318,1.318,0.0,4.441,4.441,0.0,10000
DODa,Ours (80K WP),ours,80000,0.0,1.72,1.72,0.0,3.467,3.467,35.0,10000
DODa,Ours (110K WP),ours,110000,0.0,1.68,1.68,0.0,3.551,3.551,-34.8,10000
DODa,Ours (32K BPE),ours,32000,0.0,1.901,1.901,0.0,3.145,3.145,-44.2,10000
Darija-Wiki,DarijaBERT-ar,external,80000,1.56,2.502,1.562,3.382,2.589,3.381,0.0,10000
Darija-Wiki,DarijaBERT-az,external,110000,1.858,2.219,1.858,2.829,2.946,2.829,0.0,10000
Darija-Wiki,DarijaBERT-mix,external,160000,1.502,2.142,1.503,3.512,3.04,3.511,0.0,10000
Darija-Wiki,Ours (80K WP),ours,80000,1.839,3.265,1.841,2.883,2.07,2.882,-17.9,10000
Darija-Wiki,Ours (110K WP),ours,110000,1.797,3.184,1.799,2.951,2.124,2.95,3.2,10000
Darija-Wiki,Ours (32K BPE),ours,32000,2.102,3.805,2.104,2.519,1.743,2.518,-40.0,10000
Atlaset,DarijaBERT-ar,external,80000,1.407,2.056,1.41,4.051,1.803,4.041,0.0,10000
Atlaset,DarijaBERT-az,external,110000,1.772,1.918,1.773,3.22,1.965,3.215,0.0,10000
Atlaset,DarijaBERT-mix,external,160000,1.379,1.922,1.381,4.131,1.963,4.122,0.0,10000
Atlaset,Ours (80K WP),ours,80000,1.618,2.293,1.621,3.594,1.771,3.586,-15.0,10000
Atlaset,Ours (110K WP),ours,110000,1.566,2.183,1.569,3.714,1.846,3.706,11.5,10000
Atlaset,Ours (32K BPE),ours,32000,1.864,2.571,1.867,3.119,1.492,3.112,-35.2,10000