| name,type,vocab_size,time_100k_chars,tokens_per_second_100k,model_size_mb,unknown_rate,avg_tokens_per_doc,avg_token_length,median_token_length,max_token_length,word_coverage,compression_ratio,unique_tokens_used,vocabulary_usage,top_tokens,top_token_freq,rare_token_ratio |
| bbpe_25k,BBPE,25000,0.08356547355651855,257103.79042450906,2.4044103622436523,0.0,748.822,7.796199897973083,7.0,61,1.0,4.279625865693048,19990,0.7996,"['.', ',', 'ĠâĢĶ', '-', ':', 'Ġда', 'ĠбелÓĻн', 'ĠдÓĻ', '!', 'Ġ«']","[20580, 19456, 4561, 4077, 2813, 2728, 2485, 2285, 1864, 1864]",0.15887943971985993 |
| wp_50k,WordPiece,50000,0.05166983604431152,378750.95990660717,1.3837089538574219,0.0,685.8,4.344056576261301,4.0,30,1.0,4.672904636920385,30188,0.60376,"['.', ',', '—', '-', ':', 'да', 'белән', 'дә', '«', '!']","[20885, 19721, 4748, 4432, 2972, 2722, 2468, 2281, 1975, 1947]",0.2781237577845502 |
| bpe_50k,BPE,50000,0.08070063591003418,247420.60300812745,5.065661430358887,0.0,696.054,8.388668120576853,8.0,61,1.0,4.604065201837789,30107,0.60214,"['.', ',', 'ĠâĢĶ', '-', ':', 'Ġда', 'ĠбелÓĻн', 'ĠдÓĻ', 'Ġ«', '!']","[20578, 19453, 4556, 4054, 2812, 2711, 2469, 2284, 1874, 1862]",0.2897664994851696 |
| bbpe_fixed_50k,BPE_fixed,50000,0.0562291145324707,315921.74530405947,3.993025779724121,0.0,619.666,5.173235258994361,5.0,32,1.0,5.171621486413649,32068,0.64136,"['▁—', '▁белән', '▁да', '▁дә', '▁һәм', '▁иде.', '▁бер', '▁ул', '▁дип', '▁–']","[4356, 2379, 2140, 1862, 1535, 1116, 1091, 1090, 1000, 908]",0.2763502557066234 |
| bbpe_50k,BBPE,50000,0.08784890174865723,227322.13610519317,5.047884941101074,0.0,696.454,8.38241434466598,8.0,61,1.0,4.601420912221052,30049,0.60098,"['.', ',', 'ĠâĢĶ', '-', ':', 'Ġда', 'ĠбелÓĻн', 'ĠдÓĻ', 'Ġ«', '!']","[20578, 19453, 4556, 4054, 2812, 2711, 2469, 2284, 1863, 1862]",0.2870311824020766 |
| bpe_fixed_50k,BPE_fixed,50000,0.057566165924072266,337246.7088672603,3.8777475357055664,0.0014830942091103511,674.266,4.092767542779852,4.0,30,0.6354397643096147,4.752839383863342,28498,0.56996,"['.', ',', '—', '-', 'да', ':', 'белән', 'дә', '«', '!']","[20885, 19724, 4750, 4432, 2998, 2972, 2618, 2483, 1990, 1948]",0.29777528247596324 |
| bpe_50k_freq5,BPE,50000,0.0881190299987793,226591.23687898743,5.065661430358887,0.0,696.054,8.388668120576853,8.0,61,1.0,4.604065201837789,30107,0.60214,"['.', ',', 'ĠâĢĶ', '-', ':', 'Ġда', 'ĠбелÓĻн', 'ĠдÓĻ', 'Ġ«', '!']","[20578, 19453, 4556, 4054, 2812, 2711, 2469, 2284, 1874, 1862]",0.2897664994851696 |
| wp_25k,WordPiece,25000,0.042369842529296875,496272.7908526155,0.6622514724731445,0.0,735.806,4.184752502697722,4.0,30,1.0,4.3553300734160905,19895,0.7958,"['.', ',', '—', '-', ':', 'да', 'белән', 'дә', '«', '!']","[20887, 19722, 4753, 4437, 2972, 2739, 2486, 2288, 1976, 1949]",0.1456144759989947 |
| uni_50k,Unigram,50000,0.10636377334594727,189622.8327139213,3.413966178894043,0.0,697.688,4.594715689534577,4.0,16,1.0,4.593282384102923,29572,0.59144,"['.', ',', '▁', '▁—', '▁да', '▁белән', '▁дә', ':', '!', '▁иде']","[19939, 19165, 8524, 4617, 2781, 2483, 2329, 1825, 1754, 1658]",0.29061274178276747 |
| uni_25k,Unigram,25000,0.08225703239440918,260403.26737408372,1.6880054473876953,0.0,745.354,4.300879850379819,4.0,16,1.0,4.29953820600681,19610,0.7844,"['.', ',', '▁', '▁—', '▁да', '▁белән', '▁дә', ':', '!', '▁иде']","[19521, 18909, 8112, 4635, 2795, 2488, 2352, 1834, 1833, 1671]",0.16471188169301376 |
|
|
