script_1/flexitok--bpe_script_Slav_16000_overlap.json

{"1": {"ratio_to_total_tokens": 0.6859382366808109, "expected_training_ratio_in_superset": 0.06235802151643735, "num_tokens": 11639}, "2": {"ratio_to_total_tokens": 0.054926921263554924, "expected_training_ratio_in_superset": 0.009986712957009987, "num_tokens": 932}, "3": {"ratio_to_total_tokens": 0.032826496935407824, "expected_training_ratio_in_superset": 0.008952680982383953, "num_tokens": 557}, "4": {"ratio_to_total_tokens": 0.03052805280528053, "expected_training_ratio_in_superset": 0.011101110111011102, "num_tokens": 518}, "5": {"ratio_to_total_tokens": 0.034653465346534656, "expected_training_ratio_in_superset": 0.01575157515751575, "num_tokens": 588}, "6": {"ratio_to_total_tokens": 0.025047147571900047, "expected_training_ratio_in_superset": 0.013662080493763662, "num_tokens": 425}, "7": {"ratio_to_total_tokens": 0.02157001414427157, "expected_training_ratio_in_superset": 0.013726372637263727, "num_tokens": 366}, "8": {"ratio_to_total_tokens": 0.011786892975011787, "expected_training_ratio_in_superset": 0.008572285800008572, "num_tokens": 200}, "9": {"ratio_to_total_tokens": 0.004832626119754832, "expected_training_ratio_in_superset": 0.003953966825253954, "num_tokens": 82}, "10": {"ratio_to_total_tokens": 0.004243281471004243, "expected_training_ratio_in_superset": 0.0038575286100038576, "num_tokens": 72}, "11": {"ratio_to_total_tokens": 0.09364686468646864, "expected_training_ratio_in_superset": 0.09364686468646864, "num_tokens": 1589}, "total_training_compared_to_full_model": 0.24556919977712055}