script_1/flexitok--bpe_script_CmJp_16000_overlap.json

{"1": {"ratio_to_total_tokens": 0.8629117959617428, "expected_training_ratio_in_superset": 0.07844652690561298, "num_tokens": 15428}, "2": {"ratio_to_total_tokens": 0.0031321662285362715, "expected_training_ratio_in_superset": 0.0005694847688247766, "num_tokens": 56}, "3": {"ratio_to_total_tokens": 0.003244029308126853, "expected_training_ratio_in_superset": 0.0008847352658527779, "num_tokens": 58}, "4": {"ratio_to_total_tokens": 0.004138933944851502, "expected_training_ratio_in_superset": 0.0015050668890369098, "num_tokens": 74}, "5": {"ratio_to_total_tokens": 0.004306728564237373, "expected_training_ratio_in_superset": 0.0019576038928351695, "num_tokens": 77}, "6": {"ratio_to_total_tokens": 0.006432127076458415, "expected_training_ratio_in_superset": 0.0035084329507954985, "num_tokens": 115}, "7": {"ratio_to_total_tokens": 0.005257564740757313, "expected_training_ratio_in_superset": 0.003345723016845563, "num_tokens": 94}, "8": {"ratio_to_total_tokens": 0.011577828737625147, "expected_training_ratio_in_superset": 0.0084202390819092, "num_tokens": 207}, "9": {"ratio_to_total_tokens": 0.004866043962190279, "expected_training_ratio_in_superset": 0.003981308696337502, "num_tokens": 87}, "10": {"ratio_to_total_tokens": 0.005257564740757313, "expected_training_ratio_in_superset": 0.0047796043097793756, "num_tokens": 94}, "11": {"ratio_to_total_tokens": 0.08887521673471671, "expected_training_ratio_in_superset": 0.08887521673471671, "num_tokens": 1589}, "total_training_compared_to_full_model": 0.19627394251254646}