script_1/flexitok--bpe_script_Germ_32000_overlap.json

{"1": {"ratio_to_total_tokens": 0.665023581317511, "expected_training_ratio_in_superset": 0.060456689210682826, "num_tokens": 21856}, "2": {"ratio_to_total_tokens": 0.1039099345808611, "expected_training_ratio_in_superset": 0.01889271537833838, "num_tokens": 3415}, "3": {"ratio_to_total_tokens": 0.06922257720979766, "expected_training_ratio_in_superset": 0.018878884693581177, "num_tokens": 2275}, "4": {"ratio_to_total_tokens": 0.044089456869009586, "expected_training_ratio_in_superset": 0.016032529770548937, "num_tokens": 1449}, "5": {"ratio_to_total_tokens": 0.02647193062528526, "expected_training_ratio_in_superset": 0.012032695738766025, "num_tokens": 870}, "6": {"ratio_to_total_tokens": 0.016948121101475734, "expected_training_ratio_in_superset": 0.009244429691714037, "num_tokens": 557}, "7": {"ratio_to_total_tokens": 0.012657842689791572, "expected_training_ratio_in_superset": 0.008054990802594636, "num_tokens": 416}, "8": {"ratio_to_total_tokens": 0.00724174653887114, "expected_training_ratio_in_superset": 0.005266724755542647, "num_tokens": 238}, "9": {"ratio_to_total_tokens": 0.0030427506465845124, "expected_training_ratio_in_superset": 0.0024895232562964194, "num_tokens": 100}, "10": {"ratio_to_total_tokens": 0.0030427506465845124, "expected_training_ratio_in_superset": 0.0027661369514404657, "num_tokens": 100}, "11": {"ratio_to_total_tokens": 0.048349307774227905, "expected_training_ratio_in_superset": 0.048349307774227905, "num_tokens": 1589}, "total_training_compared_to_full_model": 0.20246462802373347}