Upload fineweb2_hq/flexitok--bpe_ind_Latn_8000_overlap.json with huggingface_hub

fineweb2_hq/flexitok--bpe_ind_Latn_8000_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.594875, "expected_training_ratio_in_superset": 0.02832738095238095, "num_tokens": 4759}, "2": {"ratio_to_total_tokens": 0.065, "expected_training_ratio_in_superset": 0.00619047619047619, "num_tokens": 520}, "3": {"ratio_to_total_tokens": 0.038375, "expected_training_ratio_in_superset": 0.0054821428571428564, "num_tokens": 307}, "4": {"ratio_to_total_tokens": 0.031875, "expected_training_ratio_in_superset": 0.006071428571428571, "num_tokens": 255}, "5": {"ratio_to_total_tokens": 0.023375, "expected_training_ratio_in_superset": 0.00556547619047619, "num_tokens": 187}, "6": {"ratio_to_total_tokens": 0.019, "expected_training_ratio_in_superset": 0.005428571428571428, "num_tokens": 152}, "7": {"ratio_to_total_tokens": 0.015375, "expected_training_ratio_in_superset": 0.005125, "num_tokens": 123}, "8": {"ratio_to_total_tokens": 0.0165, "expected_training_ratio_in_superset": 0.006285714285714286, "num_tokens": 132}, "9": {"ratio_to_total_tokens": 0.016625, "expected_training_ratio_in_superset": 0.007125, "num_tokens": 133}, "10": {"ratio_to_total_tokens": 0.012875, "expected_training_ratio_in_superset": 0.00613095238095238, "num_tokens": 103}, "11": {"ratio_to_total_tokens": 0.0135, "expected_training_ratio_in_superset": 0.007071428571428572, "num_tokens": 108}, "12": {"ratio_to_total_tokens": 0.015, "expected_training_ratio_in_superset": 0.008571428571428572, "num_tokens": 120}, "13": {"ratio_to_total_tokens": 0.012125, "expected_training_ratio_in_superset": 0.007505952380952381, "num_tokens": 97}, "14": {"ratio_to_total_tokens": 0.011375, "expected_training_ratio_in_superset": 0.007583333333333333, "num_tokens": 91}, "15": {"ratio_to_total_tokens": 0.010625, "expected_training_ratio_in_superset": 0.007589285714285714, "num_tokens": 85}, "16": {"ratio_to_total_tokens": 0.011875, "expected_training_ratio_in_superset": 0.009047619047619047, "num_tokens": 95}, "17": {"ratio_to_total_tokens": 0.011125, "expected_training_ratio_in_superset": 0.009005952380952382, "num_tokens": 89}, "18": {"ratio_to_total_tokens": 0.01125, "expected_training_ratio_in_superset": 0.009642857142857142, "num_tokens": 90}, "19": {"ratio_to_total_tokens": 0.00775, "expected_training_ratio_in_superset": 0.007011904761904762, "num_tokens": 62}, "20": {"ratio_to_total_tokens": 0.00725, "expected_training_ratio_in_superset": 0.006904761904761904, "num_tokens": 58}, "21": {"ratio_to_total_tokens": 0.05425, "expected_training_ratio_in_superset": 0.05425, "num_tokens": 434}, "total_training_compared_to_full_model": 0.21591666666666667}