Upload fineweb2_hq/flexitok--bpe_cmn_Hani_8000_overlap.json with huggingface_hub

fineweb2_hq/flexitok--bpe_cmn_Hani_8000_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.33625, "expected_training_ratio_in_superset": 0.016011904761904762, "num_tokens": 2690}, "2": {"ratio_to_total_tokens": 0.389625, "expected_training_ratio_in_superset": 0.03710714285714285, "num_tokens": 3117}, "3": {"ratio_to_total_tokens": 0.031875, "expected_training_ratio_in_superset": 0.004553571428571428, "num_tokens": 255}, "4": {"ratio_to_total_tokens": 0.0225, "expected_training_ratio_in_superset": 0.004285714285714286, "num_tokens": 180}, "5": {"ratio_to_total_tokens": 0.015, "expected_training_ratio_in_superset": 0.0035714285714285713, "num_tokens": 120}, "6": {"ratio_to_total_tokens": 0.012375, "expected_training_ratio_in_superset": 0.0035357142857142857, "num_tokens": 99}, "7": {"ratio_to_total_tokens": 0.012625, "expected_training_ratio_in_superset": 0.004208333333333333, "num_tokens": 101}, "8": {"ratio_to_total_tokens": 0.0115, "expected_training_ratio_in_superset": 0.00438095238095238, "num_tokens": 92}, "9": {"ratio_to_total_tokens": 0.0095, "expected_training_ratio_in_superset": 0.004071428571428571, "num_tokens": 76}, "10": {"ratio_to_total_tokens": 0.007, "expected_training_ratio_in_superset": 0.003333333333333333, "num_tokens": 56}, "11": {"ratio_to_total_tokens": 0.0105, "expected_training_ratio_in_superset": 0.0055, "num_tokens": 84}, "12": {"ratio_to_total_tokens": 0.011, "expected_training_ratio_in_superset": 0.006285714285714286, "num_tokens": 88}, "13": {"ratio_to_total_tokens": 0.0085, "expected_training_ratio_in_superset": 0.005261904761904762, "num_tokens": 68}, "14": {"ratio_to_total_tokens": 0.00925, "expected_training_ratio_in_superset": 0.006166666666666666, "num_tokens": 74}, "15": {"ratio_to_total_tokens": 0.008875, "expected_training_ratio_in_superset": 0.006339285714285715, "num_tokens": 71}, "16": {"ratio_to_total_tokens": 0.011125, "expected_training_ratio_in_superset": 0.008476190476190476, "num_tokens": 89}, "17": {"ratio_to_total_tokens": 0.011875, "expected_training_ratio_in_superset": 0.009613095238095238, "num_tokens": 95}, "18": {"ratio_to_total_tokens": 0.011375, "expected_training_ratio_in_superset": 0.00975, "num_tokens": 91}, "19": {"ratio_to_total_tokens": 0.007875, "expected_training_ratio_in_superset": 0.007125, "num_tokens": 63}, "20": {"ratio_to_total_tokens": 0.007125, "expected_training_ratio_in_superset": 0.0067857142857142855, "num_tokens": 57}, "21": {"ratio_to_total_tokens": 0.05425, "expected_training_ratio_in_superset": 0.05425, "num_tokens": 434}, "total_training_compared_to_full_model": 0.21061309523809524}