Upload fineweb2_hq/flexitok--bpe_ell_Grek_8000_overlap.json with huggingface_hub

fineweb2_hq/flexitok--bpe_ell_Grek_8000_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.897125, "expected_training_ratio_in_superset": 0.042720238095238096, "num_tokens": 7177}, "2": {"ratio_to_total_tokens": 0.001375, "expected_training_ratio_in_superset": 0.00013095238095238093, "num_tokens": 11}, "3": {"ratio_to_total_tokens": 0.001, "expected_training_ratio_in_superset": 0.00014285714285714284, "num_tokens": 8}, "4": {"ratio_to_total_tokens": 0.000375, "expected_training_ratio_in_superset": 7.142857142857142e-05, "num_tokens": 3}, "5": {"ratio_to_total_tokens": 0.000625, "expected_training_ratio_in_superset": 0.0001488095238095238, "num_tokens": 5}, "6": {"ratio_to_total_tokens": 0.0005, "expected_training_ratio_in_superset": 0.00014285714285714284, "num_tokens": 4}, "7": {"ratio_to_total_tokens": 0.0005, "expected_training_ratio_in_superset": 0.00016666666666666666, "num_tokens": 4}, "8": {"ratio_to_total_tokens": 0.0005, "expected_training_ratio_in_superset": 0.00019047619047619045, "num_tokens": 4}, "9": {"ratio_to_total_tokens": 0.000625, "expected_training_ratio_in_superset": 0.00026785714285714287, "num_tokens": 5}, "10": {"ratio_to_total_tokens": 0.00075, "expected_training_ratio_in_superset": 0.0003571428571428571, "num_tokens": 6}, "11": {"ratio_to_total_tokens": 0.001125, "expected_training_ratio_in_superset": 0.0005892857142857143, "num_tokens": 9}, "12": {"ratio_to_total_tokens": 0.002, "expected_training_ratio_in_superset": 0.0011428571428571427, "num_tokens": 16}, "13": {"ratio_to_total_tokens": 0.002375, "expected_training_ratio_in_superset": 0.0014702380952380954, "num_tokens": 19}, "14": {"ratio_to_total_tokens": 0.002, "expected_training_ratio_in_superset": 0.0013333333333333333, "num_tokens": 16}, "15": {"ratio_to_total_tokens": 0.002875, "expected_training_ratio_in_superset": 0.002053571428571429, "num_tokens": 23}, "16": {"ratio_to_total_tokens": 0.003875, "expected_training_ratio_in_superset": 0.002952380952380952, "num_tokens": 31}, "17": {"ratio_to_total_tokens": 0.003875, "expected_training_ratio_in_superset": 0.0031369047619047618, "num_tokens": 31}, "18": {"ratio_to_total_tokens": 0.01075, "expected_training_ratio_in_superset": 0.009214285714285713, "num_tokens": 86}, "19": {"ratio_to_total_tokens": 0.006375, "expected_training_ratio_in_superset": 0.005767857142857143, "num_tokens": 51}, "20": {"ratio_to_total_tokens": 0.007125, "expected_training_ratio_in_superset": 0.0067857142857142855, "num_tokens": 57}, "21": {"ratio_to_total_tokens": 0.05425, "expected_training_ratio_in_superset": 0.05425, "num_tokens": 434}, "total_training_compared_to_full_model": 0.13303571428571428}