mod_tokenizers/flexitok--mod-tokenizers-rtl_3digit_overlap.json

{"6": {"ratio_to_total_tokens": 0.8937437934458788, "expected_training_ratio_in_superset": 0.5958291956305859, "num_tokens": 900}, "8": {"ratio_to_total_tokens": 0.08937437934458789, "expected_training_ratio_in_superset": 0.07944389275074479, "num_tokens": 90}, "9": {"ratio_to_total_tokens": 0.016881827209533268, "expected_training_ratio_in_superset": 0.016881827209533268, "num_tokens": 17}, "total_training_compared_to_full_model": 0.692154915590864}