Upload mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-rtl_3digit_overlap.json with huggingface_hub
Browse files
mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-rtl_3digit_overlap.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"4": {"ratio_to_total_tokens": 0.023593466424682397, "expected_training_ratio_in_superset": 0.01048598507763662, "num_tokens": 26}, "6": {"ratio_to_total_tokens": 0.8793103448275862, "expected_training_ratio_in_superset": 0.5862068965517241, "num_tokens": 969}, "8": {"ratio_to_total_tokens": 0.08166969147005444, "expected_training_ratio_in_superset": 0.07259528130671507, "num_tokens": 90}, "9": {"ratio_to_total_tokens": 0.015426497277676952, "expected_training_ratio_in_superset": 0.015426497277676952, "num_tokens": 17}, "total_training_compared_to_full_model": 0.6847146602137527}
|