Upload mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-ltr_5digit_overlap.json with huggingface_hub
Browse files
mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-ltr_5digit_overlap.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"2": {"ratio_to_total_tokens": 0.9081231932803665, "expected_training_ratio_in_superset": 0.20180515406230368, "num_tokens": 99899}, "4": {"ratio_to_total_tokens": 0.08209552206243295, "expected_training_ratio_in_superset": 0.03648689869441465, "num_tokens": 9031}, "6": {"ratio_to_total_tokens": 0.008808610439430577, "expected_training_ratio_in_superset": 0.005872406959620384, "num_tokens": 969}, "8": {"ratio_to_total_tokens": 0.0008181371925167718, "expected_training_ratio_in_superset": 0.0007272330600149083, "num_tokens": 90}, "9": {"ratio_to_total_tokens": 0.00015453702525316802, "expected_training_ratio_in_superset": 0.00015453702525316802, "num_tokens": 17}, "total_training_compared_to_full_model": 0.24504622980160679}
|