Upload llama_and_16k/flexitok--llama-3.2-1b-tokenizer_overlap.json with huggingface_hub

llama_and_16k/flexitok--llama-3.2-1b-tokenizer_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.6702532435129741, "expected_training_ratio_in_superset": 0.03191682111966543, "num_tokens": 85964}, "2": {"ratio_to_total_tokens": 0.18182385229540918, "expected_training_ratio_in_superset": 0.01731655736146754, "num_tokens": 23320}, "3": {"ratio_to_total_tokens": 0.06777070858283434, "expected_training_ratio_in_superset": 0.00968152979754776, "num_tokens": 8692}, "4": {"ratio_to_total_tokens": 0.017948478043912176, "expected_training_ratio_in_superset": 0.003418757722649938, "num_tokens": 2302}, "5": {"ratio_to_total_tokens": 0.010564807884231538, "expected_training_ratio_in_superset": 0.0025154304486265564, "num_tokens": 1355}, "6": {"ratio_to_total_tokens": 0.006962637225548902, "expected_training_ratio_in_superset": 0.0019893249215854007, "num_tokens": 893}, "7": {"ratio_to_total_tokens": 0.005161551896207585, "expected_training_ratio_in_superset": 0.0017205172987358615, "num_tokens": 662}, "8": {"ratio_to_total_tokens": 0.004194735528942116, "expected_training_ratio_in_superset": 0.001597994487216044, "num_tokens": 538}, "9": {"ratio_to_total_tokens": 0.003251309880239521, "expected_training_ratio_in_superset": 0.0013934185201026517, "num_tokens": 417}, "10": {"ratio_to_total_tokens": 0.003181137724550898, "expected_training_ratio_in_superset": 0.00151482748788138, "num_tokens": 408}, "11": {"ratio_to_total_tokens": 0.0023624625748502994, "expected_training_ratio_in_superset": 0.001237480396350157, "num_tokens": 303}, "12": {"ratio_to_total_tokens": 0.0025106037924151696, "expected_training_ratio_in_superset": 0.001434630738522954, "num_tokens": 322}, "13": {"ratio_to_total_tokens": 0.0020817739520958083, "expected_training_ratio_in_superset": 0.0012887172084402622, "num_tokens": 267}, "14": {"ratio_to_total_tokens": 0.0018400698602794412, "expected_training_ratio_in_superset": 0.001226713240186294, "num_tokens": 236}, "15": {"ratio_to_total_tokens": 0.0029862150698602794, "expected_training_ratio_in_superset": 0.0021330107641859136, "num_tokens": 383}, "16": {"ratio_to_total_tokens": 0.0010525823353293414, "expected_training_ratio_in_superset": 0.0008019674935842599, "num_tokens": 135}, "17": {"ratio_to_total_tokens": 0.001465818363273453, "expected_training_ratio_in_superset": 0.0011866148655070813, "num_tokens": 188}, "18": {"ratio_to_total_tokens": 0.0007251122754491018, "expected_training_ratio_in_superset": 0.0006215248075278015, "num_tokens": 93}, "19": {"ratio_to_total_tokens": 0.0007874875249500998, "expected_training_ratio_in_superset": 0.0007124887130500903, "num_tokens": 101}, "20": {"ratio_to_total_tokens": 0.0007017215568862275, "expected_training_ratio_in_superset": 0.00066830624465355, "num_tokens": 90}, "21": {"ratio_to_total_tokens": 0.012373690119760479, "expected_training_ratio_in_superset": 0.012373690119760479, "num_tokens": 1587}, "total_training_compared_to_full_model": 0.0967503237572474}