{ "run": "perloop_s0", "step": 73180, "params": 26642148, "by_k": { "1": { "loss": { "story": 1.984, "maths": 1.1458, "qa": 3.4102, "mixed": 2.7296, "explain": 0.4829, "compose": 1.9691, "wordmath": 1.1635 }, "mean_loss": 1.8407, "maths_accuracy": 0.492, "maths_levels": { "level_2": 0.9286, "level_5": 0.8095, "level_6": 0.4578, "level_7": 0.253, "level_8": 0.2771, "level_9": 0.2169, "all": 0.492 }, "mixed_topics": { "story": 2.1804, "maths": 1.4385, "qa": 3.8899 }, "mixed_accuracy": 0.106, "explain_accuracy": 0.9167, "compose_accuracy": 0.44, "gsm8k_accuracy": 0.02, "tflops_per_token": 5.814784e-05 }, "2": { "loss": { "story": 1.9264, "maths": 1.1397, "qa": 3.317, "mixed": 2.6892, "explain": 0.4779, "compose": 1.9725, "wordmath": 1.1172 }, "mean_loss": 1.8057, "maths_accuracy": 0.564, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.6145, "level_7": 0.3253, "level_8": 0.4217, "level_9": 0.2289, "all": 0.564 }, "mixed_topics": { "story": 2.1417, "maths": 1.4498, "qa": 3.8197 }, "mixed_accuracy": 0.108, "tflops_per_token": 8.909824e-05 }, "3": { "loss": { "story": 1.9266, "maths": 1.1399, "qa": 3.3138, "mixed": 2.6985, "explain": 0.4777, "compose": 1.9856, "wordmath": 1.1168 }, "mean_loss": 1.8084, "maths_accuracy": 0.556, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5663, "level_7": 0.3133, "level_8": 0.4217, "level_9": 0.241, "all": 0.556 }, "mixed_topics": { "story": 2.1522, "maths": 1.4529, "qa": 3.8316 }, "mixed_accuracy": 0.108, "explain_accuracy": 0.9278, "compose_accuracy": 0.5, "gsm8k_accuracy": 0.02, "tflops_per_token": 0.00012004864 }, "4": { "loss": { "story": 1.9285, "maths": 1.1403, "qa": 3.317, "mixed": 2.6917, "explain": 0.4776, "compose": 1.9698, "wordmath": 1.119 }, "mean_loss": 1.8063, "maths_accuracy": 0.56, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5783, "level_7": 0.3253, "level_8": 0.4096, "level_9": 0.253, "all": 0.56 }, "mixed_topics": { "story": 2.1447, "maths": 1.4487, "qa": 3.8231 }, "mixed_accuracy": 0.102, "tflops_per_token": 0.00015099904 } }, "by_mean": { "2.0": { "loss": { "story": 1.9428, "maths": 1.14, "qa": 3.3436, "mixed": 2.7042, "explain": 0.4788, "compose": 1.9783, "wordmath": 1.127 }, "mean_loss": 1.8164, "maths_accuracy": 0.56, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5663, "level_7": 0.3614, "level_8": 0.3976, "level_9": 0.241, "all": 0.56 }, "mixed_topics": { "story": 2.1576, "maths": 1.4495, "qa": 3.8421 }, "mixed_accuracy": 0.104, "mean_runs": 2.299, "mean_runs_by_dataset": { "story": 1.826, "maths": 3.082, "qa": 1.852, "mixed": 2.16, "explain": 2.598, "compose": 2.488, "wordmath": 2.088 }, "mean_runs_paid": 2.299, "run_counts": [ 651, 1346, 1451, 136 ], "activation": { "story": [ 181, 21, 186, 124 ], "maths": [ 0, 512, 0, 0 ], "qa": [ 207, 51, 143, 111 ], "mixed": [ 128, 156, 121, 107 ], "explain": [ 2, 332, 54, 124 ], "compose": [ 47, 274, 93, 98 ], "wordmath": [ 136, 116, 157, 103 ] }, "mutual_info_bits": 0.4067, "mutual_info_shuffled": 0.0036, "mean_runs_by_expert": [ 1.525, 3.093, 2.166, 1.523 ], "tflops_per_token": 9.835572571428572e-05 }, "2.5": { "loss": { "story": 1.9267, "maths": 1.1401, "qa": 3.3158, "mixed": 2.6921, "explain": 0.4781, "compose": 1.9743, "wordmath": 1.1171 }, "mean_loss": 1.8063, "maths_accuracy": 0.566, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5783, "level_7": 0.3494, "level_8": 0.4217, "level_9": 0.253, "all": 0.566 }, "mixed_topics": { "story": 2.1447, "maths": 1.4507, "qa": 3.8236 }, "mixed_accuracy": 0.11, "mean_runs": 2.796, "mean_runs_by_dataset": { "story": 2.303, "maths": 3.613, "qa": 2.346, "mixed": 2.65, "explain": 3.129, "compose": 2.955, "wordmath": 2.578 }, "mean_runs_paid": 2.796, "run_counts": [ 2, 1610, 1088, 884 ], "activation": { "story": [ 181, 21, 186, 124 ], "maths": [ 0, 512, 0, 0 ], "qa": [ 207, 51, 143, 111 ], "mixed": [ 128, 156, 121, 107 ], "explain": [ 2, 332, 54, 124 ], "compose": [ 47, 274, 93, 98 ], "wordmath": [ 136, 116, 157, 103 ] }, "mutual_info_bits": 0.4067, "mutual_info_shuffled": 0.0036, "mean_runs_by_expert": [ 1.999, 3.605, 2.676, 1.999 ], "tflops_per_token": 0.00011374456857142857 }, "3.0": { "loss": { "story": 1.9267, "maths": 1.1403, "qa": 3.3145, "mixed": 2.6941, "explain": 0.4778, "compose": 1.9743, "wordmath": 1.1175 }, "mean_loss": 1.8065, "maths_accuracy": 0.56, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5783, "level_7": 0.3253, "level_8": 0.4096, "level_9": 0.253, "all": 0.56 }, "mixed_topics": { "story": 2.1477, "maths": 1.4509, "qa": 3.8257 }, "mixed_accuracy": 0.108, "mean_runs": 3.272, "mean_runs_by_dataset": { "story": 2.85, "maths": 4.0, "qa": 2.861, "mixed": 3.127, "explain": 3.557, "compose": 3.436, "wordmath": 3.074 }, "mean_runs_paid": 3.272, "run_counts": [ 0, 623, 1363, 1598 ], "activation": { "story": [ 181, 21, 186, 124 ], "maths": [ 0, 512, 0, 0 ], "qa": [ 207, 51, 143, 111 ], "mixed": [ 128, 156, 121, 107 ], "explain": [ 2, 332, 54, 124 ], "compose": [ 47, 274, 93, 98 ], "wordmath": [ 136, 116, 157, 103 ] }, "mutual_info_bits": 0.4067, "mutual_info_shuffled": 0.0036, "mean_runs_by_expert": [ 2.549, 4.0, 3.18, 2.54 ], "tflops_per_token": 0.00012846846142857145 } }, "targets": [ 1.907, 3.5, 2.584, 1.907 ], "cycle": { "loss": { "story": 1.9396, "maths": 1.1414, "qa": 3.3372, "mixed": 2.7, "explain": 0.4785, "compose": 1.9743, "wordmath": 1.1289 }, "mean_loss": 1.8143, "loss_by_token_runs": [ 1.8274, 1.8131, 1.8073, 1.8085 ] }, "learned": { "loss": { "story": 1.9266, "maths": 1.1403, "qa": 3.3153, "mixed": 2.6928, "explain": 0.4782, "compose": 1.9772, "wordmath": 1.1169 }, "mean_loss": 1.8068, "maths_accuracy": 0.56, "maths_levels": { "level_2": 0.9524, "level_5": 0.8333, "level_6": 0.5783, "level_7": 0.3253, "level_8": 0.4096, "level_9": 0.253, "all": 0.56 }, "mixed_topics": { "story": 2.1458, "maths": 1.4507, "qa": 3.8244 }, "mixed_accuracy": 0.108, "explain_accuracy": 0.9278, "compose_accuracy": 0.4933, "gsm8k_accuracy": 0.04, "mean_runs": 2.885, "mean_runs_by_dataset": { "story": 2.404, "maths": 4.0, "qa": 2.379, "mixed": 2.648, "explain": 3.332, "compose": 2.887, "wordmath": 2.545 }, "mean_runs_paid": 2.885, "run_counts": [ 39, 1327, 1225, 993 ], "activation": { "story": [ 181, 21, 186, 124 ], "maths": [ 0, 512, 0, 0 ], "qa": [ 207, 51, 143, 111 ], "mixed": [ 128, 156, 121, 107 ], "explain": [ 2, 332, 54, 124 ], "compose": [ 47, 274, 93, 98 ], "wordmath": [ 136, 116, 157, 103 ] }, "mutual_info_bits": 0.4067, "mutual_info_shuffled": 0.0036, "mean_runs_by_expert": [ 2.0, 3.679, 3.0, 1.945 ], "tflops_per_token": 0.00011649072571428571 }, "policy": { "best_fixed_k": 2, "per_dataset_k": { "story": 2, "maths": 2, "qa": 3, "mixed": 2, "explain": 4, "compose": 4, "wordmath": 3 }, "per_dataset_mean_k": 2.857142857142857, "learned_mean_runs": 2.885, "tflops": { "learned": 0.00011649072571428571, "best_fixed": 8.909824e-05, "per_dataset": 0.00011562715428571429 }, "vs_best_fixed": { "difference": 0.00105, "ci_low": 0.00072, "ci_high": 0.00138, "ours_better": false, "at_no_more_compute": false }, "vs_per_dataset": { "difference": 0.00201, "ci_low": 0.00165, "ci_high": 0.00236, "ours_better": false, "at_no_more_compute": false }, "note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone." } }