{ "run": "perloop_s2", "step": 73180, "params": 26642148, "by_k": { "1": { "loss": { "story": 1.9353, "maths": 1.1398, "qa": 3.3466, "mixed": 2.6523, "explain": 0.486, "compose": 1.8771, "wordmath": 1.143 }, "mean_loss": 1.7971, "maths_accuracy": 0.616, "maths_levels": { "level_2": 0.9286, "level_5": 0.9167, "level_6": 0.6265, "level_7": 0.3494, "level_8": 0.506, "level_9": 0.3614, "all": 0.616 }, "mixed_topics": { "story": 2.1173, "maths": 1.396, "qa": 3.7833 }, "mixed_accuracy": 0.094, "explain_accuracy": 0.7333, "compose_accuracy": 0.4933, "gsm8k_accuracy": 0.0267, "tflops_per_token": 5.814784e-05 }, "2": { "loss": { "story": 1.8635, "maths": 1.1333, "qa": 3.2347, "mixed": 2.6026, "explain": 0.4792, "compose": 1.8624, "wordmath": 1.0623 }, "mean_loss": 1.7483, "maths_accuracy": 0.678, "maths_levels": { "level_2": 0.9286, "level_5": 0.9167, "level_6": 0.6386, "level_7": 0.4578, "level_8": 0.6627, "level_9": 0.4578, "all": 0.678 }, "mixed_topics": { "story": 2.0779, "maths": 1.3799, "qa": 3.7061 }, "mixed_accuracy": 0.15, "tflops_per_token": 8.909824e-05 }, "3": { "loss": { "story": 1.8633, "maths": 1.133, "qa": 3.2257, "mixed": 2.6044, "explain": 0.4792, "compose": 1.8672, "wordmath": 1.0589 }, "mean_loss": 1.7474, "maths_accuracy": 0.674, "maths_levels": { "level_2": 0.9048, "level_5": 0.9405, "level_6": 0.6627, "level_7": 0.4337, "level_8": 0.6506, "level_9": 0.4458, "all": 0.674 }, "mixed_topics": { "story": 2.0831, "maths": 1.3827, "qa": 3.705 }, "mixed_accuracy": 0.146, "explain_accuracy": 0.8056, "compose_accuracy": 0.6267, "gsm8k_accuracy": 0.0333, "tflops_per_token": 0.00012004864 }, "4": { "loss": { "story": 1.8676, "maths": 1.1335, "qa": 3.2307, "mixed": 2.613, "explain": 0.4795, "compose": 1.8752, "wordmath": 1.0609 }, "mean_loss": 1.7515, "maths_accuracy": 0.676, "maths_levels": { "level_2": 0.9167, "level_5": 0.881, "level_6": 0.7108, "level_7": 0.4458, "level_8": 0.6386, "level_9": 0.4578, "all": 0.676 }, "mixed_topics": { "story": 2.0873, "maths": 1.3898, "qa": 3.7184 }, "mixed_accuracy": 0.14, "tflops_per_token": 0.00015099904 } }, "by_mean": { "2.0": { "loss": { "story": 1.8984, "maths": 1.1331, "qa": 3.2684, "mixed": 2.6168, "explain": 0.4801, "compose": 1.8648, "wordmath": 1.0753 }, "mean_loss": 1.7624, "maths_accuracy": 0.674, "maths_levels": { "level_2": 0.9048, "level_5": 0.9405, "level_6": 0.6627, "level_7": 0.4337, "level_8": 0.6386, "level_9": 0.4578, "all": 0.674 }, "mixed_topics": { "story": 2.09, "maths": 1.3813, "qa": 3.7294 }, "mixed_accuracy": 0.134, "mean_runs": 2.147, "mean_runs_by_dataset": { "story": 1.551, "maths": 2.748, "qa": 1.82, "mixed": 1.994, "explain": 2.496, "compose": 2.309, "wordmath": 2.113 }, "mean_runs_paid": 2.147, "run_counts": [ 735, 1586, 1263, 0 ], "activation": { "story": [ 119, 0, 2, 391 ], "maths": [ 0, 0, 512, 0 ], "qa": [ 217, 0, 49, 246 ], "mixed": [ 179, 0, 127, 206 ], "explain": [ 63, 0, 379, 70 ], "compose": [ 141, 0, 272, 99 ], "wordmath": [ 191, 0, 171, 150 ] }, "mutual_info_bits": 0.4327, "mutual_info_shuffled": 0.0023, "mean_runs_by_expert": [ 2.125, null, 2.76, 1.367 ], "tflops_per_token": 9.365789714285714e-05 }, "2.5": { "loss": { "story": 1.8694, "maths": 1.1331, "qa": 3.2368, "mixed": 2.6055, "explain": 0.4799, "compose": 1.8656, "wordmath": 1.0623 }, "mean_loss": 1.7504, "maths_accuracy": 0.682, "maths_levels": { "level_2": 0.9167, "level_5": 0.9286, "level_6": 0.6867, "level_7": 0.4578, "level_8": 0.6506, "level_9": 0.4458, "all": 0.682 }, "mixed_topics": { "story": 2.0809, "maths": 1.3811, "qa": 3.7103 }, "mixed_accuracy": 0.144, "mean_runs": 2.654, "mean_runs_by_dataset": { "story": 2.055, "maths": 3.258, "qa": 2.33, "mixed": 2.5, "explain": 2.971, "compose": 2.814, "wordmath": 2.648 }, "mean_runs_paid": 2.654, "run_counts": [ 142, 1339, 1721, 382 ], "activation": { "story": [ 119, 0, 2, 391 ], "maths": [ 0, 0, 512, 0 ], "qa": [ 217, 0, 49, 246 ], "mixed": [ 179, 0, 127, 206 ], "explain": [ 63, 0, 379, 70 ], "compose": [ 141, 0, 272, 99 ], "wordmath": [ 191, 0, 171, 150 ] }, "mutual_info_bits": 0.4327, "mutual_info_shuffled": 0.0023, "mean_runs_by_expert": [ 2.649, null, 3.253, 1.878 ], "tflops_per_token": 0.00010933171857142857 }, "3.0": { "loss": { "story": 1.8634, "maths": 1.1334, "qa": 3.2286, "mixed": 2.6058, "explain": 0.4795, "compose": 1.8684, "wordmath": 1.0599 }, "mean_loss": 1.7484, "maths_accuracy": 0.67, "maths_levels": { "level_2": 0.9167, "level_5": 0.8929, "level_6": 0.7108, "level_7": 0.4096, "level_8": 0.6386, "level_9": 0.4458, "all": 0.67 }, "mixed_topics": { "story": 2.0829, "maths": 1.3831, "qa": 3.7082 }, "mixed_accuracy": 0.142, "mean_runs": 3.147, "mean_runs_by_dataset": { "story": 2.551, "maths": 3.748, "qa": 2.82, "mixed": 2.994, "explain": 3.496, "compose": 3.309, "wordmath": 3.113 }, "mean_runs_paid": 3.147, "run_counts": [ 0, 735, 1586, 1263 ], "activation": { "story": [ 119, 0, 2, 391 ], "maths": [ 0, 0, 512, 0 ], "qa": [ 217, 0, 49, 246 ], "mixed": [ 179, 0, 127, 206 ], "explain": [ 63, 0, 379, 70 ], "compose": [ 141, 0, 272, 99 ], "wordmath": [ 191, 0, 171, 150 ] }, "mutual_info_bits": 0.4327, "mutual_info_shuffled": 0.0023, "mean_runs_by_expert": [ 3.125, null, 3.76, 2.367 ], "tflops_per_token": 0.00012460829714285715 } }, "targets": [ 2.62, 1.867, 3.246, 1.867 ], "cycle": { "loss": { "story": 1.8808, "maths": 1.1348, "qa": 3.2582, "mixed": 2.6168, "explain": 0.4811, "compose": 1.87, "wordmath": 1.0815 }, "mean_loss": 1.7605, "loss_by_token_runs": [ 1.7907, 1.7508, 1.7444, 1.7549 ] }, "learned": { "loss": { "story": 1.8634, "maths": 1.133, "qa": 3.2295, "mixed": 2.6033, "explain": 0.4793, "compose": 1.8647, "wordmath": 1.0596 }, "mean_loss": 1.7475, "maths_accuracy": 0.674, "maths_levels": { "level_2": 0.9048, "level_5": 0.9405, "level_6": 0.6627, "level_7": 0.4337, "level_8": 0.6506, "level_9": 0.4458, "all": 0.674 }, "mixed_topics": { "story": 2.0801, "maths": 1.3803, "qa": 3.7058 }, "mixed_accuracy": 0.15, "explain_accuracy": 0.8056, "compose_accuracy": 0.62, "gsm8k_accuracy": 0.02, "mean_runs": 2.596, "mean_runs_by_dataset": { "story": 2.1, "maths": 3.0, "qa": 2.516, "mixed": 2.447, "explain": 2.803, "compose": 2.615, "wordmath": 2.691 }, "mean_runs_paid": 2.596, "run_counts": [ 0, 1448, 2136, 0 ], "activation": { "story": [ 119, 0, 2, 391 ], "maths": [ 0, 0, 512, 0 ], "qa": [ 217, 0, 49, 246 ], "mixed": [ 179, 0, 127, 206 ], "explain": [ 63, 0, 379, 70 ], "compose": [ 141, 0, 272, 99 ], "wordmath": [ 191, 0, 171, 150 ] }, "mutual_info_bits": 0.4327, "mutual_info_shuffled": 0.0023, "mean_runs_by_expert": [ 2.686, null, 3.0, 2.0 ], "tflops_per_token": 0.0001075441257142857 }, "policy": { "best_fixed_k": 3, "per_dataset_k": { "story": 3, "maths": 3, "qa": 3, "mixed": 2, "explain": 2, "compose": 2, "wordmath": 3 }, "per_dataset_mean_k": 2.5714285714285716, "learned_mean_runs": 2.596, "tflops": { "learned": 0.0001075441257142857, "best_fixed": 0.00012004864, "per_dataset": 0.00010678418285714287 }, "vs_best_fixed": { "difference": 0.00016, "ci_low": -0.00015, "ci_high": 0.00047, "ours_better": false, "at_no_more_compute": true }, "vs_per_dataset": { "difference": 0.0011, "ci_low": 0.00079, "ci_high": 0.00142, "ours_better": false, "at_no_more_compute": false }, "note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone." } }