spidory / perloop_s2 /results.json
Davidow's picture
Paper models: 10 variants x 3 seeds, fp32 safetensors
038a279 verified
Raw History Blame Contribute Delete
11.5 kB
{
"run": "perloop_s2",
"step": 73180,
"params": 26642148,
"by_k": {
"1": {
"loss": {
"story": 1.9353,
"maths": 1.1398,
"qa": 3.3466,
"mixed": 2.6523,
"explain": 0.486,
"compose": 1.8771,
"wordmath": 1.143
},
"mean_loss": 1.7971,
"maths_accuracy": 0.616,
"maths_levels": {
"level_2": 0.9286,
"level_5": 0.9167,
"level_6": 0.6265,
"level_7": 0.3494,
"level_8": 0.506,
"level_9": 0.3614,
"all": 0.616
},
"mixed_topics": {
"story": 2.1173,
"maths": 1.396,
"qa": 3.7833
},
"mixed_accuracy": 0.094,
"explain_accuracy": 0.7333,
"compose_accuracy": 0.4933,
"gsm8k_accuracy": 0.0267,
"tflops_per_token": 5.814784e-05
},
"2": {
"loss": {
"story": 1.8635,
"maths": 1.1333,
"qa": 3.2347,
"mixed": 2.6026,
"explain": 0.4792,
"compose": 1.8624,
"wordmath": 1.0623
},
"mean_loss": 1.7483,
"maths_accuracy": 0.678,
"maths_levels": {
"level_2": 0.9286,
"level_5": 0.9167,
"level_6": 0.6386,
"level_7": 0.4578,
"level_8": 0.6627,
"level_9": 0.4578,
"all": 0.678
},
"mixed_topics": {
"story": 2.0779,
"maths": 1.3799,
"qa": 3.7061
},
"mixed_accuracy": 0.15,
"tflops_per_token": 8.909824e-05
},
"3": {
"loss": {
"story": 1.8633,
"maths": 1.133,
"qa": 3.2257,
"mixed": 2.6044,
"explain": 0.4792,
"compose": 1.8672,
"wordmath": 1.0589
},
"mean_loss": 1.7474,
"maths_accuracy": 0.674,
"maths_levels": {
"level_2": 0.9048,
"level_5": 0.9405,
"level_6": 0.6627,
"level_7": 0.4337,
"level_8": 0.6506,
"level_9": 0.4458,
"all": 0.674
},
"mixed_topics": {
"story": 2.0831,
"maths": 1.3827,
"qa": 3.705
},
"mixed_accuracy": 0.146,
"explain_accuracy": 0.8056,
"compose_accuracy": 0.6267,
"gsm8k_accuracy": 0.0333,
"tflops_per_token": 0.00012004864
},
"4": {
"loss": {
"story": 1.8676,
"maths": 1.1335,
"qa": 3.2307,
"mixed": 2.613,
"explain": 0.4795,
"compose": 1.8752,
"wordmath": 1.0609
},
"mean_loss": 1.7515,
"maths_accuracy": 0.676,
"maths_levels": {
"level_2": 0.9167,
"level_5": 0.881,
"level_6": 0.7108,
"level_7": 0.4458,
"level_8": 0.6386,
"level_9": 0.4578,
"all": 0.676
},
"mixed_topics": {
"story": 2.0873,
"maths": 1.3898,
"qa": 3.7184
},
"mixed_accuracy": 0.14,
"tflops_per_token": 0.00015099904
}
},
"by_mean": {
"2.0": {
"loss": {
"story": 1.8984,
"maths": 1.1331,
"qa": 3.2684,
"mixed": 2.6168,
"explain": 0.4801,
"compose": 1.8648,
"wordmath": 1.0753
},
"mean_loss": 1.7624,
"maths_accuracy": 0.674,
"maths_levels": {
"level_2": 0.9048,
"level_5": 0.9405,
"level_6": 0.6627,
"level_7": 0.4337,
"level_8": 0.6386,
"level_9": 0.4578,
"all": 0.674
},
"mixed_topics": {
"story": 2.09,
"maths": 1.3813,
"qa": 3.7294
},
"mixed_accuracy": 0.134,
"mean_runs": 2.147,
"mean_runs_by_dataset": {
"story": 1.551,
"maths": 2.748,
"qa": 1.82,
"mixed": 1.994,
"explain": 2.496,
"compose": 2.309,
"wordmath": 2.113
},
"mean_runs_paid": 2.147,
"run_counts": [
735,
1586,
1263,
0
],
"activation": {
"story": [
119,
0,
2,
391
],
"maths": [
0,
0,
512,
0
],
"qa": [
217,
0,
49,
246
],
"mixed": [
179,
0,
127,
206
],
"explain": [
63,
0,
379,
70
],
"compose": [
141,
0,
272,
99
],
"wordmath": [
191,
0,
171,
150
]
},
"mutual_info_bits": 0.4327,
"mutual_info_shuffled": 0.0023,
"mean_runs_by_expert": [
2.125,
null,
2.76,
1.367
],
"tflops_per_token": 9.365789714285714e-05
},
"2.5": {
"loss": {
"story": 1.8694,
"maths": 1.1331,
"qa": 3.2368,
"mixed": 2.6055,
"explain": 0.4799,
"compose": 1.8656,
"wordmath": 1.0623
},
"mean_loss": 1.7504,
"maths_accuracy": 0.682,
"maths_levels": {
"level_2": 0.9167,
"level_5": 0.9286,
"level_6": 0.6867,
"level_7": 0.4578,
"level_8": 0.6506,
"level_9": 0.4458,
"all": 0.682
},
"mixed_topics": {
"story": 2.0809,
"maths": 1.3811,
"qa": 3.7103
},
"mixed_accuracy": 0.144,
"mean_runs": 2.654,
"mean_runs_by_dataset": {
"story": 2.055,
"maths": 3.258,
"qa": 2.33,
"mixed": 2.5,
"explain": 2.971,
"compose": 2.814,
"wordmath": 2.648
},
"mean_runs_paid": 2.654,
"run_counts": [
142,
1339,
1721,
382
],
"activation": {
"story": [
119,
0,
2,
391
],
"maths": [
0,
0,
512,
0
],
"qa": [
217,
0,
49,
246
],
"mixed": [
179,
0,
127,
206
],
"explain": [
63,
0,
379,
70
],
"compose": [
141,
0,
272,
99
],
"wordmath": [
191,
0,
171,
150
]
},
"mutual_info_bits": 0.4327,
"mutual_info_shuffled": 0.0023,
"mean_runs_by_expert": [
2.649,
null,
3.253,
1.878
],
"tflops_per_token": 0.00010933171857142857
},
"3.0": {
"loss": {
"story": 1.8634,
"maths": 1.1334,
"qa": 3.2286,
"mixed": 2.6058,
"explain": 0.4795,
"compose": 1.8684,
"wordmath": 1.0599
},
"mean_loss": 1.7484,
"maths_accuracy": 0.67,
"maths_levels": {
"level_2": 0.9167,
"level_5": 0.8929,
"level_6": 0.7108,
"level_7": 0.4096,
"level_8": 0.6386,
"level_9": 0.4458,
"all": 0.67
},
"mixed_topics": {
"story": 2.0829,
"maths": 1.3831,
"qa": 3.7082
},
"mixed_accuracy": 0.142,
"mean_runs": 3.147,
"mean_runs_by_dataset": {
"story": 2.551,
"maths": 3.748,
"qa": 2.82,
"mixed": 2.994,
"explain": 3.496,
"compose": 3.309,
"wordmath": 3.113
},
"mean_runs_paid": 3.147,
"run_counts": [
0,
735,
1586,
1263
],
"activation": {
"story": [
119,
0,
2,
391
],
"maths": [
0,
0,
512,
0
],
"qa": [
217,
0,
49,
246
],
"mixed": [
179,
0,
127,
206
],
"explain": [
63,
0,
379,
70
],
"compose": [
141,
0,
272,
99
],
"wordmath": [
191,
0,
171,
150
]
},
"mutual_info_bits": 0.4327,
"mutual_info_shuffled": 0.0023,
"mean_runs_by_expert": [
3.125,
null,
3.76,
2.367
],
"tflops_per_token": 0.00012460829714285715
}
},
"targets": [
2.62,
1.867,
3.246,
1.867
],
"cycle": {
"loss": {
"story": 1.8808,
"maths": 1.1348,
"qa": 3.2582,
"mixed": 2.6168,
"explain": 0.4811,
"compose": 1.87,
"wordmath": 1.0815
},
"mean_loss": 1.7605,
"loss_by_token_runs": [
1.7907,
1.7508,
1.7444,
1.7549
]
},
"learned": {
"loss": {
"story": 1.8634,
"maths": 1.133,
"qa": 3.2295,
"mixed": 2.6033,
"explain": 0.4793,
"compose": 1.8647,
"wordmath": 1.0596
},
"mean_loss": 1.7475,
"maths_accuracy": 0.674,
"maths_levels": {
"level_2": 0.9048,
"level_5": 0.9405,
"level_6": 0.6627,
"level_7": 0.4337,
"level_8": 0.6506,
"level_9": 0.4458,
"all": 0.674
},
"mixed_topics": {
"story": 2.0801,
"maths": 1.3803,
"qa": 3.7058
},
"mixed_accuracy": 0.15,
"explain_accuracy": 0.8056,
"compose_accuracy": 0.62,
"gsm8k_accuracy": 0.02,
"mean_runs": 2.596,
"mean_runs_by_dataset": {
"story": 2.1,
"maths": 3.0,
"qa": 2.516,
"mixed": 2.447,
"explain": 2.803,
"compose": 2.615,
"wordmath": 2.691
},
"mean_runs_paid": 2.596,
"run_counts": [
0,
1448,
2136,
0
],
"activation": {
"story": [
119,
0,
2,
391
],
"maths": [
0,
0,
512,
0
],
"qa": [
217,
0,
49,
246
],
"mixed": [
179,
0,
127,
206
],
"explain": [
63,
0,
379,
70
],
"compose": [
141,
0,
272,
99
],
"wordmath": [
191,
0,
171,
150
]
},
"mutual_info_bits": 0.4327,
"mutual_info_shuffled": 0.0023,
"mean_runs_by_expert": [
2.686,
null,
3.0,
2.0
],
"tflops_per_token": 0.0001075441257142857
},
"policy": {
"best_fixed_k": 3,
"per_dataset_k": {
"story": 3,
"maths": 3,
"qa": 3,
"mixed": 2,
"explain": 2,
"compose": 2,
"wordmath": 3
},
"per_dataset_mean_k": 2.5714285714285716,
"learned_mean_runs": 2.596,
"tflops": {
"learned": 0.0001075441257142857,
"best_fixed": 0.00012004864,
"per_dataset": 0.00010678418285714287
},
"vs_best_fixed": {
"difference": 0.00016,
"ci_low": -0.00015,
"ci_high": 0.00047,
"ours_better": false,
"at_no_more_compute": true
},
"vs_per_dataset": {
"difference": 0.0011,
"ci_low": 0.00079,
"ci_high": 0.00142,
"ours_better": false,
"at_no_more_compute": false
},
"note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone."
}
}