spidory / perloop_s0 /results.json
Davidow's picture
Paper models: 10 variants x 3 seeds, fp32 safetensors
038a279 verified
Raw History Blame Contribute Delete
11.5 kB
{
"run": "perloop_s0",
"step": 73180,
"params": 26642148,
"by_k": {
"1": {
"loss": {
"story": 1.984,
"maths": 1.1458,
"qa": 3.4102,
"mixed": 2.7296,
"explain": 0.4829,
"compose": 1.9691,
"wordmath": 1.1635
},
"mean_loss": 1.8407,
"maths_accuracy": 0.492,
"maths_levels": {
"level_2": 0.9286,
"level_5": 0.8095,
"level_6": 0.4578,
"level_7": 0.253,
"level_8": 0.2771,
"level_9": 0.2169,
"all": 0.492
},
"mixed_topics": {
"story": 2.1804,
"maths": 1.4385,
"qa": 3.8899
},
"mixed_accuracy": 0.106,
"explain_accuracy": 0.9167,
"compose_accuracy": 0.44,
"gsm8k_accuracy": 0.02,
"tflops_per_token": 5.814784e-05
},
"2": {
"loss": {
"story": 1.9264,
"maths": 1.1397,
"qa": 3.317,
"mixed": 2.6892,
"explain": 0.4779,
"compose": 1.9725,
"wordmath": 1.1172
},
"mean_loss": 1.8057,
"maths_accuracy": 0.564,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.6145,
"level_7": 0.3253,
"level_8": 0.4217,
"level_9": 0.2289,
"all": 0.564
},
"mixed_topics": {
"story": 2.1417,
"maths": 1.4498,
"qa": 3.8197
},
"mixed_accuracy": 0.108,
"tflops_per_token": 8.909824e-05
},
"3": {
"loss": {
"story": 1.9266,
"maths": 1.1399,
"qa": 3.3138,
"mixed": 2.6985,
"explain": 0.4777,
"compose": 1.9856,
"wordmath": 1.1168
},
"mean_loss": 1.8084,
"maths_accuracy": 0.556,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5663,
"level_7": 0.3133,
"level_8": 0.4217,
"level_9": 0.241,
"all": 0.556
},
"mixed_topics": {
"story": 2.1522,
"maths": 1.4529,
"qa": 3.8316
},
"mixed_accuracy": 0.108,
"explain_accuracy": 0.9278,
"compose_accuracy": 0.5,
"gsm8k_accuracy": 0.02,
"tflops_per_token": 0.00012004864
},
"4": {
"loss": {
"story": 1.9285,
"maths": 1.1403,
"qa": 3.317,
"mixed": 2.6917,
"explain": 0.4776,
"compose": 1.9698,
"wordmath": 1.119
},
"mean_loss": 1.8063,
"maths_accuracy": 0.56,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5783,
"level_7": 0.3253,
"level_8": 0.4096,
"level_9": 0.253,
"all": 0.56
},
"mixed_topics": {
"story": 2.1447,
"maths": 1.4487,
"qa": 3.8231
},
"mixed_accuracy": 0.102,
"tflops_per_token": 0.00015099904
}
},
"by_mean": {
"2.0": {
"loss": {
"story": 1.9428,
"maths": 1.14,
"qa": 3.3436,
"mixed": 2.7042,
"explain": 0.4788,
"compose": 1.9783,
"wordmath": 1.127
},
"mean_loss": 1.8164,
"maths_accuracy": 0.56,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5663,
"level_7": 0.3614,
"level_8": 0.3976,
"level_9": 0.241,
"all": 0.56
},
"mixed_topics": {
"story": 2.1576,
"maths": 1.4495,
"qa": 3.8421
},
"mixed_accuracy": 0.104,
"mean_runs": 2.299,
"mean_runs_by_dataset": {
"story": 1.826,
"maths": 3.082,
"qa": 1.852,
"mixed": 2.16,
"explain": 2.598,
"compose": 2.488,
"wordmath": 2.088
},
"mean_runs_paid": 2.299,
"run_counts": [
651,
1346,
1451,
136
],
"activation": {
"story": [
181,
21,
186,
124
],
"maths": [
0,
512,
0,
0
],
"qa": [
207,
51,
143,
111
],
"mixed": [
128,
156,
121,
107
],
"explain": [
2,
332,
54,
124
],
"compose": [
47,
274,
93,
98
],
"wordmath": [
136,
116,
157,
103
]
},
"mutual_info_bits": 0.4067,
"mutual_info_shuffled": 0.0036,
"mean_runs_by_expert": [
1.525,
3.093,
2.166,
1.523
],
"tflops_per_token": 9.835572571428572e-05
},
"2.5": {
"loss": {
"story": 1.9267,
"maths": 1.1401,
"qa": 3.3158,
"mixed": 2.6921,
"explain": 0.4781,
"compose": 1.9743,
"wordmath": 1.1171
},
"mean_loss": 1.8063,
"maths_accuracy": 0.566,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5783,
"level_7": 0.3494,
"level_8": 0.4217,
"level_9": 0.253,
"all": 0.566
},
"mixed_topics": {
"story": 2.1447,
"maths": 1.4507,
"qa": 3.8236
},
"mixed_accuracy": 0.11,
"mean_runs": 2.796,
"mean_runs_by_dataset": {
"story": 2.303,
"maths": 3.613,
"qa": 2.346,
"mixed": 2.65,
"explain": 3.129,
"compose": 2.955,
"wordmath": 2.578
},
"mean_runs_paid": 2.796,
"run_counts": [
2,
1610,
1088,
884
],
"activation": {
"story": [
181,
21,
186,
124
],
"maths": [
0,
512,
0,
0
],
"qa": [
207,
51,
143,
111
],
"mixed": [
128,
156,
121,
107
],
"explain": [
2,
332,
54,
124
],
"compose": [
47,
274,
93,
98
],
"wordmath": [
136,
116,
157,
103
]
},
"mutual_info_bits": 0.4067,
"mutual_info_shuffled": 0.0036,
"mean_runs_by_expert": [
1.999,
3.605,
2.676,
1.999
],
"tflops_per_token": 0.00011374456857142857
},
"3.0": {
"loss": {
"story": 1.9267,
"maths": 1.1403,
"qa": 3.3145,
"mixed": 2.6941,
"explain": 0.4778,
"compose": 1.9743,
"wordmath": 1.1175
},
"mean_loss": 1.8065,
"maths_accuracy": 0.56,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5783,
"level_7": 0.3253,
"level_8": 0.4096,
"level_9": 0.253,
"all": 0.56
},
"mixed_topics": {
"story": 2.1477,
"maths": 1.4509,
"qa": 3.8257
},
"mixed_accuracy": 0.108,
"mean_runs": 3.272,
"mean_runs_by_dataset": {
"story": 2.85,
"maths": 4.0,
"qa": 2.861,
"mixed": 3.127,
"explain": 3.557,
"compose": 3.436,
"wordmath": 3.074
},
"mean_runs_paid": 3.272,
"run_counts": [
0,
623,
1363,
1598
],
"activation": {
"story": [
181,
21,
186,
124
],
"maths": [
0,
512,
0,
0
],
"qa": [
207,
51,
143,
111
],
"mixed": [
128,
156,
121,
107
],
"explain": [
2,
332,
54,
124
],
"compose": [
47,
274,
93,
98
],
"wordmath": [
136,
116,
157,
103
]
},
"mutual_info_bits": 0.4067,
"mutual_info_shuffled": 0.0036,
"mean_runs_by_expert": [
2.549,
4.0,
3.18,
2.54
],
"tflops_per_token": 0.00012846846142857145
}
},
"targets": [
1.907,
3.5,
2.584,
1.907
],
"cycle": {
"loss": {
"story": 1.9396,
"maths": 1.1414,
"qa": 3.3372,
"mixed": 2.7,
"explain": 0.4785,
"compose": 1.9743,
"wordmath": 1.1289
},
"mean_loss": 1.8143,
"loss_by_token_runs": [
1.8274,
1.8131,
1.8073,
1.8085
]
},
"learned": {
"loss": {
"story": 1.9266,
"maths": 1.1403,
"qa": 3.3153,
"mixed": 2.6928,
"explain": 0.4782,
"compose": 1.9772,
"wordmath": 1.1169
},
"mean_loss": 1.8068,
"maths_accuracy": 0.56,
"maths_levels": {
"level_2": 0.9524,
"level_5": 0.8333,
"level_6": 0.5783,
"level_7": 0.3253,
"level_8": 0.4096,
"level_9": 0.253,
"all": 0.56
},
"mixed_topics": {
"story": 2.1458,
"maths": 1.4507,
"qa": 3.8244
},
"mixed_accuracy": 0.108,
"explain_accuracy": 0.9278,
"compose_accuracy": 0.4933,
"gsm8k_accuracy": 0.04,
"mean_runs": 2.885,
"mean_runs_by_dataset": {
"story": 2.404,
"maths": 4.0,
"qa": 2.379,
"mixed": 2.648,
"explain": 3.332,
"compose": 2.887,
"wordmath": 2.545
},
"mean_runs_paid": 2.885,
"run_counts": [
39,
1327,
1225,
993
],
"activation": {
"story": [
181,
21,
186,
124
],
"maths": [
0,
512,
0,
0
],
"qa": [
207,
51,
143,
111
],
"mixed": [
128,
156,
121,
107
],
"explain": [
2,
332,
54,
124
],
"compose": [
47,
274,
93,
98
],
"wordmath": [
136,
116,
157,
103
]
},
"mutual_info_bits": 0.4067,
"mutual_info_shuffled": 0.0036,
"mean_runs_by_expert": [
2.0,
3.679,
3.0,
1.945
],
"tflops_per_token": 0.00011649072571428571
},
"policy": {
"best_fixed_k": 2,
"per_dataset_k": {
"story": 2,
"maths": 2,
"qa": 3,
"mixed": 2,
"explain": 4,
"compose": 4,
"wordmath": 3
},
"per_dataset_mean_k": 2.857142857142857,
"learned_mean_runs": 2.885,
"tflops": {
"learned": 0.00011649072571428571,
"best_fixed": 8.909824e-05,
"per_dataset": 0.00011562715428571429
},
"vs_best_fixed": {
"difference": 0.00105,
"ci_low": 0.00072,
"ci_high": 0.00138,
"ours_better": false,
"at_no_more_compute": false
},
"vs_per_dataset": {
"difference": 0.00201,
"ci_low": 0.00165,
"ci_high": 0.00236,
"ours_better": false,
"at_no_more_compute": false
},
"note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone."
}
}