Download perloop_s2/results.json from Davidow/spidory: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/Davidow/spidory/resolve/main/perloop_s2/results.json
- Command line
-
hf download hf://Davidow/spidory/perloop_s2/results.json
-
curl -L -o results.json https://huggingface.co/Davidow/spidory/resolve/main/perloop_s2/results.json
11.5 kB
| { | |
| "run": "perloop_s2", | |
| "step": 73180, | |
| "params": 26642148, | |
| "by_k": { | |
| "1": { | |
| "loss": { | |
| "story": 1.9353, | |
| "maths": 1.1398, | |
| "qa": 3.3466, | |
| "mixed": 2.6523, | |
| "explain": 0.486, | |
| "compose": 1.8771, | |
| "wordmath": 1.143 | |
| }, | |
| "mean_loss": 1.7971, | |
| "maths_accuracy": 0.616, | |
| "maths_levels": { | |
| "level_2": 0.9286, | |
| "level_5": 0.9167, | |
| "level_6": 0.6265, | |
| "level_7": 0.3494, | |
| "level_8": 0.506, | |
| "level_9": 0.3614, | |
| "all": 0.616 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1173, | |
| "maths": 1.396, | |
| "qa": 3.7833 | |
| }, | |
| "mixed_accuracy": 0.094, | |
| "explain_accuracy": 0.7333, | |
| "compose_accuracy": 0.4933, | |
| "gsm8k_accuracy": 0.0267, | |
| "tflops_per_token": 5.814784e-05 | |
| }, | |
| "2": { | |
| "loss": { | |
| "story": 1.8635, | |
| "maths": 1.1333, | |
| "qa": 3.2347, | |
| "mixed": 2.6026, | |
| "explain": 0.4792, | |
| "compose": 1.8624, | |
| "wordmath": 1.0623 | |
| }, | |
| "mean_loss": 1.7483, | |
| "maths_accuracy": 0.678, | |
| "maths_levels": { | |
| "level_2": 0.9286, | |
| "level_5": 0.9167, | |
| "level_6": 0.6386, | |
| "level_7": 0.4578, | |
| "level_8": 0.6627, | |
| "level_9": 0.4578, | |
| "all": 0.678 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0779, | |
| "maths": 1.3799, | |
| "qa": 3.7061 | |
| }, | |
| "mixed_accuracy": 0.15, | |
| "tflops_per_token": 8.909824e-05 | |
| }, | |
| "3": { | |
| "loss": { | |
| "story": 1.8633, | |
| "maths": 1.133, | |
| "qa": 3.2257, | |
| "mixed": 2.6044, | |
| "explain": 0.4792, | |
| "compose": 1.8672, | |
| "wordmath": 1.0589 | |
| }, | |
| "mean_loss": 1.7474, | |
| "maths_accuracy": 0.674, | |
| "maths_levels": { | |
| "level_2": 0.9048, | |
| "level_5": 0.9405, | |
| "level_6": 0.6627, | |
| "level_7": 0.4337, | |
| "level_8": 0.6506, | |
| "level_9": 0.4458, | |
| "all": 0.674 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0831, | |
| "maths": 1.3827, | |
| "qa": 3.705 | |
| }, | |
| "mixed_accuracy": 0.146, | |
| "explain_accuracy": 0.8056, | |
| "compose_accuracy": 0.6267, | |
| "gsm8k_accuracy": 0.0333, | |
| "tflops_per_token": 0.00012004864 | |
| }, | |
| "4": { | |
| "loss": { | |
| "story": 1.8676, | |
| "maths": 1.1335, | |
| "qa": 3.2307, | |
| "mixed": 2.613, | |
| "explain": 0.4795, | |
| "compose": 1.8752, | |
| "wordmath": 1.0609 | |
| }, | |
| "mean_loss": 1.7515, | |
| "maths_accuracy": 0.676, | |
| "maths_levels": { | |
| "level_2": 0.9167, | |
| "level_5": 0.881, | |
| "level_6": 0.7108, | |
| "level_7": 0.4458, | |
| "level_8": 0.6386, | |
| "level_9": 0.4578, | |
| "all": 0.676 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0873, | |
| "maths": 1.3898, | |
| "qa": 3.7184 | |
| }, | |
| "mixed_accuracy": 0.14, | |
| "tflops_per_token": 0.00015099904 | |
| } | |
| }, | |
| "by_mean": { | |
| "2.0": { | |
| "loss": { | |
| "story": 1.8984, | |
| "maths": 1.1331, | |
| "qa": 3.2684, | |
| "mixed": 2.6168, | |
| "explain": 0.4801, | |
| "compose": 1.8648, | |
| "wordmath": 1.0753 | |
| }, | |
| "mean_loss": 1.7624, | |
| "maths_accuracy": 0.674, | |
| "maths_levels": { | |
| "level_2": 0.9048, | |
| "level_5": 0.9405, | |
| "level_6": 0.6627, | |
| "level_7": 0.4337, | |
| "level_8": 0.6386, | |
| "level_9": 0.4578, | |
| "all": 0.674 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.09, | |
| "maths": 1.3813, | |
| "qa": 3.7294 | |
| }, | |
| "mixed_accuracy": 0.134, | |
| "mean_runs": 2.147, | |
| "mean_runs_by_dataset": { | |
| "story": 1.551, | |
| "maths": 2.748, | |
| "qa": 1.82, | |
| "mixed": 1.994, | |
| "explain": 2.496, | |
| "compose": 2.309, | |
| "wordmath": 2.113 | |
| }, | |
| "mean_runs_paid": 2.147, | |
| "run_counts": [ | |
| 735, | |
| 1586, | |
| 1263, | |
| 0 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 119, | |
| 0, | |
| 2, | |
| 391 | |
| ], | |
| "maths": [ | |
| 0, | |
| 0, | |
| 512, | |
| 0 | |
| ], | |
| "qa": [ | |
| 217, | |
| 0, | |
| 49, | |
| 246 | |
| ], | |
| "mixed": [ | |
| 179, | |
| 0, | |
| 127, | |
| 206 | |
| ], | |
| "explain": [ | |
| 63, | |
| 0, | |
| 379, | |
| 70 | |
| ], | |
| "compose": [ | |
| 141, | |
| 0, | |
| 272, | |
| 99 | |
| ], | |
| "wordmath": [ | |
| 191, | |
| 0, | |
| 171, | |
| 150 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4327, | |
| "mutual_info_shuffled": 0.0023, | |
| "mean_runs_by_expert": [ | |
| 2.125, | |
| null, | |
| 2.76, | |
| 1.367 | |
| ], | |
| "tflops_per_token": 9.365789714285714e-05 | |
| }, | |
| "2.5": { | |
| "loss": { | |
| "story": 1.8694, | |
| "maths": 1.1331, | |
| "qa": 3.2368, | |
| "mixed": 2.6055, | |
| "explain": 0.4799, | |
| "compose": 1.8656, | |
| "wordmath": 1.0623 | |
| }, | |
| "mean_loss": 1.7504, | |
| "maths_accuracy": 0.682, | |
| "maths_levels": { | |
| "level_2": 0.9167, | |
| "level_5": 0.9286, | |
| "level_6": 0.6867, | |
| "level_7": 0.4578, | |
| "level_8": 0.6506, | |
| "level_9": 0.4458, | |
| "all": 0.682 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0809, | |
| "maths": 1.3811, | |
| "qa": 3.7103 | |
| }, | |
| "mixed_accuracy": 0.144, | |
| "mean_runs": 2.654, | |
| "mean_runs_by_dataset": { | |
| "story": 2.055, | |
| "maths": 3.258, | |
| "qa": 2.33, | |
| "mixed": 2.5, | |
| "explain": 2.971, | |
| "compose": 2.814, | |
| "wordmath": 2.648 | |
| }, | |
| "mean_runs_paid": 2.654, | |
| "run_counts": [ | |
| 142, | |
| 1339, | |
| 1721, | |
| 382 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 119, | |
| 0, | |
| 2, | |
| 391 | |
| ], | |
| "maths": [ | |
| 0, | |
| 0, | |
| 512, | |
| 0 | |
| ], | |
| "qa": [ | |
| 217, | |
| 0, | |
| 49, | |
| 246 | |
| ], | |
| "mixed": [ | |
| 179, | |
| 0, | |
| 127, | |
| 206 | |
| ], | |
| "explain": [ | |
| 63, | |
| 0, | |
| 379, | |
| 70 | |
| ], | |
| "compose": [ | |
| 141, | |
| 0, | |
| 272, | |
| 99 | |
| ], | |
| "wordmath": [ | |
| 191, | |
| 0, | |
| 171, | |
| 150 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4327, | |
| "mutual_info_shuffled": 0.0023, | |
| "mean_runs_by_expert": [ | |
| 2.649, | |
| null, | |
| 3.253, | |
| 1.878 | |
| ], | |
| "tflops_per_token": 0.00010933171857142857 | |
| }, | |
| "3.0": { | |
| "loss": { | |
| "story": 1.8634, | |
| "maths": 1.1334, | |
| "qa": 3.2286, | |
| "mixed": 2.6058, | |
| "explain": 0.4795, | |
| "compose": 1.8684, | |
| "wordmath": 1.0599 | |
| }, | |
| "mean_loss": 1.7484, | |
| "maths_accuracy": 0.67, | |
| "maths_levels": { | |
| "level_2": 0.9167, | |
| "level_5": 0.8929, | |
| "level_6": 0.7108, | |
| "level_7": 0.4096, | |
| "level_8": 0.6386, | |
| "level_9": 0.4458, | |
| "all": 0.67 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0829, | |
| "maths": 1.3831, | |
| "qa": 3.7082 | |
| }, | |
| "mixed_accuracy": 0.142, | |
| "mean_runs": 3.147, | |
| "mean_runs_by_dataset": { | |
| "story": 2.551, | |
| "maths": 3.748, | |
| "qa": 2.82, | |
| "mixed": 2.994, | |
| "explain": 3.496, | |
| "compose": 3.309, | |
| "wordmath": 3.113 | |
| }, | |
| "mean_runs_paid": 3.147, | |
| "run_counts": [ | |
| 0, | |
| 735, | |
| 1586, | |
| 1263 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 119, | |
| 0, | |
| 2, | |
| 391 | |
| ], | |
| "maths": [ | |
| 0, | |
| 0, | |
| 512, | |
| 0 | |
| ], | |
| "qa": [ | |
| 217, | |
| 0, | |
| 49, | |
| 246 | |
| ], | |
| "mixed": [ | |
| 179, | |
| 0, | |
| 127, | |
| 206 | |
| ], | |
| "explain": [ | |
| 63, | |
| 0, | |
| 379, | |
| 70 | |
| ], | |
| "compose": [ | |
| 141, | |
| 0, | |
| 272, | |
| 99 | |
| ], | |
| "wordmath": [ | |
| 191, | |
| 0, | |
| 171, | |
| 150 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4327, | |
| "mutual_info_shuffled": 0.0023, | |
| "mean_runs_by_expert": [ | |
| 3.125, | |
| null, | |
| 3.76, | |
| 2.367 | |
| ], | |
| "tflops_per_token": 0.00012460829714285715 | |
| } | |
| }, | |
| "targets": [ | |
| 2.62, | |
| 1.867, | |
| 3.246, | |
| 1.867 | |
| ], | |
| "cycle": { | |
| "loss": { | |
| "story": 1.8808, | |
| "maths": 1.1348, | |
| "qa": 3.2582, | |
| "mixed": 2.6168, | |
| "explain": 0.4811, | |
| "compose": 1.87, | |
| "wordmath": 1.0815 | |
| }, | |
| "mean_loss": 1.7605, | |
| "loss_by_token_runs": [ | |
| 1.7907, | |
| 1.7508, | |
| 1.7444, | |
| 1.7549 | |
| ] | |
| }, | |
| "learned": { | |
| "loss": { | |
| "story": 1.8634, | |
| "maths": 1.133, | |
| "qa": 3.2295, | |
| "mixed": 2.6033, | |
| "explain": 0.4793, | |
| "compose": 1.8647, | |
| "wordmath": 1.0596 | |
| }, | |
| "mean_loss": 1.7475, | |
| "maths_accuracy": 0.674, | |
| "maths_levels": { | |
| "level_2": 0.9048, | |
| "level_5": 0.9405, | |
| "level_6": 0.6627, | |
| "level_7": 0.4337, | |
| "level_8": 0.6506, | |
| "level_9": 0.4458, | |
| "all": 0.674 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.0801, | |
| "maths": 1.3803, | |
| "qa": 3.7058 | |
| }, | |
| "mixed_accuracy": 0.15, | |
| "explain_accuracy": 0.8056, | |
| "compose_accuracy": 0.62, | |
| "gsm8k_accuracy": 0.02, | |
| "mean_runs": 2.596, | |
| "mean_runs_by_dataset": { | |
| "story": 2.1, | |
| "maths": 3.0, | |
| "qa": 2.516, | |
| "mixed": 2.447, | |
| "explain": 2.803, | |
| "compose": 2.615, | |
| "wordmath": 2.691 | |
| }, | |
| "mean_runs_paid": 2.596, | |
| "run_counts": [ | |
| 0, | |
| 1448, | |
| 2136, | |
| 0 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 119, | |
| 0, | |
| 2, | |
| 391 | |
| ], | |
| "maths": [ | |
| 0, | |
| 0, | |
| 512, | |
| 0 | |
| ], | |
| "qa": [ | |
| 217, | |
| 0, | |
| 49, | |
| 246 | |
| ], | |
| "mixed": [ | |
| 179, | |
| 0, | |
| 127, | |
| 206 | |
| ], | |
| "explain": [ | |
| 63, | |
| 0, | |
| 379, | |
| 70 | |
| ], | |
| "compose": [ | |
| 141, | |
| 0, | |
| 272, | |
| 99 | |
| ], | |
| "wordmath": [ | |
| 191, | |
| 0, | |
| 171, | |
| 150 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4327, | |
| "mutual_info_shuffled": 0.0023, | |
| "mean_runs_by_expert": [ | |
| 2.686, | |
| null, | |
| 3.0, | |
| 2.0 | |
| ], | |
| "tflops_per_token": 0.0001075441257142857 | |
| }, | |
| "policy": { | |
| "best_fixed_k": 3, | |
| "per_dataset_k": { | |
| "story": 3, | |
| "maths": 3, | |
| "qa": 3, | |
| "mixed": 2, | |
| "explain": 2, | |
| "compose": 2, | |
| "wordmath": 3 | |
| }, | |
| "per_dataset_mean_k": 2.5714285714285716, | |
| "learned_mean_runs": 2.596, | |
| "tflops": { | |
| "learned": 0.0001075441257142857, | |
| "best_fixed": 0.00012004864, | |
| "per_dataset": 0.00010678418285714287 | |
| }, | |
| "vs_best_fixed": { | |
| "difference": 0.00016, | |
| "ci_low": -0.00015, | |
| "ci_high": 0.00047, | |
| "ours_better": false, | |
| "at_no_more_compute": true | |
| }, | |
| "vs_per_dataset": { | |
| "difference": 0.0011, | |
| "ci_low": 0.00079, | |
| "ci_high": 0.00142, | |
| "ours_better": false, | |
| "at_no_more_compute": false | |
| }, | |
| "note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone." | |
| } | |
| } |