Download perloop_s0/results.json from Davidow/spidory: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/Davidow/spidory/resolve/main/perloop_s0/results.json
- Command line
-
hf download hf://Davidow/spidory/perloop_s0/results.json
-
curl -L -o results.json https://huggingface.co/Davidow/spidory/resolve/main/perloop_s0/results.json
11.5 kB
| { | |
| "run": "perloop_s0", | |
| "step": 73180, | |
| "params": 26642148, | |
| "by_k": { | |
| "1": { | |
| "loss": { | |
| "story": 1.984, | |
| "maths": 1.1458, | |
| "qa": 3.4102, | |
| "mixed": 2.7296, | |
| "explain": 0.4829, | |
| "compose": 1.9691, | |
| "wordmath": 1.1635 | |
| }, | |
| "mean_loss": 1.8407, | |
| "maths_accuracy": 0.492, | |
| "maths_levels": { | |
| "level_2": 0.9286, | |
| "level_5": 0.8095, | |
| "level_6": 0.4578, | |
| "level_7": 0.253, | |
| "level_8": 0.2771, | |
| "level_9": 0.2169, | |
| "all": 0.492 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1804, | |
| "maths": 1.4385, | |
| "qa": 3.8899 | |
| }, | |
| "mixed_accuracy": 0.106, | |
| "explain_accuracy": 0.9167, | |
| "compose_accuracy": 0.44, | |
| "gsm8k_accuracy": 0.02, | |
| "tflops_per_token": 5.814784e-05 | |
| }, | |
| "2": { | |
| "loss": { | |
| "story": 1.9264, | |
| "maths": 1.1397, | |
| "qa": 3.317, | |
| "mixed": 2.6892, | |
| "explain": 0.4779, | |
| "compose": 1.9725, | |
| "wordmath": 1.1172 | |
| }, | |
| "mean_loss": 1.8057, | |
| "maths_accuracy": 0.564, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.6145, | |
| "level_7": 0.3253, | |
| "level_8": 0.4217, | |
| "level_9": 0.2289, | |
| "all": 0.564 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1417, | |
| "maths": 1.4498, | |
| "qa": 3.8197 | |
| }, | |
| "mixed_accuracy": 0.108, | |
| "tflops_per_token": 8.909824e-05 | |
| }, | |
| "3": { | |
| "loss": { | |
| "story": 1.9266, | |
| "maths": 1.1399, | |
| "qa": 3.3138, | |
| "mixed": 2.6985, | |
| "explain": 0.4777, | |
| "compose": 1.9856, | |
| "wordmath": 1.1168 | |
| }, | |
| "mean_loss": 1.8084, | |
| "maths_accuracy": 0.556, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5663, | |
| "level_7": 0.3133, | |
| "level_8": 0.4217, | |
| "level_9": 0.241, | |
| "all": 0.556 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1522, | |
| "maths": 1.4529, | |
| "qa": 3.8316 | |
| }, | |
| "mixed_accuracy": 0.108, | |
| "explain_accuracy": 0.9278, | |
| "compose_accuracy": 0.5, | |
| "gsm8k_accuracy": 0.02, | |
| "tflops_per_token": 0.00012004864 | |
| }, | |
| "4": { | |
| "loss": { | |
| "story": 1.9285, | |
| "maths": 1.1403, | |
| "qa": 3.317, | |
| "mixed": 2.6917, | |
| "explain": 0.4776, | |
| "compose": 1.9698, | |
| "wordmath": 1.119 | |
| }, | |
| "mean_loss": 1.8063, | |
| "maths_accuracy": 0.56, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5783, | |
| "level_7": 0.3253, | |
| "level_8": 0.4096, | |
| "level_9": 0.253, | |
| "all": 0.56 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1447, | |
| "maths": 1.4487, | |
| "qa": 3.8231 | |
| }, | |
| "mixed_accuracy": 0.102, | |
| "tflops_per_token": 0.00015099904 | |
| } | |
| }, | |
| "by_mean": { | |
| "2.0": { | |
| "loss": { | |
| "story": 1.9428, | |
| "maths": 1.14, | |
| "qa": 3.3436, | |
| "mixed": 2.7042, | |
| "explain": 0.4788, | |
| "compose": 1.9783, | |
| "wordmath": 1.127 | |
| }, | |
| "mean_loss": 1.8164, | |
| "maths_accuracy": 0.56, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5663, | |
| "level_7": 0.3614, | |
| "level_8": 0.3976, | |
| "level_9": 0.241, | |
| "all": 0.56 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1576, | |
| "maths": 1.4495, | |
| "qa": 3.8421 | |
| }, | |
| "mixed_accuracy": 0.104, | |
| "mean_runs": 2.299, | |
| "mean_runs_by_dataset": { | |
| "story": 1.826, | |
| "maths": 3.082, | |
| "qa": 1.852, | |
| "mixed": 2.16, | |
| "explain": 2.598, | |
| "compose": 2.488, | |
| "wordmath": 2.088 | |
| }, | |
| "mean_runs_paid": 2.299, | |
| "run_counts": [ | |
| 651, | |
| 1346, | |
| 1451, | |
| 136 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 181, | |
| 21, | |
| 186, | |
| 124 | |
| ], | |
| "maths": [ | |
| 0, | |
| 512, | |
| 0, | |
| 0 | |
| ], | |
| "qa": [ | |
| 207, | |
| 51, | |
| 143, | |
| 111 | |
| ], | |
| "mixed": [ | |
| 128, | |
| 156, | |
| 121, | |
| 107 | |
| ], | |
| "explain": [ | |
| 2, | |
| 332, | |
| 54, | |
| 124 | |
| ], | |
| "compose": [ | |
| 47, | |
| 274, | |
| 93, | |
| 98 | |
| ], | |
| "wordmath": [ | |
| 136, | |
| 116, | |
| 157, | |
| 103 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4067, | |
| "mutual_info_shuffled": 0.0036, | |
| "mean_runs_by_expert": [ | |
| 1.525, | |
| 3.093, | |
| 2.166, | |
| 1.523 | |
| ], | |
| "tflops_per_token": 9.835572571428572e-05 | |
| }, | |
| "2.5": { | |
| "loss": { | |
| "story": 1.9267, | |
| "maths": 1.1401, | |
| "qa": 3.3158, | |
| "mixed": 2.6921, | |
| "explain": 0.4781, | |
| "compose": 1.9743, | |
| "wordmath": 1.1171 | |
| }, | |
| "mean_loss": 1.8063, | |
| "maths_accuracy": 0.566, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5783, | |
| "level_7": 0.3494, | |
| "level_8": 0.4217, | |
| "level_9": 0.253, | |
| "all": 0.566 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1447, | |
| "maths": 1.4507, | |
| "qa": 3.8236 | |
| }, | |
| "mixed_accuracy": 0.11, | |
| "mean_runs": 2.796, | |
| "mean_runs_by_dataset": { | |
| "story": 2.303, | |
| "maths": 3.613, | |
| "qa": 2.346, | |
| "mixed": 2.65, | |
| "explain": 3.129, | |
| "compose": 2.955, | |
| "wordmath": 2.578 | |
| }, | |
| "mean_runs_paid": 2.796, | |
| "run_counts": [ | |
| 2, | |
| 1610, | |
| 1088, | |
| 884 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 181, | |
| 21, | |
| 186, | |
| 124 | |
| ], | |
| "maths": [ | |
| 0, | |
| 512, | |
| 0, | |
| 0 | |
| ], | |
| "qa": [ | |
| 207, | |
| 51, | |
| 143, | |
| 111 | |
| ], | |
| "mixed": [ | |
| 128, | |
| 156, | |
| 121, | |
| 107 | |
| ], | |
| "explain": [ | |
| 2, | |
| 332, | |
| 54, | |
| 124 | |
| ], | |
| "compose": [ | |
| 47, | |
| 274, | |
| 93, | |
| 98 | |
| ], | |
| "wordmath": [ | |
| 136, | |
| 116, | |
| 157, | |
| 103 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4067, | |
| "mutual_info_shuffled": 0.0036, | |
| "mean_runs_by_expert": [ | |
| 1.999, | |
| 3.605, | |
| 2.676, | |
| 1.999 | |
| ], | |
| "tflops_per_token": 0.00011374456857142857 | |
| }, | |
| "3.0": { | |
| "loss": { | |
| "story": 1.9267, | |
| "maths": 1.1403, | |
| "qa": 3.3145, | |
| "mixed": 2.6941, | |
| "explain": 0.4778, | |
| "compose": 1.9743, | |
| "wordmath": 1.1175 | |
| }, | |
| "mean_loss": 1.8065, | |
| "maths_accuracy": 0.56, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5783, | |
| "level_7": 0.3253, | |
| "level_8": 0.4096, | |
| "level_9": 0.253, | |
| "all": 0.56 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1477, | |
| "maths": 1.4509, | |
| "qa": 3.8257 | |
| }, | |
| "mixed_accuracy": 0.108, | |
| "mean_runs": 3.272, | |
| "mean_runs_by_dataset": { | |
| "story": 2.85, | |
| "maths": 4.0, | |
| "qa": 2.861, | |
| "mixed": 3.127, | |
| "explain": 3.557, | |
| "compose": 3.436, | |
| "wordmath": 3.074 | |
| }, | |
| "mean_runs_paid": 3.272, | |
| "run_counts": [ | |
| 0, | |
| 623, | |
| 1363, | |
| 1598 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 181, | |
| 21, | |
| 186, | |
| 124 | |
| ], | |
| "maths": [ | |
| 0, | |
| 512, | |
| 0, | |
| 0 | |
| ], | |
| "qa": [ | |
| 207, | |
| 51, | |
| 143, | |
| 111 | |
| ], | |
| "mixed": [ | |
| 128, | |
| 156, | |
| 121, | |
| 107 | |
| ], | |
| "explain": [ | |
| 2, | |
| 332, | |
| 54, | |
| 124 | |
| ], | |
| "compose": [ | |
| 47, | |
| 274, | |
| 93, | |
| 98 | |
| ], | |
| "wordmath": [ | |
| 136, | |
| 116, | |
| 157, | |
| 103 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4067, | |
| "mutual_info_shuffled": 0.0036, | |
| "mean_runs_by_expert": [ | |
| 2.549, | |
| 4.0, | |
| 3.18, | |
| 2.54 | |
| ], | |
| "tflops_per_token": 0.00012846846142857145 | |
| } | |
| }, | |
| "targets": [ | |
| 1.907, | |
| 3.5, | |
| 2.584, | |
| 1.907 | |
| ], | |
| "cycle": { | |
| "loss": { | |
| "story": 1.9396, | |
| "maths": 1.1414, | |
| "qa": 3.3372, | |
| "mixed": 2.7, | |
| "explain": 0.4785, | |
| "compose": 1.9743, | |
| "wordmath": 1.1289 | |
| }, | |
| "mean_loss": 1.8143, | |
| "loss_by_token_runs": [ | |
| 1.8274, | |
| 1.8131, | |
| 1.8073, | |
| 1.8085 | |
| ] | |
| }, | |
| "learned": { | |
| "loss": { | |
| "story": 1.9266, | |
| "maths": 1.1403, | |
| "qa": 3.3153, | |
| "mixed": 2.6928, | |
| "explain": 0.4782, | |
| "compose": 1.9772, | |
| "wordmath": 1.1169 | |
| }, | |
| "mean_loss": 1.8068, | |
| "maths_accuracy": 0.56, | |
| "maths_levels": { | |
| "level_2": 0.9524, | |
| "level_5": 0.8333, | |
| "level_6": 0.5783, | |
| "level_7": 0.3253, | |
| "level_8": 0.4096, | |
| "level_9": 0.253, | |
| "all": 0.56 | |
| }, | |
| "mixed_topics": { | |
| "story": 2.1458, | |
| "maths": 1.4507, | |
| "qa": 3.8244 | |
| }, | |
| "mixed_accuracy": 0.108, | |
| "explain_accuracy": 0.9278, | |
| "compose_accuracy": 0.4933, | |
| "gsm8k_accuracy": 0.04, | |
| "mean_runs": 2.885, | |
| "mean_runs_by_dataset": { | |
| "story": 2.404, | |
| "maths": 4.0, | |
| "qa": 2.379, | |
| "mixed": 2.648, | |
| "explain": 3.332, | |
| "compose": 2.887, | |
| "wordmath": 2.545 | |
| }, | |
| "mean_runs_paid": 2.885, | |
| "run_counts": [ | |
| 39, | |
| 1327, | |
| 1225, | |
| 993 | |
| ], | |
| "activation": { | |
| "story": [ | |
| 181, | |
| 21, | |
| 186, | |
| 124 | |
| ], | |
| "maths": [ | |
| 0, | |
| 512, | |
| 0, | |
| 0 | |
| ], | |
| "qa": [ | |
| 207, | |
| 51, | |
| 143, | |
| 111 | |
| ], | |
| "mixed": [ | |
| 128, | |
| 156, | |
| 121, | |
| 107 | |
| ], | |
| "explain": [ | |
| 2, | |
| 332, | |
| 54, | |
| 124 | |
| ], | |
| "compose": [ | |
| 47, | |
| 274, | |
| 93, | |
| 98 | |
| ], | |
| "wordmath": [ | |
| 136, | |
| 116, | |
| 157, | |
| 103 | |
| ] | |
| }, | |
| "mutual_info_bits": 0.4067, | |
| "mutual_info_shuffled": 0.0036, | |
| "mean_runs_by_expert": [ | |
| 2.0, | |
| 3.679, | |
| 3.0, | |
| 1.945 | |
| ], | |
| "tflops_per_token": 0.00011649072571428571 | |
| }, | |
| "policy": { | |
| "best_fixed_k": 2, | |
| "per_dataset_k": { | |
| "story": 2, | |
| "maths": 2, | |
| "qa": 3, | |
| "mixed": 2, | |
| "explain": 4, | |
| "compose": 4, | |
| "wordmath": 3 | |
| }, | |
| "per_dataset_mean_k": 2.857142857142857, | |
| "learned_mean_runs": 2.885, | |
| "tflops": { | |
| "learned": 0.00011649072571428571, | |
| "best_fixed": 8.909824e-05, | |
| "per_dataset": 0.00011562715428571429 | |
| }, | |
| "vs_best_fixed": { | |
| "difference": 0.00105, | |
| "ci_low": 0.00072, | |
| "ci_high": 0.00138, | |
| "ours_better": false, | |
| "at_no_more_compute": false | |
| }, | |
| "vs_per_dataset": { | |
| "difference": 0.00201, | |
| "ci_low": 0.00165, | |
| "ci_high": 0.00236, | |
| "ours_better": false, | |
| "at_no_more_compute": false | |
| }, | |
| "note": "A win counts only with ours_better and at_no_more_compute. The mix variant always computes every run, and its fixed-run baselines decode states it never saw alone." | |
| } | |
| } |