| { |
| "problems": 256, |
| "samples": 512, |
| "acc": 0.34375, |
| "acc_se": 0.0264731412167569, |
| "cut_share": 0.6171875, |
| "mean_tokens": 23300.6796875, |
| "closed_share": 0.42578125, |
| "p_finish": 0.3828125, |
| "p_correct_given_finished": 0.8979591836734694, |
| "p_correct_given_cut": 0.0, |
| "no_answer_finished_share": 0.01171875, |
| "loop_end_share_of_cut": 0.8829113924050633, |
| "tail_dup8_cut_mean": 0.8536547468354431, |
| "median_tokens_correct": 5700.5, |
| "by_source": { |
| "gsm8k": { |
| "problems": 128, |
| "samples": 256, |
| "acc": 0.46484375, |
| "acc_se": 0.039287919952830035, |
| "cut_share": 0.48828125, |
| "mean_tokens": 19775.83203125 |
| }, |
| "math": { |
| "problems": 128, |
| "samples": 256, |
| "acc": 0.22265625, |
| "acc_se": 0.03224717766273672, |
| "cut_share": 0.74609375, |
| "mean_tokens": 26825.52734375 |
| } |
| }, |
| "run": { |
| "name": "dev/base+rep1.1", |
| "model": "base", |
| "data": "dev256", |
| "n": 2, |
| "shards": 1, |
| "est": 14, |
| "set": [ |
| "repetition_penalty=1.1" |
| ] |
| }, |
| "sampling": { |
| "temperature": 0.6, |
| "top_p": 0.95, |
| "top_k": 20, |
| "presence_penalty": 1.5, |
| "repetition_penalty": 1.1, |
| "max_tokens": 32768, |
| "thinking": true |
| }, |
| "edit_note": null, |
| "tokens_total": 11929948, |
| "gpu_seconds": 2025, |
| "tok_per_s_per_gpu": 6028 |
| } |
