| { |
| "step": 12000, |
| "parameters": 958692, |
| "lm_eval_version": "0.4.12", |
| "evaluation_dtype": "bfloat16", |
| "softmax_dtype": "float32", |
| "evaluation_autocast": false, |
| "num_fewshot": 0, |
| "bos_prefix": true, |
| "leaderboard_accuracy": { |
| "arc_easy": 0.29292929292929293, |
| "arc_challenge": 0.2175767918088737, |
| "hellaswag": 0.29346743676558457, |
| "piqa": 0.5462459194776932 |
| }, |
| "lm_eval_metric_policy": "acc_norm when available, otherwise acc", |
| "selection_metric": "validation.normalized_bpb (lower is better)", |
| "arithmark_2.0": { |
| "acc": 0.2744, |
| "correct": 686, |
| "total": 2500, |
| "by_operator_count": { |
| "1": { |
| "acc": 0.2552, |
| "correct": 319, |
| "total": 1250 |
| }, |
| "2": { |
| "acc": 0.30666666666666664, |
| "correct": 230, |
| "total": 750 |
| }, |
| "3": { |
| "acc": 0.274, |
| "correct": 137, |
| "total": 500 |
| } |
| } |
| }, |
| "arc_average": 0.2552530423690833, |
| "open_slm_leaderboard_average": 0.34234159965309024, |
| "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4", |
| "validation": { |
| "loss": 1.1765458586014317, |
| "perplexity": 3.2431525260156, |
| "normalized_bpb": 1.6973968755827331, |
| "tokens": 1144831, |
| "normalized_utf8_bytes": 1144831, |
| "window": 2048, |
| "stride": 1024 |
| } |
| } |
