| { |
| "step": 40000, |
| "parameters": 7849161, |
| "lm_eval_version": "0.4.12", |
| "num_fewshot": 0, |
| "leaderboard_accuracy": { |
| "arc_easy": 0.35353535353535354, |
| "arc_challenge": 0.23293515358361774, |
| "hellaswag": 0.2819159529974109, |
| "piqa": 0.5821545157780196 |
| }, |
| "lm_eval_metric_policy": "acc_norm when available, otherwise acc", |
| "selection_metric": "open_slm_leaderboard_average", |
| "arithmark_2.0": { |
| "acc": 0.2912, |
| "correct": 728, |
| "total": 2500, |
| "by_operator_count": { |
| "1": { |
| "acc": 0.2848, |
| "correct": 356, |
| "total": 1250 |
| }, |
| "2": { |
| "acc": 0.3253333333333333, |
| "correct": 244, |
| "total": 750 |
| }, |
| "3": { |
| "acc": 0.256, |
| "correct": 128, |
| "total": 500 |
| } |
| } |
| }, |
| "arc_average": 0.29323525355948565, |
| "open_slm_leaderboard_average": 0.36212643058372906, |
| "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4", |
| "validation": { |
| "loss": 2.9902732021203575, |
| "perplexity": 19.89111604438146, |
| "normalized_bpb": 1.3520133424151615, |
| "tokens": 358911, |
| "normalized_utf8_bytes": 1145226 |
| } |
| } |
