benchmark_results.json

{
  "step": 40000,
  "parameters": 7849161,
  "lm_eval_version": "0.4.12",
  "num_fewshot": 0,
  "leaderboard_accuracy": {
    "arc_easy": 0.35353535353535354,
    "arc_challenge": 0.23293515358361774,
    "hellaswag": 0.2819159529974109,
    "piqa": 0.5821545157780196
  },
  "lm_eval_metric_policy": "acc_norm when available, otherwise acc",
  "selection_metric": "open_slm_leaderboard_average",
  "arithmark_2.0": {
    "acc": 0.2912,
    "correct": 728,
    "total": 2500,
    "by_operator_count": {
      "1": {
        "acc": 0.2848,
        "correct": 356,
        "total": 1250
      },
      "2": {
        "acc": 0.3253333333333333,
        "correct": 244,
        "total": 750
      },
      "3": {
        "acc": 0.256,
        "correct": 128,
        "total": 500
      }
    }
  },
  "arc_average": 0.29323525355948565,
  "open_slm_leaderboard_average": 0.36212643058372906,
  "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4",
  "validation": {
    "loss": 2.9902732021203575,
    "perplexity": 19.89111604438146,
    "normalized_bpb": 1.3520133424151615,
    "tokens": 358911,
    "normalized_utf8_bytes": 1145226
  }
}