{
  "step": 12000,
  "parameters": 958692,
  "lm_eval_version": "0.4.12",
  "evaluation_dtype": "bfloat16",
  "softmax_dtype": "float32",
  "evaluation_autocast": false,
  "num_fewshot": 0,
  "bos_prefix": true,
  "leaderboard_accuracy": {
    "arc_easy": 0.29292929292929293,
    "arc_challenge": 0.2175767918088737,
    "hellaswag": 0.29346743676558457,
    "piqa": 0.5462459194776932
  },
  "lm_eval_metric_policy": "acc_norm when available, otherwise acc",
  "selection_metric": "validation.normalized_bpb (lower is better)",
  "arithmark_2.0": {
    "acc": 0.2744,
    "correct": 686,
    "total": 2500,
    "by_operator_count": {
      "1": {
        "acc": 0.2552,
        "correct": 319,
        "total": 1250
      },
      "2": {
        "acc": 0.30666666666666664,
        "correct": 230,
        "total": 750
      },
      "3": {
        "acc": 0.274,
        "correct": 137,
        "total": 500
      }
    }
  },
  "arc_average": 0.2552530423690833,
  "open_slm_leaderboard_average": 0.34234159965309024,
  "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4",
  "validation": {
    "loss": 1.1765458586014317,
    "perplexity": 3.2431525260156,
    "normalized_bpb": 1.6973968755827331,
    "tokens": 1144831,
    "normalized_utf8_bytes": 1144831,
    "window": 2048,
    "stride": 1024
  }
}