{
  "acc": 36.1,
  "acc_norm": 36.1,
  "categories": {
    "elementary_school_math_continuation::addition::grades_1_2::easy": {
      "acc": 26.5625,
      "acc_norm": 26.5625,
      "norm_correct": 34,
      "raw_correct": 34,
      "total": 128
    },
    "elementary_school_math_continuation::comparison::grades_2_3::medium": {
      "acc": 15.909090909090908,
      "acc_norm": 15.909090909090908,
      "norm_correct": 7,
      "raw_correct": 7,
      "total": 44
    },
    "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
      "acc": 37.5,
      "acc_norm": 37.5,
      "norm_correct": 18,
      "raw_correct": 18,
      "total": 48
    },
    "elementary_school_math_continuation::data::grades_2_3::easy": {
      "acc": 27.906976744186046,
      "acc_norm": 27.906976744186046,
      "norm_correct": 12,
      "raw_correct": 12,
      "total": 43
    },
    "elementary_school_math_continuation::division::grades_3_4::medium": {
      "acc": 37.03703703703704,
      "acc_norm": 37.03703703703704,
      "norm_correct": 20,
      "raw_correct": 20,
      "total": 54
    },
    "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
      "acc": 24.0,
      "acc_norm": 26.0,
      "norm_correct": 13,
      "raw_correct": 12,
      "total": 50
    },
    "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
      "acc": 53.84615384615385,
      "acc_norm": 53.84615384615385,
      "norm_correct": 28,
      "raw_correct": 28,
      "total": 52
    },
    "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
      "acc": 46.666666666666664,
      "acc_norm": 46.666666666666664,
      "norm_correct": 21,
      "raw_correct": 21,
      "total": 45
    },
    "elementary_school_math_continuation::measurement::grades_2_3::easy": {
      "acc": 36.84210526315789,
      "acc_norm": 36.84210526315789,
      "norm_correct": 28,
      "raw_correct": 28,
      "total": 76
    },
    "elementary_school_math_continuation::money::grades_3_4::medium": {
      "acc": 32.8125,
      "acc_norm": 32.8125,
      "norm_correct": 21,
      "raw_correct": 21,
      "total": 64
    },
    "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
      "acc": 44.5945945945946,
      "acc_norm": 44.5945945945946,
      "norm_correct": 33,
      "raw_correct": 33,
      "total": 74
    },
    "elementary_school_math_continuation::patterns::grades_3_4::medium": {
      "acc": 28.30188679245283,
      "acc_norm": 28.30188679245283,
      "norm_correct": 15,
      "raw_correct": 15,
      "total": 53
    },
    "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
      "acc": 26.495726495726498,
      "acc_norm": 25.64102564102564,
      "norm_correct": 30,
      "raw_correct": 31,
      "total": 117
    },
    "elementary_school_math_continuation::time::grades_2_3::easy": {
      "acc": 92.72727272727272,
      "acc_norm": 92.72727272727272,
      "norm_correct": 51,
      "raw_correct": 51,
      "total": 55
    },
    "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
      "acc": 26.08695652173913,
      "acc_norm": 26.08695652173913,
      "norm_correct": 12,
      "raw_correct": 12,
      "total": 46
    },
    "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
      "acc": 42.10526315789473,
      "acc_norm": 42.10526315789473,
      "norm_correct": 8,
      "raw_correct": 8,
      "total": 19
    },
    "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
      "acc": 31.25,
      "acc_norm": 31.25,
      "norm_correct": 10,
      "raw_correct": 10,
      "total": 32
    }
  },
  "frost_evaluation": {
    "checkpoint": "D:\\llm\\frost\\artifacts\\runs\\natural20b_pilot_muon_b32\\checkpoint-tokens-0500000000",
    "data_sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435",
    "dataset": "AxiomicLabs/Arithmark-3.0",
    "dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
    "device": "cuda",
    "evaluator_sha256": "09667e1ea05ecd8b91dbfea14a47365b555c72622afed9fc284c127ecd5e1bcd",
    "limit": null,
    "precision": "bfloat16",
    "tokenizer": "D:\\llm\\frost\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit"
  },
  "norm_correct": 361,
  "raw_correct": 361,
  "timing": {
    "evaluation_seconds": 1.190516500002559,
    "examples_per_second": 839.9715585612215,
    "tokenization_seconds": 0.3197372000031464
  },
  "total": 1000
}