{
  "step": 1000,
  "parameters": 5094112,
  "lm_eval_version": "0.4.12",
  "evaluation_dtype": "bfloat16",
  "softmax_dtype": "float32",
  "evaluation_autocast": false,
  "num_fewshot": 0,
  "model_context": 1024,
  "training_context": 1024,
  "lm_eval_context": 1024,
  "lm_eval_bos_prefix": true,
  "arithmark_3.0_bos_prefix": true,
  "benchmark_accuracy": {
    "arc_easy": 0.31186868686868685,
    "arc_challenge": 0.2167235494880546,
    "hellaswag": 0.2700657239593706,
    "piqa": 0.544069640914037
  },
  "lm_eval_metric_policy": "acc_norm when available, otherwise acc",
  "intelligence_index": 4.880960858768836,
  "average_accuracy_percent": 34.48578707629446,
  "open_slm_scores": {
    "leaderboard": "AxiomicLabs/Open_SLM_Leaderboard",
    "leaderboard_revision": "2fbaaa164009c6c0d201ddf245878f5942cc8628",
    "intelligence_index": 4.880960858768836,
    "average_accuracy_percent": 34.48578707629446,
    "components_percent": {
      "hellaswag": 27.006572395937063,
      "combined_arc": 26.429611817837074,
      "piqa": 54.4069640914037,
      "arithmark_3.0": 30.099999999999998
    },
    "chance_normalized_components": {
      "hellaswag": 2.6754298612494174,
      "combined_arc": 1.9061490904494318,
      "piqa": 8.813928182807402,
      "arithmark_3.0": 6.799999999999997
    },
    "intelligence_index_weights": {
      "hellaswag": 1.0,
      "combined_arc": 1.0,
      "piqa": 1.0,
      "arithmark_3.0": 0.65
    }
  },
  "checkpoint_policy": "every completed evaluation is committed regardless of whether validation BPB is best; rolling full-precision model, optimizer, RNG, stream cursor, and prefetched batches are stored separately",
  "best_tracking_metric": "validation.normalized_bpb@1024 (lower is better)",
  "best_validation": {
    "normalized_bpb": 1.6285278718575018,
    "step": 1000
  },
  "arithmark_3.0": {
    "dataset": "AxiomicLabs/Arithmark-3.0",
    "dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746",
    "primary_metric": "acc_norm",
    "metric_unit": "fraction",
    "acc": 0.327,
    "acc_norm": 0.301,
    "raw_correct": 327,
    "normalized_correct": 301,
    "total": 1000,
    "max_context": 1024,
    "by_category": {
      "elementary_school_math_continuation::addition::grades_1_2::easy": {
        "acc": 0.2421875,
        "acc_norm": 0.25,
        "raw_correct": 31,
        "normalized_correct": 32,
        "total": 128
      },
      "elementary_school_math_continuation::comparison::grades_2_3::medium": {
        "acc": 0.29545454545454547,
        "acc_norm": 0.29545454545454547,
        "raw_correct": 13,
        "normalized_correct": 13,
        "total": 44
      },
      "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": {
        "acc": 0.3333333333333333,
        "acc_norm": 0.2708333333333333,
        "raw_correct": 16,
        "normalized_correct": 13,
        "total": 48
      },
      "elementary_school_math_continuation::data::grades_2_3::easy": {
        "acc": 0.2558139534883721,
        "acc_norm": 0.23255813953488372,
        "raw_correct": 11,
        "normalized_correct": 10,
        "total": 43
      },
      "elementary_school_math_continuation::division::grades_3_4::medium": {
        "acc": 0.4074074074074074,
        "acc_norm": 0.4074074074074074,
        "raw_correct": 22,
        "normalized_correct": 22,
        "total": 54
      },
      "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": {
        "acc": 0.24,
        "acc_norm": 0.24,
        "raw_correct": 12,
        "normalized_correct": 12,
        "total": 50
      },
      "elementary_school_math_continuation::geometry_area::grades_4_5::medium": {
        "acc": 0.4230769230769231,
        "acc_norm": 0.3269230769230769,
        "raw_correct": 22,
        "normalized_correct": 17,
        "total": 52
      },
      "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": {
        "acc": 0.4,
        "acc_norm": 0.4222222222222222,
        "raw_correct": 18,
        "normalized_correct": 19,
        "total": 45
      },
      "elementary_school_math_continuation::measurement::grades_2_3::easy": {
        "acc": 0.27631578947368424,
        "acc_norm": 0.25,
        "raw_correct": 21,
        "normalized_correct": 19,
        "total": 76
      },
      "elementary_school_math_continuation::money::grades_3_4::medium": {
        "acc": 0.28125,
        "acc_norm": 0.265625,
        "raw_correct": 18,
        "normalized_correct": 17,
        "total": 64
      },
      "elementary_school_math_continuation::multiplication::grades_3_4::medium": {
        "acc": 0.36486486486486486,
        "acc_norm": 0.2972972972972973,
        "raw_correct": 27,
        "normalized_correct": 22,
        "total": 74
      },
      "elementary_school_math_continuation::patterns::grades_3_4::medium": {
        "acc": 0.20754716981132076,
        "acc_norm": 0.22641509433962265,
        "raw_correct": 11,
        "normalized_correct": 12,
        "total": 53
      },
      "elementary_school_math_continuation::subtraction::grades_1_2::easy": {
        "acc": 0.2222222222222222,
        "acc_norm": 0.2222222222222222,
        "raw_correct": 26,
        "normalized_correct": 26,
        "total": 117
      },
      "elementary_school_math_continuation::time::grades_2_3::easy": {
        "acc": 1.0,
        "acc_norm": 0.7818181818181819,
        "raw_correct": 55,
        "normalized_correct": 43,
        "total": 55
      },
      "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": {
        "acc": 0.21739130434782608,
        "acc_norm": 0.1956521739130435,
        "raw_correct": 10,
        "normalized_correct": 9,
        "total": 46
      },
      "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": {
        "acc": 0.21052631578947367,
        "acc_norm": 0.2631578947368421,
        "raw_correct": 4,
        "normalized_correct": 5,
        "total": 19
      },
      "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": {
        "acc": 0.3125,
        "acc_norm": 0.3125,
        "raw_correct": 10,
        "normalized_correct": 10,
        "total": 32
      }
    },
    "choice_batch_size": 64
  },
  "validation": {
    "loss": 3.6017290896745604,
    "perplexity": 36.66157081431394,
    "normalized_bpb": 1.6285278718575018,
    "tokens": 359037,
    "normalized_utf8_bytes": 1145591,
    "window": 1024,
    "stride": 512
  }
}