| { |
| "step": 60000, |
| "parameters": 5094112, |
| "lm_eval_version": "0.4.12", |
| "evaluation_dtype": "bfloat16", |
| "softmax_dtype": "float32", |
| "evaluation_autocast": false, |
| "num_fewshot": 0, |
| "model_context": 1024, |
| "training_context": 1024, |
| "lm_eval_context": 1024, |
| "lm_eval_bos_prefix": true, |
| "arithmark_3.0_bos_prefix": true, |
| "benchmark_accuracy": { |
| "arc_easy": 0.335016835016835, |
| "arc_challenge": 0.22696245733788395, |
| "hellaswag": 0.2786297550288787, |
| "piqa": 0.5729053318824809 |
| }, |
| "lm_eval_metric_policy": "acc_norm when available, otherwise acc", |
| "intelligence_index": 6.837530558172039, |
| "average_accuracy_percent": 35.26311832721798, |
| "open_slm_scores": { |
| "leaderboard": "AxiomicLabs/Open_SLM_Leaderboard", |
| "leaderboard_revision": "2fbaaa164009c6c0d201ddf245878f5942cc8628", |
| "intelligence_index": 6.837530558172039, |
| "average_accuracy_percent": 35.26311832721798, |
| "components_percent": { |
| "hellaswag": 27.86297550288787, |
| "combined_arc": 28.09896461773595, |
| "piqa": 57.29053318824809, |
| "arithmark_3.0": 27.800000000000004 |
| }, |
| "chance_normalized_components": { |
| "hellaswag": 3.8173006705171613, |
| "combined_arc": 4.13195282364793, |
| "piqa": 14.581066376496183, |
| "arithmark_3.0": 3.7333333333333396 |
| }, |
| "intelligence_index_weights": { |
| "hellaswag": 1.0, |
| "combined_arc": 1.0, |
| "piqa": 1.0, |
| "arithmark_3.0": 0.65 |
| } |
| }, |
| "checkpoint_policy": "every completed evaluation is committed regardless of whether validation BPB is best; rolling full-precision model, optimizer, RNG, stream cursor, and prefetched batches are stored separately", |
| "best_tracking_metric": "validation.normalized_bpb@1024 (lower is better)", |
| "best_validation": { |
| "normalized_bpb": 1.4126762791486454, |
| "step": 60000 |
| }, |
| "arithmark_3.0": { |
| "dataset": "AxiomicLabs/Arithmark-3.0", |
| "dataset_revision": "6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746", |
| "primary_metric": "acc_norm", |
| "metric_unit": "fraction", |
| "acc": 0.317, |
| "acc_norm": 0.278, |
| "raw_correct": 317, |
| "normalized_correct": 278, |
| "total": 1000, |
| "max_context": 1024, |
| "by_category": { |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { |
| "acc": 0.21875, |
| "acc_norm": 0.21875, |
| "raw_correct": 28, |
| "normalized_correct": 28, |
| "total": 128 |
| }, |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { |
| "acc": 0.18181818181818182, |
| "acc_norm": 0.20454545454545456, |
| "raw_correct": 8, |
| "normalized_correct": 9, |
| "total": 44 |
| }, |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { |
| "acc": 0.16666666666666666, |
| "acc_norm": 0.14583333333333334, |
| "raw_correct": 8, |
| "normalized_correct": 7, |
| "total": 48 |
| }, |
| "elementary_school_math_continuation::data::grades_2_3::easy": { |
| "acc": 0.18604651162790697, |
| "acc_norm": 0.18604651162790697, |
| "raw_correct": 8, |
| "normalized_correct": 8, |
| "total": 43 |
| }, |
| "elementary_school_math_continuation::division::grades_3_4::medium": { |
| "acc": 0.3888888888888889, |
| "acc_norm": 0.3888888888888889, |
| "raw_correct": 21, |
| "normalized_correct": 21, |
| "total": 54 |
| }, |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { |
| "acc": 0.22, |
| "acc_norm": 0.22, |
| "raw_correct": 11, |
| "normalized_correct": 11, |
| "total": 50 |
| }, |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { |
| "acc": 0.46153846153846156, |
| "acc_norm": 0.40384615384615385, |
| "raw_correct": 24, |
| "normalized_correct": 21, |
| "total": 52 |
| }, |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { |
| "acc": 0.5333333333333333, |
| "acc_norm": 0.5333333333333333, |
| "raw_correct": 24, |
| "normalized_correct": 24, |
| "total": 45 |
| }, |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { |
| "acc": 0.34210526315789475, |
| "acc_norm": 0.2894736842105263, |
| "raw_correct": 26, |
| "normalized_correct": 22, |
| "total": 76 |
| }, |
| "elementary_school_math_continuation::money::grades_3_4::medium": { |
| "acc": 0.265625, |
| "acc_norm": 0.234375, |
| "raw_correct": 17, |
| "normalized_correct": 15, |
| "total": 64 |
| }, |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { |
| "acc": 0.36486486486486486, |
| "acc_norm": 0.3108108108108108, |
| "raw_correct": 27, |
| "normalized_correct": 23, |
| "total": 74 |
| }, |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { |
| "acc": 0.18867924528301888, |
| "acc_norm": 0.1509433962264151, |
| "raw_correct": 10, |
| "normalized_correct": 8, |
| "total": 53 |
| }, |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { |
| "acc": 0.2222222222222222, |
| "acc_norm": 0.18803418803418803, |
| "raw_correct": 26, |
| "normalized_correct": 22, |
| "total": 117 |
| }, |
| "elementary_school_math_continuation::time::grades_2_3::easy": { |
| "acc": 1.0, |
| "acc_norm": 0.6363636363636364, |
| "raw_correct": 55, |
| "normalized_correct": 35, |
| "total": 55 |
| }, |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { |
| "acc": 0.2391304347826087, |
| "acc_norm": 0.2391304347826087, |
| "raw_correct": 11, |
| "normalized_correct": 11, |
| "total": 46 |
| }, |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { |
| "acc": 0.21052631578947367, |
| "acc_norm": 0.21052631578947367, |
| "raw_correct": 4, |
| "normalized_correct": 4, |
| "total": 19 |
| }, |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { |
| "acc": 0.28125, |
| "acc_norm": 0.28125, |
| "raw_correct": 9, |
| "normalized_correct": 9, |
| "total": 32 |
| } |
| }, |
| "choice_batch_size": 64 |
| }, |
| "validation": { |
| "loss": 3.124341521462218, |
| "perplexity": 22.744913124279417, |
| "normalized_bpb": 1.4126762791486454, |
| "tokens": 359037, |
| "normalized_utf8_bytes": 1145591, |
| "window": 1024, |
| "stride": 512 |
| } |
| } |
