{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.8, "eval_steps": 100, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "entropy": 1.0750614512711762, "epoch": 0.016, "grad_norm": 162.0, "learning_rate": 6.000000000000001e-07, "loss": 41.6591, "mean_token_accuracy": 0.6122298210859298, "num_tokens": 338352.0, "step": 10 }, { "entropy": 1.0839035604149103, "epoch": 0.032, "grad_norm": 155.0, "learning_rate": 1.2666666666666669e-06, "loss": 41.6397, "mean_token_accuracy": 0.6166786661371588, "num_tokens": 671193.0, "step": 20 }, { "entropy": 1.0894881710410118, "epoch": 0.048, "grad_norm": 132.0, "learning_rate": 1.9333333333333336e-06, "loss": 41.0461, "mean_token_accuracy": 0.6147234473377466, "num_tokens": 1001008.0, "step": 30 }, { "entropy": 1.138040168955922, "epoch": 0.064, "grad_norm": 126.0, "learning_rate": 2.6e-06, "loss": 41.8193, "mean_token_accuracy": 0.6091248352080584, "num_tokens": 1331208.0, "step": 40 }, { "entropy": 1.1673120155930519, "epoch": 0.08, "grad_norm": 114.5, "learning_rate": 3.266666666666667e-06, "loss": 39.7083, "mean_token_accuracy": 0.6180085053667426, "num_tokens": 1667532.0, "step": 50 }, { "entropy": 1.19383430108428, "epoch": 0.096, "grad_norm": 91.0, "learning_rate": 3.9333333333333335e-06, "loss": 37.505, "mean_token_accuracy": 0.6295596646144986, "num_tokens": 2007176.0, "step": 60 }, { "entropy": 1.354680097848177, "epoch": 0.112, "grad_norm": 78.5, "learning_rate": 4.600000000000001e-06, "loss": 37.0959, "mean_token_accuracy": 0.6256176400929689, "num_tokens": 2342259.0, "step": 70 }, { "entropy": 1.6070489153265952, "epoch": 0.128, "grad_norm": 55.0, "learning_rate": 5.2666666666666665e-06, "loss": 36.6385, "mean_token_accuracy": 0.6223620869219303, "num_tokens": 2663987.0, "step": 80 }, { "entropy": 1.682814510539174, "epoch": 0.144, "grad_norm": 37.0, "learning_rate": 5.933333333333335e-06, "loss": 32.9488, "mean_token_accuracy": 0.6449975268915296, "num_tokens": 2997824.0, "step": 90 }, { "entropy": 1.7247826386243106, "epoch": 0.16, "grad_norm": 32.0, "learning_rate": 6.600000000000001e-06, "loss": 30.8315, "mean_token_accuracy": 0.6568072270601988, "num_tokens": 3330597.0, "step": 100 }, { "epoch": 0.16, "eval_biology_entropy": 3.8070496501922606, "eval_biology_loss": 4.702436447143555, "eval_biology_mean_token_accuracy": 0.3792178168296814, "eval_biology_num_tokens": 3330597.0, "eval_biology_runtime": 57.2423, "eval_biology_samples_per_second": 8.735, "eval_biology_steps_per_second": 2.184, "step": 100 }, { "epoch": 0.16, "eval_chemistry_entropy": 2.312845594406128, "eval_chemistry_loss": 2.5579307079315186, "eval_chemistry_mean_token_accuracy": 0.5897436618804932, "eval_chemistry_num_tokens": 3330597.0, "eval_chemistry_runtime": 71.9394, "eval_chemistry_samples_per_second": 6.95, "eval_chemistry_steps_per_second": 1.738, "step": 100 }, { "epoch": 0.16, "eval_math_entropy": 1.3987004208564757, "eval_math_loss": 1.9521548748016357, "eval_math_mean_token_accuracy": 0.6628584680557251, "eval_math_num_tokens": 3330597.0, "eval_math_runtime": 74.5287, "eval_math_samples_per_second": 6.709, "eval_math_steps_per_second": 1.677, "step": 100 }, { "epoch": 0.16, "eval_physics_entropy": 1.725432638168335, "eval_physics_loss": 1.8779996633529663, "eval_physics_mean_token_accuracy": 0.6590415091514588, "eval_physics_num_tokens": 3330597.0, "eval_physics_runtime": 85.2266, "eval_physics_samples_per_second": 5.867, "eval_physics_steps_per_second": 1.467, "step": 100 }, { "entropy": 1.7152024820446967, "epoch": 0.176, "grad_norm": 26.875, "learning_rate": 7.266666666666668e-06, "loss": 29.6512, "mean_token_accuracy": 0.6657887665554881, "num_tokens": 3658264.0, "step": 110 }, { "entropy": 1.622603354603052, "epoch": 0.192, "grad_norm": 19.375, "learning_rate": 7.933333333333334e-06, "loss": 28.3031, "mean_token_accuracy": 0.6764084205031395, "num_tokens": 3995568.0, "step": 120 }, { "entropy": 1.555612001568079, "epoch": 0.208, "grad_norm": 19.875, "learning_rate": 8.6e-06, "loss": 26.6387, "mean_token_accuracy": 0.6902163974940777, "num_tokens": 4321436.0, "step": 130 }, { "entropy": 1.512053081765771, "epoch": 0.224, "grad_norm": 17.875, "learning_rate": 9.266666666666667e-06, "loss": 26.0858, "mean_token_accuracy": 0.694270471855998, "num_tokens": 4648491.0, "step": 140 }, { "entropy": 1.4869094472378492, "epoch": 0.24, "grad_norm": 19.75, "learning_rate": 9.933333333333334e-06, "loss": 25.4417, "mean_token_accuracy": 0.6989999961107969, "num_tokens": 4986175.0, "step": 150 }, { "entropy": 1.4632187210023404, "epoch": 0.256, "grad_norm": 19.375, "learning_rate": 1.0600000000000002e-05, "loss": 25.0091, "mean_token_accuracy": 0.7029873106628657, "num_tokens": 5329320.0, "step": 160 }, { "entropy": 1.4810095187276602, "epoch": 0.272, "grad_norm": 20.25, "learning_rate": 1.1266666666666668e-05, "loss": 25.2607, "mean_token_accuracy": 0.7004018183797598, "num_tokens": 5658796.0, "step": 170 }, { "entropy": 1.4340823762118817, "epoch": 0.288, "grad_norm": 18.25, "learning_rate": 1.1933333333333335e-05, "loss": 24.6883, "mean_token_accuracy": 0.7079170107841491, "num_tokens": 5980201.0, "step": 180 }, { "entropy": 1.4303428437560797, "epoch": 0.304, "grad_norm": 22.125, "learning_rate": 1.2600000000000001e-05, "loss": 24.4539, "mean_token_accuracy": 0.7080751962959766, "num_tokens": 6298858.0, "step": 190 }, { "entropy": 1.399438175559044, "epoch": 0.32, "grad_norm": 18.875, "learning_rate": 1.3266666666666668e-05, "loss": 23.9145, "mean_token_accuracy": 0.7128454186022282, "num_tokens": 6622798.0, "step": 200 }, { "epoch": 0.32, "eval_biology_entropy": 3.444120632171631, "eval_biology_loss": 4.076927661895752, "eval_biology_mean_token_accuracy": 0.42938668751716613, "eval_biology_num_tokens": 6622798.0, "eval_biology_runtime": 57.3705, "eval_biology_samples_per_second": 8.715, "eval_biology_steps_per_second": 2.179, "step": 200 }, { "epoch": 0.32, "eval_chemistry_entropy": 1.8959890480041504, "eval_chemistry_loss": 2.08449649810791, "eval_chemistry_mean_token_accuracy": 0.6399502665996551, "eval_chemistry_num_tokens": 6622798.0, "eval_chemistry_runtime": 71.8716, "eval_chemistry_samples_per_second": 6.957, "eval_chemistry_steps_per_second": 1.739, "step": 200 }, { "epoch": 0.32, "eval_math_entropy": 1.2845887546539307, "eval_math_loss": 1.6543570756912231, "eval_math_mean_token_accuracy": 0.6954182968139648, "eval_math_num_tokens": 6622798.0, "eval_math_runtime": 74.7507, "eval_math_samples_per_second": 6.689, "eval_math_steps_per_second": 1.672, "step": 200 }, { "epoch": 0.32, "eval_physics_entropy": 1.3866976137161255, "eval_physics_loss": 1.4872251749038696, "eval_physics_mean_token_accuracy": 0.7125752453804016, "eval_physics_num_tokens": 6622798.0, "eval_physics_runtime": 85.9206, "eval_physics_samples_per_second": 5.819, "eval_physics_steps_per_second": 1.455, "step": 200 }, { "entropy": 1.3804268836975098, "epoch": 0.336, "grad_norm": 18.5, "learning_rate": 1.3933333333333334e-05, "loss": 23.6643, "mean_token_accuracy": 0.714659295603633, "num_tokens": 6953611.0, "step": 210 }, { "entropy": 1.3354483414441347, "epoch": 0.352, "grad_norm": 18.125, "learning_rate": 1.46e-05, "loss": 22.9062, "mean_token_accuracy": 0.722674485296011, "num_tokens": 7290378.0, "step": 220 }, { "entropy": 1.3261277750134468, "epoch": 0.368, "grad_norm": 18.5, "learning_rate": 1.5266666666666667e-05, "loss": 22.7576, "mean_token_accuracy": 0.7249834384769202, "num_tokens": 7621082.0, "step": 230 }, { "entropy": 1.3795554626733064, "epoch": 0.384, "grad_norm": 17.875, "learning_rate": 1.5933333333333336e-05, "loss": 23.5828, "mean_token_accuracy": 0.7127871386706829, "num_tokens": 7955570.0, "step": 240 }, { "entropy": 1.3256618689745665, "epoch": 0.4, "grad_norm": 20.0, "learning_rate": 1.66e-05, "loss": 22.7361, "mean_token_accuracy": 0.7222114279866219, "num_tokens": 8291049.0, "step": 250 }, { "entropy": 1.298596952483058, "epoch": 0.416, "grad_norm": 22.0, "learning_rate": 1.726666666666667e-05, "loss": 22.2416, "mean_token_accuracy": 0.7284637104719878, "num_tokens": 8627310.0, "step": 260 }, { "entropy": 1.311914747953415, "epoch": 0.432, "grad_norm": 19.375, "learning_rate": 1.7933333333333333e-05, "loss": 22.4984, "mean_token_accuracy": 0.7248033240437508, "num_tokens": 8958371.0, "step": 270 }, { "entropy": 1.2482076751068234, "epoch": 0.448, "grad_norm": 18.0, "learning_rate": 1.86e-05, "loss": 21.4775, "mean_token_accuracy": 0.7356370355933904, "num_tokens": 9294388.0, "step": 280 }, { "entropy": 1.2598353076726199, "epoch": 0.464, "grad_norm": 20.5, "learning_rate": 1.926666666666667e-05, "loss": 21.5863, "mean_token_accuracy": 0.7334068417549133, "num_tokens": 9620366.0, "step": 290 }, { "entropy": 1.2661810796707869, "epoch": 0.48, "grad_norm": 19.5, "learning_rate": 1.9933333333333334e-05, "loss": 21.8336, "mean_token_accuracy": 0.7298611674457789, "num_tokens": 9955431.0, "step": 300 }, { "epoch": 0.48, "eval_biology_entropy": 3.3976770553588866, "eval_biology_loss": 3.904201030731201, "eval_biology_mean_token_accuracy": 0.43922377967834475, "eval_biology_num_tokens": 9955431.0, "eval_biology_runtime": 57.4472, "eval_biology_samples_per_second": 8.704, "eval_biology_steps_per_second": 2.176, "step": 300 }, { "epoch": 0.48, "eval_chemistry_entropy": 1.8206102285385133, "eval_chemistry_loss": 1.956045389175415, "eval_chemistry_mean_token_accuracy": 0.6531638257503509, "eval_chemistry_num_tokens": 9955431.0, "eval_chemistry_runtime": 71.9347, "eval_chemistry_samples_per_second": 6.951, "eval_chemistry_steps_per_second": 1.738, "step": 300 }, { "epoch": 0.48, "eval_math_entropy": 1.2509758601188659, "eval_math_loss": 1.5806307792663574, "eval_math_mean_token_accuracy": 0.7045728435516357, "eval_math_num_tokens": 9955431.0, "eval_math_runtime": 75.0056, "eval_math_samples_per_second": 6.666, "eval_math_steps_per_second": 1.667, "step": 300 }, { "epoch": 0.48, "eval_physics_entropy": 1.304166310787201, "eval_physics_loss": 1.3740965127944946, "eval_physics_mean_token_accuracy": 0.7282451548576355, "eval_physics_num_tokens": 9955431.0, "eval_physics_runtime": 85.2346, "eval_physics_samples_per_second": 5.866, "eval_physics_steps_per_second": 1.467, "step": 300 }, { "entropy": 1.2894000235944987, "epoch": 0.496, "grad_norm": 18.875, "learning_rate": 1.9933333333333334e-05, "loss": 22.125, "mean_token_accuracy": 0.7272629871964454, "num_tokens": 10289883.0, "step": 310 }, { "entropy": 1.258072443306446, "epoch": 0.512, "grad_norm": 20.5, "learning_rate": 1.985925925925926e-05, "loss": 21.7519, "mean_token_accuracy": 0.7323984511196613, "num_tokens": 10619468.0, "step": 320 }, { "entropy": 1.2734836369752884, "epoch": 0.528, "grad_norm": 16.375, "learning_rate": 1.9785185185185187e-05, "loss": 21.9114, "mean_token_accuracy": 0.7292184431105853, "num_tokens": 10944904.0, "step": 330 }, { "entropy": 1.2483941707760096, "epoch": 0.544, "grad_norm": 19.25, "learning_rate": 1.971111111111111e-05, "loss": 21.5727, "mean_token_accuracy": 0.7335125673562288, "num_tokens": 11266409.0, "step": 340 }, { "entropy": 1.2357387099415065, "epoch": 0.56, "grad_norm": 16.625, "learning_rate": 1.963703703703704e-05, "loss": 21.1173, "mean_token_accuracy": 0.7381447162479162, "num_tokens": 11605544.0, "step": 350 }, { "entropy": 1.2104782849550246, "epoch": 0.576, "grad_norm": 19.25, "learning_rate": 1.9562962962962964e-05, "loss": 20.9845, "mean_token_accuracy": 0.740512777864933, "num_tokens": 11931715.0, "step": 360 }, { "entropy": 1.2183124080300332, "epoch": 0.592, "grad_norm": 19.375, "learning_rate": 1.948888888888889e-05, "loss": 21.0928, "mean_token_accuracy": 0.7366720300167799, "num_tokens": 12257243.0, "step": 370 }, { "entropy": 1.2145392388105392, "epoch": 0.608, "grad_norm": 16.625, "learning_rate": 1.9414814814814817e-05, "loss": 20.8598, "mean_token_accuracy": 0.740410690754652, "num_tokens": 12583154.0, "step": 380 }, { "entropy": 1.2197224445641042, "epoch": 0.624, "grad_norm": 17.5, "learning_rate": 1.9340740740740743e-05, "loss": 21.1959, "mean_token_accuracy": 0.7364733774214983, "num_tokens": 12905392.0, "step": 390 }, { "entropy": 1.2305814437568188, "epoch": 0.64, "grad_norm": 18.625, "learning_rate": 1.926666666666667e-05, "loss": 21.2803, "mean_token_accuracy": 0.7369419563561678, "num_tokens": 13232198.0, "step": 400 }, { "epoch": 0.64, "eval_biology_entropy": 3.2428079328536987, "eval_biology_loss": 3.81221866607666, "eval_biology_mean_token_accuracy": 0.44962965536117555, "eval_biology_num_tokens": 13232198.0, "eval_biology_runtime": 57.9921, "eval_biology_samples_per_second": 8.622, "eval_biology_steps_per_second": 2.155, "step": 400 }, { "epoch": 0.64, "eval_chemistry_entropy": 1.71326642370224, "eval_chemistry_loss": 1.8857945203781128, "eval_chemistry_mean_token_accuracy": 0.6629956634044647, "eval_chemistry_num_tokens": 13232198.0, "eval_chemistry_runtime": 72.1018, "eval_chemistry_samples_per_second": 6.935, "eval_chemistry_steps_per_second": 1.734, "step": 400 }, { "epoch": 0.64, "eval_math_entropy": 1.1944113321304322, "eval_math_loss": 1.5440242290496826, "eval_math_mean_token_accuracy": 0.7086545734405517, "eval_math_num_tokens": 13232198.0, "eval_math_runtime": 75.1501, "eval_math_samples_per_second": 6.653, "eval_math_steps_per_second": 1.663, "step": 400 }, { "epoch": 0.64, "eval_physics_entropy": 1.211931779384613, "eval_physics_loss": 1.3088711500167847, "eval_physics_mean_token_accuracy": 0.737203718662262, "eval_physics_num_tokens": 13232198.0, "eval_physics_runtime": 85.7441, "eval_physics_samples_per_second": 5.831, "eval_physics_steps_per_second": 1.458, "step": 400 }, { "entropy": 1.2212170030921699, "epoch": 0.656, "grad_norm": 17.875, "learning_rate": 1.9192592592592593e-05, "loss": 20.9332, "mean_token_accuracy": 0.7387504555284977, "num_tokens": 13575902.0, "step": 410 }, { "entropy": 1.1882116444408894, "epoch": 0.672, "grad_norm": 20.25, "learning_rate": 1.911851851851852e-05, "loss": 20.6082, "mean_token_accuracy": 0.743332651630044, "num_tokens": 13895997.0, "step": 420 }, { "entropy": 1.207483420521021, "epoch": 0.688, "grad_norm": 18.0, "learning_rate": 1.9044444444444446e-05, "loss": 20.654, "mean_token_accuracy": 0.7427127715200186, "num_tokens": 14234888.0, "step": 430 }, { "entropy": 1.2052442319691181, "epoch": 0.704, "grad_norm": 19.0, "learning_rate": 1.8970370370370372e-05, "loss": 20.8893, "mean_token_accuracy": 0.7387381002306939, "num_tokens": 14567908.0, "step": 440 }, { "entropy": 1.181990123167634, "epoch": 0.72, "grad_norm": 21.625, "learning_rate": 1.8896296296296295e-05, "loss": 20.4926, "mean_token_accuracy": 0.7439979754388333, "num_tokens": 14882927.0, "step": 450 }, { "entropy": 1.1967087093740703, "epoch": 0.736, "grad_norm": 20.375, "learning_rate": 1.8822222222222225e-05, "loss": 20.5403, "mean_token_accuracy": 0.7425415780395269, "num_tokens": 15217342.0, "step": 460 }, { "entropy": 1.1514527130872012, "epoch": 0.752, "grad_norm": 16.75, "learning_rate": 1.874814814814815e-05, "loss": 19.9152, "mean_token_accuracy": 0.7488316483795643, "num_tokens": 15547710.0, "step": 470 }, { "entropy": 1.2158014349639417, "epoch": 0.768, "grad_norm": 21.375, "learning_rate": 1.8674074074074075e-05, "loss": 21.052, "mean_token_accuracy": 0.7373728647828102, "num_tokens": 15877177.0, "step": 480 }, { "entropy": 1.1517657596617936, "epoch": 0.784, "grad_norm": 16.125, "learning_rate": 1.86e-05, "loss": 19.9164, "mean_token_accuracy": 0.7503607623279095, "num_tokens": 16219640.0, "step": 490 }, { "entropy": 1.180898468568921, "epoch": 0.8, "grad_norm": 21.75, "learning_rate": 1.8525925925925928e-05, "loss": 20.3875, "mean_token_accuracy": 0.7441588096320629, "num_tokens": 16548261.0, "step": 500 }, { "epoch": 0.8, "eval_biology_entropy": 3.2802926797866823, "eval_biology_loss": 3.785775899887085, "eval_biology_mean_token_accuracy": 0.4522908911705017, "eval_biology_num_tokens": 16548261.0, "eval_biology_runtime": 58.1484, "eval_biology_samples_per_second": 8.599, "eval_biology_steps_per_second": 2.15, "step": 500 }, { "epoch": 0.8, "eval_chemistry_entropy": 1.712562572002411, "eval_chemistry_loss": 1.8553112745285034, "eval_chemistry_mean_token_accuracy": 0.6659346613883972, "eval_chemistry_num_tokens": 16548261.0, "eval_chemistry_runtime": 71.5095, "eval_chemistry_samples_per_second": 6.992, "eval_chemistry_steps_per_second": 1.748, "step": 500 }, { "epoch": 0.8, "eval_math_entropy": 1.1794157824516296, "eval_math_loss": 1.5209077596664429, "eval_math_mean_token_accuracy": 0.7121200709342956, "eval_math_num_tokens": 16548261.0, "eval_math_runtime": 74.5115, "eval_math_samples_per_second": 6.71, "eval_math_steps_per_second": 1.678, "step": 500 }, { "epoch": 0.8, "eval_physics_entropy": 1.1884380350112915, "eval_physics_loss": 1.2718405723571777, "eval_physics_mean_token_accuracy": 0.742140392780304, "eval_physics_num_tokens": 16548261.0, "eval_physics_runtime": 85.7525, "eval_physics_samples_per_second": 5.831, "eval_physics_steps_per_second": 1.458, "step": 500 } ], "logging_steps": 10, "max_steps": 3000, "num_input_tokens_seen": 0, "num_train_epochs": 5, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 1.652179086470234e+18, "train_batch_size": 4, "trial_name": null, "trial_params": null }