{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.8, "eval_steps": 100, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "entropy": 1.1280501924455166, "epoch": 0.016, "grad_norm": 161.0, "learning_rate": 6.000000000000001e-07, "loss": 36.0968, "mean_token_accuracy": 0.6165979273617268, "num_tokens": 280941.0, "step": 10 }, { "entropy": 1.1705728020519017, "epoch": 0.032, "grad_norm": 146.0, "learning_rate": 1.2666666666666669e-06, "loss": 36.5572, "mean_token_accuracy": 0.6121777599677444, "num_tokens": 558056.0, "step": 20 }, { "entropy": 1.1892314109951259, "epoch": 0.048, "grad_norm": 135.0, "learning_rate": 1.9333333333333336e-06, "loss": 35.5667, "mean_token_accuracy": 0.6162483215332031, "num_tokens": 836753.0, "step": 30 }, { "entropy": 1.2923907738178968, "epoch": 0.064, "grad_norm": 106.0, "learning_rate": 2.6e-06, "loss": 34.4747, "mean_token_accuracy": 0.6145394820719957, "num_tokens": 1126446.0, "step": 40 }, { "entropy": 1.4465828079730272, "epoch": 0.08, "grad_norm": 86.5, "learning_rate": 3.266666666666667e-06, "loss": 32.2616, "mean_token_accuracy": 0.6272102266550064, "num_tokens": 1413596.0, "step": 50 }, { "entropy": 1.5344239331781864, "epoch": 0.096, "grad_norm": 54.25, "learning_rate": 3.9333333333333335e-06, "loss": 28.8152, "mean_token_accuracy": 0.6551170501857996, "num_tokens": 1701193.0, "step": 60 }, { "entropy": 1.5569457441568375, "epoch": 0.112, "grad_norm": 45.75, "learning_rate": 4.600000000000001e-06, "loss": 26.6094, "mean_token_accuracy": 0.6667167004197836, "num_tokens": 1979232.0, "step": 70 }, { "entropy": 1.4980619736015797, "epoch": 0.128, "grad_norm": 34.0, "learning_rate": 5.2666666666666665e-06, "loss": 25.0735, "mean_token_accuracy": 0.6823939502239227, "num_tokens": 2274177.0, "step": 80 }, { "entropy": 1.4427384316921235, "epoch": 0.144, "grad_norm": 29.75, "learning_rate": 5.933333333333335e-06, "loss": 23.2103, "mean_token_accuracy": 0.6962548345327377, "num_tokens": 2548445.0, "step": 90 }, { "entropy": 1.3565625827759504, "epoch": 0.16, "grad_norm": 30.875, "learning_rate": 6.600000000000001e-06, "loss": 21.4845, "mean_token_accuracy": 0.7108539894223214, "num_tokens": 2824418.0, "step": 100 }, { "epoch": 0.16, "eval_biology_entropy": 3.535919818878174, "eval_biology_loss": 5.02457332611084, "eval_biology_mean_token_accuracy": 0.3542800180912018, "eval_biology_num_tokens": 2824418.0, "eval_biology_runtime": 49.6437, "eval_biology_samples_per_second": 10.072, "eval_biology_steps_per_second": 2.518, "step": 100 }, { "epoch": 0.16, "eval_chemistry_entropy": 2.5280555896759034, "eval_chemistry_loss": 2.8887903690338135, "eval_chemistry_mean_token_accuracy": 0.5300401592254639, "eval_chemistry_num_tokens": 2824418.0, "eval_chemistry_runtime": 61.5471, "eval_chemistry_samples_per_second": 8.124, "eval_chemistry_steps_per_second": 2.031, "step": 100 }, { "epoch": 0.16, "eval_math_entropy": 1.3736235461235047, "eval_math_loss": 1.3470827341079712, "eval_math_mean_token_accuracy": 0.7081305732727051, "eval_math_num_tokens": 2824418.0, "eval_math_runtime": 63.1282, "eval_math_samples_per_second": 7.92, "eval_math_steps_per_second": 1.98, "step": 100 }, { "epoch": 0.16, "eval_physics_entropy": 2.064271595954895, "eval_physics_loss": 2.1881864070892334, "eval_physics_mean_token_accuracy": 0.6055651969909668, "eval_physics_num_tokens": 2824418.0, "eval_physics_runtime": 71.8769, "eval_physics_samples_per_second": 6.956, "eval_physics_steps_per_second": 1.739, "step": 100 }, { "entropy": 1.3322576396167278, "epoch": 0.176, "grad_norm": 23.25, "learning_rate": 7.266666666666668e-06, "loss": 20.9108, "mean_token_accuracy": 0.7151085637509823, "num_tokens": 3110313.0, "step": 110 }, { "entropy": 1.3247066088020802, "epoch": 0.192, "grad_norm": 23.625, "learning_rate": 7.933333333333334e-06, "loss": 20.6356, "mean_token_accuracy": 0.7142969533801079, "num_tokens": 3394170.0, "step": 120 }, { "entropy": 1.256456857919693, "epoch": 0.208, "grad_norm": 21.125, "learning_rate": 8.6e-06, "loss": 19.9583, "mean_token_accuracy": 0.7226813279092312, "num_tokens": 3673600.0, "step": 130 }, { "entropy": 1.2233942866325378, "epoch": 0.224, "grad_norm": 23.5, "learning_rate": 9.266666666666667e-06, "loss": 19.4829, "mean_token_accuracy": 0.7258789885789156, "num_tokens": 3953732.0, "step": 140 }, { "entropy": 1.1788023415952922, "epoch": 0.24, "grad_norm": 22.625, "learning_rate": 9.933333333333334e-06, "loss": 18.6617, "mean_token_accuracy": 0.735758152604103, "num_tokens": 4243655.0, "step": 150 }, { "entropy": 1.1358939699828625, "epoch": 0.256, "grad_norm": 21.25, "learning_rate": 1.0600000000000002e-05, "loss": 18.067, "mean_token_accuracy": 0.7414251279085875, "num_tokens": 4531471.0, "step": 160 }, { "entropy": 1.1413200695067645, "epoch": 0.272, "grad_norm": 18.75, "learning_rate": 1.1266666666666668e-05, "loss": 18.2054, "mean_token_accuracy": 0.740831508487463, "num_tokens": 4810284.0, "step": 170 }, { "entropy": 1.125382575765252, "epoch": 0.288, "grad_norm": 17.875, "learning_rate": 1.1933333333333335e-05, "loss": 17.7696, "mean_token_accuracy": 0.7443546559661627, "num_tokens": 5095104.0, "step": 180 }, { "entropy": 1.1154642656445504, "epoch": 0.304, "grad_norm": 17.125, "learning_rate": 1.2600000000000001e-05, "loss": 17.7511, "mean_token_accuracy": 0.7442465547472239, "num_tokens": 5383732.0, "step": 190 }, { "entropy": 1.0828186847269534, "epoch": 0.32, "grad_norm": 19.875, "learning_rate": 1.3266666666666668e-05, "loss": 17.2671, "mean_token_accuracy": 0.7503975994884968, "num_tokens": 5676334.0, "step": 200 }, { "epoch": 0.32, "eval_biology_entropy": 3.6005834312438965, "eval_biology_loss": 4.470179557800293, "eval_biology_mean_token_accuracy": 0.3823731031417847, "eval_biology_num_tokens": 5676334.0, "eval_biology_runtime": 49.5338, "eval_biology_samples_per_second": 10.094, "eval_biology_steps_per_second": 2.524, "step": 200 }, { "epoch": 0.32, "eval_chemistry_entropy": 2.351666664123535, "eval_chemistry_loss": 2.5015459060668945, "eval_chemistry_mean_token_accuracy": 0.5612157773971558, "eval_chemistry_num_tokens": 5676334.0, "eval_chemistry_runtime": 61.4417, "eval_chemistry_samples_per_second": 8.138, "eval_chemistry_steps_per_second": 2.034, "step": 200 }, { "epoch": 0.32, "eval_math_entropy": 1.1107734112739562, "eval_math_loss": 1.1035394668579102, "eval_math_mean_token_accuracy": 0.744808388710022, "eval_math_num_tokens": 5676334.0, "eval_math_runtime": 63.0675, "eval_math_samples_per_second": 7.928, "eval_math_steps_per_second": 1.982, "step": 200 }, { "epoch": 0.32, "eval_physics_entropy": 1.8196128549575805, "eval_physics_loss": 1.8889656066894531, "eval_physics_mean_token_accuracy": 0.6360980491638184, "eval_physics_num_tokens": 5676334.0, "eval_physics_runtime": 71.7937, "eval_physics_samples_per_second": 6.964, "eval_physics_steps_per_second": 1.741, "step": 200 }, { "entropy": 1.0973670542240144, "epoch": 0.336, "grad_norm": 19.0, "learning_rate": 1.3933333333333334e-05, "loss": 17.6111, "mean_token_accuracy": 0.7442069798707962, "num_tokens": 5958480.0, "step": 210 }, { "entropy": 1.064722668007016, "epoch": 0.352, "grad_norm": 18.25, "learning_rate": 1.46e-05, "loss": 16.8683, "mean_token_accuracy": 0.752158772200346, "num_tokens": 6242161.0, "step": 220 }, { "entropy": 1.0617528311908244, "epoch": 0.368, "grad_norm": 20.5, "learning_rate": 1.5266666666666667e-05, "loss": 16.9997, "mean_token_accuracy": 0.751146937161684, "num_tokens": 6523679.0, "step": 230 }, { "entropy": 1.0443669583648443, "epoch": 0.384, "grad_norm": 19.625, "learning_rate": 1.5933333333333336e-05, "loss": 16.6591, "mean_token_accuracy": 0.7551476296037436, "num_tokens": 6810978.0, "step": 240 }, { "entropy": 1.0753562293946743, "epoch": 0.4, "grad_norm": 17.875, "learning_rate": 1.66e-05, "loss": 17.4353, "mean_token_accuracy": 0.7485508523881436, "num_tokens": 7096903.0, "step": 250 }, { "entropy": 1.0416356481611728, "epoch": 0.416, "grad_norm": 17.375, "learning_rate": 1.726666666666667e-05, "loss": 16.623, "mean_token_accuracy": 0.7539484884589911, "num_tokens": 7377181.0, "step": 260 }, { "entropy": 1.0091398146003485, "epoch": 0.432, "grad_norm": 20.375, "learning_rate": 1.7933333333333333e-05, "loss": 16.1553, "mean_token_accuracy": 0.7619809154421091, "num_tokens": 7650523.0, "step": 270 }, { "entropy": 0.9927664395421744, "epoch": 0.448, "grad_norm": 20.75, "learning_rate": 1.86e-05, "loss": 15.9549, "mean_token_accuracy": 0.7623589888215065, "num_tokens": 7936788.0, "step": 280 }, { "entropy": 0.9981700140982867, "epoch": 0.464, "grad_norm": 21.75, "learning_rate": 1.926666666666667e-05, "loss": 16.1264, "mean_token_accuracy": 0.761608887091279, "num_tokens": 8223147.0, "step": 290 }, { "entropy": 1.0235895983874799, "epoch": 0.48, "grad_norm": 19.375, "learning_rate": 1.9933333333333334e-05, "loss": 16.2896, "mean_token_accuracy": 0.7603169117122889, "num_tokens": 8506030.0, "step": 300 }, { "epoch": 0.48, "eval_biology_entropy": 3.66986682510376, "eval_biology_loss": 4.148224353790283, "eval_biology_mean_token_accuracy": 0.3987880692481995, "eval_biology_num_tokens": 8506030.0, "eval_biology_runtime": 49.5301, "eval_biology_samples_per_second": 10.095, "eval_biology_steps_per_second": 2.524, "step": 300 }, { "epoch": 0.48, "eval_chemistry_entropy": 2.309468500137329, "eval_chemistry_loss": 2.3434462547302246, "eval_chemistry_mean_token_accuracy": 0.5782731935977936, "eval_chemistry_num_tokens": 8506030.0, "eval_chemistry_runtime": 61.4332, "eval_chemistry_samples_per_second": 8.139, "eval_chemistry_steps_per_second": 2.035, "step": 300 }, { "epoch": 0.48, "eval_math_entropy": 1.007653223514557, "eval_math_loss": 1.0127896070480347, "eval_math_mean_token_accuracy": 0.7593745794296265, "eval_math_num_tokens": 8506030.0, "eval_math_runtime": 63.0217, "eval_math_samples_per_second": 7.934, "eval_math_steps_per_second": 1.983, "step": 300 }, { "epoch": 0.48, "eval_physics_entropy": 1.7608964786529542, "eval_physics_loss": 1.7697628736495972, "eval_physics_mean_token_accuracy": 0.6516191956996917, "eval_physics_num_tokens": 8506030.0, "eval_physics_runtime": 71.8045, "eval_physics_samples_per_second": 6.963, "eval_physics_steps_per_second": 1.741, "step": 300 }, { "entropy": 0.9751624643802643, "epoch": 0.496, "grad_norm": 27.0, "learning_rate": 1.9933333333333334e-05, "loss": 15.6472, "mean_token_accuracy": 0.7660828057676554, "num_tokens": 8788726.0, "step": 310 }, { "entropy": 0.9841579314321279, "epoch": 0.512, "grad_norm": 19.5, "learning_rate": 1.985925925925926e-05, "loss": 15.7341, "mean_token_accuracy": 0.765475382655859, "num_tokens": 9078039.0, "step": 320 }, { "entropy": 0.9567954104393721, "epoch": 0.528, "grad_norm": 22.875, "learning_rate": 1.9785185185185187e-05, "loss": 15.3963, "mean_token_accuracy": 0.7705613534897566, "num_tokens": 9373860.0, "step": 330 }, { "entropy": 0.9622254882007837, "epoch": 0.544, "grad_norm": 20.625, "learning_rate": 1.971111111111111e-05, "loss": 15.6284, "mean_token_accuracy": 0.7682077161967754, "num_tokens": 9660940.0, "step": 340 }, { "entropy": 1.013891428336501, "epoch": 0.56, "grad_norm": 28.875, "learning_rate": 1.963703703703704e-05, "loss": 16.1123, "mean_token_accuracy": 0.758896803110838, "num_tokens": 9932302.0, "step": 350 }, { "entropy": 0.9643709316849709, "epoch": 0.576, "grad_norm": 20.0, "learning_rate": 1.9562962962962964e-05, "loss": 15.4925, "mean_token_accuracy": 0.7669644717127084, "num_tokens": 10215462.0, "step": 360 }, { "entropy": 0.9496656563133001, "epoch": 0.592, "grad_norm": 18.5, "learning_rate": 1.948888888888889e-05, "loss": 15.3772, "mean_token_accuracy": 0.7689070332795381, "num_tokens": 10504396.0, "step": 370 }, { "entropy": 0.9635139308869839, "epoch": 0.608, "grad_norm": 17.5, "learning_rate": 1.9414814814814817e-05, "loss": 15.4665, "mean_token_accuracy": 0.768926914036274, "num_tokens": 10793126.0, "step": 380 }, { "entropy": 0.9394328493624926, "epoch": 0.624, "grad_norm": 19.375, "learning_rate": 1.9340740740740743e-05, "loss": 15.1417, "mean_token_accuracy": 0.7703750677406788, "num_tokens": 11081768.0, "step": 390 }, { "entropy": 0.930184380710125, "epoch": 0.64, "grad_norm": 18.25, "learning_rate": 1.926666666666667e-05, "loss": 14.8657, "mean_token_accuracy": 0.7726314965635538, "num_tokens": 11370320.0, "step": 400 }, { "epoch": 0.64, "eval_biology_entropy": 3.578768089294434, "eval_biology_loss": 4.099727630615234, "eval_biology_mean_token_accuracy": 0.4042117977142334, "eval_biology_num_tokens": 11370320.0, "eval_biology_runtime": 49.5083, "eval_biology_samples_per_second": 10.099, "eval_biology_steps_per_second": 2.525, "step": 400 }, { "epoch": 0.64, "eval_chemistry_entropy": 2.1963028135299685, "eval_chemistry_loss": 2.250903367996216, "eval_chemistry_mean_token_accuracy": 0.588011109828949, "eval_chemistry_num_tokens": 11370320.0, "eval_chemistry_runtime": 61.4213, "eval_chemistry_samples_per_second": 8.14, "eval_chemistry_steps_per_second": 2.035, "step": 400 }, { "epoch": 0.64, "eval_math_entropy": 0.9574988222122193, "eval_math_loss": 0.9642173051834106, "eval_math_mean_token_accuracy": 0.7676870808601379, "eval_math_num_tokens": 11370320.0, "eval_math_runtime": 62.9918, "eval_math_samples_per_second": 7.938, "eval_math_steps_per_second": 1.984, "step": 400 }, { "epoch": 0.64, "eval_physics_entropy": 1.6637140741348266, "eval_physics_loss": 1.7104088068008423, "eval_physics_mean_token_accuracy": 0.6602483534812927, "eval_physics_num_tokens": 11370320.0, "eval_physics_runtime": 71.7644, "eval_physics_samples_per_second": 6.967, "eval_physics_steps_per_second": 1.742, "step": 400 }, { "entropy": 0.908730499073863, "epoch": 0.656, "grad_norm": 16.625, "learning_rate": 1.9192592592592593e-05, "loss": 14.6477, "mean_token_accuracy": 0.7755995105952025, "num_tokens": 11657835.0, "step": 410 }, { "entropy": 0.9500968560576439, "epoch": 0.672, "grad_norm": 18.25, "learning_rate": 1.911851851851852e-05, "loss": 15.2383, "mean_token_accuracy": 0.769153194501996, "num_tokens": 11949262.0, "step": 420 }, { "entropy": 0.9080810055136681, "epoch": 0.688, "grad_norm": 15.625, "learning_rate": 1.9044444444444446e-05, "loss": 14.6139, "mean_token_accuracy": 0.7764004003256559, "num_tokens": 12227640.0, "step": 430 }, { "entropy": 0.9170140367001295, "epoch": 0.704, "grad_norm": 17.625, "learning_rate": 1.8970370370370372e-05, "loss": 14.7655, "mean_token_accuracy": 0.7752966023981571, "num_tokens": 12516641.0, "step": 440 }, { "entropy": 0.9368526700884103, "epoch": 0.72, "grad_norm": 19.25, "learning_rate": 1.8896296296296295e-05, "loss": 15.134, "mean_token_accuracy": 0.7706776697188615, "num_tokens": 12793343.0, "step": 450 }, { "entropy": 0.9261821806430817, "epoch": 0.736, "grad_norm": 17.5, "learning_rate": 1.8822222222222225e-05, "loss": 14.833, "mean_token_accuracy": 0.7745088890194893, "num_tokens": 13077981.0, "step": 460 }, { "entropy": 0.9040410481393337, "epoch": 0.752, "grad_norm": 19.75, "learning_rate": 1.874814814814815e-05, "loss": 14.4907, "mean_token_accuracy": 0.7791126936674118, "num_tokens": 13358957.0, "step": 470 }, { "entropy": 0.8739457067102194, "epoch": 0.768, "grad_norm": 15.8125, "learning_rate": 1.8674074074074075e-05, "loss": 14.112, "mean_token_accuracy": 0.7853138707578182, "num_tokens": 13653412.0, "step": 480 }, { "entropy": 0.946398203074932, "epoch": 0.784, "grad_norm": 19.875, "learning_rate": 1.86e-05, "loss": 15.145, "mean_token_accuracy": 0.7723897945135831, "num_tokens": 13940856.0, "step": 490 }, { "entropy": 0.9361972361803055, "epoch": 0.8, "grad_norm": 19.625, "learning_rate": 1.8525925925925928e-05, "loss": 15.2105, "mean_token_accuracy": 0.7712519209831953, "num_tokens": 14230754.0, "step": 500 }, { "epoch": 0.8, "eval_biology_entropy": 3.714933115005493, "eval_biology_loss": 3.970184564590454, "eval_biology_mean_token_accuracy": 0.41206827068328855, "eval_biology_num_tokens": 14230754.0, "eval_biology_runtime": 49.5281, "eval_biology_samples_per_second": 10.095, "eval_biology_steps_per_second": 2.524, "step": 500 }, { "epoch": 0.8, "eval_chemistry_entropy": 2.237494807243347, "eval_chemistry_loss": 2.1822235584259033, "eval_chemistry_mean_token_accuracy": 0.5956678519248962, "eval_chemistry_num_tokens": 14230754.0, "eval_chemistry_runtime": 61.0757, "eval_chemistry_samples_per_second": 8.187, "eval_chemistry_steps_per_second": 2.047, "step": 500 }, { "epoch": 0.8, "eval_math_entropy": 0.9503358464241027, "eval_math_loss": 0.9351838827133179, "eval_math_mean_token_accuracy": 0.7728546581268311, "eval_math_num_tokens": 14230754.0, "eval_math_runtime": 63.0753, "eval_math_samples_per_second": 7.927, "eval_math_steps_per_second": 1.982, "step": 500 }, { "epoch": 0.8, "eval_physics_entropy": 1.6920014672279358, "eval_physics_loss": 1.671562671661377, "eval_physics_mean_token_accuracy": 0.6658579094409942, "eval_physics_num_tokens": 14230754.0, "eval_physics_runtime": 71.8921, "eval_physics_samples_per_second": 6.955, "eval_physics_steps_per_second": 1.739, "step": 500 } ], "logging_steps": 10, "max_steps": 3000, "num_input_tokens_seen": 0, "num_train_epochs": 5, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 1.448451560178856e+18, "train_batch_size": 4, "trial_name": null, "trial_params": null }