| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9893390191897655, | |
| "eval_steps": 60, | |
| "global_step": 174, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 604.4832763671875, | |
| "epoch": 0.017057569296375266, | |
| "grad_norm": 0.4691731333732605, | |
| "kl": 0.0, | |
| "learning_rate": 1.6666666666666665e-07, | |
| "loss": 0.0291, | |
| "reward": 0.628348246216774, | |
| "reward_std": 0.34620763547718525, | |
| "rewards/accuracy_reward": 0.628348246216774, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 607.7860174179077, | |
| "epoch": 0.08528784648187633, | |
| "grad_norm": 0.9751039147377014, | |
| "kl": 0.0001576244831085205, | |
| "learning_rate": 8.333333333333334e-07, | |
| "loss": 0.0137, | |
| "reward": 0.6099330596625805, | |
| "reward_std": 0.35781326424330473, | |
| "rewards/accuracy_reward": 0.6099330596625805, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 591.4216735839843, | |
| "epoch": 0.17057569296375266, | |
| "grad_norm": 0.31423789262771606, | |
| "kl": 0.0007061958312988281, | |
| "learning_rate": 1.6666666666666669e-06, | |
| "loss": 0.0366, | |
| "reward": 0.6220982402563096, | |
| "reward_std": 0.34715419709682466, | |
| "rewards/accuracy_reward": 0.6220982402563096, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 615.2261436462402, | |
| "epoch": 0.255863539445629, | |
| "grad_norm": 0.427952378988266, | |
| "kl": 0.003865623474121094, | |
| "learning_rate": 2.5e-06, | |
| "loss": 0.0603, | |
| "reward": 0.6941964596509933, | |
| "reward_std": 0.30072550252079966, | |
| "rewards/accuracy_reward": 0.6941964596509933, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 613.013639831543, | |
| "epoch": 0.3411513859275053, | |
| "grad_norm": 0.2186485081911087, | |
| "kl": 0.05426101684570313, | |
| "learning_rate": 2.9987834972573546e-06, | |
| "loss": 0.0751, | |
| "reward": 0.742857176065445, | |
| "reward_std": 0.23966127894818784, | |
| "rewards/accuracy_reward": 0.742857176065445, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 607.6678848266602, | |
| "epoch": 0.42643923240938164, | |
| "grad_norm": 0.2839179039001465, | |
| "kl": 0.00532073974609375, | |
| "learning_rate": 2.9851204919417464e-06, | |
| "loss": 0.0546, | |
| "reward": 0.7526786044239998, | |
| "reward_std": 0.19872783049941062, | |
| "rewards/accuracy_reward": 0.7526786044239998, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 601.8375274658204, | |
| "epoch": 0.511727078891258, | |
| "grad_norm": 0.13062593340873718, | |
| "kl": 0.0034595489501953124, | |
| "learning_rate": 2.956412726139078e-06, | |
| "loss": 0.0385, | |
| "reward": 0.7564732477068901, | |
| "reward_std": 0.17959171254187822, | |
| "rewards/accuracy_reward": 0.7564732477068901, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 597.1578399658204, | |
| "epoch": 0.5970149253731343, | |
| "grad_norm": 0.35054460167884827, | |
| "kl": 0.005078506469726562, | |
| "learning_rate": 2.9129510189868974e-06, | |
| "loss": 0.0341, | |
| "reward": 0.7522321745753289, | |
| "reward_std": 0.16307315276935697, | |
| "rewards/accuracy_reward": 0.7522321745753289, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 581.5973480224609, | |
| "epoch": 0.6823027718550106, | |
| "grad_norm": 0.11960138380527496, | |
| "kl": 0.00443115234375, | |
| "learning_rate": 2.8551756519155732e-06, | |
| "loss": 0.0368, | |
| "reward": 0.7622768253087997, | |
| "reward_std": 0.17930736346170306, | |
| "rewards/accuracy_reward": 0.7622768253087997, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 596.7301620483398, | |
| "epoch": 0.767590618336887, | |
| "grad_norm": 0.11841974407434464, | |
| "kl": 0.005435562133789063, | |
| "learning_rate": 2.7836719084521715e-06, | |
| "loss": 0.0353, | |
| "reward": 0.7607143238186836, | |
| "reward_std": 0.18814852144569158, | |
| "rewards/accuracy_reward": 0.7607143238186836, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 585.3933280944824, | |
| "epoch": 0.8528784648187633, | |
| "grad_norm": 0.10040715336799622, | |
| "kl": 0.0057315826416015625, | |
| "learning_rate": 2.699164145105252e-06, | |
| "loss": 0.025, | |
| "reward": 0.7537946745753288, | |
| "reward_std": 0.16595598962157965, | |
| "rewards/accuracy_reward": 0.7537946745753288, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 577.9147621154785, | |
| "epoch": 0.9381663113006397, | |
| "grad_norm": 0.24622774124145508, | |
| "kl": 0.007873153686523438, | |
| "learning_rate": 2.602508453394493e-06, | |
| "loss": 0.0338, | |
| "reward": 0.787723246216774, | |
| "reward_std": 0.18768559023737907, | |
| "rewards/accuracy_reward": 0.787723246216774, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 1.0341151385927505, | |
| "grad_norm": 0.15860386192798615, | |
| "learning_rate": 2.4946839873611927e-06, | |
| "loss": 0.0344, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.0341151385927505, | |
| "eval_clip_ratio": 0.0, | |
| "eval_completion_length": 563.3263092041016, | |
| "eval_kl": 0.007824897766113281, | |
| "eval_loss": 0.012524000369012356, | |
| "eval_reward": 0.6738281548023224, | |
| "eval_reward_std": 0.2058188347145915, | |
| "eval_rewards/accuracy_reward": 0.6738281548023224, | |
| "eval_runtime": 420.7271, | |
| "eval_samples_per_second": 1.188, | |
| "eval_steps_per_second": 0.012, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 558.5659664154052, | |
| "epoch": 1.1194029850746268, | |
| "grad_norm": 15.58234977722168, | |
| "kl": 0.015855026245117188, | |
| "learning_rate": 2.3767830444148337e-06, | |
| "loss": 0.0307, | |
| "reward": 0.778013427555561, | |
| "reward_std": 0.17006983272731305, | |
| "rewards/accuracy_reward": 0.778013427555561, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 548.0241348266602, | |
| "epoch": 1.2046908315565032, | |
| "grad_norm": 0.21251261234283447, | |
| "kl": 0.00769195556640625, | |
| "learning_rate": 2.25e-06, | |
| "loss": 0.0296, | |
| "reward": 0.7854911044239998, | |
| "reward_std": 0.16244285507127643, | |
| "rewards/accuracy_reward": 0.7854911044239998, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 563.5969009399414, | |
| "epoch": 1.2899786780383795, | |
| "grad_norm": 0.2231518030166626, | |
| "kl": 0.01058197021484375, | |
| "learning_rate": 2.1156192081791355e-06, | |
| "loss": 0.0301, | |
| "reward": 0.7732143193483353, | |
| "reward_std": 0.19672715626657009, | |
| "rewards/accuracy_reward": 0.7732143193483353, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 550.5730125427247, | |
| "epoch": 1.375266524520256, | |
| "grad_norm": 0.20531561970710754, | |
| "kl": 0.010945892333984375, | |
| "learning_rate": 1.975001990702209e-06, | |
| "loss": 0.0303, | |
| "reward": 0.7774553924798966, | |
| "reward_std": 0.1880516828969121, | |
| "rewards/accuracy_reward": 0.7774553924798966, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 559.7966743469239, | |
| "epoch": 1.4605543710021323, | |
| "grad_norm": 0.33798426389694214, | |
| "kl": 0.01414337158203125, | |
| "learning_rate": 1.829572846368326e-06, | |
| "loss": 0.04, | |
| "reward": 0.7678571820259095, | |
| "reward_std": 0.2084697039797902, | |
| "rewards/accuracy_reward": 0.7678571820259095, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 579.8114044189454, | |
| "epoch": 1.5458422174840085, | |
| "grad_norm": 0.6812328696250916, | |
| "kl": 0.029425048828125, | |
| "learning_rate": 1.6808050203829845e-06, | |
| "loss": 0.074, | |
| "reward": 0.736160746216774, | |
| "reward_std": 0.23041144628077745, | |
| "rewards/accuracy_reward": 0.736160746216774, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 606.0049392700196, | |
| "epoch": 1.6311300639658848, | |
| "grad_norm": 2.7374346256256104, | |
| "kl": 0.090008544921875, | |
| "learning_rate": 1.5302055798981605e-06, | |
| "loss": 0.1251, | |
| "reward": 0.6779018133878708, | |
| "reward_std": 0.30644247531890867, | |
| "rewards/accuracy_reward": 0.6779018133878708, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 638.524806213379, | |
| "epoch": 1.716417910447761, | |
| "grad_norm": 5.840439319610596, | |
| "kl": 0.35400390625, | |
| "learning_rate": 1.3793001469249112e-06, | |
| "loss": 0.1864, | |
| "reward": 0.6270089536905289, | |
| "reward_std": 0.331193171441555, | |
| "rewards/accuracy_reward": 0.6270089536905289, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 650.1977951049805, | |
| "epoch": 1.8017057569296375, | |
| "grad_norm": 14.723860740661621, | |
| "kl": 0.818017578125, | |
| "learning_rate": 1.2296174432791415e-06, | |
| "loss": 0.1931, | |
| "reward": 0.5450893104076385, | |
| "reward_std": 0.33599049374461176, | |
| "rewards/accuracy_reward": 0.5450893104076385, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 552.9535926818847, | |
| "epoch": 1.886993603411514, | |
| "grad_norm": 26.323904037475586, | |
| "kl": 1.0900390625, | |
| "learning_rate": 1.0826738041253211e-06, | |
| "loss": 0.1206, | |
| "reward": 0.5149553798139095, | |
| "reward_std": 0.3531858079135418, | |
| "rewards/accuracy_reward": 0.5149553798139095, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 505.44801330566406, | |
| "epoch": 1.9722814498933903, | |
| "grad_norm": 15.791183471679688, | |
| "kl": 0.76748046875, | |
| "learning_rate": 9.399578170010685e-07, | |
| "loss": 0.0274, | |
| "reward": 0.48549109175801275, | |
| "reward_std": 0.3420914553105831, | |
| "rewards/accuracy_reward": 0.48549109175801275, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 2.068230277185501, | |
| "grad_norm": 14.470078468322754, | |
| "learning_rate": 8.029152419343472e-07, | |
| "loss": -0.0002, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.068230277185501, | |
| "eval_clip_ratio": 0.0, | |
| "eval_completion_length": 477.81743812561035, | |
| "eval_kl": 0.83087158203125, | |
| "eval_loss": 0.014172928407788277, | |
| "eval_reward": 0.4252232350409031, | |
| "eval_reward_std": 0.2951010586693883, | |
| "eval_rewards/accuracy_reward": 0.4252232350409031, | |
| "eval_runtime": 449.8696, | |
| "eval_samples_per_second": 1.111, | |
| "eval_steps_per_second": 0.011, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 477.33952713012695, | |
| "epoch": 2.1535181236673773, | |
| "grad_norm": 20.696521759033203, | |
| "kl": 0.8743408203125, | |
| "learning_rate": 6.729343654174626e-07, | |
| "loss": 0.0312, | |
| "reward": 0.4702009156346321, | |
| "reward_std": 0.3341362001374364, | |
| "rewards/accuracy_reward": 0.4702009156346321, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 519.6768089294434, | |
| "epoch": 2.2388059701492535, | |
| "grad_norm": 32.883033752441406, | |
| "kl": 1.52685546875, | |
| "learning_rate": 5.513319366069343e-07, | |
| "loss": 0.0803, | |
| "reward": 0.46651787906885145, | |
| "reward_std": 0.31465277783572676, | |
| "rewards/accuracy_reward": 0.46651787906885145, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 538.6719017028809, | |
| "epoch": 2.3240938166311302, | |
| "grad_norm": 10.882832527160645, | |
| "kl": 1.92373046875, | |
| "learning_rate": 4.3933982822017883e-07, | |
| "loss": 0.1152, | |
| "reward": 0.43080359026789666, | |
| "reward_std": 0.29358932226896284, | |
| "rewards/accuracy_reward": 0.43080359026789666, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 530.0337310791016, | |
| "epoch": 2.4093816631130065, | |
| "grad_norm": 17.36768341064453, | |
| "kl": 1.71376953125, | |
| "learning_rate": 3.380925572585183e-07, | |
| "loss": 0.1217, | |
| "reward": 0.44866073280572893, | |
| "reward_std": 0.31872403249144554, | |
| "rewards/accuracy_reward": 0.44866073280572893, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.4377464294433, | |
| "epoch": 2.4946695095948828, | |
| "grad_norm": 8.74123764038086, | |
| "kl": 1.97939453125, | |
| "learning_rate": 2.4861579197570804e-07, | |
| "loss": 0.1345, | |
| "reward": 0.5015625268220901, | |
| "reward_std": 0.323883581161499, | |
| "rewards/accuracy_reward": 0.5015625268220901, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 530.0359634399414, | |
| "epoch": 2.579957356076759, | |
| "grad_norm": 11.574440002441406, | |
| "kl": 1.9712890625, | |
| "learning_rate": 1.718159615201853e-07, | |
| "loss": 0.1274, | |
| "reward": 0.4540178768336773, | |
| "reward_std": 0.3195581167936325, | |
| "rewards/accuracy_reward": 0.4540178768336773, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 528.8886405944825, | |
| "epoch": 2.6652452025586353, | |
| "grad_norm": 16.876115798950195, | |
| "kl": 1.59755859375, | |
| "learning_rate": 1.0847107350878571e-07, | |
| "loss": 0.1166, | |
| "reward": 0.4720982313156128, | |
| "reward_std": 0.33180556632578373, | |
| "rewards/accuracy_reward": 0.4720982313156128, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 525.8712371826172, | |
| "epoch": 2.750533049040512, | |
| "grad_norm": 8.542109489440918, | |
| "kl": 2.288671875, | |
| "learning_rate": 5.922283255294164e-08, | |
| "loss": 0.1575, | |
| "reward": 0.4640625201165676, | |
| "reward_std": 0.32937051728367805, | |
| "rewards/accuracy_reward": 0.4640625201165676, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 515.5904273986816, | |
| "epoch": 2.835820895522388, | |
| "grad_norm": 59.300743103027344, | |
| "kl": 2.4236328125, | |
| "learning_rate": 2.4570139579284723e-08, | |
| "loss": 0.1451, | |
| "reward": 0.4558035910129547, | |
| "reward_std": 0.3251536630094051, | |
| "rewards/accuracy_reward": 0.4558035910129547, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 529.5910972595215, | |
| "epoch": 2.9211087420042645, | |
| "grad_norm": 11.339794158935547, | |
| "kl": 2.040234375, | |
| "learning_rate": 4.864037798685106e-09, | |
| "loss": 0.1336, | |
| "reward": 0.4558035910129547, | |
| "reward_std": 0.3135051976889372, | |
| "rewards/accuracy_reward": 0.4558035910129547, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.6137437820435, | |
| "epoch": 2.9893390191897655, | |
| "kl": 2.185546875, | |
| "reward": 0.43164064455777407, | |
| "reward_std": 0.3127579181455076, | |
| "rewards/accuracy_reward": 0.43164064455777407, | |
| "step": 174, | |
| "total_flos": 0.0, | |
| "train_loss": 0.07567721740583536, | |
| "train_runtime": 24349.6607, | |
| "train_samples_per_second": 0.924, | |
| "train_steps_per_second": 0.007 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 174, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |