| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.0012675024260788625, | |
| "eval_steps": 500, | |
| "global_step": 8, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0001584378032598578, | |
| "grad_norm": 4.8242447519442067e-05, | |
| "learning_rate": 0.0029998415465061004, | |
| "loss": 0.0, | |
| "loss/policy_avg": 7.299240678548813e-08, | |
| "objective/entropy": 67.43842315673828, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": 1.5219240188598633, | |
| "objective/scores": 1.52203369140625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 1, | |
| "timer/active_model_forward": 0.7594873309135437, | |
| "timer/calc_advantages": 31.33674430847168, | |
| "timer/calc_loss": 0.774461030960083, | |
| "timer/generate": 28.869869232177734, | |
| "timer/get_reward": 0.5197552442550659, | |
| "timer/model_forward": 0.7690438628196716, | |
| "timer/ref_model_forward": 0.7777600884437561, | |
| "timer/training_step": 34.545597076416016, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0003168756065197156, | |
| "grad_norm": 0.00026670348597690463, | |
| "learning_rate": 0.002999683093012201, | |
| "loss": -0.0, | |
| "loss/policy_avg": -1.7043203115463257e-07, | |
| "objective/entropy": 118.22715759277344, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -3.824268341064453, | |
| "objective/scores": -3.82421875, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 2, | |
| "timer/active_model_forward": 0.6569761037826538, | |
| "timer/calc_advantages": 30.551788330078125, | |
| "timer/calc_loss": 0.6658673286437988, | |
| "timer/generate": 28.41728973388672, | |
| "timer/get_reward": 0.43157756328582764, | |
| "timer/model_forward": 0.6641299724578857, | |
| "timer/ref_model_forward": 0.67349773645401, | |
| "timer/training_step": 33.37031936645508, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0004753134097795734, | |
| "grad_norm": 0.00013700829003937542, | |
| "learning_rate": 0.0029995246395183015, | |
| "loss": -0.0, | |
| "loss/policy_avg": -2.9243528842926025e-07, | |
| "objective/entropy": 1081.7860107421875, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -5.476104736328125, | |
| "objective/scores": -5.476806640625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 3, | |
| "timer/active_model_forward": 0.9020705223083496, | |
| "timer/calc_advantages": 32.11817169189453, | |
| "timer/calc_loss": 0.9118615984916687, | |
| "timer/generate": 29.300296783447266, | |
| "timer/get_reward": 0.5959209203720093, | |
| "timer/model_forward": 0.9100804924964905, | |
| "timer/ref_model_forward": 0.9205952882766724, | |
| "timer/training_step": 35.83306121826172, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0006337512130394313, | |
| "grad_norm": 7.692693907301873e-05, | |
| "learning_rate": 0.002999366186024402, | |
| "loss": -0.0, | |
| "loss/policy_avg": -2.0489096641540527e-07, | |
| "objective/entropy": 1181.2274169921875, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -5.0478363037109375, | |
| "objective/scores": -5.04833984375, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 4, | |
| "timer/active_model_forward": 0.8427364826202393, | |
| "timer/calc_advantages": 31.744182586669922, | |
| "timer/calc_loss": 0.8533791899681091, | |
| "timer/generate": 29.096416473388672, | |
| "timer/get_reward": 0.5531487464904785, | |
| "timer/model_forward": 0.851642906665802, | |
| "timer/ref_model_forward": 0.8617820143699646, | |
| "timer/training_step": 35.25868225097656, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.000792189016299289, | |
| "grad_norm": 3.911158273695037e-05, | |
| "learning_rate": 0.0029992077325305022, | |
| "loss": -0.0, | |
| "loss/policy_avg": -2.1513551473617554e-07, | |
| "objective/entropy": 1185.438232421875, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -4.198909759521484, | |
| "objective/scores": -4.198486328125, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 5, | |
| "timer/active_model_forward": 0.791987955570221, | |
| "timer/calc_advantages": 31.324127197265625, | |
| "timer/calc_loss": 0.8030194044113159, | |
| "timer/generate": 28.832427978515625, | |
| "timer/get_reward": 0.5202938914299011, | |
| "timer/model_forward": 0.8012962937355042, | |
| "timer/ref_model_forward": 0.8093616366386414, | |
| "timer/training_step": 34.65721130371094, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0009506268195591468, | |
| "grad_norm": 2.5278823159169406e-05, | |
| "learning_rate": 0.002999049279036603, | |
| "loss": -0.0, | |
| "loss/policy_avg": -1.2619420886039734e-07, | |
| "objective/entropy": 528.8932495117188, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -3.1182708740234375, | |
| "objective/scores": -3.1180419921875, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 6, | |
| "timer/active_model_forward": 0.7996934652328491, | |
| "timer/calc_advantages": 31.30746078491211, | |
| "timer/calc_loss": 0.8107499480247498, | |
| "timer/generate": 28.788341522216797, | |
| "timer/get_reward": 0.5261460542678833, | |
| "timer/model_forward": 0.8090219497680664, | |
| "timer/ref_model_forward": 0.8171602487564087, | |
| "timer/training_step": 34.65579605102539, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0011090646228190046, | |
| "grad_norm": 1.0207173772869282e-06, | |
| "learning_rate": 0.0029988908255427034, | |
| "loss": -0.0, | |
| "loss/policy_avg": -3.1781382858753204e-08, | |
| "objective/entropy": 1.5396817922592163, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -1.2192144393920898, | |
| "objective/scores": -1.2193603515625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 7, | |
| "timer/active_model_forward": 1.0039308071136475, | |
| "timer/calc_advantages": 32.610069274902344, | |
| "timer/calc_loss": 1.0154139995574951, | |
| "timer/generate": 29.52051544189453, | |
| "timer/get_reward": 0.6604768633842468, | |
| "timer/model_forward": 1.0136762857437134, | |
| "timer/ref_model_forward": 1.0217972993850708, | |
| "timer/training_step": 36.71036911010742, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| }, | |
| { | |
| "epoch": 0.0012675024260788625, | |
| "grad_norm": 1.4357982536239433e-06, | |
| "learning_rate": 0.0029987323720488037, | |
| "loss": -0.0, | |
| "loss/policy_avg": -7.380731403827667e-08, | |
| "objective/entropy": 2.126983404159546, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": -1.3588581085205078, | |
| "objective/scores": -1.35894775390625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "step": 8, | |
| "timer/active_model_forward": 1.0782428979873657, | |
| "timer/calc_advantages": 33.10707092285156, | |
| "timer/calc_loss": 1.0924396514892578, | |
| "timer/generate": 29.794893264770508, | |
| "timer/get_reward": 0.7108417749404907, | |
| "timer/model_forward": 1.0906702280044556, | |
| "timer/ref_model_forward": 1.0941966772079468, | |
| "timer/training_step": 37.46529769897461, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 18933, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 8, | |
| "total_flos": 4025485044940800.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |