| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.003802507278236587, |
| "eval_steps": 500, |
| "global_step": 24, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0001584378032598578, |
| "grad_norm": 4.8242447519442067e-05, |
| "learning_rate": 0.0029998415465061004, |
| "loss": 0.0, |
| "loss/policy_avg": 7.299240678548813e-08, |
| "objective/entropy": 67.43842315673828, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": 1.5219240188598633, |
| "objective/scores": 1.52203369140625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 1, |
| "timer/active_model_forward": 0.7594873309135437, |
| "timer/calc_advantages": 31.33674430847168, |
| "timer/calc_loss": 0.774461030960083, |
| "timer/generate": 28.869869232177734, |
| "timer/get_reward": 0.5197552442550659, |
| "timer/model_forward": 0.7690438628196716, |
| "timer/ref_model_forward": 0.7777600884437561, |
| "timer/training_step": 34.545597076416016, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0003168756065197156, |
| "grad_norm": 0.00026670348597690463, |
| "learning_rate": 0.002999683093012201, |
| "loss": -0.0, |
| "loss/policy_avg": -1.7043203115463257e-07, |
| "objective/entropy": 118.22715759277344, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -3.824268341064453, |
| "objective/scores": -3.82421875, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 2, |
| "timer/active_model_forward": 0.6569761037826538, |
| "timer/calc_advantages": 30.551788330078125, |
| "timer/calc_loss": 0.6658673286437988, |
| "timer/generate": 28.41728973388672, |
| "timer/get_reward": 0.43157756328582764, |
| "timer/model_forward": 0.6641299724578857, |
| "timer/ref_model_forward": 0.67349773645401, |
| "timer/training_step": 33.37031936645508, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0004753134097795734, |
| "grad_norm": 0.00013700829003937542, |
| "learning_rate": 0.0029995246395183015, |
| "loss": -0.0, |
| "loss/policy_avg": -2.9243528842926025e-07, |
| "objective/entropy": 1081.7860107421875, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -5.476104736328125, |
| "objective/scores": -5.476806640625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 3, |
| "timer/active_model_forward": 0.9020705223083496, |
| "timer/calc_advantages": 32.11817169189453, |
| "timer/calc_loss": 0.9118615984916687, |
| "timer/generate": 29.300296783447266, |
| "timer/get_reward": 0.5959209203720093, |
| "timer/model_forward": 0.9100804924964905, |
| "timer/ref_model_forward": 0.9205952882766724, |
| "timer/training_step": 35.83306121826172, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0006337512130394313, |
| "grad_norm": 7.692693907301873e-05, |
| "learning_rate": 0.002999366186024402, |
| "loss": -0.0, |
| "loss/policy_avg": -2.0489096641540527e-07, |
| "objective/entropy": 1181.2274169921875, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -5.0478363037109375, |
| "objective/scores": -5.04833984375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 4, |
| "timer/active_model_forward": 0.8427364826202393, |
| "timer/calc_advantages": 31.744182586669922, |
| "timer/calc_loss": 0.8533791899681091, |
| "timer/generate": 29.096416473388672, |
| "timer/get_reward": 0.5531487464904785, |
| "timer/model_forward": 0.851642906665802, |
| "timer/ref_model_forward": 0.8617820143699646, |
| "timer/training_step": 35.25868225097656, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.000792189016299289, |
| "grad_norm": 3.911158273695037e-05, |
| "learning_rate": 0.0029992077325305022, |
| "loss": -0.0, |
| "loss/policy_avg": -2.1513551473617554e-07, |
| "objective/entropy": 1185.438232421875, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -4.198909759521484, |
| "objective/scores": -4.198486328125, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 5, |
| "timer/active_model_forward": 0.791987955570221, |
| "timer/calc_advantages": 31.324127197265625, |
| "timer/calc_loss": 0.8030194044113159, |
| "timer/generate": 28.832427978515625, |
| "timer/get_reward": 0.5202938914299011, |
| "timer/model_forward": 0.8012962937355042, |
| "timer/ref_model_forward": 0.8093616366386414, |
| "timer/training_step": 34.65721130371094, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0009506268195591468, |
| "grad_norm": 2.5278823159169406e-05, |
| "learning_rate": 0.002999049279036603, |
| "loss": -0.0, |
| "loss/policy_avg": -1.2619420886039734e-07, |
| "objective/entropy": 528.8932495117188, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -3.1182708740234375, |
| "objective/scores": -3.1180419921875, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 6, |
| "timer/active_model_forward": 0.7996934652328491, |
| "timer/calc_advantages": 31.30746078491211, |
| "timer/calc_loss": 0.8107499480247498, |
| "timer/generate": 28.788341522216797, |
| "timer/get_reward": 0.5261460542678833, |
| "timer/model_forward": 0.8090219497680664, |
| "timer/ref_model_forward": 0.8171602487564087, |
| "timer/training_step": 34.65579605102539, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0011090646228190046, |
| "grad_norm": 1.0207173772869282e-06, |
| "learning_rate": 0.0029988908255427034, |
| "loss": -0.0, |
| "loss/policy_avg": -3.1781382858753204e-08, |
| "objective/entropy": 1.5396817922592163, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.2192144393920898, |
| "objective/scores": -1.2193603515625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 7, |
| "timer/active_model_forward": 1.0039308071136475, |
| "timer/calc_advantages": 32.610069274902344, |
| "timer/calc_loss": 1.0154139995574951, |
| "timer/generate": 29.52051544189453, |
| "timer/get_reward": 0.6604768633842468, |
| "timer/model_forward": 1.0136762857437134, |
| "timer/ref_model_forward": 1.0217972993850708, |
| "timer/training_step": 36.71036911010742, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0012675024260788625, |
| "grad_norm": 1.4357982536239433e-06, |
| "learning_rate": 0.0029987323720488037, |
| "loss": -0.0, |
| "loss/policy_avg": -7.380731403827667e-08, |
| "objective/entropy": 2.126983404159546, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.3588581085205078, |
| "objective/scores": -1.35894775390625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 8, |
| "timer/active_model_forward": 1.0782428979873657, |
| "timer/calc_advantages": 33.10707092285156, |
| "timer/calc_loss": 1.0924396514892578, |
| "timer/generate": 29.794893264770508, |
| "timer/get_reward": 0.7108417749404907, |
| "timer/model_forward": 1.0906702280044556, |
| "timer/ref_model_forward": 1.0941966772079468, |
| "timer/training_step": 37.46529769897461, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0014259402293387202, |
| "grad_norm": 1.58626687607466e-06, |
| "learning_rate": 0.002998573918554904, |
| "loss": -0.0, |
| "loss/policy_avg": -5.851325113326311e-08, |
| "objective/entropy": 5.8357086181640625, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.3656387329101562, |
| "objective/scores": -1.36566162109375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 9, |
| "timer/active_model_forward": 0.9783480167388916, |
| "timer/calc_advantages": 32.70989990234375, |
| "timer/calc_loss": 0.9892486333847046, |
| "timer/generate": 29.658161163330078, |
| "timer/get_reward": 0.6437245011329651, |
| "timer/model_forward": 0.9874842166900635, |
| "timer/ref_model_forward": 0.9964737892150879, |
| "timer/training_step": 36.70344924926758, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.001584378032598578, |
| "grad_norm": 1.8057213537758798e-06, |
| "learning_rate": 0.002998415465061005, |
| "loss": -0.0, |
| "loss/policy_avg": -4.391768015921116e-08, |
| "objective/entropy": 37.01044464111328, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.1269941329956055, |
| "objective/scores": -1.126983642578125, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 10, |
| "timer/active_model_forward": 0.7927595973014832, |
| "timer/calc_advantages": 31.501224517822266, |
| "timer/calc_loss": 0.8029042482376099, |
| "timer/generate": 28.97046661376953, |
| "timer/get_reward": 0.5210747718811035, |
| "timer/model_forward": 0.8011468052864075, |
| "timer/ref_model_forward": 0.8100461363792419, |
| "timer/training_step": 34.83835983276367, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0017428158358584358, |
| "grad_norm": 1.3606485254058498e-06, |
| "learning_rate": 0.002998257011567105, |
| "loss": -0.0, |
| "loss/policy_avg": -6.2282197177410126e-09, |
| "objective/entropy": 82.50379943847656, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -0.20352506637573242, |
| "objective/scores": -0.20351409912109375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 11, |
| "timer/active_model_forward": 0.8447611331939697, |
| "timer/calc_advantages": 31.68113899230957, |
| "timer/calc_loss": 0.8556842803955078, |
| "timer/generate": 29.030838012695312, |
| "timer/get_reward": 0.5556706786155701, |
| "timer/model_forward": 0.853936493396759, |
| "timer/ref_model_forward": 0.8623838424682617, |
| "timer/training_step": 35.189205169677734, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0019012536391182935, |
| "grad_norm": 1.5168055824688054e-06, |
| "learning_rate": 0.0029980985580732055, |
| "loss": -0.0, |
| "loss/policy_avg": -5.324545782059431e-08, |
| "objective/entropy": 43.37924575805664, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.273842453956604, |
| "objective/scores": -1.2738037109375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 12, |
| "timer/active_model_forward": 0.6771436929702759, |
| "timer/calc_advantages": 30.566415786743164, |
| "timer/calc_loss": 0.6872404217720032, |
| "timer/generate": 28.381263732910156, |
| "timer/get_reward": 0.4441335201263428, |
| "timer/model_forward": 0.6854845881462097, |
| "timer/ref_model_forward": 0.6946568489074707, |
| "timer/training_step": 33.479454040527344, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0020596914423781513, |
| "grad_norm": 9.032095817929076e-07, |
| "learning_rate": 0.0029979401045793063, |
| "loss": -0.0, |
| "loss/policy_avg": -5.2401446737349033e-08, |
| "objective/entropy": 43.8469123840332, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.2306889295578003, |
| "objective/scores": -1.230804443359375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 13, |
| "timer/active_model_forward": 0.8871802091598511, |
| "timer/calc_advantages": 31.831775665283203, |
| "timer/calc_loss": 0.8963775634765625, |
| "timer/generate": 29.063201904296875, |
| "timer/get_reward": 0.5812451839447021, |
| "timer/model_forward": 0.8946273326873779, |
| "timer/ref_model_forward": 0.9052593111991882, |
| "timer/training_step": 35.49396514892578, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.002218129245638009, |
| "grad_norm": 1.8163491404266097e-06, |
| "learning_rate": 0.0029977816510854067, |
| "loss": -0.0, |
| "loss/policy_avg": -6.629852578043938e-08, |
| "objective/entropy": 46.85642623901367, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.4318056106567383, |
| "objective/scores": -1.431884765625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 14, |
| "timer/active_model_forward": 0.7645549774169922, |
| "timer/calc_advantages": 31.06265640258789, |
| "timer/calc_loss": 0.775362491607666, |
| "timer/generate": 28.632740020751953, |
| "timer/get_reward": 0.5032914876937866, |
| "timer/model_forward": 0.7736347317695618, |
| "timer/ref_model_forward": 0.7821637988090515, |
| "timer/training_step": 34.282867431640625, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.002376567048897867, |
| "grad_norm": 1.1021771797459223e-06, |
| "learning_rate": 0.002997623197591507, |
| "loss": -0.0, |
| "loss/policy_avg": -4.4819898903369904e-08, |
| "objective/entropy": 83.53251647949219, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -0.6715226173400879, |
| "objective/scores": -0.671539306640625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 15, |
| "timer/active_model_forward": 1.1367318630218506, |
| "timer/calc_advantages": 33.59859848022461, |
| "timer/calc_loss": 1.2175476551055908, |
| "timer/generate": 30.091690063476562, |
| "timer/get_reward": 0.7473501563072205, |
| "timer/model_forward": 1.2155828475952148, |
| "timer/ref_model_forward": 1.1532230377197266, |
| "timer/training_step": 38.26678466796875, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.002535004852157725, |
| "grad_norm": 1.2331845482549397e-06, |
| "learning_rate": 0.0029974647440976073, |
| "loss": -0.0, |
| "loss/policy_avg": -1.6909325495362282e-08, |
| "objective/entropy": 119.47362518310547, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -0.5125551223754883, |
| "objective/scores": -0.5126724243164062, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 16, |
| "timer/active_model_forward": 0.9046459197998047, |
| "timer/calc_advantages": 32.02373123168945, |
| "timer/calc_loss": 0.914932906627655, |
| "timer/generate": 29.190690994262695, |
| "timer/get_reward": 0.5946121215820312, |
| "timer/model_forward": 0.9131810665130615, |
| "timer/ref_model_forward": 0.922136127948761, |
| "timer/training_step": 35.75535202026367, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0026934426554175825, |
| "grad_norm": 2.2155286387715023e-06, |
| "learning_rate": 0.002997306290603708, |
| "loss": -0.0, |
| "loss/policy_avg": -4.8712536226958036e-08, |
| "objective/entropy": 121.45540618896484, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.4430209398269653, |
| "objective/scores": -1.443115234375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 17, |
| "timer/active_model_forward": 0.7191606163978577, |
| "timer/calc_advantages": 31.075271606445312, |
| "timer/calc_loss": 0.7287229895591736, |
| "timer/generate": 28.78363037109375, |
| "timer/get_reward": 0.4706084430217743, |
| "timer/model_forward": 0.7269375324249268, |
| "timer/ref_model_forward": 0.737448513507843, |
| "timer/training_step": 34.119632720947266, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0028518804586774404, |
| "grad_norm": 8.308816177304834e-07, |
| "learning_rate": 0.0029971478371098085, |
| "loss": -0.0, |
| "loss/policy_avg": -1.1129304766654968e-07, |
| "objective/entropy": 96.97429656982422, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -2.8110389709472656, |
| "objective/scores": -2.811279296875, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 18, |
| "timer/active_model_forward": 0.8409894704818726, |
| "timer/calc_advantages": 31.73868179321289, |
| "timer/calc_loss": 0.8524103164672852, |
| "timer/generate": 29.05769920349121, |
| "timer/get_reward": 0.5517525672912598, |
| "timer/model_forward": 0.8506196737289429, |
| "timer/ref_model_forward": 0.8585314154624939, |
| "timer/training_step": 35.25176239013672, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0030103182619372983, |
| "grad_norm": 3.799691228323354e-07, |
| "learning_rate": 0.002996989383615909, |
| "loss": -0.0, |
| "loss/policy_avg": -9.569339454174042e-08, |
| "objective/entropy": 85.54360961914062, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -2.591951370239258, |
| "objective/scores": -2.591796875, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 19, |
| "timer/active_model_forward": 0.8125832080841064, |
| "timer/calc_advantages": 31.4530029296875, |
| "timer/calc_loss": 0.8229714632034302, |
| "timer/generate": 28.860492706298828, |
| "timer/get_reward": 0.5346251130104065, |
| "timer/model_forward": 0.8212224245071411, |
| "timer/ref_model_forward": 0.8305649757385254, |
| "timer/training_step": 34.838218688964844, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.003168756065197156, |
| "grad_norm": 3.7777222132717725e-06, |
| "learning_rate": 0.002996830930122009, |
| "loss": -0.0, |
| "loss/policy_avg": -7.898779585957527e-08, |
| "objective/entropy": 78.87588500976562, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -2.1325230598449707, |
| "objective/scores": -2.13275146484375, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 20, |
| "timer/active_model_forward": 1.0649389028549194, |
| "timer/calc_advantages": 33.12159729003906, |
| "timer/calc_loss": 1.0808004140853882, |
| "timer/generate": 29.827653884887695, |
| "timer/get_reward": 0.7013706564903259, |
| "timer/model_forward": 1.0790588855743408, |
| "timer/ref_model_forward": 1.0825612545013428, |
| "timer/training_step": 37.4581413269043, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0033271938684570138, |
| "grad_norm": 2.958485652015952e-07, |
| "learning_rate": 0.00299667247662811, |
| "loss": -0.0, |
| "loss/policy_avg": -6.577465683221817e-08, |
| "objective/entropy": 26.52663803100586, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.5283641815185547, |
| "objective/scores": -1.52838134765625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 21, |
| "timer/active_model_forward": 0.8408429622650146, |
| "timer/calc_advantages": 31.583477020263672, |
| "timer/calc_loss": 0.8517074584960938, |
| "timer/generate": 28.94709014892578, |
| "timer/get_reward": 0.5524436235427856, |
| "timer/model_forward": 0.8499797582626343, |
| "timer/ref_model_forward": 0.859445333480835, |
| "timer/training_step": 35.0816535949707, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0034856316717168717, |
| "grad_norm": 2.1688055085178348e-07, |
| "learning_rate": 0.0029965140231342103, |
| "loss": -0.0, |
| "loss/policy_avg": -7.275957614183426e-08, |
| "objective/entropy": 5.097399711608887, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.3907923698425293, |
| "objective/scores": -1.390869140625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 22, |
| "timer/active_model_forward": 0.7821312546730042, |
| "timer/calc_advantages": 31.17966079711914, |
| "timer/calc_loss": 0.7932944297790527, |
| "timer/generate": 28.714157104492188, |
| "timer/get_reward": 0.5111904144287109, |
| "timer/model_forward": 0.7915641069412231, |
| "timer/ref_model_forward": 0.8004368543624878, |
| "timer/training_step": 34.46521759033203, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.0036440694749767296, |
| "grad_norm": 1.0821660367810182e-07, |
| "learning_rate": 0.0029963555696403106, |
| "loss": -0.0, |
| "loss/policy_avg": -5.2619725465774536e-08, |
| "objective/entropy": 0.7932516932487488, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.3286209106445312, |
| "objective/scores": -1.3287353515625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 23, |
| "timer/active_model_forward": 0.7814183235168457, |
| "timer/calc_advantages": 31.230453491210938, |
| "timer/calc_loss": 0.7906038761138916, |
| "timer/generate": 28.76888656616211, |
| "timer/get_reward": 0.511726438999176, |
| "timer/model_forward": 0.7888429164886475, |
| "timer/ref_model_forward": 0.7980221509933472, |
| "timer/training_step": 34.51930236816406, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| }, |
| { |
| "epoch": 0.003802507278236587, |
| "grad_norm": 7.358872267104744e-08, |
| "learning_rate": 0.0029961971161464114, |
| "loss": -0.0, |
| "loss/policy_avg": -4.1211023926734924e-08, |
| "objective/entropy": 0.42043036222457886, |
| "objective/kl": 0.0, |
| "objective/rlhf_reward": -1.3438358306884766, |
| "objective/scores": -1.34381103515625, |
| "policy/approxkl_avg": 0.0, |
| "policy/clipfrac_avg": 0.0, |
| "step": 24, |
| "timer/active_model_forward": 1.0947436094284058, |
| "timer/calc_advantages": 33.27772903442383, |
| "timer/calc_loss": 1.1070483922958374, |
| "timer/generate": 29.915428161621094, |
| "timer/get_reward": 0.7210176587104797, |
| "timer/model_forward": 1.1053131818771362, |
| "timer/ref_model_forward": 1.1129379272460938, |
| "timer/training_step": 37.71331024169922, |
| "val/num_eos_tokens": 0.0, |
| "val/ratio": 1.0 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 18933, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 8, |
| "total_flos": 1.2330858923163648e+16, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
