Invalid JSON: Unexpected token 'N', ..."tio_var": NaN
},"... is not valid JSON
| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.002535004852157725, | |
| "eval_steps": 500, | |
| "global_step": 16, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0001584378032598578, | |
| "grad_norm": 6.941703031770885e-05, | |
| "learning_rate": 2.9998415465061005e-05, | |
| "loss": 0.0, | |
| "loss/policy_avg": 9.534414857625961e-08, | |
| "objective/entropy": 66.7407455444336, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": 1.9394512176513672, | |
| "objective/scores": 1.9393310546875, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "policy/entropy_avg": 0.501089334487915, | |
| "step": 1, | |
| "timer/calc_advantages": 1.838552713394165, | |
| "timer/calc_loss": 0.718826174736023, | |
| "timer/get_reward": 0.5057681202888489, | |
| "timer/training_step": 5.023805141448975, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0003168756065197156, | |
| "grad_norm": 8.962108612060547, | |
| "learning_rate": 2.999683093012201e-05, | |
| "loss": 0.0079, | |
| "loss/policy_avg": 0.007912077941000462, | |
| "objective/entropy": 58.23992919921875, | |
| "objective/kl": 0.006753697991371155, | |
| "objective/rlhf_reward": 2.58967924118042, | |
| "objective/scores": 2.59033203125, | |
| "policy/approxkl_avg": 0.143972247838974, | |
| "policy/clipfrac_avg": 0.376953125, | |
| "policy/entropy_avg": 0.437211811542511, | |
| "step": 2, | |
| "timer/calc_advantages": 1.693800687789917, | |
| "timer/calc_loss": 0.6173452734947205, | |
| "timer/get_reward": 0.431838721036911, | |
| "timer/training_step": 4.488475322723389, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0006933212280273, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0004753134097795734, | |
| "grad_norm": 8.80933666229248, | |
| "learning_rate": 2.9995246395183014e-05, | |
| "loss": 0.0087, | |
| "loss/policy_avg": 0.008671796880662441, | |
| "objective/entropy": 60.747886657714844, | |
| "objective/kl": 0.19268979132175446, | |
| "objective/rlhf_reward": 2.797236442565918, | |
| "objective/scores": 2.8162841796875, | |
| "policy/approxkl_avg": 0.22207684814929962, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.47127196192741394, | |
| "step": 3, | |
| "timer/calc_advantages": 2.117703437805176, | |
| "timer/calc_loss": 0.8592336773872375, | |
| "timer/get_reward": 0.5989187955856323, | |
| "timer/training_step": 5.804616928100586, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 0.9997344613075256, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0006337512130394313, | |
| "grad_norm": 15.397004127502441, | |
| "learning_rate": 2.999366186024402e-05, | |
| "loss": 0.0181, | |
| "loss/policy_avg": 0.018090050667524338, | |
| "objective/entropy": 59.366294860839844, | |
| "objective/kl": 0.16402865946292877, | |
| "objective/rlhf_reward": 2.7970707416534424, | |
| "objective/scores": 2.8134765625, | |
| "policy/approxkl_avg": 0.30915290117263794, | |
| "policy/clipfrac_avg": 0.41796875, | |
| "policy/entropy_avg": 0.4661800265312195, | |
| "step": 4, | |
| "timer/calc_advantages": 2.018901824951172, | |
| "timer/calc_loss": 0.8049512505531311, | |
| "timer/get_reward": 0.5547392964363098, | |
| "timer/training_step": 5.506021022796631, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0003275871276855, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.000792189016299289, | |
| "grad_norm": 6.137847231002524e-05, | |
| "learning_rate": 2.9992077325305024e-05, | |
| "loss": 0.0, | |
| "loss/policy_avg": 1.0040821507573128e-07, | |
| "objective/entropy": 61.850738525390625, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": 2.3658785820007324, | |
| "objective/scores": 2.36578369140625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "policy/entropy_avg": 0.48403242230415344, | |
| "step": 5, | |
| "timer/calc_advantages": 1.889434576034546, | |
| "timer/calc_loss": 0.7489800453186035, | |
| "timer/get_reward": 0.5218558311462402, | |
| "timer/training_step": 5.19569206237793, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0009506268195591468, | |
| "grad_norm": 3.129560947418213, | |
| "learning_rate": 2.9990492790366028e-05, | |
| "loss": 0.002, | |
| "loss/policy_avg": 0.002019597217440605, | |
| "objective/entropy": 64.70206451416016, | |
| "objective/kl": -0.011846143752336502, | |
| "objective/rlhf_reward": 2.4429714679718018, | |
| "objective/scores": 2.44146728515625, | |
| "policy/approxkl_avg": 0.05796036496758461, | |
| "policy/clipfrac_avg": 0.224609375, | |
| "policy/entropy_avg": 0.4975966215133667, | |
| "step": 6, | |
| "timer/calc_advantages": 1.939255714416504, | |
| "timer/calc_loss": 0.7629643082618713, | |
| "timer/get_reward": 0.5286913514137268, | |
| "timer/training_step": 5.261943340301514, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0005271434783936, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0011090646228190046, | |
| "grad_norm": 4.28013277053833, | |
| "learning_rate": 2.9988908255427033e-05, | |
| "loss": 0.0037, | |
| "loss/policy_avg": 0.0036908092442899942, | |
| "objective/entropy": 68.02294158935547, | |
| "objective/kl": 0.15080219507217407, | |
| "objective/rlhf_reward": 3.149240016937256, | |
| "objective/scores": 3.1646728515625, | |
| "policy/approxkl_avg": 0.11497651040554047, | |
| "policy/clipfrac_avg": 0.25, | |
| "policy/entropy_avg": 0.5161693692207336, | |
| "step": 7, | |
| "timer/calc_advantages": 2.2946014404296875, | |
| "timer/calc_loss": 0.96665358543396, | |
| "timer/get_reward": 0.6638086438179016, | |
| "timer/training_step": 6.369439125061035, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 0.99953293800354, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0012675024260788625, | |
| "grad_norm": 8.370040893554688, | |
| "learning_rate": 2.9987323720488037e-05, | |
| "loss": 0.0078, | |
| "loss/policy_avg": 0.00783943198621273, | |
| "objective/entropy": 66.24089813232422, | |
| "objective/kl": 0.0030039921402931213, | |
| "objective/rlhf_reward": 3.1140072345733643, | |
| "objective/scores": 3.1142501831054688, | |
| "policy/approxkl_avg": 0.13499276340007782, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.4966353476047516, | |
| "step": 8, | |
| "timer/calc_advantages": 2.4299309253692627, | |
| "timer/calc_loss": 1.046256422996521, | |
| "timer/get_reward": 0.7162653207778931, | |
| "timer/training_step": 6.773098945617676, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0011930465698242, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0014259402293387202, | |
| "grad_norm": 5.985812094877474e-05, | |
| "learning_rate": 2.9985739185549042e-05, | |
| "loss": 0.0, | |
| "loss/policy_avg": 1.3085082173347473e-07, | |
| "objective/entropy": 57.91233825683594, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": 2.9793567657470703, | |
| "objective/scores": 2.9791259765625, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "policy/entropy_avg": 0.4229130744934082, | |
| "step": 9, | |
| "timer/calc_advantages": 2.2175822257995605, | |
| "timer/calc_loss": 0.940474271774292, | |
| "timer/get_reward": 0.6452049016952515, | |
| "timer/training_step": 6.192902565002441, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.001584378032598578, | |
| "grad_norm": 2.061887502670288, | |
| "learning_rate": 2.9984154650610046e-05, | |
| "loss": 0.0012, | |
| "loss/policy_avg": 0.00118393381126225, | |
| "objective/entropy": 57.78638458251953, | |
| "objective/kl": 0.016596710309386253, | |
| "objective/rlhf_reward": 2.493161916732788, | |
| "objective/scores": 2.494873046875, | |
| "policy/approxkl_avg": 0.04327381029725075, | |
| "policy/clipfrac_avg": 0.220703125, | |
| "policy/entropy_avg": 0.45412811636924744, | |
| "step": 10, | |
| "timer/calc_advantages": 1.9425208568572998, | |
| "timer/calc_loss": 0.7499428987503052, | |
| "timer/get_reward": 0.5237178802490234, | |
| "timer/training_step": 5.241105556488037, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0002281665802002, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0017428158358584358, | |
| "grad_norm": 3.437352180480957, | |
| "learning_rate": 2.998257011567105e-05, | |
| "loss": 0.003, | |
| "loss/policy_avg": 0.003013810608536005, | |
| "objective/entropy": 62.60158920288086, | |
| "objective/kl": 0.04900990426540375, | |
| "objective/rlhf_reward": 3.26180362701416, | |
| "objective/scores": 3.2667236328125, | |
| "policy/approxkl_avg": 0.08036807179450989, | |
| "policy/clipfrac_avg": 0.2734375, | |
| "policy/entropy_avg": 0.49370765686035156, | |
| "step": 11, | |
| "timer/calc_advantages": 2.010678768157959, | |
| "timer/calc_loss": 0.8044981956481934, | |
| "timer/get_reward": 0.5571302771568298, | |
| "timer/training_step": 5.486995220184326, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0001283884048462, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0019012536391182935, | |
| "grad_norm": 3.6680727005004883, | |
| "learning_rate": 2.9980985580732055e-05, | |
| "loss": 0.0026, | |
| "loss/policy_avg": 0.002624134300276637, | |
| "objective/entropy": 70.5216064453125, | |
| "objective/kl": 0.039649978280067444, | |
| "objective/rlhf_reward": 1.9454692602157593, | |
| "objective/scores": 1.94927978515625, | |
| "policy/approxkl_avg": 0.07172106206417084, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5483217239379883, | |
| "step": 12, | |
| "timer/calc_advantages": 1.708000898361206, | |
| "timer/calc_loss": 0.6391787528991699, | |
| "timer/get_reward": 0.4455040693283081, | |
| "timer/training_step": 4.587984085083008, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0003334283828735, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.0020596914423781513, | |
| "grad_norm": 5.551970753003843e-05, | |
| "learning_rate": 2.997940104579306e-05, | |
| "loss": 0.0, | |
| "loss/policy_avg": 6.216578185558319e-08, | |
| "objective/entropy": 58.013153076171875, | |
| "objective/kl": 0.0, | |
| "objective/rlhf_reward": 1.9868602752685547, | |
| "objective/scores": 1.9864273071289062, | |
| "policy/approxkl_avg": 0.0, | |
| "policy/clipfrac_avg": 0.0, | |
| "policy/entropy_avg": 0.44822999835014343, | |
| "step": 13, | |
| "timer/calc_advantages": 2.0904719829559326, | |
| "timer/calc_loss": 0.8480838537216187, | |
| "timer/get_reward": 0.587557315826416, | |
| "timer/training_step": 5.7307024002075195, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.002218129245638009, | |
| "grad_norm": 2.7803940773010254, | |
| "learning_rate": 2.9977816510854064e-05, | |
| "loss": 0.0015, | |
| "loss/policy_avg": 0.0014500978868454695, | |
| "objective/entropy": 69.96898651123047, | |
| "objective/kl": 0.06745042651891708, | |
| "objective/rlhf_reward": 2.469928741455078, | |
| "objective/scores": 2.477020263671875, | |
| "policy/approxkl_avg": 0.053128380328416824, | |
| "policy/clipfrac_avg": 0.2734375, | |
| "policy/entropy_avg": 0.5138126015663147, | |
| "step": 14, | |
| "timer/calc_advantages": 1.917910099029541, | |
| "timer/calc_loss": 0.7225819230079651, | |
| "timer/get_reward": 0.5053159594535828, | |
| "timer/training_step": 5.0998334884643555, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 0.9998296499252319, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.002376567048897867, | |
| "grad_norm": 2.532649278640747, | |
| "learning_rate": 2.997623197591507e-05, | |
| "loss": 0.0018, | |
| "loss/policy_avg": 0.0017822147347033024, | |
| "objective/entropy": 58.39247131347656, | |
| "objective/kl": -0.011621108278632164, | |
| "objective/rlhf_reward": 2.397026538848877, | |
| "objective/scores": 2.39556884765625, | |
| "policy/approxkl_avg": 0.05379398912191391, | |
| "policy/clipfrac_avg": 0.248046875, | |
| "policy/entropy_avg": 0.4527440071105957, | |
| "step": 15, | |
| "timer/calc_advantages": 2.565062999725342, | |
| "timer/calc_loss": 1.1112425327301025, | |
| "timer/get_reward": 0.7521659731864929, | |
| "timer/training_step": 7.156440734863281, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0004982948303223, | |
| "val/ratio_var": NaN | |
| }, | |
| { | |
| "epoch": 0.002535004852157725, | |
| "grad_norm": 5.747053623199463, | |
| "learning_rate": 2.9974647440976074e-05, | |
| "loss": 0.0039, | |
| "loss/policy_avg": 0.0039013477507978678, | |
| "objective/entropy": 58.41150665283203, | |
| "objective/kl": -0.03636426478624344, | |
| "objective/rlhf_reward": 2.3745107650756836, | |
| "objective/scores": 2.370819091796875, | |
| "policy/approxkl_avg": 0.07460636645555496, | |
| "policy/clipfrac_avg": 0.310546875, | |
| "policy/entropy_avg": 0.4354715943336487, | |
| "step": 16, | |
| "timer/calc_advantages": 2.135810136795044, | |
| "timer/calc_loss": 0.8651329278945923, | |
| "timer/get_reward": 0.5979053378105164, | |
| "timer/training_step": 5.837425231933594, | |
| "val/num_eos_tokens": 0.0, | |
| "val/ratio": 1.0008955001831055, | |
| "val/ratio_var": NaN | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 18933, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 8, | |
| "total_flos": 8202719718014976.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |