| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 500, |
| "global_step": 542, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0018450184501845018, |
| "grad_norm": 174.53293244150143, |
| "learning_rate": 0.0, |
| "logits/chosen": -0.28515625, |
| "logits/rejected": -0.365234375, |
| "logps/chosen": -556.0, |
| "logps/rejected": -448.0, |
| "loss": 0.6914, |
| "nll_loss": 1.453125, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.01845018450184502, |
| "grad_norm": 346.4662455777756, |
| "learning_rate": 8.181818181818182e-08, |
| "logits/chosen": -0.9115939736366272, |
| "logits/rejected": -0.8186306357383728, |
| "logps/chosen": -508.8888854980469, |
| "logps/rejected": -512.2222290039062, |
| "loss": 0.6986, |
| "nll_loss": 1.6076388359069824, |
| "rewards/accuracies": 0.2638888955116272, |
| "rewards/chosen": 0.0194566510617733, |
| "rewards/margins": 0.011149088852107525, |
| "rewards/rejected": 0.0083482526242733, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.03690036900369004, |
| "grad_norm": 136.41193032953723, |
| "learning_rate": 1.7272727272727272e-07, |
| "logits/chosen": -0.9195312261581421, |
| "logits/rejected": -0.21855469048023224, |
| "logps/chosen": -479.0, |
| "logps/rejected": -481.6000061035156, |
| "loss": 0.5387, |
| "nll_loss": 1.8468749523162842, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": 0.36181640625, |
| "rewards/margins": 0.3769287168979645, |
| "rewards/rejected": -0.015087890438735485, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.055350553505535055, |
| "grad_norm": 130.6864319952881, |
| "learning_rate": 2.636363636363636e-07, |
| "logits/chosen": -0.724609375, |
| "logits/rejected": -0.31591796875, |
| "logps/chosen": -523.0, |
| "logps/rejected": -518.0, |
| "loss": 0.259, |
| "nll_loss": 1.5070312023162842, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.501562476158142, |
| "rewards/margins": 1.534765601158142, |
| "rewards/rejected": -0.03243408352136612, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07380073800738007, |
| "grad_norm": 9.172076329512516, |
| "learning_rate": 3.545454545454545e-07, |
| "logits/chosen": -0.8726562261581421, |
| "logits/rejected": -0.361572265625, |
| "logps/chosen": -484.0, |
| "logps/rejected": -476.79998779296875, |
| "loss": 0.0942, |
| "nll_loss": 1.7843749523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7953124046325684, |
| "rewards/margins": 3.5882811546325684, |
| "rewards/rejected": -0.788769543170929, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.09225092250922509, |
| "grad_norm": 133.45013172512645, |
| "learning_rate": 4.4545454545454544e-07, |
| "logits/chosen": -0.667187511920929, |
| "logits/rejected": -0.2557128965854645, |
| "logps/chosen": -466.1000061035156, |
| "logps/rejected": -514.0, |
| "loss": 0.0757, |
| "nll_loss": 1.747656226158142, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 3.504687547683716, |
| "rewards/margins": 5.074999809265137, |
| "rewards/rejected": -1.5654296875, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11070110701107011, |
| "grad_norm": 27.938955497888248, |
| "learning_rate": 4.958932238193018e-07, |
| "logits/chosen": -0.443359375, |
| "logits/rejected": -0.11607666313648224, |
| "logps/chosen": -424.1000061035156, |
| "logps/rejected": -622.0, |
| "loss": 0.0719, |
| "nll_loss": 1.6359374523162842, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 4.342187404632568, |
| "rewards/margins": 7.287499904632568, |
| "rewards/rejected": -2.9453125, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.12915129151291513, |
| "grad_norm": 2.667667834720549, |
| "learning_rate": 4.856262833675565e-07, |
| "logits/chosen": -0.34898680448532104, |
| "logits/rejected": -0.0115966796875, |
| "logps/chosen": -544.7999877929688, |
| "logps/rejected": -636.4000244140625, |
| "loss": 0.0561, |
| "nll_loss": 1.5499999523162842, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 4.755078315734863, |
| "rewards/margins": 9.284375190734863, |
| "rewards/rejected": -4.529687404632568, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.14760147601476015, |
| "grad_norm": 49.19225093797519, |
| "learning_rate": 4.753593429158111e-07, |
| "logits/chosen": -0.33574217557907104, |
| "logits/rejected": -0.006542968563735485, |
| "logps/chosen": -437.6000061035156, |
| "logps/rejected": -614.2000122070312, |
| "loss": 0.0096, |
| "nll_loss": 1.533593773841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.714062690734863, |
| "rewards/margins": 9.884374618530273, |
| "rewards/rejected": -5.159375190734863, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.16605166051660517, |
| "grad_norm": 7.869193854585077, |
| "learning_rate": 4.6509240246406566e-07, |
| "logits/chosen": -0.44111329317092896, |
| "logits/rejected": -0.08427734673023224, |
| "logps/chosen": -421.3999938964844, |
| "logps/rejected": -608.4000244140625, |
| "loss": 0.0053, |
| "nll_loss": 1.591406226158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.628125190734863, |
| "rewards/margins": 11.384374618530273, |
| "rewards/rejected": -6.768750190734863, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.18450184501845018, |
| "grad_norm": 0.07766290165816067, |
| "learning_rate": 4.548254620123203e-07, |
| "logits/chosen": -0.3090454041957855, |
| "logits/rejected": -0.009716796688735485, |
| "logps/chosen": -431.3999938964844, |
| "logps/rejected": -543.4000244140625, |
| "loss": 0.004, |
| "nll_loss": 1.316796898841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 6.25, |
| "rewards/margins": 12.481249809265137, |
| "rewards/rejected": -6.221875190734863, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.2029520295202952, |
| "grad_norm": 2.219444743672593, |
| "learning_rate": 4.445585215605749e-07, |
| "logits/chosen": -0.2965331971645355, |
| "logits/rejected": -0.10786132514476776, |
| "logps/chosen": -410.0, |
| "logps/rejected": -554.2000122070312, |
| "loss": 0.0035, |
| "nll_loss": 1.5421874523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 4.670312404632568, |
| "rewards/margins": 12.443750381469727, |
| "rewards/rejected": -7.778124809265137, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.22140221402214022, |
| "grad_norm": 193.37089815524854, |
| "learning_rate": 4.3429158110882957e-07, |
| "logits/chosen": -0.48212891817092896, |
| "logits/rejected": 0.2857910096645355, |
| "logps/chosen": -447.6000061035156, |
| "logps/rejected": -562.7999877929688, |
| "loss": 0.0252, |
| "nll_loss": 1.564062476158142, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 5.265625, |
| "rewards/margins": 14.199999809265137, |
| "rewards/rejected": -8.940625190734863, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.23985239852398524, |
| "grad_norm": 3.835693063482046, |
| "learning_rate": 4.2402464065708417e-07, |
| "logits/chosen": -0.23652343451976776, |
| "logits/rejected": 0.23701171576976776, |
| "logps/chosen": -455.3999938964844, |
| "logps/rejected": -596.7999877929688, |
| "loss": 0.0055, |
| "nll_loss": 1.6085937023162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 5.831250190734863, |
| "rewards/margins": 15.287500381469727, |
| "rewards/rejected": -9.482812881469727, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.25830258302583026, |
| "grad_norm": 0.24648110266264378, |
| "learning_rate": 4.137577002053388e-07, |
| "logits/chosen": -0.25221556425094604, |
| "logits/rejected": 0.06816406548023224, |
| "logps/chosen": -432.79998779296875, |
| "logps/rejected": -649.7999877929688, |
| "loss": 0.0079, |
| "nll_loss": 1.424218773841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 7.181250095367432, |
| "rewards/margins": 15.356249809265137, |
| "rewards/rejected": -8.184374809265137, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.2767527675276753, |
| "grad_norm": 0.03622304658168317, |
| "learning_rate": 4.0349075975359343e-07, |
| "logits/chosen": -0.32623291015625, |
| "logits/rejected": 0.09409179538488388, |
| "logps/chosen": -373.79998779296875, |
| "logps/rejected": -513.2000122070312, |
| "loss": 0.0027, |
| "nll_loss": 1.293359398841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 6.625, |
| "rewards/margins": 13.981249809265137, |
| "rewards/rejected": -7.340624809265137, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.2952029520295203, |
| "grad_norm": 0.020076000811240717, |
| "learning_rate": 3.932238193018481e-07, |
| "logits/chosen": -0.31867676973342896, |
| "logits/rejected": 0.16098633408546448, |
| "logps/chosen": -477.0, |
| "logps/rejected": -640.4000244140625, |
| "loss": 0.0022, |
| "nll_loss": 1.346093773841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 7.65625, |
| "rewards/margins": 16.581249237060547, |
| "rewards/rejected": -8.909375190734863, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.31365313653136534, |
| "grad_norm": 0.041859392866214304, |
| "learning_rate": 3.8295687885010263e-07, |
| "logits/chosen": -0.4400878846645355, |
| "logits/rejected": 0.0843505859375, |
| "logps/chosen": -390.6000061035156, |
| "logps/rejected": -612.7999877929688, |
| "loss": 0.0027, |
| "nll_loss": 1.485937476158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 7.912499904632568, |
| "rewards/margins": 16.293750762939453, |
| "rewards/rejected": -8.393750190734863, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.33210332103321033, |
| "grad_norm": 0.05035591243969461, |
| "learning_rate": 3.726899383983573e-07, |
| "logits/chosen": -0.51611328125, |
| "logits/rejected": 0.005908203311264515, |
| "logps/chosen": -419.79998779296875, |
| "logps/rejected": -572.2000122070312, |
| "loss": 0.0016, |
| "nll_loss": 1.400781273841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 8.496874809265137, |
| "rewards/margins": 16.387500762939453, |
| "rewards/rejected": -7.909375190734863, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.3505535055350554, |
| "grad_norm": 0.10815032342544012, |
| "learning_rate": 3.624229979466119e-07, |
| "logits/chosen": -0.37841796875, |
| "logits/rejected": 0.03369140625, |
| "logps/chosen": -444.3999938964844, |
| "logps/rejected": -575.0, |
| "loss": 0.0024, |
| "nll_loss": 1.273828148841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 9.0625, |
| "rewards/margins": 14.831250190734863, |
| "rewards/rejected": -5.7578125, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.36900369003690037, |
| "grad_norm": 0.04004367261026864, |
| "learning_rate": 3.521560574948665e-07, |
| "logits/chosen": -0.40791016817092896, |
| "logits/rejected": -0.10537109524011612, |
| "logps/chosen": -368.20001220703125, |
| "logps/rejected": -501.79998779296875, |
| "loss": 0.0018, |
| "nll_loss": 1.365625023841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 8.859375, |
| "rewards/margins": 15.050000190734863, |
| "rewards/rejected": -6.193749904632568, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.3874538745387454, |
| "grad_norm": 0.008541615515180453, |
| "learning_rate": 3.418891170431211e-07, |
| "logits/chosen": -0.30952149629592896, |
| "logits/rejected": 0.10322265326976776, |
| "logps/chosen": -470.0, |
| "logps/rejected": -550.7999877929688, |
| "loss": 0.0029, |
| "nll_loss": 1.355078101158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.175000190734863, |
| "rewards/margins": 16.53125, |
| "rewards/rejected": -6.373437404632568, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.4059040590405904, |
| "grad_norm": 0.02023002394682239, |
| "learning_rate": 3.3162217659137575e-07, |
| "logits/chosen": -0.2582031190395355, |
| "logits/rejected": 0.18632812798023224, |
| "logps/chosen": -386.3999938964844, |
| "logps/rejected": -556.5999755859375, |
| "loss": 0.0031, |
| "nll_loss": 1.3464844226837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 8.5625, |
| "rewards/margins": 15.675000190734863, |
| "rewards/rejected": -7.118750095367432, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.42435424354243545, |
| "grad_norm": 37.84292558111141, |
| "learning_rate": 3.213552361396304e-07, |
| "logits/chosen": -0.41533201932907104, |
| "logits/rejected": -0.09333495795726776, |
| "logps/chosen": -491.0, |
| "logps/rejected": -661.5999755859375, |
| "loss": 0.0038, |
| "nll_loss": 1.3156249523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.203125, |
| "rewards/margins": 16.450000762939453, |
| "rewards/rejected": -6.25, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.44280442804428044, |
| "grad_norm": 0.23253825124415758, |
| "learning_rate": 3.1108829568788496e-07, |
| "logits/chosen": -0.15390625596046448, |
| "logits/rejected": 0.15219727158546448, |
| "logps/chosen": -409.29998779296875, |
| "logps/rejected": -550.0, |
| "loss": 0.0245, |
| "nll_loss": 1.2214844226837158, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 10.21875, |
| "rewards/margins": 18.28125, |
| "rewards/rejected": -8.065625190734863, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.4612546125461255, |
| "grad_norm": 0.6855044368096641, |
| "learning_rate": 3.008213552361396e-07, |
| "logits/chosen": -0.2812744081020355, |
| "logits/rejected": 0.24277344346046448, |
| "logps/chosen": -502.20001220703125, |
| "logps/rejected": -546.2000122070312, |
| "loss": 0.0018, |
| "nll_loss": 1.3703124523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.5, |
| "rewards/margins": 17.024999618530273, |
| "rewards/rejected": -6.518750190734863, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.4797047970479705, |
| "grad_norm": 0.7753136116337657, |
| "learning_rate": 2.9055441478439427e-07, |
| "logits/chosen": -0.26799315214157104, |
| "logits/rejected": 0.06092529371380806, |
| "logps/chosen": -447.79998779296875, |
| "logps/rejected": -633.7999877929688, |
| "loss": 0.0025, |
| "nll_loss": 1.353124976158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.787500381469727, |
| "rewards/margins": 16.918750762939453, |
| "rewards/rejected": -6.120312690734863, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.4981549815498155, |
| "grad_norm": 0.10339419092952733, |
| "learning_rate": 2.8028747433264887e-07, |
| "logits/chosen": -0.22612304985523224, |
| "logits/rejected": 0.2598632872104645, |
| "logps/chosen": -411.3999938964844, |
| "logps/rejected": -534.5999755859375, |
| "loss": 0.0025, |
| "nll_loss": 1.2843749523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 9.537500381469727, |
| "rewards/margins": 15.912500381469727, |
| "rewards/rejected": -6.354687690734863, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5166051660516605, |
| "grad_norm": 0.1215998717046616, |
| "learning_rate": 2.7002053388090347e-07, |
| "logits/chosen": -0.34821778535842896, |
| "logits/rejected": 0.02799682691693306, |
| "logps/chosen": -348.3999938964844, |
| "logps/rejected": -595.2000122070312, |
| "loss": 0.0015, |
| "nll_loss": 1.345312476158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.699999809265137, |
| "rewards/margins": 17.975000381469727, |
| "rewards/rejected": -7.259375095367432, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.5350553505535055, |
| "grad_norm": 0.02878650522676844, |
| "learning_rate": 2.597535934291581e-07, |
| "logits/chosen": -0.378662109375, |
| "logits/rejected": 0.10524902492761612, |
| "logps/chosen": -372.1000061035156, |
| "logps/rejected": -542.0, |
| "loss": 0.0014, |
| "nll_loss": 1.271875023841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.925000190734863, |
| "rewards/margins": 18.943750381469727, |
| "rewards/rejected": -8.015625, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.5535055350553506, |
| "grad_norm": 0.03536097923100537, |
| "learning_rate": 2.4948665297741273e-07, |
| "logits/chosen": -0.3155273497104645, |
| "logits/rejected": -0.10869140923023224, |
| "logps/chosen": -419.1000061035156, |
| "logps/rejected": -685.2000122070312, |
| "loss": 0.0025, |
| "nll_loss": 1.337890625, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.925000190734863, |
| "rewards/margins": 17.868749618530273, |
| "rewards/rejected": -6.954687595367432, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.5719557195571956, |
| "grad_norm": 0.03473546153791582, |
| "learning_rate": 2.3921971252566733e-07, |
| "logits/chosen": -0.31220704317092896, |
| "logits/rejected": 0.03559570387005806, |
| "logps/chosen": -417.1000061035156, |
| "logps/rejected": -614.0, |
| "loss": 0.0014, |
| "nll_loss": 1.290624976158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 10.956250190734863, |
| "rewards/margins": 17.387500762939453, |
| "rewards/rejected": -6.418749809265137, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.5904059040590406, |
| "grad_norm": 36.361485146275975, |
| "learning_rate": 2.2895277207392196e-07, |
| "logits/chosen": -0.28837889432907104, |
| "logits/rejected": 0.21499022841453552, |
| "logps/chosen": -387.79998779296875, |
| "logps/rejected": -581.2000122070312, |
| "loss": 0.0045, |
| "nll_loss": 1.299218773841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 11.431249618530273, |
| "rewards/margins": 17.537500381469727, |
| "rewards/rejected": -6.108593940734863, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6088560885608856, |
| "grad_norm": 0.017578308664404376, |
| "learning_rate": 2.1868583162217656e-07, |
| "logits/chosen": -0.16363525390625, |
| "logits/rejected": 0.0478515625, |
| "logps/chosen": -438.3999938964844, |
| "logps/rejected": -585.5999755859375, |
| "loss": 0.0013, |
| "nll_loss": 1.249609351158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 11.34375, |
| "rewards/margins": 17.706249237060547, |
| "rewards/rejected": -6.364062309265137, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.6273062730627307, |
| "grad_norm": 0.13503151970629174, |
| "learning_rate": 2.0841889117043122e-07, |
| "logits/chosen": -0.276611328125, |
| "logits/rejected": 0.21477051079273224, |
| "logps/chosen": -387.79998779296875, |
| "logps/rejected": -552.0, |
| "loss": 0.0014, |
| "nll_loss": 1.246484398841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.106249809265137, |
| "rewards/margins": 18.924999237060547, |
| "rewards/rejected": -6.785937309265137, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.6457564575645757, |
| "grad_norm": 0.2744112671801272, |
| "learning_rate": 1.9815195071868582e-07, |
| "logits/chosen": -0.26923829317092896, |
| "logits/rejected": 0.15705566108226776, |
| "logps/chosen": -448.6000061035156, |
| "logps/rejected": -711.0, |
| "loss": 0.0028, |
| "nll_loss": 1.2371094226837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.068750381469727, |
| "rewards/margins": 18.393749237060547, |
| "rewards/rejected": -6.315625190734863, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.6642066420664207, |
| "grad_norm": 1.288974393127278, |
| "learning_rate": 1.8788501026694045e-07, |
| "logits/chosen": -0.2408447265625, |
| "logits/rejected": 0.12333984673023224, |
| "logps/chosen": -428.20001220703125, |
| "logps/rejected": -714.4000244140625, |
| "loss": 0.0014, |
| "nll_loss": 1.2585937976837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.168749809265137, |
| "rewards/margins": 18.75, |
| "rewards/rejected": -6.59375, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.6826568265682657, |
| "grad_norm": 0.010817526846263152, |
| "learning_rate": 1.7761806981519505e-07, |
| "logits/chosen": -0.4193359315395355, |
| "logits/rejected": 0.12089844048023224, |
| "logps/chosen": -387.6000061035156, |
| "logps/rejected": -546.5999755859375, |
| "loss": 0.0013, |
| "nll_loss": 1.220312476158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.699999809265137, |
| "rewards/margins": 20.431249618530273, |
| "rewards/rejected": -7.745312690734863, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.7011070110701108, |
| "grad_norm": 0.9313842568161499, |
| "learning_rate": 1.673511293634497e-07, |
| "logits/chosen": -0.2831054627895355, |
| "logits/rejected": 0.2867675721645355, |
| "logps/chosen": -284.8999938964844, |
| "logps/rejected": -574.7999877929688, |
| "loss": 0.0015, |
| "nll_loss": 1.189062476158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 11.875, |
| "rewards/margins": 19.59375, |
| "rewards/rejected": -7.71875, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.7195571955719557, |
| "grad_norm": 0.015457285064348653, |
| "learning_rate": 1.570841889117043e-07, |
| "logits/chosen": -0.38752442598342896, |
| "logits/rejected": 0.01973876915872097, |
| "logps/chosen": -436.3999938964844, |
| "logps/rejected": -622.4000244140625, |
| "loss": 0.0106, |
| "nll_loss": 1.1261718273162842, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 11.868749618530273, |
| "rewards/margins": 17.862499237060547, |
| "rewards/rejected": -5.993750095367432, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.7380073800738007, |
| "grad_norm": 0.14413562107865965, |
| "learning_rate": 1.4681724845995894e-07, |
| "logits/chosen": -0.24350586533546448, |
| "logits/rejected": 0.20991210639476776, |
| "logps/chosen": -371.1000061035156, |
| "logps/rejected": -562.5999755859375, |
| "loss": 0.0014, |
| "nll_loss": 1.131250023841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 11.506250381469727, |
| "rewards/margins": 18.356250762939453, |
| "rewards/rejected": -6.846875190734863, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.7564575645756457, |
| "grad_norm": 0.015979900222072605, |
| "learning_rate": 1.3655030800821354e-07, |
| "logits/chosen": -0.2548828125, |
| "logits/rejected": 0.2605957090854645, |
| "logps/chosen": -368.70001220703125, |
| "logps/rejected": -534.5999755859375, |
| "loss": 0.0012, |
| "nll_loss": 1.148046851158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 13.168749809265137, |
| "rewards/margins": 20.318750381469727, |
| "rewards/rejected": -7.157812595367432, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.7749077490774908, |
| "grad_norm": 0.009958077789392873, |
| "learning_rate": 1.2628336755646817e-07, |
| "logits/chosen": -0.34492188692092896, |
| "logits/rejected": 0.02458496019244194, |
| "logps/chosen": -404.3999938964844, |
| "logps/rejected": -572.5999755859375, |
| "loss": 0.0334, |
| "nll_loss": 1.1804687976837158, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 11.949999809265137, |
| "rewards/margins": 17.006250381469727, |
| "rewards/rejected": -5.053124904632568, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.7933579335793358, |
| "grad_norm": 0.018519608671113973, |
| "learning_rate": 1.1601642710472278e-07, |
| "logits/chosen": -0.3464599549770355, |
| "logits/rejected": 0.13808593153953552, |
| "logps/chosen": -344.5, |
| "logps/rejected": -527.0, |
| "loss": 0.0015, |
| "nll_loss": 1.1984374523162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 13.162500381469727, |
| "rewards/margins": 19.318750381469727, |
| "rewards/rejected": -6.146874904632568, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.8118081180811808, |
| "grad_norm": 0.014726861292652616, |
| "learning_rate": 1.0574948665297741e-07, |
| "logits/chosen": -0.3399902284145355, |
| "logits/rejected": 0.012939453125, |
| "logps/chosen": -365.29998779296875, |
| "logps/rejected": -544.5999755859375, |
| "loss": 0.0012, |
| "nll_loss": 1.1847655773162842, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.768750190734863, |
| "rewards/margins": 20.362499237060547, |
| "rewards/rejected": -7.607812404632568, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.8302583025830258, |
| "grad_norm": 0.224159824109424, |
| "learning_rate": 9.548254620123203e-08, |
| "logits/chosen": -0.2451171875, |
| "logits/rejected": 0.18867187201976776, |
| "logps/chosen": -461.20001220703125, |
| "logps/rejected": -573.2000122070312, |
| "loss": 0.0012, |
| "nll_loss": 1.203515648841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.550000190734863, |
| "rewards/margins": 18.918750762939453, |
| "rewards/rejected": -6.364843845367432, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.8487084870848709, |
| "grad_norm": 0.3216885287305295, |
| "learning_rate": 8.521560574948664e-08, |
| "logits/chosen": -0.4150390625, |
| "logits/rejected": 0.19716796278953552, |
| "logps/chosen": -384.70001220703125, |
| "logps/rejected": -514.5999755859375, |
| "loss": 0.0014, |
| "nll_loss": 1.2429687976837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.600000381469727, |
| "rewards/margins": 19.018749237060547, |
| "rewards/rejected": -6.389843940734863, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.8671586715867159, |
| "grad_norm": 0.036868046311022036, |
| "learning_rate": 7.494866529774127e-08, |
| "logits/chosen": -0.44775390625, |
| "logits/rejected": 0.04902343824505806, |
| "logps/chosen": -364.3999938964844, |
| "logps/rejected": -635.5999755859375, |
| "loss": 0.0013, |
| "nll_loss": 1.2976562976837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 11.993749618530273, |
| "rewards/margins": 18.274999618530273, |
| "rewards/rejected": -6.285937309265137, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.8856088560885609, |
| "grad_norm": 0.5078671632789603, |
| "learning_rate": 6.468172484599589e-08, |
| "logits/chosen": -0.24858398735523224, |
| "logits/rejected": -0.013867187313735485, |
| "logps/chosen": -319.5, |
| "logps/rejected": -668.7999877929688, |
| "loss": 0.0013, |
| "nll_loss": 1.154296875, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.774999618530273, |
| "rewards/margins": 19.331249237060547, |
| "rewards/rejected": -6.546875, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.9040590405904059, |
| "grad_norm": 0.02980417558547798, |
| "learning_rate": 5.441478439425051e-08, |
| "logits/chosen": -0.26640623807907104, |
| "logits/rejected": 0.18837890028953552, |
| "logps/chosen": -387.79998779296875, |
| "logps/rejected": -545.5999755859375, |
| "loss": 0.002, |
| "nll_loss": 1.148046851158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.412500381469727, |
| "rewards/margins": 19.231250762939453, |
| "rewards/rejected": -6.824999809265137, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.922509225092251, |
| "grad_norm": 0.187006563020737, |
| "learning_rate": 4.414784394250513e-08, |
| "logits/chosen": -0.24520263075828552, |
| "logits/rejected": 0.14360351860523224, |
| "logps/chosen": -381.20001220703125, |
| "logps/rejected": -571.5999755859375, |
| "loss": 0.0012, |
| "nll_loss": 1.130468726158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.1875, |
| "rewards/margins": 19.200000762939453, |
| "rewards/rejected": -6.991406440734863, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.940959409594096, |
| "grad_norm": 0.1976693035630184, |
| "learning_rate": 3.3880903490759755e-08, |
| "logits/chosen": -0.35673826932907104, |
| "logits/rejected": -0.04693603515625, |
| "logps/chosen": -359.70001220703125, |
| "logps/rejected": -559.4000244140625, |
| "loss": 0.0017, |
| "nll_loss": 1.214453101158142, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.074999809265137, |
| "rewards/margins": 18.293750762939453, |
| "rewards/rejected": -6.229687690734863, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.959409594095941, |
| "grad_norm": 0.19235015104809136, |
| "learning_rate": 2.361396303901437e-08, |
| "logits/chosen": -0.4139648377895355, |
| "logits/rejected": 0.10092773288488388, |
| "logps/chosen": -356.79998779296875, |
| "logps/rejected": -668.4000244140625, |
| "loss": 0.0012, |
| "nll_loss": 1.1335937976837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.293749809265137, |
| "rewards/margins": 19.274999618530273, |
| "rewards/rejected": -7.0, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.977859778597786, |
| "grad_norm": 0.011034995549329435, |
| "learning_rate": 1.3347022587268993e-08, |
| "logits/chosen": -0.3798828125, |
| "logits/rejected": 0.1337890625, |
| "logps/chosen": -325.29998779296875, |
| "logps/rejected": -492.3999938964844, |
| "loss": 0.0012, |
| "nll_loss": 1.123437523841858, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.818750381469727, |
| "rewards/margins": 20.15625, |
| "rewards/rejected": -7.365624904632568, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.996309963099631, |
| "grad_norm": 0.0682979395748976, |
| "learning_rate": 3.080082135523614e-09, |
| "logits/chosen": -0.31596678495407104, |
| "logits/rejected": 0.13836669921875, |
| "logps/chosen": -375.0, |
| "logps/rejected": -526.0, |
| "loss": 0.0012, |
| "nll_loss": 1.1609375476837158, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 12.78125, |
| "rewards/margins": 19.537500381469727, |
| "rewards/rejected": -6.743750095367432, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_logits/chosen": -0.31317609548568726, |
| "eval_logits/rejected": 0.21189998090267181, |
| "eval_logps/chosen": -294.4615478515625, |
| "eval_logps/rejected": -450.1538391113281, |
| "eval_loss": 0.004863434005528688, |
| "eval_nll_loss": 1.1430288553237915, |
| "eval_rewards/accuracies": 1.0, |
| "eval_rewards/chosen": 12.75, |
| "eval_rewards/margins": 18.009614944458008, |
| "eval_rewards/rejected": -5.266225814819336, |
| "eval_runtime": 15.407, |
| "eval_samples_per_second": 6.491, |
| "eval_steps_per_second": 0.844, |
| "step": 542 |
| }, |
| { |
| "epoch": 1.0, |
| "step": 542, |
| "total_flos": 0.0, |
| "train_loss": 0.03677848608291457, |
| "train_runtime": 1919.659, |
| "train_samples_per_second": 2.257, |
| "train_steps_per_second": 0.282 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 542, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
