| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 500, |
| "global_step": 197, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 0.167724609375, |
| "epoch": 0.005089058524173028, |
| "grad_norm": 63.83088684082031, |
| "learning_rate": 5e-06, |
| "logits/chosen": -1.1478307765991018, |
| "logits/rejected": -1.1531979437304007, |
| "logps/chosen": -230.5, |
| "logps/rejected": -250.5, |
| "loss": 0.726806640625, |
| "mean_token_accuracy": 0.9508575797080994, |
| "num_tokens": 387774.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0360107421875, |
| "rewards/margins": 0.007598876953125, |
| "rewards/rejected": -0.04305267333984375, |
| "step": 1 |
| }, |
| { |
| "entropy": 0.1708984375, |
| "epoch": 0.010178117048346057, |
| "grad_norm": 58.35269546508789, |
| "learning_rate": 4.974619289340102e-06, |
| "logits/chosen": -1.1465309061137527, |
| "logits/rejected": -1.1435811743754352, |
| "logps/chosen": -202.75, |
| "logps/rejected": -220.75, |
| "loss": 0.73284912109375, |
| "mean_token_accuracy": 0.9518170654773712, |
| "num_tokens": 796045.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.01251220703125, |
| "rewards/margins": -0.01416015625, |
| "rewards/rejected": 0.00146484375, |
| "step": 2 |
| }, |
| { |
| "entropy": 0.1708984375, |
| "epoch": 0.015267175572519083, |
| "grad_norm": 52.742557525634766, |
| "learning_rate": 4.949238578680203e-06, |
| "logits/chosen": -1.22388826039798, |
| "logits/rejected": -1.2534275052969308, |
| "logps/chosen": -217.0, |
| "logps/rejected": -210.75, |
| "loss": 0.695587158203125, |
| "mean_token_accuracy": 0.9499112516641617, |
| "num_tokens": 1158499.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.03204345703125, |
| "rewards/margins": 0.07501220703125, |
| "rewards/rejected": -0.1068115234375, |
| "step": 3 |
| }, |
| { |
| "entropy": 0.175537109375, |
| "epoch": 0.020356234096692113, |
| "grad_norm": 58.555728912353516, |
| "learning_rate": 4.923857868020305e-06, |
| "logits/chosen": -1.0887703188820197, |
| "logits/rejected": -1.110711324029513, |
| "logps/chosen": -247.5, |
| "logps/rejected": -225.25, |
| "loss": 0.7191162109375, |
| "mean_token_accuracy": 0.9449715912342072, |
| "num_tokens": 1526178.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.0124664306640625, |
| "rewards/margins": 0.05133056640625, |
| "rewards/rejected": -0.0640869140625, |
| "step": 4 |
| }, |
| { |
| "entropy": 0.181884765625, |
| "epoch": 0.02544529262086514, |
| "grad_norm": 57.25611114501953, |
| "learning_rate": 4.898477157360406e-06, |
| "logits/chosen": -1.1241738300809683, |
| "logits/rejected": -1.1185368980659924, |
| "logps/chosen": -197.25, |
| "logps/rejected": -204.5, |
| "loss": 0.746917724609375, |
| "mean_token_accuracy": 0.9447457045316696, |
| "num_tokens": 1898739.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.036865234375, |
| "rewards/margins": 0.0240478515625, |
| "rewards/rejected": -0.06103515625, |
| "step": 5 |
| }, |
| { |
| "entropy": 0.172119140625, |
| "epoch": 0.030534351145038167, |
| "grad_norm": 55.632781982421875, |
| "learning_rate": 4.873096446700508e-06, |
| "logits/chosen": -1.118042467736593, |
| "logits/rejected": -1.1188858755620001, |
| "logps/chosen": -195.5, |
| "logps/rejected": -213.5, |
| "loss": 0.706695556640625, |
| "mean_token_accuracy": 0.9515382200479507, |
| "num_tokens": 2280846.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.0267333984375, |
| "rewards/margins": 0.051971435546875, |
| "rewards/rejected": -0.07904052734375, |
| "step": 6 |
| }, |
| { |
| "entropy": 0.165283203125, |
| "epoch": 0.035623409669211195, |
| "grad_norm": 58.1217155456543, |
| "learning_rate": 4.84771573604061e-06, |
| "logits/chosen": -1.2056221529623334, |
| "logits/rejected": -1.1698524733043854, |
| "logps/chosen": -203.75, |
| "logps/rejected": -237.5, |
| "loss": 0.778106689453125, |
| "mean_token_accuracy": 0.9536973237991333, |
| "num_tokens": 2661565.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.047119140625, |
| "rewards/margins": -0.0155029296875, |
| "rewards/rejected": -0.031982421875, |
| "step": 7 |
| }, |
| { |
| "entropy": 0.16015625, |
| "epoch": 0.04071246819338423, |
| "grad_norm": 57.468605041503906, |
| "learning_rate": 4.822335025380711e-06, |
| "logits/chosen": -1.285398958255684, |
| "logits/rejected": -1.2651885553215019, |
| "logps/chosen": -185.25, |
| "logps/rejected": -195.0, |
| "loss": 0.76959228515625, |
| "mean_token_accuracy": 0.9562991708517075, |
| "num_tokens": 3011121.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.07479095458984375, |
| "rewards/margins": -0.06903076171875, |
| "rewards/rejected": -0.0052642822265625, |
| "step": 8 |
| }, |
| { |
| "entropy": 0.177978515625, |
| "epoch": 0.04580152671755725, |
| "grad_norm": 61.59230041503906, |
| "learning_rate": 4.796954314720812e-06, |
| "logits/chosen": -1.1463901211868202, |
| "logits/rejected": -1.151384163231262, |
| "logps/chosen": -262.0, |
| "logps/rejected": -263.0, |
| "loss": 0.77764892578125, |
| "mean_token_accuracy": 0.9490418434143066, |
| "num_tokens": 3364659.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.0624542236328125, |
| "rewards/margins": -0.07977294921875, |
| "rewards/rejected": 0.017181396484375, |
| "step": 9 |
| }, |
| { |
| "entropy": 0.17041015625, |
| "epoch": 0.05089058524173028, |
| "grad_norm": 66.9396743774414, |
| "learning_rate": 4.771573604060914e-06, |
| "logits/chosen": -1.1312320566718426, |
| "logits/rejected": -1.114105189953177, |
| "logps/chosen": -246.25, |
| "logps/rejected": -253.5, |
| "loss": 0.784912109375, |
| "mean_token_accuracy": 0.9495462626218796, |
| "num_tokens": 3756722.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.11962890625, |
| "rewards/margins": -0.0777587890625, |
| "rewards/rejected": -0.0416259765625, |
| "step": 10 |
| }, |
| { |
| "entropy": 0.17041015625, |
| "epoch": 0.05597964376590331, |
| "grad_norm": 60.69718933105469, |
| "learning_rate": 4.746192893401016e-06, |
| "logits/chosen": -1.2082123033021486, |
| "logits/rejected": -1.240414757271401, |
| "logps/chosen": -206.75, |
| "logps/rejected": -253.0, |
| "loss": 0.77130126953125, |
| "mean_token_accuracy": 0.9514396339654922, |
| "num_tokens": 4150502.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.122711181640625, |
| "rewards/margins": -0.0660400390625, |
| "rewards/rejected": -0.056396484375, |
| "step": 11 |
| }, |
| { |
| "entropy": 0.166015625, |
| "epoch": 0.061068702290076333, |
| "grad_norm": 68.75598907470703, |
| "learning_rate": 4.7208121827411175e-06, |
| "logits/chosen": -1.1497054731535286, |
| "logits/rejected": -1.1323881415685357, |
| "logps/chosen": -233.0, |
| "logps/rejected": -214.0, |
| "loss": 0.81524658203125, |
| "mean_token_accuracy": 0.9499669224023819, |
| "num_tokens": 4548017.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.14825439453125, |
| "rewards/margins": -0.148681640625, |
| "rewards/rejected": 0.0008544921875, |
| "step": 12 |
| }, |
| { |
| "entropy": 0.171142578125, |
| "epoch": 0.06615776081424936, |
| "grad_norm": 54.791534423828125, |
| "learning_rate": 4.695431472081219e-06, |
| "logits/chosen": -1.1963053139896285, |
| "logits/rejected": -1.164336637055035, |
| "logps/chosen": -204.0, |
| "logps/rejected": -226.5, |
| "loss": 0.71929931640625, |
| "mean_token_accuracy": 0.952528104186058, |
| "num_tokens": 4902525.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0484619140625, |
| "rewards/margins": 0.0706787109375, |
| "rewards/rejected": -0.1187744140625, |
| "step": 13 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.07124681933842239, |
| "grad_norm": 55.923667907714844, |
| "learning_rate": 4.67005076142132e-06, |
| "logits/chosen": -1.227218462140038, |
| "logits/rejected": -1.2252739974581663, |
| "logps/chosen": -204.0, |
| "logps/rejected": -245.75, |
| "loss": 0.681884765625, |
| "mean_token_accuracy": 0.9524703174829483, |
| "num_tokens": 5247776.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.017791748046875, |
| "rewards/margins": 0.07401275634765625, |
| "rewards/rejected": -0.056304931640625, |
| "step": 14 |
| }, |
| { |
| "entropy": 0.16796875, |
| "epoch": 0.07633587786259542, |
| "grad_norm": 54.7180290222168, |
| "learning_rate": 4.644670050761422e-06, |
| "logits/chosen": -1.1794452967378677, |
| "logits/rejected": -1.213616516672937, |
| "logps/chosen": -208.75, |
| "logps/rejected": -199.5, |
| "loss": 0.7034912109375, |
| "mean_token_accuracy": 0.9460672587156296, |
| "num_tokens": 5614879.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.097320556640625, |
| "rewards/margins": 0.0595703125, |
| "rewards/rejected": -0.15673828125, |
| "step": 15 |
| }, |
| { |
| "entropy": 0.176513671875, |
| "epoch": 0.08142493638676845, |
| "grad_norm": 62.994319915771484, |
| "learning_rate": 4.6192893401015235e-06, |
| "logits/chosen": -1.2212638279189771, |
| "logits/rejected": -1.2088961146919388, |
| "logps/chosen": -227.25, |
| "logps/rejected": -213.0, |
| "loss": 0.83746337890625, |
| "mean_token_accuracy": 0.9471840560436249, |
| "num_tokens": 5971613.0, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.173828125, |
| "rewards/margins": -0.18463134765625, |
| "rewards/rejected": 0.01068115234375, |
| "step": 16 |
| }, |
| { |
| "entropy": 0.169677734375, |
| "epoch": 0.08651399491094147, |
| "grad_norm": 60.76310348510742, |
| "learning_rate": 4.593908629441624e-06, |
| "logits/chosen": -1.2065562340385436, |
| "logits/rejected": -1.2022840016395056, |
| "logps/chosen": -218.5, |
| "logps/rejected": -190.0, |
| "loss": 0.77734375, |
| "mean_token_accuracy": 0.9474756270647049, |
| "num_tokens": 6368369.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.067138671875, |
| "rewards/margins": -0.05255126953125, |
| "rewards/rejected": -0.0137939453125, |
| "step": 17 |
| }, |
| { |
| "entropy": 0.1748046875, |
| "epoch": 0.0916030534351145, |
| "grad_norm": 61.58626174926758, |
| "learning_rate": 4.568527918781726e-06, |
| "logits/chosen": -1.1644959212587278, |
| "logits/rejected": -1.1554610862237855, |
| "logps/chosen": -213.5, |
| "logps/rejected": -256.5, |
| "loss": 0.744384765625, |
| "mean_token_accuracy": 0.9518721997737885, |
| "num_tokens": 6749942.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.1009521484375, |
| "rewards/margins": 0.003753662109375, |
| "rewards/rejected": -0.104736328125, |
| "step": 18 |
| }, |
| { |
| "entropy": 0.170654296875, |
| "epoch": 0.09669211195928754, |
| "grad_norm": 69.10458374023438, |
| "learning_rate": 4.543147208121828e-06, |
| "logits/chosen": -1.2018930915056671, |
| "logits/rejected": -1.1882594101155102, |
| "logps/chosen": -208.75, |
| "logps/rejected": -223.0, |
| "loss": 0.7578125, |
| "mean_token_accuracy": 0.9503921419382095, |
| "num_tokens": 7150669.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.16357421875, |
| "rewards/margins": -0.0054931640625, |
| "rewards/rejected": -0.157958984375, |
| "step": 19 |
| }, |
| { |
| "entropy": 0.1708984375, |
| "epoch": 0.10178117048346055, |
| "grad_norm": 59.20901870727539, |
| "learning_rate": 4.5177664974619295e-06, |
| "logits/chosen": -1.153684045887259, |
| "logits/rejected": -1.1406723101806675, |
| "logps/chosen": -214.25, |
| "logps/rejected": -235.75, |
| "loss": 0.742584228515625, |
| "mean_token_accuracy": 0.9510545581579208, |
| "num_tokens": 7577633.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.1868896484375, |
| "rewards/margins": -0.008544921875, |
| "rewards/rejected": -0.17822265625, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.160888671875, |
| "epoch": 0.10687022900763359, |
| "grad_norm": 53.25274658203125, |
| "learning_rate": 4.492385786802031e-06, |
| "logits/chosen": -1.2115618534718147, |
| "logits/rejected": -1.1697568144140837, |
| "logps/chosen": -220.5, |
| "logps/rejected": -224.5, |
| "loss": 0.6365966796875, |
| "mean_token_accuracy": 0.954043835401535, |
| "num_tokens": 7935697.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0498046875, |
| "rewards/margins": 0.22021484375, |
| "rewards/rejected": -0.2708282470703125, |
| "step": 21 |
| }, |
| { |
| "entropy": 0.1708984375, |
| "epoch": 0.11195928753180662, |
| "grad_norm": 64.42499542236328, |
| "learning_rate": 4.467005076142132e-06, |
| "logits/chosen": -1.163452744541083, |
| "logits/rejected": -1.1574276551228084, |
| "logps/chosen": -206.75, |
| "logps/rejected": -249.0, |
| "loss": 0.7249755859375, |
| "mean_token_accuracy": 0.9557177871465683, |
| "num_tokens": 8344719.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.064208984375, |
| "rewards/margins": 0.043914794921875, |
| "rewards/rejected": -0.108154296875, |
| "step": 22 |
| }, |
| { |
| "entropy": 0.1650390625, |
| "epoch": 0.11704834605597965, |
| "grad_norm": 57.46247863769531, |
| "learning_rate": 4.441624365482234e-06, |
| "logits/chosen": -1.184751779280805, |
| "logits/rejected": -1.1601956141572203, |
| "logps/chosen": -221.5, |
| "logps/rejected": -223.5, |
| "loss": 0.725341796875, |
| "mean_token_accuracy": 0.9495507925748825, |
| "num_tokens": 8715576.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.12018585205078125, |
| "rewards/margins": 0.1531982421875, |
| "rewards/rejected": -0.27362060546875, |
| "step": 23 |
| }, |
| { |
| "entropy": 0.166259765625, |
| "epoch": 0.12213740458015267, |
| "grad_norm": 88.59230041503906, |
| "learning_rate": 4.4162436548223355e-06, |
| "logits/chosen": -1.2269864585121328, |
| "logits/rejected": -1.2167687340565112, |
| "logps/chosen": -190.5, |
| "logps/rejected": -223.25, |
| "loss": 0.616790771484375, |
| "mean_token_accuracy": 0.9541012793779373, |
| "num_tokens": 9131729.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0047607421875, |
| "rewards/margins": 0.2396240234375, |
| "rewards/rejected": -0.244873046875, |
| "step": 24 |
| }, |
| { |
| "entropy": 0.174560546875, |
| "epoch": 0.1272264631043257, |
| "grad_norm": 59.16816711425781, |
| "learning_rate": 4.390862944162436e-06, |
| "logits/chosen": -1.2085149622005857, |
| "logits/rejected": -1.1962000528044974, |
| "logps/chosen": -209.5, |
| "logps/rejected": -248.25, |
| "loss": 0.68780517578125, |
| "mean_token_accuracy": 0.9498510509729385, |
| "num_tokens": 9510884.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.14385986328125, |
| "rewards/margins": 0.06866455078125, |
| "rewards/rejected": -0.212890625, |
| "step": 25 |
| }, |
| { |
| "entropy": 0.166259765625, |
| "epoch": 0.13231552162849872, |
| "grad_norm": 58.58485412597656, |
| "learning_rate": 4.365482233502538e-06, |
| "logits/chosen": -1.2357831115334332, |
| "logits/rejected": -1.2414339305565623, |
| "logps/chosen": -205.25, |
| "logps/rejected": -210.0, |
| "loss": 0.735107421875, |
| "mean_token_accuracy": 0.9495077282190323, |
| "num_tokens": 9892399.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.144287109375, |
| "rewards/margins": 0.02099609375, |
| "rewards/rejected": -0.16522216796875, |
| "step": 26 |
| }, |
| { |
| "entropy": 0.17333984375, |
| "epoch": 0.13740458015267176, |
| "grad_norm": 51.31973648071289, |
| "learning_rate": 4.34010152284264e-06, |
| "logits/chosen": -1.1937940321742593, |
| "logits/rejected": -1.20553321824887, |
| "logps/chosen": -185.875, |
| "logps/rejected": -204.75, |
| "loss": 0.66217041015625, |
| "mean_token_accuracy": 0.9475287944078445, |
| "num_tokens": 10284784.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.072509765625, |
| "rewards/margins": 0.17462158203125, |
| "rewards/rejected": -0.247314453125, |
| "step": 27 |
| }, |
| { |
| "entropy": 0.16552734375, |
| "epoch": 0.14249363867684478, |
| "grad_norm": 61.93020248413086, |
| "learning_rate": 4.3147208121827415e-06, |
| "logits/chosen": -1.2497175985959397, |
| "logits/rejected": -1.2349540219835924, |
| "logps/chosen": -185.5, |
| "logps/rejected": -200.5, |
| "loss": 0.785888671875, |
| "mean_token_accuracy": 0.9526190459728241, |
| "num_tokens": 10640064.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.24853515625, |
| "rewards/margins": -0.0711669921875, |
| "rewards/rejected": -0.1773681640625, |
| "step": 28 |
| }, |
| { |
| "entropy": 0.168701171875, |
| "epoch": 0.1475826972010178, |
| "grad_norm": 47.82231903076172, |
| "learning_rate": 4.289340101522843e-06, |
| "logits/chosen": -1.2240958044550552, |
| "logits/rejected": -1.2331657411234425, |
| "logps/chosen": -164.75, |
| "logps/rejected": -183.5, |
| "loss": 0.64813232421875, |
| "mean_token_accuracy": 0.9512129127979279, |
| "num_tokens": 11043531.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0230712890625, |
| "rewards/margins": 0.1962890625, |
| "rewards/rejected": -0.21942138671875, |
| "step": 29 |
| }, |
| { |
| "entropy": 0.166015625, |
| "epoch": 0.15267175572519084, |
| "grad_norm": 64.59899139404297, |
| "learning_rate": 4.263959390862945e-06, |
| "logits/chosen": -1.1938851832774966, |
| "logits/rejected": -1.2063193481273322, |
| "logps/chosen": -203.375, |
| "logps/rejected": -209.25, |
| "loss": 0.75030517578125, |
| "mean_token_accuracy": 0.9482824355363846, |
| "num_tokens": 11433645.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.160888671875, |
| "rewards/margins": 0.0379638671875, |
| "rewards/rejected": -0.1988525390625, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.174560546875, |
| "epoch": 0.15776081424936386, |
| "grad_norm": 50.42904281616211, |
| "learning_rate": 4.238578680203046e-06, |
| "logits/chosen": -1.1847193683809207, |
| "logits/rejected": -1.2039758508125116, |
| "logps/chosen": -187.0, |
| "logps/rejected": -196.0, |
| "loss": 0.6805324554443359, |
| "mean_token_accuracy": 0.9458457380533218, |
| "num_tokens": 11802241.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.01849365234375, |
| "rewards/margins": 0.2666015625, |
| "rewards/rejected": -0.285400390625, |
| "step": 31 |
| }, |
| { |
| "entropy": 0.164794921875, |
| "epoch": 0.1628498727735369, |
| "grad_norm": 65.630126953125, |
| "learning_rate": 4.2131979695431475e-06, |
| "logits/chosen": -1.2242753640413924, |
| "logits/rejected": -1.2582800165640973, |
| "logps/chosen": -237.25, |
| "logps/rejected": -239.75, |
| "loss": 0.762481689453125, |
| "mean_token_accuracy": 0.9494172036647797, |
| "num_tokens": 12168742.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.256591796875, |
| "rewards/margins": 0.0673828125, |
| "rewards/rejected": -0.323486328125, |
| "step": 32 |
| }, |
| { |
| "entropy": 0.1669921875, |
| "epoch": 0.16793893129770993, |
| "grad_norm": 64.15123748779297, |
| "learning_rate": 4.187817258883249e-06, |
| "logits/chosen": -1.2054564969855512, |
| "logits/rejected": -1.2242075139011555, |
| "logps/chosen": -225.75, |
| "logps/rejected": -242.25, |
| "loss": 0.776153564453125, |
| "mean_token_accuracy": 0.9507817625999451, |
| "num_tokens": 12544019.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.259033203125, |
| "rewards/margins": -0.0609130859375, |
| "rewards/rejected": -0.1982421875, |
| "step": 33 |
| }, |
| { |
| "entropy": 0.16162109375, |
| "epoch": 0.17302798982188294, |
| "grad_norm": 59.748802185058594, |
| "learning_rate": 4.162436548223351e-06, |
| "logits/chosen": -1.1906290703191211, |
| "logits/rejected": -1.2255973378124254, |
| "logps/chosen": -185.5, |
| "logps/rejected": -189.5, |
| "loss": 0.81256103515625, |
| "mean_token_accuracy": 0.951210081577301, |
| "num_tokens": 12934809.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.261016845703125, |
| "rewards/margins": -0.1204071044921875, |
| "rewards/rejected": -0.140380859375, |
| "step": 34 |
| }, |
| { |
| "entropy": 0.160400390625, |
| "epoch": 0.178117048346056, |
| "grad_norm": 62.484031677246094, |
| "learning_rate": 4.137055837563453e-06, |
| "logits/chosen": -1.1568720249350068, |
| "logits/rejected": -1.1932806025584015, |
| "logps/chosen": -214.75, |
| "logps/rejected": -264.25, |
| "loss": 0.743255615234375, |
| "mean_token_accuracy": 0.952626571059227, |
| "num_tokens": 13333240.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.19916534423828125, |
| "rewards/margins": 0.0906982421875, |
| "rewards/rejected": -0.2904052734375, |
| "step": 35 |
| }, |
| { |
| "entropy": 0.168701171875, |
| "epoch": 0.183206106870229, |
| "grad_norm": 57.09201431274414, |
| "learning_rate": 4.1116751269035535e-06, |
| "logits/chosen": -1.244021900396991, |
| "logits/rejected": -1.2053838438463438, |
| "logps/chosen": -227.75, |
| "logps/rejected": -235.75, |
| "loss": 0.598724365234375, |
| "mean_token_accuracy": 0.9517507553100586, |
| "num_tokens": 13672135.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.099365234375, |
| "rewards/margins": 0.307861328125, |
| "rewards/rejected": -0.4072265625, |
| "step": 36 |
| }, |
| { |
| "entropy": 0.168212890625, |
| "epoch": 0.18829516539440203, |
| "grad_norm": 70.22441101074219, |
| "learning_rate": 4.086294416243655e-06, |
| "logits/chosen": -1.2293163289174895, |
| "logits/rejected": -1.2229109217299612, |
| "logps/chosen": -224.25, |
| "logps/rejected": -227.75, |
| "loss": 0.82135009765625, |
| "mean_token_accuracy": 0.9494140148162842, |
| "num_tokens": 14069079.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.0679931640625, |
| "rewards/margins": -0.12286376953125, |
| "rewards/rejected": 0.0546875, |
| "step": 37 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.19338422391857507, |
| "grad_norm": 76.26187896728516, |
| "learning_rate": 4.060913705583757e-06, |
| "logits/chosen": -1.210362411249237, |
| "logits/rejected": -1.2507633848434636, |
| "logps/chosen": -263.5, |
| "logps/rejected": -245.75, |
| "loss": 0.789825439453125, |
| "mean_token_accuracy": 0.9464226067066193, |
| "num_tokens": 14439718.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.00927734375, |
| "rewards/margins": -0.00775146484375, |
| "rewards/rejected": -0.001953125, |
| "step": 38 |
| }, |
| { |
| "entropy": 0.1767578125, |
| "epoch": 0.1984732824427481, |
| "grad_norm": 60.25480270385742, |
| "learning_rate": 4.035532994923858e-06, |
| "logits/chosen": -1.2676762543068074, |
| "logits/rejected": -1.2517246495133931, |
| "logps/chosen": -240.25, |
| "logps/rejected": -250.0, |
| "loss": 0.706390380859375, |
| "mean_token_accuracy": 0.9461905360221863, |
| "num_tokens": 14842983.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.234130859375, |
| "rewards/margins": 0.1005859375, |
| "rewards/rejected": -0.334716796875, |
| "step": 39 |
| }, |
| { |
| "entropy": 0.158447265625, |
| "epoch": 0.2035623409669211, |
| "grad_norm": 56.411930084228516, |
| "learning_rate": 4.0101522842639595e-06, |
| "logits/chosen": -1.253394155349274, |
| "logits/rejected": -1.2763997522602422, |
| "logps/chosen": -219.5, |
| "logps/rejected": -248.0, |
| "loss": 0.6590461730957031, |
| "mean_token_accuracy": 0.9516128301620483, |
| "num_tokens": 15207602.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.115966796875, |
| "rewards/margins": 0.2529296875, |
| "rewards/rejected": -0.3690185546875, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.20865139949109415, |
| "grad_norm": 50.47223663330078, |
| "learning_rate": 3.984771573604061e-06, |
| "logits/chosen": -1.221418931634342, |
| "logits/rejected": -1.240179635098703, |
| "logps/chosen": -198.75, |
| "logps/rejected": -241.0, |
| "loss": 0.61651611328125, |
| "mean_token_accuracy": 0.9533202648162842, |
| "num_tokens": 15615485.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.01708984375, |
| "rewards/margins": 0.32470703125, |
| "rewards/rejected": -0.3079833984375, |
| "step": 41 |
| }, |
| { |
| "entropy": 0.164794921875, |
| "epoch": 0.21374045801526717, |
| "grad_norm": 63.18607711791992, |
| "learning_rate": 3.959390862944163e-06, |
| "logits/chosen": -1.2487682814246732, |
| "logits/rejected": -1.2684586865119198, |
| "logps/chosen": -230.5, |
| "logps/rejected": -251.25, |
| "loss": 0.73516845703125, |
| "mean_token_accuracy": 0.9500314593315125, |
| "num_tokens": 16006587.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.0408935546875, |
| "rewards/margins": 0.07275390625, |
| "rewards/rejected": -0.11383056640625, |
| "step": 42 |
| }, |
| { |
| "entropy": 0.16796875, |
| "epoch": 0.21882951653944022, |
| "grad_norm": 57.65352249145508, |
| "learning_rate": 3.934010152284265e-06, |
| "logits/chosen": -1.3110932592597404, |
| "logits/rejected": -1.3089875033252198, |
| "logps/chosen": -223.0, |
| "logps/rejected": -271.25, |
| "loss": 0.7037353515625, |
| "mean_token_accuracy": 0.952717587351799, |
| "num_tokens": 16376034.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.1783447265625, |
| "rewards/margins": 0.10040283203125, |
| "rewards/rejected": -0.2783203125, |
| "step": 43 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.22391857506361323, |
| "grad_norm": 51.37344741821289, |
| "learning_rate": 3.9086294416243655e-06, |
| "logits/chosen": -1.293287887477467, |
| "logits/rejected": -1.2826172419774333, |
| "logps/chosen": -169.0, |
| "logps/rejected": -188.25, |
| "loss": 0.68048095703125, |
| "mean_token_accuracy": 0.9513245075941086, |
| "num_tokens": 16729880.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.14501953125, |
| "rewards/margins": 0.17425537109375, |
| "rewards/rejected": -0.31884765625, |
| "step": 44 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.22900763358778625, |
| "grad_norm": 60.11509704589844, |
| "learning_rate": 3.883248730964467e-06, |
| "logits/chosen": -1.2668976340142997, |
| "logits/rejected": -1.2750247764479736, |
| "logps/chosen": -227.0, |
| "logps/rejected": -250.0, |
| "loss": 0.6617431640625, |
| "mean_token_accuracy": 0.9519101679325104, |
| "num_tokens": 17112685.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0929718017578125, |
| "rewards/margins": 0.20703125, |
| "rewards/rejected": -0.2998046875, |
| "step": 45 |
| }, |
| { |
| "entropy": 0.16552734375, |
| "epoch": 0.2340966921119593, |
| "grad_norm": 70.05858612060547, |
| "learning_rate": 3.857868020304569e-06, |
| "logits/chosen": -1.256115313351624, |
| "logits/rejected": -1.292612336747009, |
| "logps/chosen": -203.75, |
| "logps/rejected": -204.25, |
| "loss": 0.78472900390625, |
| "mean_token_accuracy": 0.9469475299119949, |
| "num_tokens": 17464688.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1900482177734375, |
| "rewards/margins": -0.072418212890625, |
| "rewards/rejected": -0.118408203125, |
| "step": 46 |
| }, |
| { |
| "entropy": 0.16552734375, |
| "epoch": 0.23918575063613232, |
| "grad_norm": 63.72059631347656, |
| "learning_rate": 3.832487309644671e-06, |
| "logits/chosen": -1.2366188823977582, |
| "logits/rejected": -1.2068790190894856, |
| "logps/chosen": -215.75, |
| "logps/rejected": -216.25, |
| "loss": 0.70928955078125, |
| "mean_token_accuracy": 0.947726234793663, |
| "num_tokens": 17827406.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.126556396484375, |
| "rewards/margins": 0.05908203125, |
| "rewards/rejected": -0.18597412109375, |
| "step": 47 |
| }, |
| { |
| "entropy": 0.173095703125, |
| "epoch": 0.24427480916030533, |
| "grad_norm": 51.02957534790039, |
| "learning_rate": 3.8071065989847715e-06, |
| "logits/chosen": -1.2951891338286738, |
| "logits/rejected": -1.2901734123706725, |
| "logps/chosen": -186.375, |
| "logps/rejected": -190.375, |
| "loss": 0.686798095703125, |
| "mean_token_accuracy": 0.9495810121297836, |
| "num_tokens": 18168837.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.149169921875, |
| "rewards/margins": 0.10107421875, |
| "rewards/rejected": -0.25054931640625, |
| "step": 48 |
| }, |
| { |
| "entropy": 0.171142578125, |
| "epoch": 0.24936386768447838, |
| "grad_norm": 65.16993713378906, |
| "learning_rate": 3.7817258883248736e-06, |
| "logits/chosen": -1.2489111246586129, |
| "logits/rejected": -1.2595230297930642, |
| "logps/chosen": -226.25, |
| "logps/rejected": -253.5, |
| "loss": 0.6953582763671875, |
| "mean_token_accuracy": 0.9487173557281494, |
| "num_tokens": 18520757.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.05126953125, |
| "rewards/margins": 0.1453857421875, |
| "rewards/rejected": -0.197357177734375, |
| "step": 49 |
| }, |
| { |
| "entropy": 0.161865234375, |
| "epoch": 0.2544529262086514, |
| "grad_norm": 85.86858367919922, |
| "learning_rate": 3.756345177664975e-06, |
| "logits/chosen": -1.2788417924140663, |
| "logits/rejected": -1.2818688140867107, |
| "logps/chosen": -222.25, |
| "logps/rejected": -228.0, |
| "loss": 0.906097412109375, |
| "mean_token_accuracy": 0.9481759369373322, |
| "num_tokens": 18907465.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.32647705078125, |
| "rewards/margins": -0.261962890625, |
| "rewards/rejected": -0.06494140625, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.1689453125, |
| "epoch": 0.2595419847328244, |
| "grad_norm": 63.7056999206543, |
| "learning_rate": 3.7309644670050766e-06, |
| "logits/chosen": -1.24593670349563, |
| "logits/rejected": -1.2525011882667132, |
| "logps/chosen": -204.5, |
| "logps/rejected": -235.0, |
| "loss": 0.809600830078125, |
| "mean_token_accuracy": 0.9492675065994263, |
| "num_tokens": 19271716.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.22967529296875, |
| "rewards/margins": -0.108154296875, |
| "rewards/rejected": -0.12200927734375, |
| "step": 51 |
| }, |
| { |
| "entropy": 0.16943359375, |
| "epoch": 0.26463104325699743, |
| "grad_norm": 66.75394439697266, |
| "learning_rate": 3.705583756345178e-06, |
| "logits/chosen": -1.2951451816324733, |
| "logits/rejected": -1.2748576702987078, |
| "logps/chosen": -220.0, |
| "logps/rejected": -201.5, |
| "loss": 0.7838897705078125, |
| "mean_token_accuracy": 0.9473998248577118, |
| "num_tokens": 19651670.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.283111572265625, |
| "rewards/margins": -0.046630859375, |
| "rewards/rejected": -0.235687255859375, |
| "step": 52 |
| }, |
| { |
| "entropy": 0.167724609375, |
| "epoch": 0.2697201017811705, |
| "grad_norm": 68.03031921386719, |
| "learning_rate": 3.6802030456852796e-06, |
| "logits/chosen": -1.2656714275142638, |
| "logits/rejected": -1.2242606863621242, |
| "logps/chosen": -184.375, |
| "logps/rejected": -200.75, |
| "loss": 0.81591796875, |
| "mean_token_accuracy": 0.9486018866300583, |
| "num_tokens": 20048489.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.284912109375, |
| "rewards/margins": -0.12548828125, |
| "rewards/rejected": -0.158782958984375, |
| "step": 53 |
| }, |
| { |
| "entropy": 0.164306640625, |
| "epoch": 0.2748091603053435, |
| "grad_norm": 52.74068832397461, |
| "learning_rate": 3.654822335025381e-06, |
| "logits/chosen": -1.3013878893228477, |
| "logits/rejected": -1.3064163255128232, |
| "logps/chosen": -192.5, |
| "logps/rejected": -185.5, |
| "loss": 0.67987060546875, |
| "mean_token_accuracy": 0.9475405365228653, |
| "num_tokens": 20384513.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.19287109375, |
| "rewards/margins": 0.16943359375, |
| "rewards/rejected": -0.3623046875, |
| "step": 54 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.27989821882951654, |
| "grad_norm": 71.49076843261719, |
| "learning_rate": 3.629441624365482e-06, |
| "logits/chosen": -1.2508704209061283, |
| "logits/rejected": -1.2312254570289394, |
| "logps/chosen": -225.0, |
| "logps/rejected": -227.5, |
| "loss": 0.86474609375, |
| "mean_token_accuracy": 0.9473535567522049, |
| "num_tokens": 20754117.0, |
| "rewards/accuracies": 0.25, |
| "rewards/chosen": -0.34716796875, |
| "rewards/margins": -0.2258148193359375, |
| "rewards/rejected": -0.1211395263671875, |
| "step": 55 |
| }, |
| { |
| "entropy": 0.1640625, |
| "epoch": 0.28498727735368956, |
| "grad_norm": 70.81625366210938, |
| "learning_rate": 3.6040609137055843e-06, |
| "logits/chosen": -1.3251861075583002, |
| "logits/rejected": -1.32016448634709, |
| "logps/chosen": -237.75, |
| "logps/rejected": -248.0, |
| "loss": 0.78326416015625, |
| "mean_token_accuracy": 0.948715329170227, |
| "num_tokens": 21157270.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.26702880859375, |
| "rewards/margins": -0.0032958984375, |
| "rewards/rejected": -0.26409912109375, |
| "step": 56 |
| }, |
| { |
| "entropy": 0.16162109375, |
| "epoch": 0.2900763358778626, |
| "grad_norm": 57.185699462890625, |
| "learning_rate": 3.5786802030456856e-06, |
| "logits/chosen": -1.3391477581159092, |
| "logits/rejected": -1.2969421235061, |
| "logps/chosen": -167.5, |
| "logps/rejected": -206.75, |
| "loss": 0.7360153198242188, |
| "mean_token_accuracy": 0.9557931572198868, |
| "num_tokens": 21557901.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.13916015625, |
| "rewards/margins": 0.08935546875, |
| "rewards/rejected": -0.22784423828125, |
| "step": 57 |
| }, |
| { |
| "entropy": 0.167236328125, |
| "epoch": 0.2951653944020356, |
| "grad_norm": 67.23629760742188, |
| "learning_rate": 3.5532994923857873e-06, |
| "logits/chosen": -1.2344125058729924, |
| "logits/rejected": -1.2176261940150734, |
| "logps/chosen": -221.625, |
| "logps/rejected": -227.5, |
| "loss": 0.833160400390625, |
| "mean_token_accuracy": 0.9486464709043503, |
| "num_tokens": 21930023.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.220458984375, |
| "rewards/margins": -0.0969696044921875, |
| "rewards/rejected": -0.1231689453125, |
| "step": 58 |
| }, |
| { |
| "entropy": 0.158447265625, |
| "epoch": 0.30025445292620867, |
| "grad_norm": 61.011383056640625, |
| "learning_rate": 3.5279187817258886e-06, |
| "logits/chosen": -1.2608206675107227, |
| "logits/rejected": -1.2702774965163959, |
| "logps/chosen": -230.5, |
| "logps/rejected": -221.0, |
| "loss": 0.693756103515625, |
| "mean_token_accuracy": 0.9499951750040054, |
| "num_tokens": 22300544.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.32568359375, |
| "rewards/margins": 0.140350341796875, |
| "rewards/rejected": -0.466796875, |
| "step": 59 |
| }, |
| { |
| "entropy": 0.155517578125, |
| "epoch": 0.3053435114503817, |
| "grad_norm": 60.74739456176758, |
| "learning_rate": 3.5025380710659903e-06, |
| "logits/chosen": -1.2432721402471825, |
| "logits/rejected": -1.1966404831484798, |
| "logps/chosen": -188.25, |
| "logps/rejected": -196.0, |
| "loss": 0.8065185546875, |
| "mean_token_accuracy": 0.9513755738735199, |
| "num_tokens": 22699715.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.320556640625, |
| "rewards/margins": -0.060028076171875, |
| "rewards/rejected": -0.260986328125, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.17333984375, |
| "epoch": 0.3104325699745547, |
| "grad_norm": 64.06476593017578, |
| "learning_rate": 3.4771573604060916e-06, |
| "logits/chosen": -1.23914246177565, |
| "logits/rejected": -1.2589912793052047, |
| "logps/chosen": -208.25, |
| "logps/rejected": -225.25, |
| "loss": 0.77069091796875, |
| "mean_token_accuracy": 0.9473724365234375, |
| "num_tokens": 23067391.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.082275390625, |
| "rewards/margins": 0.08203125, |
| "rewards/rejected": -0.166015625, |
| "step": 61 |
| }, |
| { |
| "entropy": 0.1689453125, |
| "epoch": 0.3155216284987277, |
| "grad_norm": 48.07145309448242, |
| "learning_rate": 3.451776649746193e-06, |
| "logits/chosen": -1.2898154412235743, |
| "logits/rejected": -1.2884235645949569, |
| "logps/chosen": -205.25, |
| "logps/rejected": -217.125, |
| "loss": 0.5599517822265625, |
| "mean_token_accuracy": 0.947222501039505, |
| "num_tokens": 23396708.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.03094482421875, |
| "rewards/margins": 0.41748046875, |
| "rewards/rejected": -0.38671875, |
| "step": 62 |
| }, |
| { |
| "entropy": 0.1640625, |
| "epoch": 0.32061068702290074, |
| "grad_norm": 65.72078704833984, |
| "learning_rate": 3.4263959390862946e-06, |
| "logits/chosen": -1.2525791988685302, |
| "logits/rejected": -1.2465728013183064, |
| "logps/chosen": -226.5, |
| "logps/rejected": -198.75, |
| "loss": 0.852203369140625, |
| "mean_token_accuracy": 0.9457215964794159, |
| "num_tokens": 23771644.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.45166015625, |
| "rewards/margins": -0.1153564453125, |
| "rewards/rejected": -0.335205078125, |
| "step": 63 |
| }, |
| { |
| "entropy": 0.16943359375, |
| "epoch": 0.3256997455470738, |
| "grad_norm": 70.23268127441406, |
| "learning_rate": 3.4010152284263963e-06, |
| "logits/chosen": -1.2732464634523972, |
| "logits/rejected": -1.2514687027641094, |
| "logps/chosen": -235.75, |
| "logps/rejected": -247.25, |
| "loss": 0.781402587890625, |
| "mean_token_accuracy": 0.9476925879716873, |
| "num_tokens": 24154169.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.297607421875, |
| "rewards/margins": -0.0504150390625, |
| "rewards/rejected": -0.246826171875, |
| "step": 64 |
| }, |
| { |
| "entropy": 0.1630859375, |
| "epoch": 0.33078880407124683, |
| "grad_norm": 75.50202941894531, |
| "learning_rate": 3.375634517766498e-06, |
| "logits/chosen": -1.2872747032412404, |
| "logits/rejected": -1.2523587882327665, |
| "logps/chosen": -243.0, |
| "logps/rejected": -231.0, |
| "loss": 0.81689453125, |
| "mean_token_accuracy": 0.9463559240102768, |
| "num_tokens": 24486129.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.22308349609375, |
| "rewards/margins": -0.103271484375, |
| "rewards/rejected": -0.11993408203125, |
| "step": 65 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.33587786259541985, |
| "grad_norm": 65.13658142089844, |
| "learning_rate": 3.3502538071065993e-06, |
| "logits/chosen": -1.2898683758825709, |
| "logits/rejected": -1.2849647133801598, |
| "logps/chosen": -220.25, |
| "logps/rejected": -229.5, |
| "loss": 0.7967147827148438, |
| "mean_token_accuracy": 0.9507217556238174, |
| "num_tokens": 24857818.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.2047119140625, |
| "rewards/margins": 0.05828857421875, |
| "rewards/rejected": -0.263427734375, |
| "step": 66 |
| }, |
| { |
| "entropy": 0.165283203125, |
| "epoch": 0.34096692111959287, |
| "grad_norm": 61.869327545166016, |
| "learning_rate": 3.3248730964467006e-06, |
| "logits/chosen": -1.2517319440883818, |
| "logits/rejected": -1.2434228301098385, |
| "logps/chosen": -205.625, |
| "logps/rejected": -222.25, |
| "loss": 0.697723388671875, |
| "mean_token_accuracy": 0.9499448090791702, |
| "num_tokens": 25270495.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.1441650390625, |
| "rewards/margins": 0.16796875, |
| "rewards/rejected": -0.31201171875, |
| "step": 67 |
| }, |
| { |
| "entropy": 0.156005859375, |
| "epoch": 0.3460559796437659, |
| "grad_norm": 69.01359558105469, |
| "learning_rate": 3.2994923857868023e-06, |
| "logits/chosen": -1.2596454913995498, |
| "logits/rejected": -1.2822667709063957, |
| "logps/chosen": -238.75, |
| "logps/rejected": -219.25, |
| "loss": 0.788330078125, |
| "mean_token_accuracy": 0.9476147145032883, |
| "num_tokens": 25639922.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.2823486328125, |
| "rewards/margins": -0.060791015625, |
| "rewards/rejected": -0.2213134765625, |
| "step": 68 |
| }, |
| { |
| "entropy": 0.170166015625, |
| "epoch": 0.3511450381679389, |
| "grad_norm": 65.33216094970703, |
| "learning_rate": 3.2741116751269036e-06, |
| "logits/chosen": -1.2824587990174645, |
| "logits/rejected": -1.2777971502557461, |
| "logps/chosen": -199.25, |
| "logps/rejected": -190.5, |
| "loss": 0.833953857421875, |
| "mean_token_accuracy": 0.945390522480011, |
| "num_tokens": 26010115.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.238037109375, |
| "rewards/margins": -0.1005859375, |
| "rewards/rejected": -0.1373443603515625, |
| "step": 69 |
| }, |
| { |
| "entropy": 0.162841796875, |
| "epoch": 0.356234096692112, |
| "grad_norm": 64.25736999511719, |
| "learning_rate": 3.2487309644670053e-06, |
| "logits/chosen": -1.332777326624499, |
| "logits/rejected": -1.3164871807406162, |
| "logps/chosen": -175.625, |
| "logps/rejected": -220.75, |
| "loss": 0.727630615234375, |
| "mean_token_accuracy": 0.950916975736618, |
| "num_tokens": 26385610.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.1827850341796875, |
| "rewards/margins": 0.056396484375, |
| "rewards/rejected": -0.2392578125, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.1611328125, |
| "epoch": 0.361323155216285, |
| "grad_norm": 57.03504943847656, |
| "learning_rate": 3.223350253807107e-06, |
| "logits/chosen": -1.2973189023839775, |
| "logits/rejected": -1.2664155194693132, |
| "logps/chosen": -195.75, |
| "logps/rejected": -225.25, |
| "loss": 0.6987152099609375, |
| "mean_token_accuracy": 0.9523173123598099, |
| "num_tokens": 26772574.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.239013671875, |
| "rewards/margins": 0.1385498046875, |
| "rewards/rejected": -0.37744140625, |
| "step": 71 |
| }, |
| { |
| "entropy": 0.151123046875, |
| "epoch": 0.366412213740458, |
| "grad_norm": 106.73540496826172, |
| "learning_rate": 3.1979695431472087e-06, |
| "logits/chosen": -1.3368664755840716, |
| "logits/rejected": -1.3149258298106234, |
| "logps/chosen": -207.75, |
| "logps/rejected": -193.75, |
| "loss": 0.76409912109375, |
| "mean_token_accuracy": 0.9506115764379501, |
| "num_tokens": 27153798.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.32080078125, |
| "rewards/margins": -0.001861572265625, |
| "rewards/rejected": -0.318359375, |
| "step": 72 |
| }, |
| { |
| "entropy": 0.162841796875, |
| "epoch": 0.37150127226463103, |
| "grad_norm": 43.361202239990234, |
| "learning_rate": 3.17258883248731e-06, |
| "logits/chosen": -1.292466410490634, |
| "logits/rejected": -1.2795658979837796, |
| "logps/chosen": -174.25, |
| "logps/rejected": -169.0, |
| "loss": 0.6114501953125, |
| "mean_token_accuracy": 0.9446200430393219, |
| "num_tokens": 27531852.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.11895751953125, |
| "rewards/margins": 0.2860107421875, |
| "rewards/rejected": -0.40478515625, |
| "step": 73 |
| }, |
| { |
| "entropy": 0.165283203125, |
| "epoch": 0.37659033078880405, |
| "grad_norm": 258.8431701660156, |
| "learning_rate": 3.1472081218274113e-06, |
| "logits/chosen": -1.255416233032602, |
| "logits/rejected": -1.2341775737485738, |
| "logps/chosen": -251.25, |
| "logps/rejected": -240.5, |
| "loss": 0.8010711669921875, |
| "mean_token_accuracy": 0.9430906176567078, |
| "num_tokens": 27903266.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.41162109375, |
| "rewards/margins": 0.0003662109375, |
| "rewards/rejected": -0.4114990234375, |
| "step": 74 |
| }, |
| { |
| "entropy": 0.1513671875, |
| "epoch": 0.3816793893129771, |
| "grad_norm": 59.12688064575195, |
| "learning_rate": 3.121827411167513e-06, |
| "logits/chosen": -1.3708437654075616, |
| "logits/rejected": -1.3749631542345564, |
| "logps/chosen": -184.25, |
| "logps/rejected": -207.75, |
| "loss": 0.7006683349609375, |
| "mean_token_accuracy": 0.9533696621656418, |
| "num_tokens": 28291715.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.008056640625, |
| "rewards/margins": 0.16162109375, |
| "rewards/rejected": -0.169677734375, |
| "step": 75 |
| }, |
| { |
| "entropy": 0.168212890625, |
| "epoch": 0.38676844783715014, |
| "grad_norm": 70.00326538085938, |
| "learning_rate": 3.0964467005076143e-06, |
| "logits/chosen": -1.2278840581970798, |
| "logits/rejected": -1.252243618402488, |
| "logps/chosen": -241.25, |
| "logps/rejected": -222.25, |
| "loss": 0.7878265380859375, |
| "mean_token_accuracy": 0.9443240016698837, |
| "num_tokens": 28640895.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.22607421875, |
| "rewards/margins": -0.0015869140625, |
| "rewards/rejected": -0.2257080078125, |
| "step": 76 |
| }, |
| { |
| "entropy": 0.147216796875, |
| "epoch": 0.39185750636132316, |
| "grad_norm": 52.42808151245117, |
| "learning_rate": 3.071065989847716e-06, |
| "logits/chosen": -1.3088794461049085, |
| "logits/rejected": -1.2848204770716634, |
| "logps/chosen": -174.375, |
| "logps/rejected": -186.75, |
| "loss": 0.730010986328125, |
| "mean_token_accuracy": 0.9527680724859238, |
| "num_tokens": 29031104.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.24163818359375, |
| "rewards/margins": 0.07861328125, |
| "rewards/rejected": -0.31982421875, |
| "step": 77 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.3969465648854962, |
| "grad_norm": 66.45071411132812, |
| "learning_rate": 3.0456852791878177e-06, |
| "logits/chosen": -1.2249955483759087, |
| "logits/rejected": -1.2137169350336674, |
| "logps/chosen": -216.0, |
| "logps/rejected": -232.0, |
| "loss": 0.7823486328125, |
| "mean_token_accuracy": 0.9497682005167007, |
| "num_tokens": 29411929.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.2440185546875, |
| "rewards/margins": -0.053955078125, |
| "rewards/rejected": -0.1904296875, |
| "step": 78 |
| }, |
| { |
| "entropy": 0.169189453125, |
| "epoch": 0.4020356234096692, |
| "grad_norm": 71.31132507324219, |
| "learning_rate": 3.0203045685279194e-06, |
| "logits/chosen": -1.311240796195369, |
| "logits/rejected": -1.3000907150950454, |
| "logps/chosen": -202.0, |
| "logps/rejected": -218.75, |
| "loss": 0.727569580078125, |
| "mean_token_accuracy": 0.9468916058540344, |
| "num_tokens": 29765589.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.289306640625, |
| "rewards/margins": 0.17138671875, |
| "rewards/rejected": -0.460205078125, |
| "step": 79 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.4071246819338422, |
| "grad_norm": 64.0061264038086, |
| "learning_rate": 2.9949238578680207e-06, |
| "logits/chosen": -1.3101197475281547, |
| "logits/rejected": -1.2970568708741963, |
| "logps/chosen": -220.25, |
| "logps/rejected": -211.5, |
| "loss": 0.729522705078125, |
| "mean_token_accuracy": 0.9487328827381134, |
| "num_tokens": 30163265.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.21240234375, |
| "rewards/margins": 0.0606689453125, |
| "rewards/rejected": -0.273193359375, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.4122137404580153, |
| "grad_norm": 65.17308807373047, |
| "learning_rate": 2.969543147208122e-06, |
| "logits/chosen": -1.3028268301410426, |
| "logits/rejected": -1.3046930569719115, |
| "logps/chosen": -221.0, |
| "logps/rejected": -223.25, |
| "loss": 0.70806884765625, |
| "mean_token_accuracy": 0.9474814534187317, |
| "num_tokens": 30509121.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.2777099609375, |
| "rewards/margins": 0.07781982421875, |
| "rewards/rejected": -0.35546875, |
| "step": 81 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.4173027989821883, |
| "grad_norm": 50.35979080200195, |
| "learning_rate": 2.9441624365482237e-06, |
| "logits/chosen": -1.2836960568317857, |
| "logits/rejected": -1.306755689640039, |
| "logps/chosen": -194.25, |
| "logps/rejected": -214.75, |
| "loss": 0.6282501220703125, |
| "mean_token_accuracy": 0.9473407566547394, |
| "num_tokens": 30917508.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.16790771484375, |
| "rewards/margins": 0.30712890625, |
| "rewards/rejected": -0.47509765625, |
| "step": 82 |
| }, |
| { |
| "entropy": 0.156494140625, |
| "epoch": 0.4223918575063613, |
| "grad_norm": 55.4305419921875, |
| "learning_rate": 2.918781725888325e-06, |
| "logits/chosen": -1.3347602124239275, |
| "logits/rejected": -1.297045195641747, |
| "logps/chosen": -168.0, |
| "logps/rejected": -182.5, |
| "loss": 0.702239990234375, |
| "mean_token_accuracy": 0.9523750394582748, |
| "num_tokens": 31290108.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.17498779296875, |
| "rewards/margins": 0.1236572265625, |
| "rewards/rejected": -0.298828125, |
| "step": 83 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.42748091603053434, |
| "grad_norm": 72.86205291748047, |
| "learning_rate": 2.8934010152284262e-06, |
| "logits/chosen": -1.2560234762562408, |
| "logits/rejected": -1.2293338403613354, |
| "logps/chosen": -207.75, |
| "logps/rejected": -219.0, |
| "loss": 0.7640380859375, |
| "mean_token_accuracy": 0.9502075463533401, |
| "num_tokens": 31677755.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.6025390625, |
| "rewards/margins": 0.078369140625, |
| "rewards/rejected": -0.6806640625, |
| "step": 84 |
| }, |
| { |
| "entropy": 0.16015625, |
| "epoch": 0.43256997455470736, |
| "grad_norm": 60.04010009765625, |
| "learning_rate": 2.8680203045685284e-06, |
| "logits/chosen": -1.3011174292040961, |
| "logits/rejected": -1.2539586523088446, |
| "logps/chosen": -210.5, |
| "logps/rejected": -211.25, |
| "loss": 0.669281005859375, |
| "mean_token_accuracy": 0.9515846818685532, |
| "num_tokens": 32041342.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1500244140625, |
| "rewards/margins": 0.138671875, |
| "rewards/rejected": -0.28857421875, |
| "step": 85 |
| }, |
| { |
| "entropy": 0.1572265625, |
| "epoch": 0.43765903307888043, |
| "grad_norm": 65.90755462646484, |
| "learning_rate": 2.8426395939086297e-06, |
| "logits/chosen": -1.2890849091113363, |
| "logits/rejected": -1.2671961630083879, |
| "logps/chosen": -212.0, |
| "logps/rejected": -212.0, |
| "loss": 0.8037109375, |
| "mean_token_accuracy": 0.9498841613531113, |
| "num_tokens": 32426894.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.328125, |
| "rewards/margins": -0.07586669921875, |
| "rewards/rejected": -0.2518310546875, |
| "step": 86 |
| }, |
| { |
| "entropy": 0.160888671875, |
| "epoch": 0.44274809160305345, |
| "grad_norm": 64.37542724609375, |
| "learning_rate": 2.8172588832487314e-06, |
| "logits/chosen": -1.2516031645716161, |
| "logits/rejected": -1.2446139578943387, |
| "logps/chosen": -238.5, |
| "logps/rejected": -244.5, |
| "loss": 0.734710693359375, |
| "mean_token_accuracy": 0.9513941407203674, |
| "num_tokens": 32747265.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.337158203125, |
| "rewards/margins": 0.085693359375, |
| "rewards/rejected": -0.42431640625, |
| "step": 87 |
| }, |
| { |
| "entropy": 0.164306640625, |
| "epoch": 0.44783715012722647, |
| "grad_norm": 64.61540222167969, |
| "learning_rate": 2.7918781725888327e-06, |
| "logits/chosen": -1.3129108468652448, |
| "logits/rejected": -1.3204735199009792, |
| "logps/chosen": -208.5, |
| "logps/rejected": -216.5, |
| "loss": 0.674041748046875, |
| "mean_token_accuracy": 0.9465741366147995, |
| "num_tokens": 33124296.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.34832763671875, |
| "rewards/margins": 0.1859130859375, |
| "rewards/rejected": -0.533203125, |
| "step": 88 |
| }, |
| { |
| "entropy": 0.166748046875, |
| "epoch": 0.4529262086513995, |
| "grad_norm": 62.742977142333984, |
| "learning_rate": 2.7664974619289344e-06, |
| "logits/chosen": -1.2541273019597161, |
| "logits/rejected": -1.2661804942223065, |
| "logps/chosen": -202.25, |
| "logps/rejected": -204.0, |
| "loss": 0.775146484375, |
| "mean_token_accuracy": 0.9480140656232834, |
| "num_tokens": 33498040.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.30126953125, |
| "rewards/margins": -0.014892578125, |
| "rewards/rejected": -0.28759765625, |
| "step": 89 |
| }, |
| { |
| "entropy": 0.1611328125, |
| "epoch": 0.4580152671755725, |
| "grad_norm": 70.147216796875, |
| "learning_rate": 2.7411167512690357e-06, |
| "logits/chosen": -1.2708255735449616, |
| "logits/rejected": -1.2573322500480986, |
| "logps/chosen": -226.0, |
| "logps/rejected": -218.5, |
| "loss": 0.7926025390625, |
| "mean_token_accuracy": 0.9451356083154678, |
| "num_tokens": 33845280.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.323974609375, |
| "rewards/margins": -0.10205078125, |
| "rewards/rejected": -0.2225341796875, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.15625, |
| "epoch": 0.4631043256997455, |
| "grad_norm": 63.841976165771484, |
| "learning_rate": 2.715736040609137e-06, |
| "logits/chosen": -1.326140821141854, |
| "logits/rejected": -1.3591119533293379, |
| "logps/chosen": -200.25, |
| "logps/rejected": -205.0, |
| "loss": 0.790740966796875, |
| "mean_token_accuracy": 0.9486561864614487, |
| "num_tokens": 34242803.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.4453125, |
| "rewards/margins": -0.0655517578125, |
| "rewards/rejected": -0.380126953125, |
| "step": 91 |
| }, |
| { |
| "entropy": 0.16015625, |
| "epoch": 0.4681933842239186, |
| "grad_norm": 73.43412780761719, |
| "learning_rate": 2.6903553299492387e-06, |
| "logits/chosen": -1.252702561138765, |
| "logits/rejected": -1.2816603315955593, |
| "logps/chosen": -250.25, |
| "logps/rejected": -238.75, |
| "loss": 0.90521240234375, |
| "mean_token_accuracy": 0.9443905353546143, |
| "num_tokens": 34628251.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.48974609375, |
| "rewards/margins": -0.2421875, |
| "rewards/rejected": -0.2470703125, |
| "step": 92 |
| }, |
| { |
| "entropy": 0.171142578125, |
| "epoch": 0.4732824427480916, |
| "grad_norm": 63.70939254760742, |
| "learning_rate": 2.6649746192893404e-06, |
| "logits/chosen": -1.254758430715978, |
| "logits/rejected": -1.2633837670267376, |
| "logps/chosen": -201.5, |
| "logps/rejected": -184.25, |
| "loss": 0.7750244140625, |
| "mean_token_accuracy": 0.9433987885713577, |
| "num_tokens": 35007915.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.306884765625, |
| "rewards/margins": -0.08544921875, |
| "rewards/rejected": -0.220947265625, |
| "step": 93 |
| }, |
| { |
| "entropy": 0.162109375, |
| "epoch": 0.47837150127226463, |
| "grad_norm": 73.37699890136719, |
| "learning_rate": 2.639593908629442e-06, |
| "logits/chosen": -1.2285106824311274, |
| "logits/rejected": -1.2533075679141386, |
| "logps/chosen": -218.75, |
| "logps/rejected": -242.25, |
| "loss": 0.84307861328125, |
| "mean_token_accuracy": 0.947221428155899, |
| "num_tokens": 35387530.0, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.4287109375, |
| "rewards/margins": -0.1712646484375, |
| "rewards/rejected": -0.25738525390625, |
| "step": 94 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.48346055979643765, |
| "grad_norm": 51.54319381713867, |
| "learning_rate": 2.6142131979695434e-06, |
| "logits/chosen": -1.276040442817514, |
| "logits/rejected": -1.2760191663203924, |
| "logps/chosen": -208.5, |
| "logps/rejected": -240.0, |
| "loss": 0.6256561279296875, |
| "mean_token_accuracy": 0.9505224972963333, |
| "num_tokens": 35732544.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.03521728515625, |
| "rewards/margins": 0.306884765625, |
| "rewards/rejected": -0.3427734375, |
| "step": 95 |
| }, |
| { |
| "entropy": 0.165283203125, |
| "epoch": 0.48854961832061067, |
| "grad_norm": 64.21118927001953, |
| "learning_rate": 2.588832487309645e-06, |
| "logits/chosen": -1.2820478858157205, |
| "logits/rejected": -1.2736327297402361, |
| "logps/chosen": -211.5, |
| "logps/rejected": -197.0, |
| "loss": 0.8376922607421875, |
| "mean_token_accuracy": 0.9446868598461151, |
| "num_tokens": 36116587.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.236328125, |
| "rewards/margins": -0.08966064453125, |
| "rewards/rejected": -0.146087646484375, |
| "step": 96 |
| }, |
| { |
| "entropy": 0.15869140625, |
| "epoch": 0.49363867684478374, |
| "grad_norm": 60.89461135864258, |
| "learning_rate": 2.5634517766497464e-06, |
| "logits/chosen": -1.2462152995908475, |
| "logits/rejected": -1.2528073331416407, |
| "logps/chosen": -232.0, |
| "logps/rejected": -247.25, |
| "loss": 0.69921875, |
| "mean_token_accuracy": 0.9487164616584778, |
| "num_tokens": 36525364.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1297607421875, |
| "rewards/margins": 0.209716796875, |
| "rewards/rejected": -0.3388671875, |
| "step": 97 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.49872773536895676, |
| "grad_norm": 58.411373138427734, |
| "learning_rate": 2.5380710659898476e-06, |
| "logits/chosen": -1.2857249020723578, |
| "logits/rejected": -1.273766229120395, |
| "logps/chosen": -194.0, |
| "logps/rejected": -214.5, |
| "loss": 0.718597412109375, |
| "mean_token_accuracy": 0.949738010764122, |
| "num_tokens": 36924408.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.32177734375, |
| "rewards/margins": 0.137939453125, |
| "rewards/rejected": -0.459716796875, |
| "step": 98 |
| }, |
| { |
| "entropy": 0.155029296875, |
| "epoch": 0.5038167938931297, |
| "grad_norm": 65.45690155029297, |
| "learning_rate": 2.5126903553299493e-06, |
| "logits/chosen": -1.3141924885583736, |
| "logits/rejected": -1.3078939917330479, |
| "logps/chosen": -222.25, |
| "logps/rejected": -223.75, |
| "loss": 0.7358856201171875, |
| "mean_token_accuracy": 0.951283186674118, |
| "num_tokens": 37320690.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.2308349609375, |
| "rewards/margins": 0.076904296875, |
| "rewards/rejected": -0.308990478515625, |
| "step": 99 |
| }, |
| { |
| "entropy": 0.175048828125, |
| "epoch": 0.5089058524173028, |
| "grad_norm": 64.2022705078125, |
| "learning_rate": 2.487309644670051e-06, |
| "logits/chosen": -1.2262440668616592, |
| "logits/rejected": -1.23488919059254, |
| "logps/chosen": -232.75, |
| "logps/rejected": -212.375, |
| "loss": 0.8157958984375, |
| "mean_token_accuracy": 0.9397356808185577, |
| "num_tokens": 37699769.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.3427734375, |
| "rewards/margins": -0.0829620361328125, |
| "rewards/rejected": -0.2600555419921875, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.154541015625, |
| "epoch": 0.5139949109414759, |
| "grad_norm": 65.66177368164062, |
| "learning_rate": 2.4619289340101523e-06, |
| "logits/chosen": -1.318018712738355, |
| "logits/rejected": -1.2828259768178074, |
| "logps/chosen": -190.25, |
| "logps/rejected": -202.5, |
| "loss": 0.87933349609375, |
| "mean_token_accuracy": 0.9535940736532211, |
| "num_tokens": 38075154.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.4375, |
| "rewards/margins": -0.1856842041015625, |
| "rewards/rejected": -0.2518310546875, |
| "step": 101 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.5190839694656488, |
| "grad_norm": 63.887428283691406, |
| "learning_rate": 2.436548223350254e-06, |
| "logits/chosen": -1.2580108193399442, |
| "logits/rejected": -1.2583606455369734, |
| "logps/chosen": -215.25, |
| "logps/rejected": -235.5, |
| "loss": 0.760650634765625, |
| "mean_token_accuracy": 0.9483280181884766, |
| "num_tokens": 38449417.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.2845458984375, |
| "rewards/margins": 0.07366943359375, |
| "rewards/rejected": -0.3583984375, |
| "step": 102 |
| }, |
| { |
| "entropy": 0.161865234375, |
| "epoch": 0.5241730279898219, |
| "grad_norm": 73.05874633789062, |
| "learning_rate": 2.4111675126903553e-06, |
| "logits/chosen": -1.2512428178163009, |
| "logits/rejected": -1.2793649019025604, |
| "logps/chosen": -232.5, |
| "logps/rejected": -254.5, |
| "loss": 0.8672332763671875, |
| "mean_token_accuracy": 0.9479686319828033, |
| "num_tokens": 38849315.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.39599609375, |
| "rewards/margins": -0.0771484375, |
| "rewards/rejected": -0.317626953125, |
| "step": 103 |
| }, |
| { |
| "entropy": 0.150634765625, |
| "epoch": 0.5292620865139949, |
| "grad_norm": 59.51718521118164, |
| "learning_rate": 2.385786802030457e-06, |
| "logits/chosen": -1.3055115561137152, |
| "logits/rejected": -1.2614773898545937, |
| "logps/chosen": -214.0, |
| "logps/rejected": -234.0, |
| "loss": 0.663818359375, |
| "mean_token_accuracy": 0.9556228965520859, |
| "num_tokens": 39246450.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.277099609375, |
| "rewards/margins": 0.1888427734375, |
| "rewards/rejected": -0.465087890625, |
| "step": 104 |
| }, |
| { |
| "entropy": 0.159423828125, |
| "epoch": 0.5343511450381679, |
| "grad_norm": 47.1342658996582, |
| "learning_rate": 2.3604060913705588e-06, |
| "logits/chosen": -1.349871941358015, |
| "logits/rejected": -1.3362314439717622, |
| "logps/chosen": -193.5, |
| "logps/rejected": -208.0, |
| "loss": 0.577880859375, |
| "mean_token_accuracy": 0.9497708082199097, |
| "num_tokens": 39635764.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0133056640625, |
| "rewards/margins": 0.435546875, |
| "rewards/rejected": -0.44970703125, |
| "step": 105 |
| }, |
| { |
| "entropy": 0.166748046875, |
| "epoch": 0.539440203562341, |
| "grad_norm": 58.260555267333984, |
| "learning_rate": 2.33502538071066e-06, |
| "logits/chosen": -1.3027896006955104, |
| "logits/rejected": -1.2992721109006562, |
| "logps/chosen": -216.75, |
| "logps/rejected": -191.75, |
| "loss": 0.73553466796875, |
| "mean_token_accuracy": 0.9484568685293198, |
| "num_tokens": 39969463.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.2978515625, |
| "rewards/margins": 0.049072265625, |
| "rewards/rejected": -0.347412109375, |
| "step": 106 |
| }, |
| { |
| "entropy": 0.16064453125, |
| "epoch": 0.544529262086514, |
| "grad_norm": 62.18130111694336, |
| "learning_rate": 2.3096446700507618e-06, |
| "logits/chosen": -1.2421911175554698, |
| "logits/rejected": -1.3233783198097135, |
| "logps/chosen": -200.5, |
| "logps/rejected": -207.75, |
| "loss": 0.804779052734375, |
| "mean_token_accuracy": 0.946388453245163, |
| "num_tokens": 40350675.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.3505859375, |
| "rewards/margins": -0.074981689453125, |
| "rewards/rejected": -0.275146484375, |
| "step": 107 |
| }, |
| { |
| "entropy": 0.159912109375, |
| "epoch": 0.549618320610687, |
| "grad_norm": 62.705894470214844, |
| "learning_rate": 2.284263959390863e-06, |
| "logits/chosen": -1.315684198825371, |
| "logits/rejected": -1.3131661685802256, |
| "logps/chosen": -188.5, |
| "logps/rejected": -195.375, |
| "loss": 0.77838134765625, |
| "mean_token_accuracy": 0.9471890181303024, |
| "num_tokens": 40724174.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.4033203125, |
| "rewards/margins": -0.06640625, |
| "rewards/rejected": -0.336669921875, |
| "step": 108 |
| }, |
| { |
| "entropy": 0.1669921875, |
| "epoch": 0.55470737913486, |
| "grad_norm": 66.83467102050781, |
| "learning_rate": 2.2588832487309648e-06, |
| "logits/chosen": -1.2883205988766404, |
| "logits/rejected": -1.308848106252747, |
| "logps/chosen": -258.0, |
| "logps/rejected": -261.25, |
| "loss": 0.7769012451171875, |
| "mean_token_accuracy": 0.9474173188209534, |
| "num_tokens": 41115013.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.186798095703125, |
| "rewards/margins": 0.14091873168945312, |
| "rewards/rejected": -0.327880859375, |
| "step": 109 |
| }, |
| { |
| "entropy": 0.163818359375, |
| "epoch": 0.5597964376590331, |
| "grad_norm": 67.90021514892578, |
| "learning_rate": 2.233502538071066e-06, |
| "logits/chosen": -1.2429459761599158, |
| "logits/rejected": -1.237265815477489, |
| "logps/chosen": -230.0, |
| "logps/rejected": -218.5, |
| "loss": 0.807220458984375, |
| "mean_token_accuracy": 0.9451711028814316, |
| "num_tokens": 41488642.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.345703125, |
| "rewards/margins": -0.065887451171875, |
| "rewards/rejected": -0.2799072265625, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.15771484375, |
| "epoch": 0.5648854961832062, |
| "grad_norm": 69.03768920898438, |
| "learning_rate": 2.2081218274111678e-06, |
| "logits/chosen": -1.262367353042372, |
| "logits/rejected": -1.2231653630458967, |
| "logps/chosen": -223.25, |
| "logps/rejected": -202.75, |
| "loss": 0.890869140625, |
| "mean_token_accuracy": 0.9496006816625595, |
| "num_tokens": 41863681.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.42919921875, |
| "rewards/margins": -0.21875, |
| "rewards/rejected": -0.2099609375, |
| "step": 111 |
| }, |
| { |
| "entropy": 0.1640625, |
| "epoch": 0.5699745547073791, |
| "grad_norm": 65.44580841064453, |
| "learning_rate": 2.182741116751269e-06, |
| "logits/chosen": -1.230319660623082, |
| "logits/rejected": -1.251510805200625, |
| "logps/chosen": -253.25, |
| "logps/rejected": -244.5, |
| "loss": 0.7418975830078125, |
| "mean_token_accuracy": 0.9484907984733582, |
| "num_tokens": 42189294.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.26123046875, |
| "rewards/margins": 0.162109375, |
| "rewards/rejected": -0.42236328125, |
| "step": 112 |
| }, |
| { |
| "entropy": 0.166015625, |
| "epoch": 0.5750636132315522, |
| "grad_norm": 56.990074157714844, |
| "learning_rate": 2.1573604060913707e-06, |
| "logits/chosen": -1.228292510885018, |
| "logits/rejected": -1.2380661145880048, |
| "logps/chosen": -188.75, |
| "logps/rejected": -210.25, |
| "loss": 0.7041778564453125, |
| "mean_token_accuracy": 0.9472708106040955, |
| "num_tokens": 42566983.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.39013671875, |
| "rewards/margins": 0.166259765625, |
| "rewards/rejected": -0.5556640625, |
| "step": 113 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.5801526717557252, |
| "grad_norm": 73.63758850097656, |
| "learning_rate": 2.1319796954314725e-06, |
| "logits/chosen": -1.2304216080221377, |
| "logits/rejected": -1.2626504818763358, |
| "logps/chosen": -242.75, |
| "logps/rejected": -261.25, |
| "loss": 0.784637451171875, |
| "mean_token_accuracy": 0.9489694982767105, |
| "num_tokens": 42913983.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.3817138671875, |
| "rewards/margins": 0.06597900390625, |
| "rewards/rejected": -0.4482421875, |
| "step": 114 |
| }, |
| { |
| "entropy": 0.161865234375, |
| "epoch": 0.5852417302798982, |
| "grad_norm": 62.614219665527344, |
| "learning_rate": 2.1065989847715737e-06, |
| "logits/chosen": -1.3461430935037932, |
| "logits/rejected": -1.3234544320772617, |
| "logps/chosen": -185.5, |
| "logps/rejected": -206.25, |
| "loss": 0.7844467163085938, |
| "mean_token_accuracy": 0.9493400007486343, |
| "num_tokens": 43279452.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.310546875, |
| "rewards/margins": 0.0008544921875, |
| "rewards/rejected": -0.3109893798828125, |
| "step": 115 |
| }, |
| { |
| "entropy": 0.166748046875, |
| "epoch": 0.5903307888040712, |
| "grad_norm": 73.2928695678711, |
| "learning_rate": 2.0812182741116755e-06, |
| "logits/chosen": -1.2476711193063883, |
| "logits/rejected": -1.229951835621045, |
| "logps/chosen": -241.75, |
| "logps/rejected": -224.25, |
| "loss": 0.85125732421875, |
| "mean_token_accuracy": 0.9468885958194733, |
| "num_tokens": 43674027.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.483642578125, |
| "rewards/margins": -0.1435546875, |
| "rewards/rejected": -0.3389892578125, |
| "step": 116 |
| }, |
| { |
| "entropy": 0.166259765625, |
| "epoch": 0.5954198473282443, |
| "grad_norm": 66.14990234375, |
| "learning_rate": 2.0558375634517767e-06, |
| "logits/chosen": -1.1910507876796015, |
| "logits/rejected": -1.1934789413274294, |
| "logps/chosen": -231.25, |
| "logps/rejected": -230.5, |
| "loss": 0.78106689453125, |
| "mean_token_accuracy": 0.950650081038475, |
| "num_tokens": 44047350.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.2755126953125, |
| "rewards/margins": 0.078369140625, |
| "rewards/rejected": -0.354248046875, |
| "step": 117 |
| }, |
| { |
| "entropy": 0.15478515625, |
| "epoch": 0.6005089058524173, |
| "grad_norm": 52.3381233215332, |
| "learning_rate": 2.0304568527918785e-06, |
| "logits/chosen": -1.2695932504330432, |
| "logits/rejected": -1.2736707511822316, |
| "logps/chosen": -181.75, |
| "logps/rejected": -203.25, |
| "loss": 0.661346435546875, |
| "mean_token_accuracy": 0.950680673122406, |
| "num_tokens": 44459928.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.265625, |
| "rewards/margins": 0.265625, |
| "rewards/rejected": -0.53076171875, |
| "step": 118 |
| }, |
| { |
| "entropy": 0.159912109375, |
| "epoch": 0.6055979643765903, |
| "grad_norm": 49.49362564086914, |
| "learning_rate": 2.0050761421319797e-06, |
| "logits/chosen": -1.316048034713927, |
| "logits/rejected": -1.3241544765179663, |
| "logps/chosen": -168.0, |
| "logps/rejected": -182.0, |
| "loss": 0.6475830078125, |
| "mean_token_accuracy": 0.9487863481044769, |
| "num_tokens": 44836514.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.25537109375, |
| "rewards/margins": 0.240570068359375, |
| "rewards/rejected": -0.49462890625, |
| "step": 119 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.6106870229007634, |
| "grad_norm": 74.30496215820312, |
| "learning_rate": 1.9796954314720814e-06, |
| "logits/chosen": -1.2736612451556142, |
| "logits/rejected": -1.2686709039286261, |
| "logps/chosen": -208.25, |
| "logps/rejected": -245.0, |
| "loss": 0.751373291015625, |
| "mean_token_accuracy": 0.9507658183574677, |
| "num_tokens": 45174916.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.353515625, |
| "rewards/margins": 0.05206298828125, |
| "rewards/rejected": -0.4052734375, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.1650390625, |
| "epoch": 0.6157760814249363, |
| "grad_norm": 64.3706283569336, |
| "learning_rate": 1.9543147208121827e-06, |
| "logits/chosen": -1.270622788271707, |
| "logits/rejected": -1.2759695510336173, |
| "logps/chosen": -203.0, |
| "logps/rejected": -201.0, |
| "loss": 0.756500244140625, |
| "mean_token_accuracy": 0.9468925297260284, |
| "num_tokens": 45574505.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1610107421875, |
| "rewards/margins": 0.05908203125, |
| "rewards/rejected": -0.220703125, |
| "step": 121 |
| }, |
| { |
| "entropy": 0.169921875, |
| "epoch": 0.6208651399491094, |
| "grad_norm": 85.26793670654297, |
| "learning_rate": 1.9289340101522844e-06, |
| "logits/chosen": -1.2705119265476728, |
| "logits/rejected": -1.2609238111160443, |
| "logps/chosen": -246.25, |
| "logps/rejected": -230.75, |
| "loss": 0.944061279296875, |
| "mean_token_accuracy": 0.9459521919488907, |
| "num_tokens": 45911388.0, |
| "rewards/accuracies": 0.25, |
| "rewards/chosen": -0.48193359375, |
| "rewards/margins": -0.3070068359375, |
| "rewards/rejected": -0.1748046875, |
| "step": 122 |
| }, |
| { |
| "entropy": 0.16064453125, |
| "epoch": 0.6259541984732825, |
| "grad_norm": 63.35386657714844, |
| "learning_rate": 1.9035532994923857e-06, |
| "logits/chosen": -1.1881255673183586, |
| "logits/rejected": -1.2287232149134593, |
| "logps/chosen": -229.25, |
| "logps/rejected": -229.25, |
| "loss": 0.6989974975585938, |
| "mean_token_accuracy": 0.949111670255661, |
| "num_tokens": 46263528.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.358184814453125, |
| "rewards/margins": 0.215087890625, |
| "rewards/rejected": -0.57421875, |
| "step": 123 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.6310432569974554, |
| "grad_norm": 63.35451126098633, |
| "learning_rate": 1.8781725888324874e-06, |
| "logits/chosen": -1.2589330922736834, |
| "logits/rejected": -1.2295728762675209, |
| "logps/chosen": -230.0, |
| "logps/rejected": -241.25, |
| "loss": 0.821014404296875, |
| "mean_token_accuracy": 0.9476511925458908, |
| "num_tokens": 46617770.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.27587890625, |
| "rewards/margins": 0.0260009765625, |
| "rewards/rejected": -0.302093505859375, |
| "step": 124 |
| }, |
| { |
| "entropy": 0.15673828125, |
| "epoch": 0.6361323155216285, |
| "grad_norm": 64.3207778930664, |
| "learning_rate": 1.852791878172589e-06, |
| "logits/chosen": -1.2831963765459964, |
| "logits/rejected": -1.2945043114039763, |
| "logps/chosen": -208.0, |
| "logps/rejected": -215.5, |
| "loss": 0.807525634765625, |
| "mean_token_accuracy": 0.9477943778038025, |
| "num_tokens": 46994298.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.33453369140625, |
| "rewards/margins": -0.073486328125, |
| "rewards/rejected": -0.260833740234375, |
| "step": 125 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.6412213740458015, |
| "grad_norm": 55.1900749206543, |
| "learning_rate": 1.8274111675126904e-06, |
| "logits/chosen": -1.2910271166985685, |
| "logits/rejected": -1.2747680953691005, |
| "logps/chosen": -183.75, |
| "logps/rejected": -211.0, |
| "loss": 0.7557106018066406, |
| "mean_token_accuracy": 0.9515034556388855, |
| "num_tokens": 47389706.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.15460205078125, |
| "rewards/margins": 0.17578125, |
| "rewards/rejected": -0.3310546875, |
| "step": 126 |
| }, |
| { |
| "entropy": 0.1728515625, |
| "epoch": 0.6463104325699746, |
| "grad_norm": 56.20667266845703, |
| "learning_rate": 1.8020304568527921e-06, |
| "logits/chosen": -1.2981477149706437, |
| "logits/rejected": -1.2675109223021184, |
| "logps/chosen": -222.0, |
| "logps/rejected": -232.5, |
| "loss": 0.598602294921875, |
| "mean_token_accuracy": 0.9497958421707153, |
| "num_tokens": 47781197.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.27587890625, |
| "rewards/margins": 0.317626953125, |
| "rewards/rejected": -0.59326171875, |
| "step": 127 |
| }, |
| { |
| "entropy": 0.156005859375, |
| "epoch": 0.6513994910941476, |
| "grad_norm": 56.498069763183594, |
| "learning_rate": 1.7766497461928936e-06, |
| "logits/chosen": -1.2623762821907927, |
| "logits/rejected": -1.274385917719584, |
| "logps/chosen": -212.0, |
| "logps/rejected": -233.75, |
| "loss": 0.679718017578125, |
| "mean_token_accuracy": 0.9534448385238647, |
| "num_tokens": 48205592.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.3408203125, |
| "rewards/margins": 0.1522216796875, |
| "rewards/rejected": -0.493408203125, |
| "step": 128 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.6564885496183206, |
| "grad_norm": 64.94985961914062, |
| "learning_rate": 1.7512690355329951e-06, |
| "logits/chosen": -1.2347368781188934, |
| "logits/rejected": -1.2598863054707647, |
| "logps/chosen": -224.0, |
| "logps/rejected": -214.25, |
| "loss": 0.808380126953125, |
| "mean_token_accuracy": 0.9472066313028336, |
| "num_tokens": 48537876.0, |
| "rewards/accuracies": 0.21875, |
| "rewards/chosen": -0.46728515625, |
| "rewards/margins": -0.0799560546875, |
| "rewards/rejected": -0.3883056640625, |
| "step": 129 |
| }, |
| { |
| "entropy": 0.16357421875, |
| "epoch": 0.6615776081424937, |
| "grad_norm": 74.20825958251953, |
| "learning_rate": 1.7258883248730964e-06, |
| "logits/chosen": -1.2947300208859263, |
| "logits/rejected": -1.2771773239258781, |
| "logps/chosen": -243.75, |
| "logps/rejected": -248.0, |
| "loss": 0.887298583984375, |
| "mean_token_accuracy": 0.9483547806739807, |
| "num_tokens": 48920092.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.42333984375, |
| "rewards/margins": -0.15228271484375, |
| "rewards/rejected": -0.2706298828125, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.169921875, |
| "epoch": 0.6666666666666666, |
| "grad_norm": 73.51078033447266, |
| "learning_rate": 1.7005076142131981e-06, |
| "logits/chosen": -1.2110933219118905, |
| "logits/rejected": -1.2308418207144254, |
| "logps/chosen": -235.75, |
| "logps/rejected": -248.75, |
| "loss": 0.8043212890625, |
| "mean_token_accuracy": 0.9493052661418915, |
| "num_tokens": 49305958.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.451171875, |
| "rewards/margins": -0.077392578125, |
| "rewards/rejected": -0.374267578125, |
| "step": 131 |
| }, |
| { |
| "entropy": 0.15869140625, |
| "epoch": 0.6717557251908397, |
| "grad_norm": 58.30695343017578, |
| "learning_rate": 1.6751269035532996e-06, |
| "logits/chosen": -1.3244597093243355, |
| "logits/rejected": -1.3132443195709225, |
| "logps/chosen": -206.0, |
| "logps/rejected": -228.0, |
| "loss": 0.668853759765625, |
| "mean_token_accuracy": 0.9519393891096115, |
| "num_tokens": 49693417.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.265869140625, |
| "rewards/margins": 0.1748046875, |
| "rewards/rejected": -0.43994140625, |
| "step": 132 |
| }, |
| { |
| "entropy": 0.160400390625, |
| "epoch": 0.6768447837150128, |
| "grad_norm": 53.58084487915039, |
| "learning_rate": 1.6497461928934011e-06, |
| "logits/chosen": -1.244864238209805, |
| "logits/rejected": -1.2545958025840016, |
| "logps/chosen": -183.625, |
| "logps/rejected": -190.375, |
| "loss": 0.700408935546875, |
| "mean_token_accuracy": 0.9522906541824341, |
| "num_tokens": 50059783.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.2659912109375, |
| "rewards/margins": 0.1514892578125, |
| "rewards/rejected": -0.4169921875, |
| "step": 133 |
| }, |
| { |
| "entropy": 0.164306640625, |
| "epoch": 0.6819338422391857, |
| "grad_norm": 68.5660171508789, |
| "learning_rate": 1.6243654822335026e-06, |
| "logits/chosen": -1.241397688917905, |
| "logits/rejected": -1.278434530141986, |
| "logps/chosen": -242.25, |
| "logps/rejected": -234.75, |
| "loss": 0.8605575561523438, |
| "mean_token_accuracy": 0.9462583363056183, |
| "num_tokens": 50413308.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.59716796875, |
| "rewards/margins": -0.11083984375, |
| "rewards/rejected": -0.48583984375, |
| "step": 134 |
| }, |
| { |
| "entropy": 0.164794921875, |
| "epoch": 0.6870229007633588, |
| "grad_norm": 64.64373016357422, |
| "learning_rate": 1.5989847715736043e-06, |
| "logits/chosen": -1.300048987756223, |
| "logits/rejected": -1.3131778176200615, |
| "logps/chosen": -192.75, |
| "logps/rejected": -190.875, |
| "loss": 0.850250244140625, |
| "mean_token_accuracy": 0.9482599049806595, |
| "num_tokens": 50756854.0, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.349609375, |
| "rewards/margins": -0.200927734375, |
| "rewards/rejected": -0.14849853515625, |
| "step": 135 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.6921119592875318, |
| "grad_norm": 60.40647506713867, |
| "learning_rate": 1.5736040609137056e-06, |
| "logits/chosen": -1.286467965109169, |
| "logits/rejected": -1.2765258255496543, |
| "logps/chosen": -209.25, |
| "logps/rejected": -207.25, |
| "loss": 0.673126220703125, |
| "mean_token_accuracy": 0.951656237244606, |
| "num_tokens": 51131502.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.367919921875, |
| "rewards/margins": 0.16021728515625, |
| "rewards/rejected": -0.52734375, |
| "step": 136 |
| }, |
| { |
| "entropy": 0.163330078125, |
| "epoch": 0.6972010178117048, |
| "grad_norm": 61.93556594848633, |
| "learning_rate": 1.5482233502538071e-06, |
| "logits/chosen": -1.3216607979661747, |
| "logits/rejected": -1.331168170680353, |
| "logps/chosen": -198.25, |
| "logps/rejected": -213.5, |
| "loss": 0.71380615234375, |
| "mean_token_accuracy": 0.9489306807518005, |
| "num_tokens": 51483225.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.2080078125, |
| "rewards/margins": 0.1612548828125, |
| "rewards/rejected": -0.36962890625, |
| "step": 137 |
| }, |
| { |
| "entropy": 0.159912109375, |
| "epoch": 0.7022900763358778, |
| "grad_norm": 63.112213134765625, |
| "learning_rate": 1.5228426395939088e-06, |
| "logits/chosen": -1.2680197798757142, |
| "logits/rejected": -1.2732023051613852, |
| "logps/chosen": -245.0, |
| "logps/rejected": -265.25, |
| "loss": 0.705902099609375, |
| "mean_token_accuracy": 0.950348436832428, |
| "num_tokens": 51849542.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.454833984375, |
| "rewards/margins": 0.133056640625, |
| "rewards/rejected": -0.58740234375, |
| "step": 138 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.7073791348600509, |
| "grad_norm": 65.93107604980469, |
| "learning_rate": 1.4974619289340103e-06, |
| "logits/chosen": -1.2531528664415568, |
| "logits/rejected": -1.2751016177936147, |
| "logps/chosen": -240.5, |
| "logps/rejected": -234.75, |
| "loss": 0.771026611328125, |
| "mean_token_accuracy": 0.9475905448198318, |
| "num_tokens": 52234352.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.31005859375, |
| "rewards/margins": 0.0687255859375, |
| "rewards/rejected": -0.378173828125, |
| "step": 139 |
| }, |
| { |
| "entropy": 0.16748046875, |
| "epoch": 0.712468193384224, |
| "grad_norm": 60.60008239746094, |
| "learning_rate": 1.4720812182741118e-06, |
| "logits/chosen": -1.284045350811356, |
| "logits/rejected": -1.3076317590229236, |
| "logps/chosen": -205.75, |
| "logps/rejected": -207.0, |
| "loss": 0.71966552734375, |
| "mean_token_accuracy": 0.9487140476703644, |
| "num_tokens": 52544764.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.2362060546875, |
| "rewards/margins": 0.08050537109375, |
| "rewards/rejected": -0.3160400390625, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.159423828125, |
| "epoch": 0.7175572519083969, |
| "grad_norm": 61.68688201904297, |
| "learning_rate": 1.4467005076142131e-06, |
| "logits/chosen": -1.2626397077576366, |
| "logits/rejected": -1.2523052221622069, |
| "logps/chosen": -207.5, |
| "logps/rejected": -230.5, |
| "loss": 0.716705322265625, |
| "mean_token_accuracy": 0.9503261744976044, |
| "num_tokens": 52948022.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.290771484375, |
| "rewards/margins": 0.0987548828125, |
| "rewards/rejected": -0.3899383544921875, |
| "step": 141 |
| }, |
| { |
| "entropy": 0.161865234375, |
| "epoch": 0.72264631043257, |
| "grad_norm": 63.84259796142578, |
| "learning_rate": 1.4213197969543148e-06, |
| "logits/chosen": -1.3211154042459343, |
| "logits/rejected": -1.324081790319996, |
| "logps/chosen": -241.5, |
| "logps/rejected": -234.25, |
| "loss": 0.76654052734375, |
| "mean_token_accuracy": 0.949401780962944, |
| "num_tokens": 53288797.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.40576171875, |
| "rewards/margins": 0.03076171875, |
| "rewards/rejected": -0.4375, |
| "step": 142 |
| }, |
| { |
| "entropy": 0.16064453125, |
| "epoch": 0.727735368956743, |
| "grad_norm": 59.44301223754883, |
| "learning_rate": 1.3959390862944163e-06, |
| "logits/chosen": -1.2445779844461344, |
| "logits/rejected": -1.2510412319436552, |
| "logps/chosen": -179.125, |
| "logps/rejected": -184.5, |
| "loss": 0.8189239501953125, |
| "mean_token_accuracy": 0.9503347277641296, |
| "num_tokens": 53650401.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.379638671875, |
| "rewards/margins": -0.0517578125, |
| "rewards/rejected": -0.327880859375, |
| "step": 143 |
| }, |
| { |
| "entropy": 0.160400390625, |
| "epoch": 0.732824427480916, |
| "grad_norm": 57.853416442871094, |
| "learning_rate": 1.3705583756345178e-06, |
| "logits/chosen": -1.2394903485408175, |
| "logits/rejected": -1.261283751051131, |
| "logps/chosen": -214.0, |
| "logps/rejected": -211.75, |
| "loss": 0.7115325927734375, |
| "mean_token_accuracy": 0.9477828443050385, |
| "num_tokens": 54054341.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.356689453125, |
| "rewards/margins": 0.125, |
| "rewards/rejected": -0.48193359375, |
| "step": 144 |
| }, |
| { |
| "entropy": 0.16455078125, |
| "epoch": 0.7379134860050891, |
| "grad_norm": 65.62645721435547, |
| "learning_rate": 1.3451776649746193e-06, |
| "logits/chosen": -1.2286020505571353, |
| "logits/rejected": -1.208450173191077, |
| "logps/chosen": -223.0, |
| "logps/rejected": -245.5, |
| "loss": 0.75048828125, |
| "mean_token_accuracy": 0.9515127539634705, |
| "num_tokens": 54427493.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.38330078125, |
| "rewards/margins": -0.000244140625, |
| "rewards/rejected": -0.38189697265625, |
| "step": 145 |
| }, |
| { |
| "entropy": 0.156494140625, |
| "epoch": 0.7430025445292621, |
| "grad_norm": 68.8692398071289, |
| "learning_rate": 1.319796954314721e-06, |
| "logits/chosen": -1.2827174915867314, |
| "logits/rejected": -1.290493482770122, |
| "logps/chosen": -218.25, |
| "logps/rejected": -206.75, |
| "loss": 0.8368377685546875, |
| "mean_token_accuracy": 0.9481632858514786, |
| "num_tokens": 54791174.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.33935546875, |
| "rewards/margins": -0.12493896484375, |
| "rewards/rejected": -0.2144775390625, |
| "step": 146 |
| }, |
| { |
| "entropy": 0.164306640625, |
| "epoch": 0.7480916030534351, |
| "grad_norm": 55.04671859741211, |
| "learning_rate": 1.2944162436548225e-06, |
| "logits/chosen": -1.2610715380959399, |
| "logits/rejected": -1.2777534346884782, |
| "logps/chosen": -166.5, |
| "logps/rejected": -155.25, |
| "loss": 0.7591552734375, |
| "mean_token_accuracy": 0.9474459439516068, |
| "num_tokens": 55190542.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.2154541015625, |
| "rewards/margins": -0.035186767578125, |
| "rewards/rejected": -0.179931640625, |
| "step": 147 |
| }, |
| { |
| "entropy": 0.15673828125, |
| "epoch": 0.7531806615776081, |
| "grad_norm": 64.17411804199219, |
| "learning_rate": 1.2690355329949238e-06, |
| "logits/chosen": -1.2668030115539317, |
| "logits/rejected": -1.2703246015537304, |
| "logps/chosen": -179.25, |
| "logps/rejected": -225.0, |
| "loss": 0.754150390625, |
| "mean_token_accuracy": 0.9535480886697769, |
| "num_tokens": 55568891.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.3115234375, |
| "rewards/margins": 0.02471923828125, |
| "rewards/rejected": -0.336669921875, |
| "step": 148 |
| }, |
| { |
| "entropy": 0.167236328125, |
| "epoch": 0.7582697201017812, |
| "grad_norm": 77.12625885009766, |
| "learning_rate": 1.2436548223350255e-06, |
| "logits/chosen": -1.246196790872148, |
| "logits/rejected": -1.2578459995620601, |
| "logps/chosen": -221.25, |
| "logps/rejected": -218.25, |
| "loss": 0.7427825927734375, |
| "mean_token_accuracy": 0.9458917081356049, |
| "num_tokens": 55926373.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1793212890625, |
| "rewards/margins": 0.071319580078125, |
| "rewards/rejected": -0.250244140625, |
| "step": 149 |
| }, |
| { |
| "entropy": 0.159912109375, |
| "epoch": 0.7633587786259542, |
| "grad_norm": 60.0277214050293, |
| "learning_rate": 1.218274111675127e-06, |
| "logits/chosen": -1.2921580835065267, |
| "logits/rejected": -1.2859186866957593, |
| "logps/chosen": -219.75, |
| "logps/rejected": -219.75, |
| "loss": 0.680755615234375, |
| "mean_token_accuracy": 0.9510093927383423, |
| "num_tokens": 56285700.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.2650146484375, |
| "rewards/margins": 0.1168212890625, |
| "rewards/rejected": -0.382568359375, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.167724609375, |
| "epoch": 0.7684478371501272, |
| "grad_norm": 63.86808395385742, |
| "learning_rate": 1.1928934010152285e-06, |
| "logits/chosen": -1.3243436768126475, |
| "logits/rejected": -1.289354653269572, |
| "logps/chosen": -237.0, |
| "logps/rejected": -260.25, |
| "loss": 0.7232666015625, |
| "mean_token_accuracy": 0.9495467245578766, |
| "num_tokens": 56701491.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.33740234375, |
| "rewards/margins": 0.0955810546875, |
| "rewards/rejected": -0.4326171875, |
| "step": 151 |
| }, |
| { |
| "entropy": 0.163818359375, |
| "epoch": 0.7735368956743003, |
| "grad_norm": 74.54528045654297, |
| "learning_rate": 1.16751269035533e-06, |
| "logits/chosen": -1.2510128515313985, |
| "logits/rejected": -1.2449032379965963, |
| "logps/chosen": -252.75, |
| "logps/rejected": -269.5, |
| "loss": 0.78497314453125, |
| "mean_token_accuracy": 0.9484715014696121, |
| "num_tokens": 57047974.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.259765625, |
| "rewards/margins": -0.009765625, |
| "rewards/rejected": -0.249755859375, |
| "step": 152 |
| }, |
| { |
| "entropy": 0.17041015625, |
| "epoch": 0.7786259541984732, |
| "grad_norm": 66.40087127685547, |
| "learning_rate": 1.1421319796954315e-06, |
| "logits/chosen": -1.2434997907611427, |
| "logits/rejected": -1.2419221284680912, |
| "logps/chosen": -233.5, |
| "logps/rejected": -222.25, |
| "loss": 0.744964599609375, |
| "mean_token_accuracy": 0.9437924772500992, |
| "num_tokens": 57392596.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.3751220703125, |
| "rewards/margins": 0.0340576171875, |
| "rewards/rejected": -0.40966796875, |
| "step": 153 |
| }, |
| { |
| "entropy": 0.164306640625, |
| "epoch": 0.7837150127226463, |
| "grad_norm": 53.339717864990234, |
| "learning_rate": 1.116751269035533e-06, |
| "logits/chosen": -1.235711641356627, |
| "logits/rejected": -1.2588020280263763, |
| "logps/chosen": -199.5, |
| "logps/rejected": -228.25, |
| "loss": 0.61297607421875, |
| "mean_token_accuracy": 0.9511121064424515, |
| "num_tokens": 57805475.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.190673828125, |
| "rewards/margins": 0.30908203125, |
| "rewards/rejected": -0.4990234375, |
| "step": 154 |
| }, |
| { |
| "entropy": 0.155517578125, |
| "epoch": 0.7888040712468194, |
| "grad_norm": 58.77641677856445, |
| "learning_rate": 1.0913705583756345e-06, |
| "logits/chosen": -1.2875096398514654, |
| "logits/rejected": -1.2338864169086345, |
| "logps/chosen": -192.25, |
| "logps/rejected": -199.75, |
| "loss": 0.72454833984375, |
| "mean_token_accuracy": 0.9548005163669586, |
| "num_tokens": 58174003.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.3203125, |
| "rewards/margins": 0.0784912109375, |
| "rewards/rejected": -0.3978271484375, |
| "step": 155 |
| }, |
| { |
| "entropy": 0.159912109375, |
| "epoch": 0.7938931297709924, |
| "grad_norm": 57.71940231323242, |
| "learning_rate": 1.0659898477157362e-06, |
| "logits/chosen": -1.2822042524364599, |
| "logits/rejected": -1.302401272896131, |
| "logps/chosen": -230.25, |
| "logps/rejected": -229.75, |
| "loss": 0.7072296142578125, |
| "mean_token_accuracy": 0.9481226801872253, |
| "num_tokens": 58513712.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.398193359375, |
| "rewards/margins": 0.172119140625, |
| "rewards/rejected": -0.5712890625, |
| "step": 156 |
| }, |
| { |
| "entropy": 0.167724609375, |
| "epoch": 0.7989821882951654, |
| "grad_norm": 58.8786506652832, |
| "learning_rate": 1.0406091370558377e-06, |
| "logits/chosen": -1.292047990535198, |
| "logits/rejected": -1.3054225527754584, |
| "logps/chosen": -210.0, |
| "logps/rejected": -242.25, |
| "loss": 0.66375732421875, |
| "mean_token_accuracy": 0.9478696882724762, |
| "num_tokens": 58896680.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.280303955078125, |
| "rewards/margins": 0.2080078125, |
| "rewards/rejected": -0.48974609375, |
| "step": 157 |
| }, |
| { |
| "entropy": 0.162841796875, |
| "epoch": 0.8040712468193384, |
| "grad_norm": 59.46760940551758, |
| "learning_rate": 1.0152284263959392e-06, |
| "logits/chosen": -1.24424356589235, |
| "logits/rejected": -1.238614479620221, |
| "logps/chosen": -221.75, |
| "logps/rejected": -230.0, |
| "loss": 0.7036838531494141, |
| "mean_token_accuracy": 0.9477695375680923, |
| "num_tokens": 59263462.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.1445770263671875, |
| "rewards/margins": 0.207275390625, |
| "rewards/rejected": -0.3521728515625, |
| "step": 158 |
| }, |
| { |
| "entropy": 0.162841796875, |
| "epoch": 0.8091603053435115, |
| "grad_norm": 65.75154113769531, |
| "learning_rate": 9.898477157360407e-07, |
| "logits/chosen": -1.2624473891183055, |
| "logits/rejected": -1.259654311209261, |
| "logps/chosen": -228.25, |
| "logps/rejected": -212.75, |
| "loss": 0.778594970703125, |
| "mean_token_accuracy": 0.944281741976738, |
| "num_tokens": 59633558.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.264617919921875, |
| "rewards/margins": -0.0078125, |
| "rewards/rejected": -0.2572021484375, |
| "step": 159 |
| }, |
| { |
| "entropy": 0.162841796875, |
| "epoch": 0.8142493638676844, |
| "grad_norm": 66.01615142822266, |
| "learning_rate": 9.644670050761422e-07, |
| "logits/chosen": -1.2357071070169634, |
| "logits/rejected": -1.2384324784738467, |
| "logps/chosen": -256.25, |
| "logps/rejected": -248.5, |
| "loss": 0.7286834716796875, |
| "mean_token_accuracy": 0.9499144405126572, |
| "num_tokens": 60016025.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.320068359375, |
| "rewards/margins": 0.130126953125, |
| "rewards/rejected": -0.449462890625, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.152587890625, |
| "epoch": 0.8193384223918575, |
| "grad_norm": 78.33038330078125, |
| "learning_rate": 9.390862944162437e-07, |
| "logits/chosen": -1.3029380474205265, |
| "logits/rejected": -1.3006044617540562, |
| "logps/chosen": -200.75, |
| "logps/rejected": -223.75, |
| "loss": 0.7779769897460938, |
| "mean_token_accuracy": 0.9547528177499771, |
| "num_tokens": 60444729.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.326904296875, |
| "rewards/margins": 0.07421875, |
| "rewards/rejected": -0.4010009765625, |
| "step": 161 |
| }, |
| { |
| "entropy": 0.161865234375, |
| "epoch": 0.8244274809160306, |
| "grad_norm": 63.31332778930664, |
| "learning_rate": 9.137055837563452e-07, |
| "logits/chosen": -1.2962333428603996, |
| "logits/rejected": -1.2906292243153006, |
| "logps/chosen": -203.75, |
| "logps/rejected": -227.5, |
| "loss": 0.84423828125, |
| "mean_token_accuracy": 0.9507206827402115, |
| "num_tokens": 60806152.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.3492431640625, |
| "rewards/margins": -0.097412109375, |
| "rewards/rejected": -0.2523193359375, |
| "step": 162 |
| }, |
| { |
| "entropy": 0.157470703125, |
| "epoch": 0.8295165394402035, |
| "grad_norm": 53.934791564941406, |
| "learning_rate": 8.883248730964468e-07, |
| "logits/chosen": -1.2821091434094367, |
| "logits/rejected": -1.2567944198943195, |
| "logps/chosen": -182.5, |
| "logps/rejected": -213.0, |
| "loss": 0.723388671875, |
| "mean_token_accuracy": 0.9540557265281677, |
| "num_tokens": 61201391.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.25140380859375, |
| "rewards/margins": 0.138916015625, |
| "rewards/rejected": -0.39111328125, |
| "step": 163 |
| }, |
| { |
| "entropy": 0.16064453125, |
| "epoch": 0.8346055979643766, |
| "grad_norm": 66.25235748291016, |
| "learning_rate": 8.629441624365482e-07, |
| "logits/chosen": -1.2868481912577527, |
| "logits/rejected": -1.3049520468495386, |
| "logps/chosen": -199.25, |
| "logps/rejected": -213.5, |
| "loss": 0.8221435546875, |
| "mean_token_accuracy": 0.9519907087087631, |
| "num_tokens": 61572941.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.24542236328125, |
| "rewards/margins": -0.0406494140625, |
| "rewards/rejected": -0.2047119140625, |
| "step": 164 |
| }, |
| { |
| "entropy": 0.1591796875, |
| "epoch": 0.8396946564885496, |
| "grad_norm": 58.21211624145508, |
| "learning_rate": 8.375634517766498e-07, |
| "logits/chosen": -1.3208748146171105, |
| "logits/rejected": -1.3522801461996157, |
| "logps/chosen": -193.75, |
| "logps/rejected": -242.5, |
| "loss": 0.65692138671875, |
| "mean_token_accuracy": 0.9525688141584396, |
| "num_tokens": 61950200.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.421875, |
| "rewards/margins": 0.1826171875, |
| "rewards/rejected": -0.60498046875, |
| "step": 165 |
| }, |
| { |
| "entropy": 0.156982421875, |
| "epoch": 0.8447837150127226, |
| "grad_norm": 66.12063598632812, |
| "learning_rate": 8.121827411167513e-07, |
| "logits/chosen": -1.301899044199523, |
| "logits/rejected": -1.2575513722113114, |
| "logps/chosen": -221.25, |
| "logps/rejected": -218.25, |
| "loss": 0.794952392578125, |
| "mean_token_accuracy": 0.9528547078371048, |
| "num_tokens": 62317017.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.30322265625, |
| "rewards/margins": -0.009185791015625, |
| "rewards/rejected": -0.2940673828125, |
| "step": 166 |
| }, |
| { |
| "entropy": 0.1611328125, |
| "epoch": 0.8498727735368957, |
| "grad_norm": 66.28915405273438, |
| "learning_rate": 7.868020304568528e-07, |
| "logits/chosen": -1.2902138038428934, |
| "logits/rejected": -1.280293306008564, |
| "logps/chosen": -230.5, |
| "logps/rejected": -249.25, |
| "loss": 0.7397384643554688, |
| "mean_token_accuracy": 0.9489684700965881, |
| "num_tokens": 62671349.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.3994140625, |
| "rewards/margins": 0.1123046875, |
| "rewards/rejected": -0.5123291015625, |
| "step": 167 |
| }, |
| { |
| "entropy": 0.150634765625, |
| "epoch": 0.8549618320610687, |
| "grad_norm": 75.95428466796875, |
| "learning_rate": 7.614213197969544e-07, |
| "logits/chosen": -1.2779956306537894, |
| "logits/rejected": -1.2907999663209873, |
| "logps/chosen": -184.5, |
| "logps/rejected": -218.0, |
| "loss": 0.876434326171875, |
| "mean_token_accuracy": 0.9518905580043793, |
| "num_tokens": 63052689.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.338134765625, |
| "rewards/margins": -0.1540069580078125, |
| "rewards/rejected": -0.18463134765625, |
| "step": 168 |
| }, |
| { |
| "entropy": 0.1669921875, |
| "epoch": 0.8600508905852418, |
| "grad_norm": 63.13179016113281, |
| "learning_rate": 7.360406091370559e-07, |
| "logits/chosen": -1.2887472754751863, |
| "logits/rejected": -1.2670022003966717, |
| "logps/chosen": -214.75, |
| "logps/rejected": -228.25, |
| "loss": 0.78338623046875, |
| "mean_token_accuracy": 0.9487744569778442, |
| "num_tokens": 63405602.0, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.312744140625, |
| "rewards/margins": -0.020751953125, |
| "rewards/rejected": -0.2926025390625, |
| "step": 169 |
| }, |
| { |
| "entropy": 0.166748046875, |
| "epoch": 0.8651399491094147, |
| "grad_norm": 69.41549682617188, |
| "learning_rate": 7.106598984771574e-07, |
| "logits/chosen": -1.2627873517587107, |
| "logits/rejected": -1.2781351035738362, |
| "logps/chosen": -219.5, |
| "logps/rejected": -236.25, |
| "loss": 0.81103515625, |
| "mean_token_accuracy": 0.9494354575872421, |
| "num_tokens": 63792124.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.3935546875, |
| "rewards/margins": -0.071044921875, |
| "rewards/rejected": -0.322509765625, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.8702290076335878, |
| "grad_norm": 59.073123931884766, |
| "learning_rate": 6.852791878172589e-07, |
| "logits/chosen": -1.291131682486795, |
| "logits/rejected": -1.3237879333881308, |
| "logps/chosen": -203.5, |
| "logps/rejected": -223.5, |
| "loss": 0.6531982421875, |
| "mean_token_accuracy": 0.9505943953990936, |
| "num_tokens": 64155911.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.1978759765625, |
| "rewards/margins": 0.2900390625, |
| "rewards/rejected": -0.48779296875, |
| "step": 171 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.8753180661577609, |
| "grad_norm": 59.49985885620117, |
| "learning_rate": 6.598984771573605e-07, |
| "logits/chosen": -1.2140906585743236, |
| "logits/rejected": -1.1976295247955515, |
| "logps/chosen": -200.0, |
| "logps/rejected": -205.25, |
| "loss": 0.706451416015625, |
| "mean_token_accuracy": 0.9494357705116272, |
| "num_tokens": 64532812.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.234375, |
| "rewards/margins": 0.100830078125, |
| "rewards/rejected": -0.33447265625, |
| "step": 172 |
| }, |
| { |
| "entropy": 0.152587890625, |
| "epoch": 0.8804071246819338, |
| "grad_norm": 52.89236831665039, |
| "learning_rate": 6.345177664974619e-07, |
| "logits/chosen": -1.3571618690737455, |
| "logits/rejected": -1.3450652315884195, |
| "logps/chosen": -174.625, |
| "logps/rejected": -204.25, |
| "loss": 0.59918212890625, |
| "mean_token_accuracy": 0.9549228101968765, |
| "num_tokens": 64943462.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.1142578125, |
| "rewards/margins": 0.2861328125, |
| "rewards/rejected": -0.400390625, |
| "step": 173 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.8854961832061069, |
| "grad_norm": 63.51266098022461, |
| "learning_rate": 6.091370558375635e-07, |
| "logits/chosen": -1.2747322101681697, |
| "logits/rejected": -1.278634245927865, |
| "logps/chosen": -250.25, |
| "logps/rejected": -267.0, |
| "loss": 0.6451339721679688, |
| "mean_token_accuracy": 0.9491883218288422, |
| "num_tokens": 65286854.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.319091796875, |
| "rewards/margins": 0.359130859375, |
| "rewards/rejected": -0.67626953125, |
| "step": 174 |
| }, |
| { |
| "entropy": 0.156494140625, |
| "epoch": 0.8905852417302799, |
| "grad_norm": 54.96915817260742, |
| "learning_rate": 5.83756345177665e-07, |
| "logits/chosen": -1.265227401251292, |
| "logits/rejected": -1.2394450329297166, |
| "logps/chosen": -169.0, |
| "logps/rejected": -184.375, |
| "loss": 0.73223876953125, |
| "mean_token_accuracy": 0.9530220776796341, |
| "num_tokens": 65705185.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.24835205078125, |
| "rewards/margins": 0.1209716796875, |
| "rewards/rejected": -0.368408203125, |
| "step": 175 |
| }, |
| { |
| "entropy": 0.1611328125, |
| "epoch": 0.8956743002544529, |
| "grad_norm": 72.51837158203125, |
| "learning_rate": 5.583756345177665e-07, |
| "logits/chosen": -1.291017567652124, |
| "logits/rejected": -1.3191793802079457, |
| "logps/chosen": -260.75, |
| "logps/rejected": -237.25, |
| "loss": 0.888946533203125, |
| "mean_token_accuracy": 0.9461717158555984, |
| "num_tokens": 66041900.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.342071533203125, |
| "rewards/margins": -0.199951171875, |
| "rewards/rejected": -0.14239501953125, |
| "step": 176 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.9007633587786259, |
| "grad_norm": 63.78479766845703, |
| "learning_rate": 5.329949238578681e-07, |
| "logits/chosen": -1.2883858909839578, |
| "logits/rejected": -1.2964681724098532, |
| "logps/chosen": -227.5, |
| "logps/rejected": -206.25, |
| "loss": 0.831298828125, |
| "mean_token_accuracy": 0.9456988126039505, |
| "num_tokens": 66397264.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.43017578125, |
| "rewards/margins": -0.098876953125, |
| "rewards/rejected": -0.33203125, |
| "step": 177 |
| }, |
| { |
| "entropy": 0.16552734375, |
| "epoch": 0.905852417302799, |
| "grad_norm": 64.87358093261719, |
| "learning_rate": 5.076142131979696e-07, |
| "logits/chosen": -1.2762111021043254, |
| "logits/rejected": -1.277499721225321, |
| "logps/chosen": -215.25, |
| "logps/rejected": -232.0, |
| "loss": 0.82025146484375, |
| "mean_token_accuracy": 0.9479079395532608, |
| "num_tokens": 66746118.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.335205078125, |
| "rewards/margins": -0.015380859375, |
| "rewards/rejected": -0.320556640625, |
| "step": 178 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.910941475826972, |
| "grad_norm": 63.49721145629883, |
| "learning_rate": 4.822335025380711e-07, |
| "logits/chosen": -1.2594111699857853, |
| "logits/rejected": -1.2441778402503918, |
| "logps/chosen": -203.75, |
| "logps/rejected": -218.0, |
| "loss": 0.7030181884765625, |
| "mean_token_accuracy": 0.949649915099144, |
| "num_tokens": 67123472.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.20211029052734375, |
| "rewards/margins": 0.1322021484375, |
| "rewards/rejected": -0.333984375, |
| "step": 179 |
| }, |
| { |
| "entropy": 0.16650390625, |
| "epoch": 0.916030534351145, |
| "grad_norm": 76.09004974365234, |
| "learning_rate": 4.568527918781726e-07, |
| "logits/chosen": -1.286043293513197, |
| "logits/rejected": -1.2636617381085613, |
| "logps/chosen": -236.25, |
| "logps/rejected": -225.0, |
| "loss": 0.9158935546875, |
| "mean_token_accuracy": 0.9456801265478134, |
| "num_tokens": 67465986.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.5185546875, |
| "rewards/margins": -0.237213134765625, |
| "rewards/rejected": -0.281982421875, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.1552734375, |
| "epoch": 0.9211195928753181, |
| "grad_norm": 63.31582260131836, |
| "learning_rate": 4.314720812182741e-07, |
| "logits/chosen": -1.251146523022114, |
| "logits/rejected": -1.2975083912016157, |
| "logps/chosen": -198.25, |
| "logps/rejected": -203.75, |
| "loss": 0.7587127685546875, |
| "mean_token_accuracy": 0.9466626644134521, |
| "num_tokens": 67864018.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.345458984375, |
| "rewards/margins": 0.0047607421875, |
| "rewards/rejected": -0.349609375, |
| "step": 181 |
| }, |
| { |
| "entropy": 0.153564453125, |
| "epoch": 0.926208651399491, |
| "grad_norm": 54.91245651245117, |
| "learning_rate": 4.0609137055837566e-07, |
| "logits/chosen": -1.2824599495241058, |
| "logits/rejected": -1.2501124767523026, |
| "logps/chosen": -194.0, |
| "logps/rejected": -206.5, |
| "loss": 0.65789794921875, |
| "mean_token_accuracy": 0.9545126259326935, |
| "num_tokens": 68229219.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.26611328125, |
| "rewards/margins": 0.213134765625, |
| "rewards/rejected": -0.47900390625, |
| "step": 182 |
| }, |
| { |
| "entropy": 0.165771484375, |
| "epoch": 0.9312977099236641, |
| "grad_norm": 60.69124221801758, |
| "learning_rate": 3.807106598984772e-07, |
| "logits/chosen": -1.2306977794006666, |
| "logits/rejected": -1.2199483238128237, |
| "logps/chosen": -212.5, |
| "logps/rejected": -250.25, |
| "loss": 0.7246932983398438, |
| "mean_token_accuracy": 0.9500255733728409, |
| "num_tokens": 68606870.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.113128662109375, |
| "rewards/margins": 0.128448486328125, |
| "rewards/rejected": -0.24212646484375, |
| "step": 183 |
| }, |
| { |
| "entropy": 0.162353515625, |
| "epoch": 0.9363867684478372, |
| "grad_norm": 64.28182983398438, |
| "learning_rate": 3.553299492385787e-07, |
| "logits/chosen": -1.2587565773941798, |
| "logits/rejected": -1.2312963361066895, |
| "logps/chosen": -212.5, |
| "logps/rejected": -213.5, |
| "loss": 0.64691162109375, |
| "mean_token_accuracy": 0.9500741362571716, |
| "num_tokens": 68978668.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.16424560546875, |
| "rewards/margins": 0.3056640625, |
| "rewards/rejected": -0.47021484375, |
| "step": 184 |
| }, |
| { |
| "entropy": 0.166015625, |
| "epoch": 0.9414758269720102, |
| "grad_norm": 57.6767692565918, |
| "learning_rate": 3.2994923857868026e-07, |
| "logits/chosen": -1.2751947302167712, |
| "logits/rejected": -1.2666428366190272, |
| "logps/chosen": -235.0, |
| "logps/rejected": -263.5, |
| "loss": 0.6388931274414062, |
| "mean_token_accuracy": 0.9481093883514404, |
| "num_tokens": 69372058.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.0687255859375, |
| "rewards/margins": 0.2685546875, |
| "rewards/rejected": -0.3369140625, |
| "step": 185 |
| }, |
| { |
| "entropy": 0.172607421875, |
| "epoch": 0.9465648854961832, |
| "grad_norm": 60.594364166259766, |
| "learning_rate": 3.0456852791878176e-07, |
| "logits/chosen": -1.2357604930632382, |
| "logits/rejected": -1.2510668876440354, |
| "logps/chosen": -225.25, |
| "logps/rejected": -216.875, |
| "loss": 0.702392578125, |
| "mean_token_accuracy": 0.9432621896266937, |
| "num_tokens": 69753477.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.2430419921875, |
| "rewards/margins": 0.139495849609375, |
| "rewards/rejected": -0.3818359375, |
| "step": 186 |
| }, |
| { |
| "entropy": 0.1611328125, |
| "epoch": 0.9516539440203562, |
| "grad_norm": 63.02968215942383, |
| "learning_rate": 2.7918781725888326e-07, |
| "logits/chosen": -1.2820828380810267, |
| "logits/rejected": -1.3018126825693557, |
| "logps/chosen": -237.0, |
| "logps/rejected": -264.5, |
| "loss": 0.7537841796875, |
| "mean_token_accuracy": 0.9488907307386398, |
| "num_tokens": 70144262.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.1806640625, |
| "rewards/margins": 0.0987548828125, |
| "rewards/rejected": -0.280029296875, |
| "step": 187 |
| }, |
| { |
| "entropy": 0.157470703125, |
| "epoch": 0.9567430025445293, |
| "grad_norm": 62.73226547241211, |
| "learning_rate": 2.538071065989848e-07, |
| "logits/chosen": -1.3243303259905526, |
| "logits/rejected": -1.3251137517496847, |
| "logps/chosen": -177.125, |
| "logps/rejected": -189.5, |
| "loss": 0.8604354858398438, |
| "mean_token_accuracy": 0.9508524388074875, |
| "num_tokens": 70522847.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.4222412109375, |
| "rewards/margins": -0.1492919921875, |
| "rewards/rejected": -0.272705078125, |
| "step": 188 |
| }, |
| { |
| "entropy": 0.158935546875, |
| "epoch": 0.9618320610687023, |
| "grad_norm": 58.20997619628906, |
| "learning_rate": 2.284263959390863e-07, |
| "logits/chosen": -1.3108909963701287, |
| "logits/rejected": -1.307712402955017, |
| "logps/chosen": -180.125, |
| "logps/rejected": -224.0, |
| "loss": 0.732025146484375, |
| "mean_token_accuracy": 0.9533163607120514, |
| "num_tokens": 70940314.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.1817626953125, |
| "rewards/margins": 0.1083984375, |
| "rewards/rejected": -0.2891845703125, |
| "step": 189 |
| }, |
| { |
| "entropy": 0.156982421875, |
| "epoch": 0.9669211195928753, |
| "grad_norm": 64.7269287109375, |
| "learning_rate": 2.0304568527918783e-07, |
| "logits/chosen": -1.3086706742163743, |
| "logits/rejected": -1.3384005897848779, |
| "logps/chosen": -246.75, |
| "logps/rejected": -256.75, |
| "loss": 0.761688232421875, |
| "mean_token_accuracy": 0.9508141130208969, |
| "num_tokens": 71285208.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.1785888671875, |
| "rewards/margins": 0.101318359375, |
| "rewards/rejected": -0.279296875, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.158935546875, |
| "epoch": 0.9720101781170484, |
| "grad_norm": 76.1530990600586, |
| "learning_rate": 1.7766497461928935e-07, |
| "logits/chosen": -1.2052610815729892, |
| "logits/rejected": -1.2051475881810254, |
| "logps/chosen": -245.0, |
| "logps/rejected": -249.25, |
| "loss": 0.84283447265625, |
| "mean_token_accuracy": 0.9478296190500259, |
| "num_tokens": 71692156.0, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.44970703125, |
| "rewards/margins": -0.0787353515625, |
| "rewards/rejected": -0.37158203125, |
| "step": 191 |
| }, |
| { |
| "entropy": 0.153564453125, |
| "epoch": 0.9770992366412213, |
| "grad_norm": 49.5556640625, |
| "learning_rate": 1.5228426395939088e-07, |
| "logits/chosen": -1.2858759972786784, |
| "logits/rejected": -1.3171012184101922, |
| "logps/chosen": -206.0, |
| "logps/rejected": -227.5, |
| "loss": 0.5926513671875, |
| "mean_token_accuracy": 0.9544648975133896, |
| "num_tokens": 72062381.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.1572265625, |
| "rewards/margins": 0.33837890625, |
| "rewards/rejected": -0.49658203125, |
| "step": 192 |
| }, |
| { |
| "entropy": 0.163818359375, |
| "epoch": 0.9821882951653944, |
| "grad_norm": 55.3388671875, |
| "learning_rate": 1.269035532994924e-07, |
| "logits/chosen": -1.2850746114469256, |
| "logits/rejected": -1.2788264737681292, |
| "logps/chosen": -187.75, |
| "logps/rejected": -178.75, |
| "loss": 0.7740325927734375, |
| "mean_token_accuracy": 0.9483982175588608, |
| "num_tokens": 72459408.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.284423828125, |
| "rewards/margins": -0.025634765625, |
| "rewards/rejected": -0.25927734375, |
| "step": 193 |
| }, |
| { |
| "entropy": 0.16015625, |
| "epoch": 0.9872773536895675, |
| "grad_norm": 62.76466751098633, |
| "learning_rate": 1.0152284263959391e-07, |
| "logits/chosen": -1.2936464840174513, |
| "logits/rejected": -1.3121440655724232, |
| "logps/chosen": -232.25, |
| "logps/rejected": -268.25, |
| "loss": 0.710235595703125, |
| "mean_token_accuracy": 0.952901765704155, |
| "num_tokens": 72853047.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.21728515625, |
| "rewards/margins": 0.15203857421875, |
| "rewards/rejected": -0.36962890625, |
| "step": 194 |
| }, |
| { |
| "entropy": 0.1572265625, |
| "epoch": 0.9923664122137404, |
| "grad_norm": 53.645687103271484, |
| "learning_rate": 7.614213197969544e-08, |
| "logits/chosen": -1.2712459537508989, |
| "logits/rejected": -1.234670637217532, |
| "logps/chosen": -163.875, |
| "logps/rejected": -177.5, |
| "loss": 0.69317626953125, |
| "mean_token_accuracy": 0.9523515999317169, |
| "num_tokens": 73277603.0, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.1658935546875, |
| "rewards/margins": 0.06201171875, |
| "rewards/rejected": -0.22802734375, |
| "step": 195 |
| }, |
| { |
| "entropy": 0.166015625, |
| "epoch": 0.9974554707379135, |
| "grad_norm": 65.67327880859375, |
| "learning_rate": 5.076142131979696e-08, |
| "logits/chosen": -1.2582488581214466, |
| "logits/rejected": -1.2718061289438034, |
| "logps/chosen": -210.0, |
| "logps/rejected": -215.5, |
| "loss": 0.739013671875, |
| "mean_token_accuracy": 0.9474593698978424, |
| "num_tokens": 73625261.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.23590087890625, |
| "rewards/margins": 0.0318603515625, |
| "rewards/rejected": -0.267578125, |
| "step": 196 |
| }, |
| { |
| "entropy": 0.15283203125, |
| "epoch": 1.0, |
| "grad_norm": 91.96894836425781, |
| "learning_rate": 2.538071065989848e-08, |
| "logits/chosen": -1.2407954088557835, |
| "logits/rejected": -1.2307852017793066, |
| "logps/chosen": -217.5, |
| "logps/rejected": -237.0, |
| "loss": 0.7696762084960938, |
| "mean_token_accuracy": 0.9491404592990875, |
| "num_tokens": 73770526.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.31298828125, |
| "rewards/margins": 0.15869140625, |
| "rewards/rejected": -0.470703125, |
| "step": 197 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 197, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.5608290532420223e+19, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
