diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,7251 @@ +{ + "best_global_step": 300, + "best_metric": 0.51461613, + "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt10_noidk__pari0.5_0602_p2prompt/v3-20250603-112113/checkpoint-300", + "epoch": 1.0075447972335743, + "eval_steps": 300, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002514932411191449, + "grad_norm": 7.572820663452148, + "learning_rate": 3.3333333333333333e-06, + "logits/chosen": -0.5129972100257874, + "logits/rejected": -0.6397510766983032, + "logps/chosen": -8.847090721130371, + "logps/rejected": -18.28310203552246, + "loss": 1.2403231859207153, + "memory(GiB)": 30.59, + "nll_loss": 0.5471758246421814, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1, + "train_speed(iter/s)": 0.010604 + }, + { + "epoch": 0.005029864822382898, + "grad_norm": 7.455193042755127, + "learning_rate": 6.666666666666667e-06, + "logits/chosen": -0.5209798812866211, + "logits/rejected": -0.6239324808120728, + "logps/chosen": -4.766471862792969, + "logps/rejected": -15.138076782226562, + "loss": 1.065555453300476, + "memory(GiB)": 33.8, + "nll_loss": 0.37240836024284363, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2, + "train_speed(iter/s)": 0.01094 + }, + { + "epoch": 0.007544797233574348, + "grad_norm": 10.782440185546875, + "learning_rate": 1e-05, + "logits/chosen": -0.4890531897544861, + "logits/rejected": -0.6535542011260986, + "logps/chosen": -4.760848522186279, + "logps/rejected": -17.92791175842285, + "loss": 1.0910710096359253, + "memory(GiB)": 33.8, + "nll_loss": 0.3938094973564148, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0034015527926385403, + "rewards/margins": -0.008013413287699223, + "rewards/rejected": 0.004611859563738108, + "step": 3, + "train_speed(iter/s)": 0.011055 + }, + { + "epoch": 0.010059729644765796, + "grad_norm": 13.736228942871094, + "learning_rate": 1.3333333333333333e-05, + "logits/chosen": -0.4501972496509552, + "logits/rejected": -0.6398910880088806, + "logps/chosen": -3.6690049171447754, + "logps/rejected": -21.246660232543945, + "loss": 1.0828931331634521, + "memory(GiB)": 37.26, + "nll_loss": 0.39956799149513245, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.006963790860027075, + "rewards/margins": 0.019915670156478882, + "rewards/rejected": -0.012951879762113094, + "step": 4, + "train_speed(iter/s)": 0.011095 + }, + { + "epoch": 0.012574662055957246, + "grad_norm": 8.89594554901123, + "learning_rate": 1.6666666666666667e-05, + "logits/chosen": -0.41603705286979675, + "logits/rejected": -0.5863696336746216, + "logps/chosen": -8.551430702209473, + "logps/rejected": -19.92246437072754, + "loss": 1.1882262229919434, + "memory(GiB)": 37.26, + "nll_loss": 0.5108650922775269, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.014113119803369045, + "rewards/margins": 0.03240397199988365, + "rewards/rejected": -0.018290851265192032, + "step": 5, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.015089594467148696, + "grad_norm": 5.3424577713012695, + "learning_rate": 2e-05, + "logits/chosen": -0.4721744656562805, + "logits/rejected": -0.5799455642700195, + "logps/chosen": -9.535165786743164, + "logps/rejected": -19.043678283691406, + "loss": 1.188051462173462, + "memory(GiB)": 37.26, + "nll_loss": 0.524101972579956, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.010117664933204651, + "rewards/margins": 0.0616740845143795, + "rewards/rejected": -0.05155642703175545, + "step": 6, + "train_speed(iter/s)": 0.011145 + }, + { + "epoch": 0.017604526878340146, + "grad_norm": 3.5404231548309326, + "learning_rate": 2.3333333333333336e-05, + "logits/chosen": -0.4064752459526062, + "logits/rejected": -0.5387755036354065, + "logps/chosen": -9.426046371459961, + "logps/rejected": -24.705936431884766, + "loss": 1.1507773399353027, + "memory(GiB)": 40.75, + "nll_loss": 0.5126925706863403, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.0260818712413311, + "rewards/margins": 0.1276150643825531, + "rewards/rejected": -0.1536969393491745, + "step": 7, + "train_speed(iter/s)": 0.011146 + }, + { + "epoch": 0.020119459289531592, + "grad_norm": 4.785594463348389, + "learning_rate": 2.6666666666666667e-05, + "logits/chosen": -0.40693986415863037, + "logits/rejected": -0.5652169585227966, + "logps/chosen": -5.92374849319458, + "logps/rejected": -20.89351463317871, + "loss": 1.02293860912323, + "memory(GiB)": 40.75, + "nll_loss": 0.39562517404556274, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.05045217275619507, + "rewards/margins": 0.1817830502986908, + "rewards/rejected": -0.23223522305488586, + "step": 8, + "train_speed(iter/s)": 0.011156 + }, + { + "epoch": 0.022634391700723042, + "grad_norm": 4.752041816711426, + "learning_rate": 3e-05, + "logits/chosen": -0.3253605365753174, + "logits/rejected": -0.5100574493408203, + "logps/chosen": -5.970067024230957, + "logps/rejected": -27.08715057373047, + "loss": 0.8663768768310547, + "memory(GiB)": 40.75, + "nll_loss": 0.36089974641799927, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04699152708053589, + "rewards/margins": 0.49801987409591675, + "rewards/rejected": -0.45102834701538086, + "step": 9, + "train_speed(iter/s)": 0.011163 + }, + { + "epoch": 0.025149324111914492, + "grad_norm": 5.771737575531006, + "learning_rate": 3.3333333333333335e-05, + "logits/chosen": -0.4344920516014099, + "logits/rejected": -0.5643672347068787, + "logps/chosen": -6.490581035614014, + "logps/rejected": -19.547592163085938, + "loss": 1.0153567790985107, + "memory(GiB)": 40.75, + "nll_loss": 0.429390549659729, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.19264209270477295, + "rewards/margins": 0.43160852789878845, + "rewards/rejected": -0.6242506504058838, + "step": 10, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.027664256523105942, + "grad_norm": 5.376907825469971, + "learning_rate": 3.6666666666666666e-05, + "logits/chosen": -0.4512503147125244, + "logits/rejected": -0.6003657579421997, + "logps/chosen": -6.500467300415039, + "logps/rejected": -27.486557006835938, + "loss": 0.9551109075546265, + "memory(GiB)": 40.75, + "nll_loss": 0.40139278769493103, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12439044564962387, + "rewards/margins": 0.5762982368469238, + "rewards/rejected": -0.7006886601448059, + "step": 11, + "train_speed(iter/s)": 0.011187 + }, + { + "epoch": 0.030179188934297392, + "grad_norm": 3.0979809761047363, + "learning_rate": 4e-05, + "logits/chosen": -0.4187595844268799, + "logits/rejected": -0.5968605279922485, + "logps/chosen": -4.248518466949463, + "logps/rejected": -26.590652465820312, + "loss": 0.7605504393577576, + "memory(GiB)": 40.75, + "nll_loss": 0.2799570858478546, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.14420613646507263, + "rewards/margins": 0.6546018123626709, + "rewards/rejected": -0.7988079190254211, + "step": 12, + "train_speed(iter/s)": 0.011196 + }, + { + "epoch": 0.03269412134548884, + "grad_norm": 9.02223014831543, + "learning_rate": 4.3333333333333334e-05, + "logits/chosen": -0.40893471240997314, + "logits/rejected": -0.5400319695472717, + "logps/chosen": -7.627485275268555, + "logps/rejected": -21.307247161865234, + "loss": 1.2244255542755127, + "memory(GiB)": 40.75, + "nll_loss": 0.6750409007072449, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.2602284550666809, + "rewards/margins": 0.49883735179901123, + "rewards/rejected": -0.7590658664703369, + "step": 13, + "train_speed(iter/s)": 0.011204 + }, + { + "epoch": 0.03520905375668029, + "grad_norm": 5.483772277832031, + "learning_rate": 4.666666666666667e-05, + "logits/chosen": -0.5219076871871948, + "logits/rejected": -0.593062698841095, + "logps/chosen": -8.136625289916992, + "logps/rejected": -17.90158462524414, + "loss": 1.2523046731948853, + "memory(GiB)": 40.75, + "nll_loss": 0.6594831347465515, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.14057493209838867, + "rewards/margins": 0.4059276878833771, + "rewards/rejected": -0.5465026497840881, + "step": 14, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.03772398616787174, + "grad_norm": 5.3392133712768555, + "learning_rate": 5e-05, + "logits/chosen": -0.40230512619018555, + "logits/rejected": -0.587448000907898, + "logps/chosen": -3.3861958980560303, + "logps/rejected": -22.72595977783203, + "loss": 0.8488077521324158, + "memory(GiB)": 40.75, + "nll_loss": 0.4007309377193451, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.02581140398979187, + "rewards/margins": 0.6975324153900146, + "rewards/rejected": -0.7233438491821289, + "step": 15, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.040238918579063185, + "grad_norm": 3.3262414932250977, + "learning_rate": 5.333333333333333e-05, + "logits/chosen": -0.342201292514801, + "logits/rejected": -0.575752854347229, + "logps/chosen": -3.893523693084717, + "logps/rejected": -28.40308952331543, + "loss": 0.8233633637428284, + "memory(GiB)": 40.75, + "nll_loss": 0.3312762379646301, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.03883545473217964, + "rewards/margins": 0.5127238035202026, + "rewards/rejected": -0.4738883376121521, + "step": 16, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.04275385099025464, + "grad_norm": 2.462789297103882, + "learning_rate": 5.666666666666667e-05, + "logits/chosen": -0.49846047163009644, + "logits/rejected": -0.6083415746688843, + "logps/chosen": -1.8747515678405762, + "logps/rejected": -16.662185668945312, + "loss": 0.7689712047576904, + "memory(GiB)": 40.75, + "nll_loss": 0.238669753074646, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.041000571101903915, + "rewards/margins": 0.40174293518066406, + "rewards/rejected": -0.36074236035346985, + "step": 17, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.045268783401446085, + "grad_norm": 4.245744228363037, + "learning_rate": 6e-05, + "logits/chosen": -0.4577159881591797, + "logits/rejected": -0.5291006565093994, + "logps/chosen": -8.686376571655273, + "logps/rejected": -17.176931381225586, + "loss": 1.2110098600387573, + "memory(GiB)": 40.75, + "nll_loss": 0.5645009875297546, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.049589186906814575, + "rewards/margins": 0.15553997457027435, + "rewards/rejected": -0.20512914657592773, + "step": 18, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.04778371581263754, + "grad_norm": 3.633544683456421, + "learning_rate": 6.333333333333333e-05, + "logits/chosen": -0.4796344041824341, + "logits/rejected": -0.5480799674987793, + "logps/chosen": -10.090361595153809, + "logps/rejected": -17.440088272094727, + "loss": 1.1647558212280273, + "memory(GiB)": 40.75, + "nll_loss": 0.5513034462928772, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.004148788750171661, + "rewards/margins": 0.22297149896621704, + "rewards/rejected": -0.2271203100681305, + "step": 19, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.050298648223828984, + "grad_norm": 3.275191307067871, + "learning_rate": 6.666666666666667e-05, + "logits/chosen": -0.46574562788009644, + "logits/rejected": -0.5685222744941711, + "logps/chosen": -5.4855055809021, + "logps/rejected": -15.27071762084961, + "loss": 1.0493377447128296, + "memory(GiB)": 40.75, + "nll_loss": 0.40973347425460815, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.02530045434832573, + "rewards/margins": 0.1835480034351349, + "rewards/rejected": -0.15824756026268005, + "step": 20, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.05281358063502043, + "grad_norm": 2.346269130706787, + "learning_rate": 7e-05, + "logits/chosen": -0.5085228085517883, + "logits/rejected": -0.6115283966064453, + "logps/chosen": -8.632377624511719, + "logps/rejected": -18.418798446655273, + "loss": 1.1822807788848877, + "memory(GiB)": 40.75, + "nll_loss": 0.6134262084960938, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.07014060765504837, + "rewards/margins": 0.30156341195106506, + "rewards/rejected": -0.2314227819442749, + "step": 21, + "train_speed(iter/s)": 0.011245 + }, + { + "epoch": 0.055328513046211884, + "grad_norm": 3.8027472496032715, + "learning_rate": 7.333333333333333e-05, + "logits/chosen": -0.4623318314552307, + "logits/rejected": -0.5286604762077332, + "logps/chosen": -4.330241680145264, + "logps/rejected": -15.440506935119629, + "loss": 0.9398723840713501, + "memory(GiB)": 40.75, + "nll_loss": 0.32075199484825134, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.009936274029314518, + "rewards/margins": 0.20203867554664612, + "rewards/rejected": -0.21197494864463806, + "step": 22, + "train_speed(iter/s)": 0.011248 + }, + { + "epoch": 0.05784344545740333, + "grad_norm": 2.3595120906829834, + "learning_rate": 7.666666666666667e-05, + "logits/chosen": -0.49782079458236694, + "logits/rejected": -0.5459281206130981, + "logps/chosen": -5.645054817199707, + "logps/rejected": -15.544858932495117, + "loss": 0.9763747453689575, + "memory(GiB)": 40.75, + "nll_loss": 0.3531709313392639, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.011355478316545486, + "rewards/margins": 0.17947493493556976, + "rewards/rejected": -0.16811946034431458, + "step": 23, + "train_speed(iter/s)": 0.011252 + }, + { + "epoch": 0.060358377868594784, + "grad_norm": 2.5567357540130615, + "learning_rate": 8e-05, + "logits/chosen": -0.52140212059021, + "logits/rejected": -0.5767900943756104, + "logps/chosen": -9.158100128173828, + "logps/rejected": -18.047792434692383, + "loss": 1.1764006614685059, + "memory(GiB)": 40.75, + "nll_loss": 0.6046724915504456, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.03944522514939308, + "rewards/margins": 0.2968653440475464, + "rewards/rejected": -0.2574201226234436, + "step": 24, + "train_speed(iter/s)": 0.011255 + }, + { + "epoch": 0.06287331027978624, + "grad_norm": 3.3667352199554443, + "learning_rate": 8.333333333333334e-05, + "logits/chosen": -0.4414227604866028, + "logits/rejected": -0.5774338245391846, + "logps/chosen": -5.001893997192383, + "logps/rejected": -18.086332321166992, + "loss": 1.0027401447296143, + "memory(GiB)": 40.75, + "nll_loss": 0.40850552916526794, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.01537637785077095, + "rewards/margins": 0.2611372172832489, + "rewards/rejected": -0.24576085805892944, + "step": 25, + "train_speed(iter/s)": 0.011255 + }, + { + "epoch": 0.06538824269097768, + "grad_norm": 2.4368274211883545, + "learning_rate": 8.666666666666667e-05, + "logits/chosen": -0.33819591999053955, + "logits/rejected": -0.5409271121025085, + "logps/chosen": -11.725374221801758, + "logps/rejected": -26.261959075927734, + "loss": 1.0086052417755127, + "memory(GiB)": 40.75, + "nll_loss": 0.5157173275947571, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07034377753734589, + "rewards/margins": 0.5667710900306702, + "rewards/rejected": -0.49642735719680786, + "step": 26, + "train_speed(iter/s)": 0.011254 + }, + { + "epoch": 0.06790317510216913, + "grad_norm": 4.366164207458496, + "learning_rate": 9e-05, + "logits/chosen": -0.3777789771556854, + "logits/rejected": -0.5889321565628052, + "logps/chosen": -5.34947395324707, + "logps/rejected": -27.725887298583984, + "loss": 0.8724238872528076, + "memory(GiB)": 40.75, + "nll_loss": 0.39934495091438293, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.022416604682803154, + "rewards/margins": 0.6145756244659424, + "rewards/rejected": -0.6369922161102295, + "step": 27, + "train_speed(iter/s)": 0.011254 + }, + { + "epoch": 0.07041810751336058, + "grad_norm": 3.8030436038970947, + "learning_rate": 9.333333333333334e-05, + "logits/chosen": -0.43299004435539246, + "logits/rejected": -0.5541425347328186, + "logps/chosen": -8.235193252563477, + "logps/rejected": -26.827539443969727, + "loss": 0.874066174030304, + "memory(GiB)": 40.75, + "nll_loss": 0.45102763175964355, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07945333421230316, + "rewards/margins": 0.8036540746688843, + "rewards/rejected": -0.8831074237823486, + "step": 28, + "train_speed(iter/s)": 0.011253 + }, + { + "epoch": 0.07293303992455202, + "grad_norm": 3.0242788791656494, + "learning_rate": 9.666666666666667e-05, + "logits/chosen": -0.37702634930610657, + "logits/rejected": -0.5286097526550293, + "logps/chosen": -6.507118225097656, + "logps/rejected": -25.659378051757812, + "loss": 0.9697012901306152, + "memory(GiB)": 40.75, + "nll_loss": 0.4811092019081116, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.04141651839017868, + "rewards/margins": 0.5634151101112366, + "rewards/rejected": -0.6048315763473511, + "step": 29, + "train_speed(iter/s)": 0.011253 + }, + { + "epoch": 0.07544797233574348, + "grad_norm": 2.460881233215332, + "learning_rate": 0.0001, + "logits/chosen": -0.371891051530838, + "logits/rejected": -0.45985037088394165, + "logps/chosen": -6.48842716217041, + "logps/rejected": -18.697708129882812, + "loss": 0.9460075497627258, + "memory(GiB)": 40.75, + "nll_loss": 0.3780769407749176, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.0511014498770237, + "rewards/margins": 0.3350966274738312, + "rewards/rejected": -0.2839951515197754, + "step": 30, + "train_speed(iter/s)": 0.011254 + }, + { + "epoch": 0.07796290474693493, + "grad_norm": 3.6821117401123047, + "learning_rate": 0.00010333333333333334, + "logits/chosen": -0.39938536286354065, + "logits/rejected": -0.49178823828697205, + "logps/chosen": -6.468450546264648, + "logps/rejected": -18.927385330200195, + "loss": 1.019504189491272, + "memory(GiB)": 40.75, + "nll_loss": 0.5059184432029724, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.0709109753370285, + "rewards/margins": 0.5467783808708191, + "rewards/rejected": -0.4758673906326294, + "step": 31, + "train_speed(iter/s)": 0.011256 + }, + { + "epoch": 0.08047783715812637, + "grad_norm": 2.8029754161834717, + "learning_rate": 0.00010666666666666667, + "logits/chosen": -0.38790467381477356, + "logits/rejected": -0.4925430417060852, + "logps/chosen": -5.981967926025391, + "logps/rejected": -28.314722061157227, + "loss": 0.9189813733100891, + "memory(GiB)": 40.75, + "nll_loss": 0.3903328776359558, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08847282826900482, + "rewards/margins": 0.501548171043396, + "rewards/rejected": -0.41307532787323, + "step": 32, + "train_speed(iter/s)": 0.011256 + }, + { + "epoch": 0.08299276956931782, + "grad_norm": 3.7524354457855225, + "learning_rate": 0.00011000000000000002, + "logits/chosen": -0.3702802360057831, + "logits/rejected": -0.4830480217933655, + "logps/chosen": -6.820221424102783, + "logps/rejected": -18.442974090576172, + "loss": 0.9390460252761841, + "memory(GiB)": 40.75, + "nll_loss": 0.4203004837036133, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.05472969263792038, + "rewards/margins": 0.4696621000766754, + "rewards/rejected": -0.41493239998817444, + "step": 33, + "train_speed(iter/s)": 0.011257 + }, + { + "epoch": 0.08550770198050928, + "grad_norm": 2.968419075012207, + "learning_rate": 0.00011333333333333334, + "logits/chosen": -0.35177576541900635, + "logits/rejected": -0.45535844564437866, + "logps/chosen": -4.456936836242676, + "logps/rejected": -22.066486358642578, + "loss": 0.8263772130012512, + "memory(GiB)": 40.75, + "nll_loss": 0.35278084874153137, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.009354054927825928, + "rewards/margins": 0.7917211651802063, + "rewards/rejected": -0.8010752201080322, + "step": 34, + "train_speed(iter/s)": 0.011259 + }, + { + "epoch": 0.08802263439170073, + "grad_norm": 2.9763481616973877, + "learning_rate": 0.00011666666666666668, + "logits/chosen": -0.2393473982810974, + "logits/rejected": -0.4172413945198059, + "logps/chosen": -4.113461017608643, + "logps/rejected": -30.265304565429688, + "loss": 0.8524877429008484, + "memory(GiB)": 40.75, + "nll_loss": 0.471011221408844, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.05692119151353836, + "rewards/margins": 1.161907434463501, + "rewards/rejected": -1.1049861907958984, + "step": 35, + "train_speed(iter/s)": 0.011258 + }, + { + "epoch": 0.09053756680289217, + "grad_norm": 2.4537270069122314, + "learning_rate": 0.00012, + "logits/chosen": -0.276807963848114, + "logits/rejected": -0.41199547052383423, + "logps/chosen": -7.090043544769287, + "logps/rejected": -29.689010620117188, + "loss": 0.7714070081710815, + "memory(GiB)": 40.75, + "nll_loss": 0.3793071210384369, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.036474838852882385, + "rewards/margins": 1.0788801908493042, + "rewards/rejected": -1.0424052476882935, + "step": 36, + "train_speed(iter/s)": 0.011257 + }, + { + "epoch": 0.09305249921408362, + "grad_norm": 3.29215145111084, + "learning_rate": 0.00012333333333333334, + "logits/chosen": -0.3102665841579437, + "logits/rejected": -0.4546661078929901, + "logps/chosen": -5.750801086425781, + "logps/rejected": -31.45724868774414, + "loss": 0.7538890838623047, + "memory(GiB)": 40.75, + "nll_loss": 0.4176124334335327, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1214592307806015, + "rewards/margins": 1.2956981658935547, + "rewards/rejected": -1.4171574115753174, + "step": 37, + "train_speed(iter/s)": 0.011259 + }, + { + "epoch": 0.09556743162527508, + "grad_norm": 3.8826475143432617, + "learning_rate": 0.00012666666666666666, + "logits/chosen": -0.37785372138023376, + "logits/rejected": -0.45433154702186584, + "logps/chosen": -8.914945602416992, + "logps/rejected": -22.28145408630371, + "loss": 0.9718711972236633, + "memory(GiB)": 40.75, + "nll_loss": 0.542230486869812, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05876921862363815, + "rewards/margins": 0.9656041860580444, + "rewards/rejected": -1.0243734121322632, + "step": 38, + "train_speed(iter/s)": 0.01126 + }, + { + "epoch": 0.09808236403646652, + "grad_norm": 10.463335990905762, + "learning_rate": 0.00013000000000000002, + "logits/chosen": -0.40348339080810547, + "logits/rejected": -0.4396521747112274, + "logps/chosen": -9.546710014343262, + "logps/rejected": -24.998611450195312, + "loss": 1.1912176609039307, + "memory(GiB)": 40.75, + "nll_loss": 0.6404471397399902, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.18508824706077576, + "rewards/margins": 0.8297311663627625, + "rewards/rejected": -1.0148195028305054, + "step": 39, + "train_speed(iter/s)": 0.01126 + }, + { + "epoch": 0.10059729644765797, + "grad_norm": 3.4194676876068115, + "learning_rate": 0.00013333333333333334, + "logits/chosen": -0.36826181411743164, + "logits/rejected": -0.5072036981582642, + "logps/chosen": -6.227029800415039, + "logps/rejected": -28.42563247680664, + "loss": 0.7728609442710876, + "memory(GiB)": 40.75, + "nll_loss": 0.3573758006095886, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.021011125296354294, + "rewards/margins": 1.0436646938323975, + "rewards/rejected": -1.0226534605026245, + "step": 40, + "train_speed(iter/s)": 0.011261 + }, + { + "epoch": 0.10311222885884942, + "grad_norm": 4.902315139770508, + "learning_rate": 0.00013666666666666666, + "logits/chosen": -0.3183041512966156, + "logits/rejected": -0.48007485270500183, + "logps/chosen": -9.719247817993164, + "logps/rejected": -22.464027404785156, + "loss": 1.06601881980896, + "memory(GiB)": 40.75, + "nll_loss": 0.5293514728546143, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.04249940067529678, + "rewards/margins": 0.45808011293411255, + "rewards/rejected": -0.4155806601047516, + "step": 41, + "train_speed(iter/s)": 0.01126 + }, + { + "epoch": 0.10562716127004086, + "grad_norm": 5.45733118057251, + "learning_rate": 0.00014, + "logits/chosen": -0.3853970468044281, + "logits/rejected": -0.4931572675704956, + "logps/chosen": -8.472291946411133, + "logps/rejected": -24.512069702148438, + "loss": 0.943773627281189, + "memory(GiB)": 40.75, + "nll_loss": 0.4035080075263977, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07248181104660034, + "rewards/margins": 0.5888606309890747, + "rewards/rejected": -0.661342442035675, + "step": 42, + "train_speed(iter/s)": 0.011261 + }, + { + "epoch": 0.10814209368123232, + "grad_norm": 3.7701737880706787, + "learning_rate": 0.00014333333333333334, + "logits/chosen": -0.24171334505081177, + "logits/rejected": -0.46347737312316895, + "logps/chosen": -3.810516119003296, + "logps/rejected": -28.03646469116211, + "loss": 0.7043237090110779, + "memory(GiB)": 40.75, + "nll_loss": 0.33659031987190247, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10358152538537979, + "rewards/margins": 1.084468960762024, + "rewards/rejected": -0.9808874726295471, + "step": 43, + "train_speed(iter/s)": 0.01126 + }, + { + "epoch": 0.11065702609242377, + "grad_norm": 5.19516658782959, + "learning_rate": 0.00014666666666666666, + "logits/chosen": -0.3297630548477173, + "logits/rejected": -0.4781489074230194, + "logps/chosen": -8.038228988647461, + "logps/rejected": -33.8675651550293, + "loss": 0.8617865443229675, + "memory(GiB)": 40.75, + "nll_loss": 0.5089200139045715, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.21755161881446838, + "rewards/margins": 1.4295934438705444, + "rewards/rejected": -1.6471449136734009, + "step": 44, + "train_speed(iter/s)": 0.01126 + }, + { + "epoch": 0.11317195850361522, + "grad_norm": 4.779160499572754, + "learning_rate": 0.00015000000000000001, + "logits/chosen": -0.49006256461143494, + "logits/rejected": -0.5443361401557922, + "logps/chosen": -9.670989990234375, + "logps/rejected": -27.758705139160156, + "loss": 0.9880771636962891, + "memory(GiB)": 40.75, + "nll_loss": 0.5687355995178223, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21813175082206726, + "rewards/margins": 1.2758864164352417, + "rewards/rejected": -1.4940180778503418, + "step": 45, + "train_speed(iter/s)": 0.011261 + }, + { + "epoch": 0.11568689091480666, + "grad_norm": 4.207818031311035, + "learning_rate": 0.00015333333333333334, + "logits/chosen": -0.3747759461402893, + "logits/rejected": -0.5024475455284119, + "logps/chosen": -6.3994646072387695, + "logps/rejected": -32.430179595947266, + "loss": 0.9543445706367493, + "memory(GiB)": 40.75, + "nll_loss": 0.5410714745521545, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.1253054440021515, + "rewards/margins": 1.216636300086975, + "rewards/rejected": -1.3419415950775146, + "step": 46, + "train_speed(iter/s)": 0.011262 + }, + { + "epoch": 0.11820182332599811, + "grad_norm": 2.187269926071167, + "learning_rate": 0.00015666666666666666, + "logits/chosen": -0.34373798966407776, + "logits/rejected": -0.5492506623268127, + "logps/chosen": -5.171298027038574, + "logps/rejected": -34.377967834472656, + "loss": 0.5840844511985779, + "memory(GiB)": 40.75, + "nll_loss": 0.2608174979686737, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04855790734291077, + "rewards/margins": 1.4605152606964111, + "rewards/rejected": -1.4119572639465332, + "step": 47, + "train_speed(iter/s)": 0.011263 + }, + { + "epoch": 0.12071675573718957, + "grad_norm": 2.492182970046997, + "learning_rate": 0.00016, + "logits/chosen": -0.3911299705505371, + "logits/rejected": -0.5427611470222473, + "logps/chosen": -5.3708953857421875, + "logps/rejected": -30.956531524658203, + "loss": 0.7430465221405029, + "memory(GiB)": 40.75, + "nll_loss": 0.3517543077468872, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.04663639888167381, + "rewards/margins": 1.1671051979064941, + "rewards/rejected": -1.2137415409088135, + "step": 48, + "train_speed(iter/s)": 0.011264 + }, + { + "epoch": 0.12323168814838101, + "grad_norm": 2.454068422317505, + "learning_rate": 0.00016333333333333334, + "logits/chosen": -0.30930572748184204, + "logits/rejected": -0.5388137698173523, + "logps/chosen": -4.821499824523926, + "logps/rejected": -33.31132125854492, + "loss": 0.6669176816940308, + "memory(GiB)": 40.75, + "nll_loss": 0.34143078327178955, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09115171432495117, + "rewards/margins": 1.3478808403015137, + "rewards/rejected": -1.2567291259765625, + "step": 49, + "train_speed(iter/s)": 0.011264 + }, + { + "epoch": 0.12574662055957248, + "grad_norm": 2.7763614654541016, + "learning_rate": 0.0001666666666666667, + "logits/chosen": -0.22302384674549103, + "logits/rejected": -0.476222962141037, + "logps/chosen": -6.5321455001831055, + "logps/rejected": -33.70460891723633, + "loss": 0.8022267818450928, + "memory(GiB)": 40.75, + "nll_loss": 0.3984905481338501, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.011320816352963448, + "rewards/margins": 1.2645773887634277, + "rewards/rejected": -1.2532566785812378, + "step": 50, + "train_speed(iter/s)": 0.011263 + }, + { + "epoch": 0.12826155297076391, + "grad_norm": 3.5810701847076416, + "learning_rate": 0.00017, + "logits/chosen": -0.2979552447795868, + "logits/rejected": -0.4668833017349243, + "logps/chosen": -7.627035140991211, + "logps/rejected": -35.17571258544922, + "loss": 0.8107961416244507, + "memory(GiB)": 40.75, + "nll_loss": 0.4284512996673584, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06179499253630638, + "rewards/margins": 1.205071210861206, + "rewards/rejected": -1.2668663263320923, + "step": 51, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.13077648538195535, + "grad_norm": 2.9454948902130127, + "learning_rate": 0.00017333333333333334, + "logits/chosen": -0.2851395010948181, + "logits/rejected": -0.4914030134677887, + "logps/chosen": -6.73934268951416, + "logps/rejected": -44.49215316772461, + "loss": 0.7192223072052002, + "memory(GiB)": 40.75, + "nll_loss": 0.3857932686805725, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06120564043521881, + "rewards/margins": 2.360339403152466, + "rewards/rejected": -2.4215452671051025, + "step": 52, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.13329141779314682, + "grad_norm": 2.907097339630127, + "learning_rate": 0.00017666666666666666, + "logits/chosen": -0.29824766516685486, + "logits/rejected": -0.5210357904434204, + "logps/chosen": -8.840283393859863, + "logps/rejected": -34.15322494506836, + "loss": 0.9132320284843445, + "memory(GiB)": 40.75, + "nll_loss": 0.5277525782585144, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.013686132617294788, + "rewards/margins": 1.6544121503829956, + "rewards/rejected": -1.6680983304977417, + "step": 53, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.13580635020433826, + "grad_norm": 3.2579548358917236, + "learning_rate": 0.00018, + "logits/chosen": -0.3622821867465973, + "logits/rejected": -0.49751946330070496, + "logps/chosen": -9.77501106262207, + "logps/rejected": -34.95624542236328, + "loss": 0.8283704519271851, + "memory(GiB)": 40.75, + "nll_loss": 0.478503942489624, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04811567813158035, + "rewards/margins": 1.6710915565490723, + "rewards/rejected": -1.6229758262634277, + "step": 54, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.1383212826155297, + "grad_norm": 6.844382286071777, + "learning_rate": 0.00018333333333333334, + "logits/chosen": -0.34049201011657715, + "logits/rejected": -0.5361011028289795, + "logps/chosen": -4.566088676452637, + "logps/rejected": -41.054141998291016, + "loss": 0.7934082746505737, + "memory(GiB)": 40.75, + "nll_loss": 0.4864325225353241, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.022840898483991623, + "rewards/margins": 2.574941873550415, + "rewards/rejected": -2.597783088684082, + "step": 55, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.14083621502672117, + "grad_norm": 5.409123420715332, + "learning_rate": 0.0001866666666666667, + "logits/chosen": -0.3723406195640564, + "logits/rejected": -0.4861374497413635, + "logps/chosen": -11.017836570739746, + "logps/rejected": -34.19987487792969, + "loss": 1.1561821699142456, + "memory(GiB)": 40.75, + "nll_loss": 0.7329697608947754, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.1480638086795807, + "rewards/margins": 2.0217599868774414, + "rewards/rejected": -2.16982364654541, + "step": 56, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.1433511474379126, + "grad_norm": 7.563216686248779, + "learning_rate": 0.00019, + "logits/chosen": -0.32556983828544617, + "logits/rejected": -0.5365452170372009, + "logps/chosen": -5.449405193328857, + "logps/rejected": -46.6372184753418, + "loss": 0.8106161952018738, + "memory(GiB)": 40.75, + "nll_loss": 0.5291056632995605, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.11195089668035507, + "rewards/margins": 2.898953676223755, + "rewards/rejected": -3.010904312133789, + "step": 57, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.14586607984910405, + "grad_norm": 9.61186408996582, + "learning_rate": 0.00019333333333333333, + "logits/chosen": -0.4079486131668091, + "logits/rejected": -0.5931285619735718, + "logps/chosen": -3.3833391666412354, + "logps/rejected": -41.95534896850586, + "loss": 0.559334397315979, + "memory(GiB)": 40.75, + "nll_loss": 0.30327463150024414, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04168446362018585, + "rewards/margins": 2.6080174446105957, + "rewards/rejected": -2.566333055496216, + "step": 58, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.14838101226029551, + "grad_norm": 3.3261170387268066, + "learning_rate": 0.00019666666666666666, + "logits/chosen": -0.4187636971473694, + "logits/rejected": -0.4730520248413086, + "logps/chosen": -8.343189239501953, + "logps/rejected": -25.519798278808594, + "loss": 1.058987021446228, + "memory(GiB)": 40.75, + "nll_loss": 0.5353435277938843, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0768323540687561, + "rewards/margins": 1.170108437538147, + "rewards/rejected": -1.2469407320022583, + "step": 59, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.15089594467148695, + "grad_norm": 3.7475709915161133, + "learning_rate": 0.0002, + "logits/chosen": -0.31574761867523193, + "logits/rejected": -0.4878937005996704, + "logps/chosen": -6.4234299659729, + "logps/rejected": -30.35470199584961, + "loss": 0.7999802827835083, + "memory(GiB)": 40.75, + "nll_loss": 0.3874775171279907, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.019132789224386215, + "rewards/margins": 1.3630902767181396, + "rewards/rejected": -1.3439574241638184, + "step": 60, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.1534108770826784, + "grad_norm": 2.8922085762023926, + "learning_rate": 0.0001999996142159566, + "logits/chosen": -0.35342201590538025, + "logits/rejected": -0.4222828149795532, + "logps/chosen": -7.674136638641357, + "logps/rejected": -26.560787200927734, + "loss": 0.7506588101387024, + "memory(GiB)": 40.75, + "nll_loss": 0.3475904166698456, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04072082042694092, + "rewards/margins": 1.29744291305542, + "rewards/rejected": -1.2567219734191895, + "step": 61, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.15592580949386986, + "grad_norm": 2.781667947769165, + "learning_rate": 0.0001999984568668029, + "logits/chosen": -0.33968424797058105, + "logits/rejected": -0.45871520042419434, + "logps/chosen": -3.5645596981048584, + "logps/rejected": -33.30908203125, + "loss": 0.6515002250671387, + "memory(GiB)": 40.75, + "nll_loss": 0.28380468487739563, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.10595045238733292, + "rewards/margins": 1.6635351181030273, + "rewards/rejected": -1.5575847625732422, + "step": 62, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.1584407419050613, + "grad_norm": 3.0299599170684814, + "learning_rate": 0.00019999652796146876, + "logits/chosen": -0.3050180971622467, + "logits/rejected": -0.4909503161907196, + "logps/chosen": -7.468794822692871, + "logps/rejected": -40.68026351928711, + "loss": 0.7424418330192566, + "memory(GiB)": 40.75, + "nll_loss": 0.41286778450012207, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.06561660766601562, + "rewards/margins": 1.9888997077941895, + "rewards/rejected": -1.9232829809188843, + "step": 63, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.16095567431625274, + "grad_norm": 2.0761804580688477, + "learning_rate": 0.0001999938275148369, + "logits/chosen": -0.33227968215942383, + "logits/rejected": -0.45812830328941345, + "logps/chosen": -4.157924652099609, + "logps/rejected": -40.23651123046875, + "loss": 0.40325966477394104, + "memory(GiB)": 40.75, + "nll_loss": 0.19691286981105804, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.10368794947862625, + "rewards/margins": 2.3141493797302246, + "rewards/rejected": -2.417837619781494, + "step": 64, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.1634706067274442, + "grad_norm": 3.9792428016662598, + "learning_rate": 0.00019999035554774314, + "logits/chosen": -0.3644486367702484, + "logits/rejected": -0.5156970024108887, + "logps/chosen": -5.496757984161377, + "logps/rejected": -43.13388442993164, + "loss": 0.759586751461029, + "memory(GiB)": 40.75, + "nll_loss": 0.5011360049247742, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.13431251049041748, + "rewards/margins": 2.4990766048431396, + "rewards/rejected": -2.6333892345428467, + "step": 65, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.16598553913863565, + "grad_norm": 3.7544219493865967, + "learning_rate": 0.00019998611208697607, + "logits/chosen": -0.3731898367404938, + "logits/rejected": -0.5181675553321838, + "logps/chosen": -4.577354431152344, + "logps/rejected": -36.74962615966797, + "loss": 0.7680931091308594, + "memory(GiB)": 40.75, + "nll_loss": 0.4070277214050293, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09029161185026169, + "rewards/margins": 1.9024454355239868, + "rewards/rejected": -1.992737054824829, + "step": 66, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.16850047154982709, + "grad_norm": 2.9526925086975098, + "learning_rate": 0.00019998109716527686, + "logits/chosen": -0.247542604804039, + "logits/rejected": -0.45712605118751526, + "logps/chosen": -3.374553918838501, + "logps/rejected": -43.25963592529297, + "loss": 0.49172505736351013, + "memory(GiB)": 40.75, + "nll_loss": 0.22025981545448303, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04232815280556679, + "rewards/margins": 2.575462579727173, + "rewards/rejected": -2.5331344604492188, + "step": 67, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.17101540396101855, + "grad_norm": 5.039420127868652, + "learning_rate": 0.00019997531082133904, + "logits/chosen": -0.3098013997077942, + "logits/rejected": -0.42491626739501953, + "logps/chosen": -7.849802494049072, + "logps/rejected": -31.89962387084961, + "loss": 1.0089820623397827, + "memory(GiB)": 40.75, + "nll_loss": 0.5222876071929932, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.09253531694412231, + "rewards/margins": 1.4570719003677368, + "rewards/rejected": -1.549607276916504, + "step": 68, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.17353033637221, + "grad_norm": 2.4941506385803223, + "learning_rate": 0.00019996875309980826, + "logits/chosen": -0.33015602827072144, + "logits/rejected": -0.4958527684211731, + "logps/chosen": -4.380959510803223, + "logps/rejected": -36.05854415893555, + "loss": 0.6548417806625366, + "memory(GiB)": 40.75, + "nll_loss": 0.38906389474868774, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07272060960531235, + "rewards/margins": 2.0416646003723145, + "rewards/rejected": -1.9689440727233887, + "step": 69, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.17604526878340146, + "grad_norm": 3.3756778240203857, + "learning_rate": 0.0001999614240512817, + "logits/chosen": -0.2773034870624542, + "logits/rejected": -0.48985666036605835, + "logps/chosen": -7.3368024826049805, + "logps/rejected": -28.274978637695312, + "loss": 0.7877169251441956, + "memory(GiB)": 40.75, + "nll_loss": 0.4065099060535431, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.1428125947713852, + "rewards/margins": 1.3781116008758545, + "rewards/rejected": -1.2352991104125977, + "step": 70, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.1785602011945929, + "grad_norm": 3.289449453353882, + "learning_rate": 0.00019995332373230808, + "logits/chosen": -0.2643814980983734, + "logits/rejected": -0.441372275352478, + "logps/chosen": -4.96966552734375, + "logps/rejected": -32.28032684326172, + "loss": 0.6629552841186523, + "memory(GiB)": 40.75, + "nll_loss": 0.2761652171611786, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.028238017112016678, + "rewards/margins": 1.3064823150634766, + "rewards/rejected": -1.2782442569732666, + "step": 71, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.18107513360578434, + "grad_norm": 2.8338823318481445, + "learning_rate": 0.00019994445220538677, + "logits/chosen": -0.2861837148666382, + "logits/rejected": -0.39666813611984253, + "logps/chosen": -6.460289001464844, + "logps/rejected": -33.8844108581543, + "loss": 0.7865106463432312, + "memory(GiB)": 40.75, + "nll_loss": 0.4185709059238434, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.03227805718779564, + "rewards/margins": 1.8171049356460571, + "rewards/rejected": -1.849382996559143, + "step": 72, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.1835900660169758, + "grad_norm": 4.286563396453857, + "learning_rate": 0.0001999348095389677, + "logits/chosen": -0.2642534673213959, + "logits/rejected": -0.4921380281448364, + "logps/chosen": -6.74280309677124, + "logps/rejected": -48.935733795166016, + "loss": 0.6765459775924683, + "memory(GiB)": 40.75, + "nll_loss": 0.4257548451423645, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.023126238957047462, + "rewards/margins": 2.3492531776428223, + "rewards/rejected": -2.3723795413970947, + "step": 73, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.18610499842816725, + "grad_norm": 6.242228984832764, + "learning_rate": 0.0001999243958074506, + "logits/chosen": -0.24496644735336304, + "logits/rejected": -0.4604867100715637, + "logps/chosen": -14.664331436157227, + "logps/rejected": -44.660789489746094, + "loss": 1.2609174251556396, + "memory(GiB)": 40.75, + "nll_loss": 0.827994704246521, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4426795244216919, + "rewards/margins": 1.524437427520752, + "rewards/rejected": -1.9671170711517334, + "step": 74, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.18861993083935868, + "grad_norm": 2.4951188564300537, + "learning_rate": 0.00019991321109118448, + "logits/chosen": -0.40645161271095276, + "logits/rejected": -0.4706539511680603, + "logps/chosen": -6.498587131500244, + "logps/rejected": -31.508909225463867, + "loss": 0.6895300149917603, + "memory(GiB)": 40.75, + "nll_loss": 0.3541663587093353, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.004417119547724724, + "rewards/margins": 1.655824899673462, + "rewards/rejected": -1.6514074802398682, + "step": 75, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.19113486325055015, + "grad_norm": 2.4047465324401855, + "learning_rate": 0.00019990125547646704, + "logits/chosen": -0.33654719591140747, + "logits/rejected": -0.49985551834106445, + "logps/chosen": -3.1749448776245117, + "logps/rejected": -31.598142623901367, + "loss": 0.6832078695297241, + "memory(GiB)": 40.75, + "nll_loss": 0.28699803352355957, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.018617400899529457, + "rewards/margins": 1.5892831087112427, + "rewards/rejected": -1.570665717124939, + "step": 76, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.1936497956617416, + "grad_norm": 2.1775262355804443, + "learning_rate": 0.00019988852905554404, + "logits/chosen": -0.2977695167064667, + "logits/rejected": -0.4677082896232605, + "logps/chosen": -5.7894110679626465, + "logps/rejected": -33.92140579223633, + "loss": 0.6954871416091919, + "memory(GiB)": 40.75, + "nll_loss": 0.35726743936538696, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11432527005672455, + "rewards/margins": 1.4882932901382446, + "rewards/rejected": -1.3739678859710693, + "step": 77, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.19616472807293303, + "grad_norm": 2.386960983276367, + "learning_rate": 0.0001998750319266084, + "logits/chosen": -0.34930819272994995, + "logits/rejected": -0.4996439814567566, + "logps/chosen": -5.451239585876465, + "logps/rejected": -30.024625778198242, + "loss": 0.8510737419128418, + "memory(GiB)": 40.75, + "nll_loss": 0.38498157262802124, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.08021941781044006, + "rewards/margins": 1.2762808799743652, + "rewards/rejected": -1.1960614919662476, + "step": 78, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.1986796604841245, + "grad_norm": 2.540059804916382, + "learning_rate": 0.00019986076419379974, + "logits/chosen": -0.2711866497993469, + "logits/rejected": -0.46172043681144714, + "logps/chosen": -7.032684803009033, + "logps/rejected": -37.4212760925293, + "loss": 0.8995335698127747, + "memory(GiB)": 40.75, + "nll_loss": 0.5719486474990845, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1289057433605194, + "rewards/margins": 1.6372534036636353, + "rewards/rejected": -1.508347749710083, + "step": 79, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.20119459289531594, + "grad_norm": 2.888524293899536, + "learning_rate": 0.00019984572596720324, + "logits/chosen": -0.4125524163246155, + "logits/rejected": -0.550200343132019, + "logps/chosen": -4.954836845397949, + "logps/rejected": -31.84545135498047, + "loss": 0.7266713976860046, + "memory(GiB)": 40.75, + "nll_loss": 0.4038025140762329, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.018991030752658844, + "rewards/margins": 1.6552242040634155, + "rewards/rejected": -1.6742154359817505, + "step": 80, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.20370952530650738, + "grad_norm": 2.54023814201355, + "learning_rate": 0.00019982991736284915, + "logits/chosen": -0.36888328194618225, + "logits/rejected": -0.48596253991127014, + "logps/chosen": -6.946263313293457, + "logps/rejected": -37.912750244140625, + "loss": 0.6610869765281677, + "memory(GiB)": 40.75, + "nll_loss": 0.3929502069950104, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.005873316898941994, + "rewards/margins": 2.2654542922973633, + "rewards/rejected": -2.2595808506011963, + "step": 81, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.20622445771769884, + "grad_norm": 5.155144214630127, + "learning_rate": 0.00019981333850271158, + "logits/chosen": -0.37043288350105286, + "logits/rejected": -0.5056601166725159, + "logps/chosen": -7.0625691413879395, + "logps/rejected": -45.226646423339844, + "loss": 0.6925163269042969, + "memory(GiB)": 40.75, + "nll_loss": 0.4076215326786041, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.03391120955348015, + "rewards/margins": 3.1254899501800537, + "rewards/rejected": -3.1594011783599854, + "step": 82, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.20873939012889028, + "grad_norm": 4.161493301391602, + "learning_rate": 0.0001997959895147077, + "logits/chosen": -0.3119804859161377, + "logits/rejected": -0.519420862197876, + "logps/chosen": -5.138388156890869, + "logps/rejected": -51.65058517456055, + "loss": 0.6154710054397583, + "memory(GiB)": 40.75, + "nll_loss": 0.3612300753593445, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08062062412500381, + "rewards/margins": 3.3395729064941406, + "rewards/rejected": -3.2589523792266846, + "step": 83, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.21125432254008172, + "grad_norm": 3.640066146850586, + "learning_rate": 0.0001997778705326968, + "logits/chosen": -0.3015586733818054, + "logits/rejected": -0.561132550239563, + "logps/chosen": -2.76611328125, + "logps/rejected": -47.526405334472656, + "loss": 0.5920863151550293, + "memory(GiB)": 40.75, + "nll_loss": 0.3293038606643677, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07201344519853592, + "rewards/margins": 2.9245572090148926, + "rewards/rejected": -2.852543592453003, + "step": 84, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.2137692549512732, + "grad_norm": 9.056829452514648, + "learning_rate": 0.00019975898169647915, + "logits/chosen": -0.3201826810836792, + "logits/rejected": -0.49801909923553467, + "logps/chosen": -8.43231201171875, + "logps/rejected": -44.97834014892578, + "loss": 0.7947379350662231, + "memory(GiB)": 40.75, + "nll_loss": 0.45788073539733887, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.03715405985713005, + "rewards/margins": 2.6956162452697754, + "rewards/rejected": -2.7327704429626465, + "step": 85, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.21628418736246463, + "grad_norm": 3.5543746948242188, + "learning_rate": 0.000199739323151795, + "logits/chosen": -0.2616378962993622, + "logits/rejected": -0.5049853920936584, + "logps/chosen": -5.9475860595703125, + "logps/rejected": -39.59865951538086, + "loss": 0.7357087135314941, + "memory(GiB)": 40.75, + "nll_loss": 0.3480994701385498, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.013387706130743027, + "rewards/margins": 2.0890746116638184, + "rewards/rejected": -2.1024622917175293, + "step": 86, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.21879911977365607, + "grad_norm": 2.5816776752471924, + "learning_rate": 0.00019971889505032334, + "logits/chosen": -0.3568105101585388, + "logits/rejected": -0.5804364681243896, + "logps/chosen": -3.9897284507751465, + "logps/rejected": -40.31490707397461, + "loss": 0.6060020923614502, + "memory(GiB)": 40.75, + "nll_loss": 0.3237951099872589, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0022782403975725174, + "rewards/margins": 2.383437156677246, + "rewards/rejected": -2.3857154846191406, + "step": 87, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.22131405218484754, + "grad_norm": 3.132185935974121, + "learning_rate": 0.00019969769754968098, + "logits/chosen": -0.28933870792388916, + "logits/rejected": -0.5249110460281372, + "logps/chosen": -5.762962818145752, + "logps/rejected": -34.77198791503906, + "loss": 0.6889637112617493, + "memory(GiB)": 40.75, + "nll_loss": 0.3635081648826599, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.13572651147842407, + "rewards/margins": 1.7912075519561768, + "rewards/rejected": -1.655480980873108, + "step": 88, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.22382898459603898, + "grad_norm": 2.8023223876953125, + "learning_rate": 0.00019967573081342103, + "logits/chosen": -0.3481064736843109, + "logits/rejected": -0.5555439591407776, + "logps/chosen": -5.762640476226807, + "logps/rejected": -35.07036209106445, + "loss": 0.6983572244644165, + "memory(GiB)": 40.75, + "nll_loss": 0.3852231502532959, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04139009863138199, + "rewards/margins": 2.027146816253662, + "rewards/rejected": -1.985756754875183, + "step": 89, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.22634391700723044, + "grad_norm": 8.448691368103027, + "learning_rate": 0.00019965299501103177, + "logits/chosen": -0.28187096118927, + "logits/rejected": -0.5138179063796997, + "logps/chosen": -11.35129165649414, + "logps/rejected": -54.19221496582031, + "loss": 0.9737562537193298, + "memory(GiB)": 40.75, + "nll_loss": 0.6502060890197754, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.1801879107952118, + "rewards/margins": 3.0160610675811768, + "rewards/rejected": -3.196249008178711, + "step": 90, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.22885884941842188, + "grad_norm": 4.455811977386475, + "learning_rate": 0.00019962949031793542, + "logits/chosen": -0.388353168964386, + "logits/rejected": -0.5416743755340576, + "logps/chosen": -7.5033183097839355, + "logps/rejected": -38.96351623535156, + "loss": 0.8328152298927307, + "memory(GiB)": 40.75, + "nll_loss": 0.5262293219566345, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.18585588037967682, + "rewards/margins": 2.3831686973571777, + "rewards/rejected": -2.5690245628356934, + "step": 91, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.23137378182961332, + "grad_norm": 5.176366806030273, + "learning_rate": 0.00019960521691548674, + "logits/chosen": -0.30729976296424866, + "logits/rejected": -0.5507268905639648, + "logps/chosen": -6.70347785949707, + "logps/rejected": -47.423580169677734, + "loss": 0.7757164835929871, + "memory(GiB)": 40.75, + "nll_loss": 0.5047218203544617, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.212627112865448, + "rewards/margins": 2.656092643737793, + "rewards/rejected": -2.8687198162078857, + "step": 92, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.2338887142408048, + "grad_norm": 2.7338004112243652, + "learning_rate": 0.0001995801749909715, + "logits/chosen": -0.310178279876709, + "logits/rejected": -0.5231863856315613, + "logps/chosen": -6.638127326965332, + "logps/rejected": -34.64179992675781, + "loss": 0.6857970952987671, + "memory(GiB)": 42.51, + "nll_loss": 0.341688871383667, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04894203692674637, + "rewards/margins": 1.8471993207931519, + "rewards/rejected": -1.7982572317123413, + "step": 93, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.23640364665199623, + "grad_norm": 2.074855327606201, + "learning_rate": 0.00019955436473760525, + "logits/chosen": -0.2659550607204437, + "logits/rejected": -0.5499891042709351, + "logps/chosen": -4.5521063804626465, + "logps/rejected": -44.503509521484375, + "loss": 0.668745219707489, + "memory(GiB)": 42.51, + "nll_loss": 0.3918987512588501, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1607407182455063, + "rewards/margins": 2.3078651428222656, + "rewards/rejected": -2.1471242904663086, + "step": 94, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.23891857906318767, + "grad_norm": 2.6431190967559814, + "learning_rate": 0.0001995277863545316, + "logits/chosen": -0.38493573665618896, + "logits/rejected": -0.5655022859573364, + "logps/chosen": -3.839203119277954, + "logps/rejected": -27.24703025817871, + "loss": 0.5949357151985168, + "memory(GiB)": 42.51, + "nll_loss": 0.2755538821220398, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.11298017203807831, + "rewards/margins": 1.4701435565948486, + "rewards/rejected": -1.357163429260254, + "step": 95, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.24143351147437914, + "grad_norm": 2.935624837875366, + "learning_rate": 0.00019950044004682092, + "logits/chosen": -0.357890248298645, + "logits/rejected": -0.5321043729782104, + "logps/chosen": -9.981462478637695, + "logps/rejected": -34.8548583984375, + "loss": 0.9003065228462219, + "memory(GiB)": 42.51, + "nll_loss": 0.5299741625785828, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.04542495310306549, + "rewards/margins": 1.7066924571990967, + "rewards/rejected": -1.6612673997879028, + "step": 96, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.24394844388557058, + "grad_norm": 2.2122833728790283, + "learning_rate": 0.0001994723260254686, + "logits/chosen": -0.2592640519142151, + "logits/rejected": -0.4961639940738678, + "logps/chosen": -4.603950500488281, + "logps/rejected": -41.2968864440918, + "loss": 0.5464479923248291, + "memory(GiB)": 42.51, + "nll_loss": 0.3016549050807953, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1378301978111267, + "rewards/margins": 2.3867669105529785, + "rewards/rejected": -2.248936653137207, + "step": 97, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.24646337629676202, + "grad_norm": 3.110325574874878, + "learning_rate": 0.0001994434445073934, + "logits/chosen": -0.25290989875793457, + "logits/rejected": -0.5424689650535583, + "logps/chosen": -3.2259504795074463, + "logps/rejected": -50.58758544921875, + "loss": 0.48420143127441406, + "memory(GiB)": 42.51, + "nll_loss": 0.24583308398723602, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06779447197914124, + "rewards/margins": 2.959743022918701, + "rewards/rejected": -2.891948699951172, + "step": 98, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.24897830870795348, + "grad_norm": 5.068027019500732, + "learning_rate": 0.00019941379571543596, + "logits/chosen": -0.2531135678291321, + "logits/rejected": -0.4778846800327301, + "logps/chosen": -4.735211372375488, + "logps/rejected": -50.30940246582031, + "loss": 0.6116160750389099, + "memory(GiB)": 42.51, + "nll_loss": 0.39248543977737427, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.009140911512076855, + "rewards/margins": 3.1750411987304688, + "rewards/rejected": -3.1841821670532227, + "step": 99, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.25149324111914495, + "grad_norm": 5.654895305633545, + "learning_rate": 0.00019938337987835688, + "logits/chosen": -0.29766467213630676, + "logits/rejected": -0.39502421021461487, + "logps/chosen": -9.370654106140137, + "logps/rejected": -37.111106872558594, + "loss": 0.9719223976135254, + "memory(GiB)": 42.51, + "nll_loss": 0.5046005249023438, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21612250804901123, + "rewards/margins": 2.0566141605377197, + "rewards/rejected": -2.2727365493774414, + "step": 100, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.2540081735303364, + "grad_norm": 2.3178176879882812, + "learning_rate": 0.00019935219723083502, + "logits/chosen": -0.3190368115901947, + "logits/rejected": -0.4808640480041504, + "logps/chosen": -4.635265350341797, + "logps/rejected": -46.21360397338867, + "loss": 0.5798269510269165, + "memory(GiB)": 42.51, + "nll_loss": 0.3243585228919983, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07786446064710617, + "rewards/margins": 3.2863645553588867, + "rewards/rejected": -3.2085001468658447, + "step": 101, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.25652310594152783, + "grad_norm": 1.5522849559783936, + "learning_rate": 0.0001993202480134658, + "logits/chosen": -0.3053315281867981, + "logits/rejected": -0.5489925742149353, + "logps/chosen": -2.3068742752075195, + "logps/rejected": -59.077171325683594, + "loss": 0.29376542568206787, + "memory(GiB)": 42.51, + "nll_loss": 0.15781843662261963, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14335167407989502, + "rewards/margins": 4.1030802726745605, + "rewards/rejected": -3.959728240966797, + "step": 102, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.25903803835271927, + "grad_norm": 3.088001012802124, + "learning_rate": 0.0001992875324727591, + "logits/chosen": -0.29225340485572815, + "logits/rejected": -0.4629225730895996, + "logps/chosen": -7.737708568572998, + "logps/rejected": -56.16609573364258, + "loss": 0.657672107219696, + "memory(GiB)": 42.51, + "nll_loss": 0.4136083722114563, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.15128585696220398, + "rewards/margins": 3.662168025970459, + "rewards/rejected": -3.8134536743164062, + "step": 103, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.2615529707639107, + "grad_norm": 16.196247100830078, + "learning_rate": 0.00019925405086113768, + "logits/chosen": -0.24154868721961975, + "logits/rejected": -0.42765212059020996, + "logps/chosen": -7.00141716003418, + "logps/rejected": -44.41450500488281, + "loss": 0.9225707054138184, + "memory(GiB)": 42.51, + "nll_loss": 0.5817552804946899, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18904252350330353, + "rewards/margins": 2.805342674255371, + "rewards/rejected": -2.994385004043579, + "step": 104, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.26406790317510215, + "grad_norm": 4.839797496795654, + "learning_rate": 0.0001992198034369349, + "logits/chosen": -0.21534261107444763, + "logits/rejected": -0.4093911051750183, + "logps/chosen": -6.268553733825684, + "logps/rejected": -44.56496047973633, + "loss": 0.7437994480133057, + "memory(GiB)": 42.51, + "nll_loss": 0.4600142240524292, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12164817750453949, + "rewards/margins": 2.817305088043213, + "rewards/rejected": -2.695657253265381, + "step": 105, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.26658283558629364, + "grad_norm": 6.186526298522949, + "learning_rate": 0.000199184790464393, + "logits/chosen": -0.28391483426094055, + "logits/rejected": -0.47835806012153625, + "logps/chosen": -4.414430618286133, + "logps/rejected": -35.84981155395508, + "loss": 0.7718313336372375, + "memory(GiB)": 42.51, + "nll_loss": 0.451696515083313, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.061221636831760406, + "rewards/margins": 2.390685558319092, + "rewards/rejected": -2.3294641971588135, + "step": 106, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.2690977679974851, + "grad_norm": 4.50436544418335, + "learning_rate": 0.00019914901221366086, + "logits/chosen": -0.23250971734523773, + "logits/rejected": -0.4437968134880066, + "logps/chosen": -6.316987037658691, + "logps/rejected": -49.236106872558594, + "loss": 0.7839150428771973, + "memory(GiB)": 42.51, + "nll_loss": 0.46380385756492615, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.052675511687994, + "rewards/margins": 3.1932520866394043, + "rewards/rejected": -3.245927572250366, + "step": 107, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.2716127004086765, + "grad_norm": 3.6184003353118896, + "learning_rate": 0.0001991124689607921, + "logits/chosen": -0.19258703291416168, + "logits/rejected": -0.43321287631988525, + "logps/chosen": -7.002651691436768, + "logps/rejected": -41.406925201416016, + "loss": 0.8718461990356445, + "memory(GiB)": 42.51, + "nll_loss": 0.5701597929000854, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0768214762210846, + "rewards/margins": 2.482198715209961, + "rewards/rejected": -2.559020519256592, + "step": 108, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.27412763281986796, + "grad_norm": 10.216064453125, + "learning_rate": 0.00019907516098774275, + "logits/chosen": -0.3110716640949249, + "logits/rejected": -0.4275364577770233, + "logps/chosen": -5.352696418762207, + "logps/rejected": -44.723453521728516, + "loss": 0.8070493340492249, + "memory(GiB)": 42.51, + "nll_loss": 0.5113809704780579, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.021823018789291382, + "rewards/margins": 3.496793508529663, + "rewards/rejected": -3.474970579147339, + "step": 109, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.2766425652310594, + "grad_norm": 4.2641143798828125, + "learning_rate": 0.00019903708858236925, + "logits/chosen": -0.17137043178081512, + "logits/rejected": -0.3559553623199463, + "logps/chosen": -8.645333290100098, + "logps/rejected": -56.14238357543945, + "loss": 0.8187519311904907, + "memory(GiB)": 42.51, + "nll_loss": 0.4967080354690552, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.22593286633491516, + "rewards/margins": 3.5679526329040527, + "rewards/rejected": -3.7938857078552246, + "step": 110, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.27915749764225084, + "grad_norm": 5.008543014526367, + "learning_rate": 0.00019899825203842613, + "logits/chosen": -0.22317875921726227, + "logits/rejected": -0.4507110118865967, + "logps/chosen": -4.583744525909424, + "logps/rejected": -52.95720672607422, + "loss": 0.6659380793571472, + "memory(GiB)": 42.51, + "nll_loss": 0.42531025409698486, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.08066091686487198, + "rewards/margins": 3.507352828979492, + "rewards/rejected": -3.426692247390747, + "step": 111, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.28167243005344234, + "grad_norm": 3.2912778854370117, + "learning_rate": 0.00019895865165556377, + "logits/chosen": -0.20796722173690796, + "logits/rejected": -0.4548439085483551, + "logps/chosen": -6.193486213684082, + "logps/rejected": -50.00212860107422, + "loss": 0.724488377571106, + "memory(GiB)": 42.51, + "nll_loss": 0.36660081148147583, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.09731753170490265, + "rewards/margins": 2.9351813793182373, + "rewards/rejected": -3.032498598098755, + "step": 112, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.2841873624646338, + "grad_norm": 2.3630876541137695, + "learning_rate": 0.00019891828773932604, + "logits/chosen": -0.29762569069862366, + "logits/rejected": -0.496326208114624, + "logps/chosen": -5.607400417327881, + "logps/rejected": -45.21501922607422, + "loss": 0.5872230529785156, + "memory(GiB)": 42.51, + "nll_loss": 0.36216530203819275, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2503179609775543, + "rewards/margins": 3.1686577796936035, + "rewards/rejected": -2.918339729309082, + "step": 113, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.2867022948758252, + "grad_norm": 2.2401249408721924, + "learning_rate": 0.0001988771606011481, + "logits/chosen": -0.2846427857875824, + "logits/rejected": -0.46442368626594543, + "logps/chosen": -7.414566993713379, + "logps/rejected": -38.23175048828125, + "loss": 0.8090199828147888, + "memory(GiB)": 42.51, + "nll_loss": 0.5426386594772339, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0931171178817749, + "rewards/margins": 2.2654707431793213, + "rewards/rejected": -2.172353744506836, + "step": 114, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.28921722728701665, + "grad_norm": 2.3249471187591553, + "learning_rate": 0.0001988352705583538, + "logits/chosen": -0.36136990785598755, + "logits/rejected": -0.49599409103393555, + "logps/chosen": -6.019715785980225, + "logps/rejected": -31.482872009277344, + "loss": 0.716136634349823, + "memory(GiB)": 42.51, + "nll_loss": 0.38180771470069885, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07284583151340485, + "rewards/margins": 1.6758012771606445, + "rewards/rejected": -1.6029551029205322, + "step": 115, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.2917321596982081, + "grad_norm": 2.0250649452209473, + "learning_rate": 0.0001987926179341533, + "logits/chosen": -0.3277466893196106, + "logits/rejected": -0.49884647130966187, + "logps/chosen": -6.582662105560303, + "logps/rejected": -34.93992614746094, + "loss": 0.6409764289855957, + "memory(GiB)": 42.51, + "nll_loss": 0.3818390965461731, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.219668447971344, + "rewards/margins": 2.2215864658355713, + "rewards/rejected": -2.001918315887451, + "step": 116, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.29424709210939953, + "grad_norm": 3.080130100250244, + "learning_rate": 0.00019874920305764068, + "logits/chosen": -0.3647570312023163, + "logits/rejected": -0.5425549149513245, + "logps/chosen": -4.569772720336914, + "logps/rejected": -29.92070198059082, + "loss": 0.733748197555542, + "memory(GiB)": 42.51, + "nll_loss": 0.3840240240097046, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.09753906726837158, + "rewards/margins": 1.5259737968444824, + "rewards/rejected": -1.4284348487854004, + "step": 117, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.29676202452059103, + "grad_norm": 3.2526772022247314, + "learning_rate": 0.00019870502626379127, + "logits/chosen": -0.39825165271759033, + "logits/rejected": -0.5752891898155212, + "logps/chosen": -6.833606243133545, + "logps/rejected": -35.04240798950195, + "loss": 0.7230013012886047, + "memory(GiB)": 42.51, + "nll_loss": 0.4058716297149658, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.1110297441482544, + "rewards/margins": 1.915820837020874, + "rewards/rejected": -2.026850461959839, + "step": 118, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.29927695693178247, + "grad_norm": 2.977874994277954, + "learning_rate": 0.00019866008789345917, + "logits/chosen": -0.4147549271583557, + "logits/rejected": -0.5979161262512207, + "logps/chosen": -5.960415363311768, + "logps/rejected": -40.34906768798828, + "loss": 0.6719368696212769, + "memory(GiB)": 42.51, + "nll_loss": 0.38052472472190857, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.015904245898127556, + "rewards/margins": 2.2843177318573, + "rewards/rejected": -2.2684133052825928, + "step": 119, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.3017918893429739, + "grad_norm": 3.058821439743042, + "learning_rate": 0.00019861438829337438, + "logits/chosen": -0.3485436737537384, + "logits/rejected": -0.5699384808540344, + "logps/chosen": -8.048928260803223, + "logps/rejected": -55.26435852050781, + "loss": 0.8104152083396912, + "memory(GiB)": 42.51, + "nll_loss": 0.5313911437988281, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.08313308656215668, + "rewards/margins": 3.3404150009155273, + "rewards/rejected": -3.257282257080078, + "step": 120, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.30430682175416535, + "grad_norm": 2.809389591217041, + "learning_rate": 0.00019856792781614054, + "logits/chosen": -0.40515586733818054, + "logits/rejected": -0.6652223467826843, + "logps/chosen": -2.280519962310791, + "logps/rejected": -50.222084045410156, + "loss": 0.45217883586883545, + "memory(GiB)": 42.51, + "nll_loss": 0.26132771372795105, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1198212057352066, + "rewards/margins": 3.28947114944458, + "rewards/rejected": -3.169650077819824, + "step": 121, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.3068217541653568, + "grad_norm": 3.070807456970215, + "learning_rate": 0.00019852070682023176, + "logits/chosen": -0.4808189272880554, + "logits/rejected": -0.6974934339523315, + "logps/chosen": -7.208076477050781, + "logps/rejected": -52.298465728759766, + "loss": 0.5899146795272827, + "memory(GiB)": 42.51, + "nll_loss": 0.42121508717536926, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16679850220680237, + "rewards/margins": 3.6304702758789062, + "rewards/rejected": -3.4636716842651367, + "step": 122, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.3093366865765483, + "grad_norm": 2.8824384212493896, + "learning_rate": 0.00019847272566999026, + "logits/chosen": -0.49758437275886536, + "logits/rejected": -0.6554095149040222, + "logps/chosen": -10.397936820983887, + "logps/rejected": -42.242393493652344, + "loss": 0.8943163156509399, + "memory(GiB)": 42.51, + "nll_loss": 0.5859109163284302, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0896286740899086, + "rewards/margins": 2.782667875289917, + "rewards/rejected": -2.6930387020111084, + "step": 123, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.3118516189877397, + "grad_norm": 2.103641986846924, + "learning_rate": 0.0001984239847356233, + "logits/chosen": -0.5333049297332764, + "logits/rejected": -0.6946154832839966, + "logps/chosen": -4.652249336242676, + "logps/rejected": -53.446041107177734, + "loss": 0.5551345944404602, + "memory(GiB)": 42.51, + "nll_loss": 0.3173026144504547, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0959150567650795, + "rewards/margins": 4.159012794494629, + "rewards/rejected": -4.063097953796387, + "step": 124, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.31436655139893116, + "grad_norm": 2.719147205352783, + "learning_rate": 0.00019837448439320027, + "logits/chosen": -0.5486172437667847, + "logits/rejected": -0.7822794318199158, + "logps/chosen": -3.360109567642212, + "logps/rejected": -77.09004211425781, + "loss": 0.4081324338912964, + "memory(GiB)": 42.51, + "nll_loss": 0.27365225553512573, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09313642978668213, + "rewards/margins": 6.103594779968262, + "rewards/rejected": -6.010458469390869, + "step": 125, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.3168814838101226, + "grad_norm": 3.821580410003662, + "learning_rate": 0.0001983242250246501, + "logits/chosen": -0.49823230504989624, + "logits/rejected": -0.746193528175354, + "logps/chosen": -8.326932907104492, + "logps/rejected": -85.3308334350586, + "loss": 0.6026039719581604, + "memory(GiB)": 42.51, + "nll_loss": 0.48026546835899353, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.15297873318195343, + "rewards/margins": 6.628488063812256, + "rewards/rejected": -6.781466484069824, + "step": 126, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.31939641622131404, + "grad_norm": 2.334726333618164, + "learning_rate": 0.00019827320701775806, + "logits/chosen": -0.4417359530925751, + "logits/rejected": -0.6943372488021851, + "logps/chosen": -5.183182716369629, + "logps/rejected": -74.23169708251953, + "loss": 0.5311453938484192, + "memory(GiB)": 42.51, + "nll_loss": 0.31170862913131714, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06631641089916229, + "rewards/margins": 5.296952724456787, + "rewards/rejected": -5.2306365966796875, + "step": 127, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.3219113486325055, + "grad_norm": 3.0242960453033447, + "learning_rate": 0.0001982214307661627, + "logits/chosen": -0.39977923035621643, + "logits/rejected": -0.6834134459495544, + "logps/chosen": -4.503589153289795, + "logps/rejected": -70.66866302490234, + "loss": 0.5889554619789124, + "memory(GiB)": 42.51, + "nll_loss": 0.3487299084663391, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.18435406684875488, + "rewards/margins": 5.4617156982421875, + "rewards/rejected": -5.277361869812012, + "step": 128, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.324426281043697, + "grad_norm": 2.191619396209717, + "learning_rate": 0.00019816889666935317, + "logits/chosen": -0.3248615860939026, + "logits/rejected": -0.6416115164756775, + "logps/chosen": -6.371191024780273, + "logps/rejected": -76.59197998046875, + "loss": 0.4878350794315338, + "memory(GiB)": 42.51, + "nll_loss": 0.33031994104385376, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13981086015701294, + "rewards/margins": 5.163226127624512, + "rewards/rejected": -5.0234150886535645, + "step": 129, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.3269412134548884, + "grad_norm": 3.211867570877075, + "learning_rate": 0.00019811560513266572, + "logits/chosen": -0.3845720887184143, + "logits/rejected": -0.6389954090118408, + "logps/chosen": -5.841503620147705, + "logps/rejected": -71.1533203125, + "loss": 0.586833655834198, + "memory(GiB)": 42.51, + "nll_loss": 0.3950338065624237, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09367645531892776, + "rewards/margins": 5.478381156921387, + "rewards/rejected": -5.384705066680908, + "step": 130, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.32945614586607985, + "grad_norm": 9.350093841552734, + "learning_rate": 0.00019806155656728084, + "logits/chosen": -0.2960500419139862, + "logits/rejected": -0.6573159098625183, + "logps/chosen": -9.058235168457031, + "logps/rejected": -103.11417388916016, + "loss": 0.9421167373657227, + "memory(GiB)": 42.51, + "nll_loss": 0.7972053289413452, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.26350751519203186, + "rewards/margins": 7.675902366638184, + "rewards/rejected": -7.939410209655762, + "step": 131, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.3319710782772713, + "grad_norm": 5.248542308807373, + "learning_rate": 0.00019800675139022006, + "logits/chosen": -0.47517600655555725, + "logits/rejected": -0.6538941264152527, + "logps/chosen": -9.417102813720703, + "logps/rejected": -74.27867126464844, + "loss": 0.6414040327072144, + "memory(GiB)": 42.51, + "nll_loss": 0.3988567888736725, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.300504595041275, + "rewards/margins": 5.568276405334473, + "rewards/rejected": -5.868781089782715, + "step": 132, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.33448601068846273, + "grad_norm": 3.997988224029541, + "learning_rate": 0.00019795119002434262, + "logits/chosen": -0.47632429003715515, + "logits/rejected": -0.6377424001693726, + "logps/chosen": -6.212442874908447, + "logps/rejected": -72.35940551757812, + "loss": 0.6233670711517334, + "memory(GiB)": 42.51, + "nll_loss": 0.36861366033554077, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04314713925123215, + "rewards/margins": 5.591425895690918, + "rewards/rejected": -5.548279285430908, + "step": 133, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.33700094309965417, + "grad_norm": 2.701449394226074, + "learning_rate": 0.00019789487289834228, + "logits/chosen": -0.35887065529823303, + "logits/rejected": -0.5933378338813782, + "logps/chosen": -3.4826135635375977, + "logps/rejected": -62.30265808105469, + "loss": 0.4813486635684967, + "memory(GiB)": 42.51, + "nll_loss": 0.28860440850257874, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1144554391503334, + "rewards/margins": 4.846682548522949, + "rewards/rejected": -4.732227325439453, + "step": 134, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.33951587551084567, + "grad_norm": 1.554006814956665, + "learning_rate": 0.000197837800446744, + "logits/chosen": -0.26935291290283203, + "logits/rejected": -0.6075947284698486, + "logps/chosen": -5.505460262298584, + "logps/rejected": -90.5344467163086, + "loss": 0.36852002143859863, + "memory(GiB)": 42.51, + "nll_loss": 0.22335653007030487, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.009416868910193443, + "rewards/margins": 6.831361770629883, + "rewards/rejected": -6.840778350830078, + "step": 135, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.3420308079220371, + "grad_norm": 2.480741024017334, + "learning_rate": 0.00019777997310990063, + "logits/chosen": -0.3000340461730957, + "logits/rejected": -0.6057184338569641, + "logps/chosen": -3.0569756031036377, + "logps/rejected": -74.86396789550781, + "loss": 0.4303661286830902, + "memory(GiB)": 42.51, + "nll_loss": 0.29833856225013733, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10127638280391693, + "rewards/margins": 5.511883735656738, + "rewards/rejected": -5.410606861114502, + "step": 136, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.34454574033322855, + "grad_norm": 4.56061315536499, + "learning_rate": 0.00019772139133398942, + "logits/chosen": -0.3569238781929016, + "logits/rejected": -0.5552278161048889, + "logps/chosen": -6.037662506103516, + "logps/rejected": -72.8807373046875, + "loss": 0.5785601139068604, + "memory(GiB)": 42.51, + "nll_loss": 0.3691541850566864, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.07385054975748062, + "rewards/margins": 5.239901542663574, + "rewards/rejected": -5.313753128051758, + "step": 137, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.34706067274442, + "grad_norm": 3.128384828567505, + "learning_rate": 0.00019766205557100868, + "logits/chosen": -0.45258647203445435, + "logits/rejected": -0.6256503462791443, + "logps/chosen": -6.615728855133057, + "logps/rejected": -54.988853454589844, + "loss": 0.6647377014160156, + "memory(GiB)": 42.51, + "nll_loss": 0.4122207760810852, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1877025067806244, + "rewards/margins": 4.346957206726074, + "rewards/rejected": -4.159254550933838, + "step": 138, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.3495756051556114, + "grad_norm": 3.3702149391174316, + "learning_rate": 0.00019760196627877422, + "logits/chosen": -0.22564947605133057, + "logits/rejected": -0.6019137501716614, + "logps/chosen": -2.935713052749634, + "logps/rejected": -91.22589111328125, + "loss": 0.4306371212005615, + "memory(GiB)": 42.51, + "nll_loss": 0.2682075500488281, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09053624421358109, + "rewards/margins": 6.857407569885254, + "rewards/rejected": -6.766871452331543, + "step": 139, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.3520905375668029, + "grad_norm": 17.25383949279785, + "learning_rate": 0.0001975411239209158, + "logits/chosen": -0.4722020626068115, + "logits/rejected": -0.5809731483459473, + "logps/chosen": -11.125019073486328, + "logps/rejected": -47.38395690917969, + "loss": 1.216600775718689, + "memory(GiB)": 42.51, + "nll_loss": 0.792304515838623, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.23399552702903748, + "rewards/margins": 3.3990795612335205, + "rewards/rejected": -3.63307523727417, + "step": 140, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.35460546997799436, + "grad_norm": 3.727832794189453, + "learning_rate": 0.0001974795289668737, + "logits/chosen": -0.35420382022857666, + "logits/rejected": -0.718546986579895, + "logps/chosen": -7.968466758728027, + "logps/rejected": -91.39598846435547, + "loss": 0.7644503712654114, + "memory(GiB)": 42.51, + "nll_loss": 0.6175839900970459, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.3464764654636383, + "rewards/margins": 6.540220737457275, + "rewards/rejected": -6.886697292327881, + "step": 141, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.3571204023891858, + "grad_norm": 13.208379745483398, + "learning_rate": 0.00019741718189189485, + "logits/chosen": -0.41622060537338257, + "logits/rejected": -0.6492832899093628, + "logps/chosen": -6.71783447265625, + "logps/rejected": -69.75479888916016, + "loss": 0.8267927169799805, + "memory(GiB)": 42.51, + "nll_loss": 0.5884179472923279, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.1379324495792389, + "rewards/margins": 4.956573963165283, + "rewards/rejected": -5.094506740570068, + "step": 142, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.35963533480037724, + "grad_norm": 4.856459617614746, + "learning_rate": 0.00019735408317702948, + "logits/chosen": -0.4186263382434845, + "logits/rejected": -0.6451130509376526, + "logps/chosen": -4.069480895996094, + "logps/rejected": -62.28482437133789, + "loss": 0.4998188018798828, + "memory(GiB)": 42.51, + "nll_loss": 0.3359423875808716, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.012309092096984386, + "rewards/margins": 4.688631057739258, + "rewards/rejected": -4.676321983337402, + "step": 143, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.3621502672115687, + "grad_norm": 4.043496608734131, + "learning_rate": 0.0001972902333091271, + "logits/chosen": -0.3931187391281128, + "logits/rejected": -0.5637274384498596, + "logps/chosen": -8.15110969543457, + "logps/rejected": -48.92616271972656, + "loss": 0.7109068632125854, + "memory(GiB)": 42.51, + "nll_loss": 0.4194295108318329, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.008020445704460144, + "rewards/margins": 3.4844448566436768, + "rewards/rejected": -3.476424217224121, + "step": 144, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.3646651996227601, + "grad_norm": 3.2291014194488525, + "learning_rate": 0.00019722563278083287, + "logits/chosen": -0.3712625503540039, + "logits/rejected": -0.6469380855560303, + "logps/chosen": -2.269747734069824, + "logps/rejected": -41.929664611816406, + "loss": 0.4003128111362457, + "memory(GiB)": 42.51, + "nll_loss": 0.182529017329216, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18503372371196747, + "rewards/margins": 2.92374849319458, + "rewards/rejected": -2.7387146949768066, + "step": 145, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.3671801320339516, + "grad_norm": 4.1039204597473145, + "learning_rate": 0.00019716028209058387, + "logits/chosen": -0.44887417554855347, + "logits/rejected": -0.6388370990753174, + "logps/chosen": -5.760115623474121, + "logps/rejected": -43.50143051147461, + "loss": 0.7498366236686707, + "memory(GiB)": 42.51, + "nll_loss": 0.48517200350761414, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.02230704203248024, + "rewards/margins": 2.708374500274658, + "rewards/rejected": -2.7306814193725586, + "step": 146, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.36969506444514305, + "grad_norm": 3.1436734199523926, + "learning_rate": 0.0001970941817426052, + "logits/chosen": -0.31916379928588867, + "logits/rejected": -0.577805757522583, + "logps/chosen": -3.4299936294555664, + "logps/rejected": -47.918155670166016, + "loss": 0.4161657989025116, + "memory(GiB)": 42.51, + "nll_loss": 0.21494346857070923, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14184261858463287, + "rewards/margins": 2.971557855606079, + "rewards/rejected": -2.8297150135040283, + "step": 147, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.3722099968563345, + "grad_norm": 2.7959487438201904, + "learning_rate": 0.00019702733224690605, + "logits/chosen": -0.43002602458000183, + "logits/rejected": -0.6337836980819702, + "logps/chosen": -8.148699760437012, + "logps/rejected": -42.15251922607422, + "loss": 0.7381757497787476, + "memory(GiB)": 42.51, + "nll_loss": 0.4981796443462372, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.013383438810706139, + "rewards/margins": 2.8139827251434326, + "rewards/rejected": -2.8273658752441406, + "step": 148, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.37472492926752593, + "grad_norm": 1.9313372373580933, + "learning_rate": 0.00019695973411927578, + "logits/chosen": -0.500542402267456, + "logits/rejected": -0.7104104161262512, + "logps/chosen": -2.929166316986084, + "logps/rejected": -54.1365966796875, + "loss": 0.42066073417663574, + "memory(GiB)": 42.51, + "nll_loss": 0.22536346316337585, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06520625203847885, + "rewards/margins": 3.698242664337158, + "rewards/rejected": -3.6330366134643555, + "step": 149, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.37723986167871737, + "grad_norm": 1.9907225370407104, + "learning_rate": 0.00019689138788127995, + "logits/chosen": -0.49561041593551636, + "logits/rejected": -0.6665260195732117, + "logps/chosen": -5.222489356994629, + "logps/rejected": -42.92105484008789, + "loss": 0.6262026429176331, + "memory(GiB)": 42.51, + "nll_loss": 0.385796457529068, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09229743480682373, + "rewards/margins": 3.072531223297119, + "rewards/rejected": -2.980233669281006, + "step": 150, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.3797547940899088, + "grad_norm": 3.5840723514556885, + "learning_rate": 0.00019682229406025635, + "logits/chosen": -0.4681642949581146, + "logits/rejected": -0.6473852396011353, + "logps/chosen": -8.059581756591797, + "logps/rejected": -54.22500228881836, + "loss": 0.6054436564445496, + "memory(GiB)": 42.51, + "nll_loss": 0.38610902428627014, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.097575843334198, + "rewards/margins": 3.8078057765960693, + "rewards/rejected": -3.7102296352386475, + "step": 151, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.3822697265011003, + "grad_norm": 4.087256908416748, + "learning_rate": 0.00019675245318931083, + "logits/chosen": -0.5369412899017334, + "logits/rejected": -0.749391496181488, + "logps/chosen": -9.42272663116455, + "logps/rejected": -59.544307708740234, + "loss": 0.6138618588447571, + "memory(GiB)": 42.51, + "nll_loss": 0.4435786008834839, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.10584679991006851, + "rewards/margins": 4.319212436676025, + "rewards/rejected": -4.425059795379639, + "step": 152, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.38478465891229174, + "grad_norm": 7.815120697021484, + "learning_rate": 0.0001966818658073133, + "logits/chosen": -0.6010515689849854, + "logits/rejected": -0.7820409536361694, + "logps/chosen": -7.579458713531494, + "logps/rejected": -62.63882064819336, + "loss": 0.709304928779602, + "memory(GiB)": 42.51, + "nll_loss": 0.5141942501068115, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.13567981123924255, + "rewards/margins": 4.736479759216309, + "rewards/rejected": -4.872159481048584, + "step": 153, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.3872995913234832, + "grad_norm": 7.4138031005859375, + "learning_rate": 0.0001966105324588934, + "logits/chosen": -0.5685268640518188, + "logits/rejected": -0.7353744506835938, + "logps/chosen": -11.372655868530273, + "logps/rejected": -61.317073822021484, + "loss": 0.8202418684959412, + "memory(GiB)": 42.51, + "nll_loss": 0.583480715751648, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.05895308405160904, + "rewards/margins": 4.791215419769287, + "rewards/rejected": -4.73226261138916, + "step": 154, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.3898145237346746, + "grad_norm": 2.7024075984954834, + "learning_rate": 0.00019653845369443657, + "logits/chosen": -0.4999295175075531, + "logits/rejected": -0.7417422533035278, + "logps/chosen": -6.620236873626709, + "logps/rejected": -70.78112030029297, + "loss": 0.6438719034194946, + "memory(GiB)": 42.51, + "nll_loss": 0.5055179595947266, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.060695331543684006, + "rewards/margins": 5.345253944396973, + "rewards/rejected": -5.284558296203613, + "step": 155, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.39232945614586606, + "grad_norm": 4.160597801208496, + "learning_rate": 0.00019646563007007952, + "logits/chosen": -0.5322343707084656, + "logits/rejected": -0.7421768307685852, + "logps/chosen": -6.158726215362549, + "logps/rejected": -47.234554290771484, + "loss": 0.6948550343513489, + "memory(GiB)": 42.51, + "nll_loss": 0.41276055574417114, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.09865140169858932, + "rewards/margins": 2.9533112049102783, + "rewards/rejected": -3.0519626140594482, + "step": 156, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.3948443885570575, + "grad_norm": 4.034154891967773, + "learning_rate": 0.00019639206214770608, + "logits/chosen": -0.5310461521148682, + "logits/rejected": -0.7443493008613586, + "logps/chosen": -3.5962212085723877, + "logps/rejected": -53.00592803955078, + "loss": 0.48415228724479675, + "memory(GiB)": 42.51, + "nll_loss": 0.2999834716320038, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.020860865712165833, + "rewards/margins": 3.5892655849456787, + "rewards/rejected": -3.5684046745300293, + "step": 157, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.397359320968249, + "grad_norm": 9.343074798583984, + "learning_rate": 0.00019631775049494285, + "logits/chosen": -0.44315972924232483, + "logits/rejected": -0.7157086730003357, + "logps/chosen": -4.082563400268555, + "logps/rejected": -64.51760864257812, + "loss": 0.5230289697647095, + "memory(GiB)": 42.51, + "nll_loss": 0.2998603284358978, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.021998610347509384, + "rewards/margins": 3.9378349781036377, + "rewards/rejected": -3.959833860397339, + "step": 158, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.39987425337944044, + "grad_norm": 2.403951644897461, + "learning_rate": 0.00019624269568515486, + "logits/chosen": -0.4427807629108429, + "logits/rejected": -0.674246072769165, + "logps/chosen": -5.679157257080078, + "logps/rejected": -46.368919372558594, + "loss": 0.5779350996017456, + "memory(GiB)": 42.51, + "nll_loss": 0.39234140515327454, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11242523044347763, + "rewards/margins": 2.7115559577941895, + "rewards/rejected": -2.599130630493164, + "step": 159, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.4023891857906319, + "grad_norm": 3.0051136016845703, + "learning_rate": 0.00019616689829744105, + "logits/chosen": -0.595456600189209, + "logits/rejected": -0.7742252945899963, + "logps/chosen": -6.239538669586182, + "logps/rejected": -36.2790641784668, + "loss": 0.6849649548530579, + "memory(GiB)": 42.51, + "nll_loss": 0.43636244535446167, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12961390614509583, + "rewards/margins": 2.334453821182251, + "rewards/rejected": -2.2048397064208984, + "step": 160, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.4049041182018233, + "grad_norm": 4.199156761169434, + "learning_rate": 0.0001960903589166299, + "logits/chosen": -0.6130431890487671, + "logits/rejected": -0.6931115388870239, + "logps/chosen": -8.752334594726562, + "logps/rejected": -42.808692932128906, + "loss": 0.6933425664901733, + "memory(GiB)": 42.51, + "nll_loss": 0.39625656604766846, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.2133105993270874, + "rewards/margins": 3.1959619522094727, + "rewards/rejected": -2.9826512336730957, + "step": 161, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.40741905061301475, + "grad_norm": 4.59311056137085, + "learning_rate": 0.00019601307813327482, + "logits/chosen": -0.5155867338180542, + "logits/rejected": -0.6933278441429138, + "logps/chosen": -3.984743595123291, + "logps/rejected": -44.87555694580078, + "loss": 0.6113356351852417, + "memory(GiB)": 42.51, + "nll_loss": 0.3314781188964844, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.04041612148284912, + "rewards/margins": 2.863827705383301, + "rewards/rejected": -2.8234119415283203, + "step": 162, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.40993398302420625, + "grad_norm": 2.7300822734832764, + "learning_rate": 0.00019593505654364965, + "logits/chosen": -0.4362912178039551, + "logits/rejected": -0.6915664076805115, + "logps/chosen": -3.529961585998535, + "logps/rejected": -61.31735610961914, + "loss": 0.4943440556526184, + "memory(GiB)": 42.51, + "nll_loss": 0.3457809090614319, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07190480828285217, + "rewards/margins": 4.230862617492676, + "rewards/rejected": -4.158957481384277, + "step": 163, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.4124489154353977, + "grad_norm": 1.3495978116989136, + "learning_rate": 0.00019585629474974415, + "logits/chosen": -0.3829024136066437, + "logits/rejected": -0.7348803877830505, + "logps/chosen": -0.20720741152763367, + "logps/rejected": -75.26912689208984, + "loss": 0.09561214596033096, + "memory(GiB)": 42.51, + "nll_loss": 0.0268473532050848, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11308196187019348, + "rewards/margins": 5.472672462463379, + "rewards/rejected": -5.359590530395508, + "step": 164, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.41496384784658913, + "grad_norm": 2.3953280448913574, + "learning_rate": 0.0001957767933592591, + "logits/chosen": -0.49378660321235657, + "logits/rejected": -0.6871320009231567, + "logps/chosen": -5.856287002563477, + "logps/rejected": -55.68489456176758, + "loss": 0.5843676924705505, + "memory(GiB)": 42.51, + "nll_loss": 0.3655776381492615, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0030907923355698586, + "rewards/margins": 4.014834403991699, + "rewards/rejected": -4.017925262451172, + "step": 165, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.41747878025778057, + "grad_norm": 5.787476062774658, + "learning_rate": 0.00019569655298560195, + "logits/chosen": -0.4845959544181824, + "logits/rejected": -0.725760281085968, + "logps/chosen": -3.9740681648254395, + "logps/rejected": -63.2627067565918, + "loss": 0.4704618752002716, + "memory(GiB)": 42.51, + "nll_loss": 0.34831976890563965, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09146178513765335, + "rewards/margins": 4.952096939086914, + "rewards/rejected": -4.860634803771973, + "step": 166, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.419993712668972, + "grad_norm": 2.719466209411621, + "learning_rate": 0.0001956155742478817, + "logits/chosen": -0.4632588326931, + "logits/rejected": -0.7149248123168945, + "logps/chosen": -2.24023175239563, + "logps/rejected": -70.20541381835938, + "loss": 0.28046005964279175, + "memory(GiB)": 42.51, + "nll_loss": 0.18644680082798004, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19088204205036163, + "rewards/margins": 5.480358123779297, + "rewards/rejected": -5.289475917816162, + "step": 167, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.42250864508016345, + "grad_norm": 3.770395517349243, + "learning_rate": 0.00019553385777090458, + "logits/chosen": -0.43829765915870667, + "logits/rejected": -0.7132550477981567, + "logps/chosen": -9.330952644348145, + "logps/rejected": -72.52558135986328, + "loss": 0.5571638941764832, + "memory(GiB)": 44.28, + "nll_loss": 0.46076467633247375, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10896007716655731, + "rewards/margins": 5.002399444580078, + "rewards/rejected": -4.893439292907715, + "step": 168, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.42502357749135494, + "grad_norm": 13.122285842895508, + "learning_rate": 0.00019545140418516873, + "logits/chosen": -0.628214418888092, + "logits/rejected": -0.7334375977516174, + "logps/chosen": -7.368585586547852, + "logps/rejected": -48.79350662231445, + "loss": 0.7424007654190063, + "memory(GiB)": 44.28, + "nll_loss": 0.5702238082885742, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07907923310995102, + "rewards/margins": 3.957150459289551, + "rewards/rejected": -3.8780713081359863, + "step": 169, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.4275385099025464, + "grad_norm": 2.2032506465911865, + "learning_rate": 0.0001953682141268598, + "logits/chosen": -0.4527589678764343, + "logits/rejected": -0.6895320415496826, + "logps/chosen": -3.53696870803833, + "logps/rejected": -64.63774871826172, + "loss": 0.35415133833885193, + "memory(GiB)": 44.28, + "nll_loss": 0.22634199261665344, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.032016728073358536, + "rewards/margins": 4.734713077545166, + "rewards/rejected": -4.702696800231934, + "step": 170, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.4300534423137378, + "grad_norm": 2.4580419063568115, + "learning_rate": 0.00019528428823784567, + "logits/chosen": -0.37069255113601685, + "logits/rejected": -0.7563280463218689, + "logps/chosen": -2.2511513233184814, + "logps/rejected": -86.21305847167969, + "loss": 0.22136372327804565, + "memory(GiB)": 44.28, + "nll_loss": 0.18806159496307373, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13967791199684143, + "rewards/margins": 6.765411853790283, + "rewards/rejected": -6.625734329223633, + "step": 171, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.43256837472492926, + "grad_norm": 2.1516947746276855, + "learning_rate": 0.0001951996271656717, + "logits/chosen": -0.5217421650886536, + "logits/rejected": -0.7590201497077942, + "logps/chosen": -4.64290714263916, + "logps/rejected": -64.5576171875, + "loss": 0.4696881175041199, + "memory(GiB)": 44.28, + "nll_loss": 0.3352178931236267, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0438573844730854, + "rewards/margins": 4.907254695892334, + "rewards/rejected": -4.863397598266602, + "step": 172, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.4350833071361207, + "grad_norm": 3.411616563796997, + "learning_rate": 0.00019511423156355577, + "logits/chosen": -0.4631859362125397, + "logits/rejected": -0.7260483503341675, + "logps/chosen": -5.667875289916992, + "logps/rejected": -75.57254028320312, + "loss": 0.35108137130737305, + "memory(GiB)": 44.28, + "nll_loss": 0.30264440178871155, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08381501585245132, + "rewards/margins": 5.755500793457031, + "rewards/rejected": -5.671685695648193, + "step": 173, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.43759823954731214, + "grad_norm": 4.192603588104248, + "learning_rate": 0.00019502810209038303, + "logits/chosen": -0.5316020250320435, + "logits/rejected": -0.6840960383415222, + "logps/chosen": -10.37562084197998, + "logps/rejected": -67.83643341064453, + "loss": 1.085424542427063, + "memory(GiB)": 44.28, + "nll_loss": 0.7863109111785889, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.0769924521446228, + "rewards/margins": 4.868200778961182, + "rewards/rejected": -4.945193767547607, + "step": 174, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.44011317195850364, + "grad_norm": 4.43406343460083, + "learning_rate": 0.00019494123941070108, + "logits/chosen": -0.6808348894119263, + "logits/rejected": -0.7925645112991333, + "logps/chosen": -7.7486252784729, + "logps/rejected": -48.33315658569336, + "loss": 0.9540278911590576, + "memory(GiB)": 44.28, + "nll_loss": 0.7564924955368042, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.1413968950510025, + "rewards/margins": 3.6723074913024902, + "rewards/rejected": -3.813704013824463, + "step": 175, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.4426281043696951, + "grad_norm": 6.438490867614746, + "learning_rate": 0.00019485364419471454, + "logits/chosen": -0.6449403762817383, + "logits/rejected": -0.7584658861160278, + "logps/chosen": -6.1986799240112305, + "logps/rejected": -49.09370422363281, + "loss": 0.6880564093589783, + "memory(GiB)": 44.28, + "nll_loss": 0.38199642300605774, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.20061062276363373, + "rewards/margins": 3.613797426223755, + "rewards/rejected": -3.814408302307129, + "step": 176, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.4451430367808865, + "grad_norm": 2.3246142864227295, + "learning_rate": 0.00019476531711828027, + "logits/chosen": -0.524186372756958, + "logits/rejected": -0.6847387552261353, + "logps/chosen": -6.170748710632324, + "logps/rejected": -58.475975036621094, + "loss": 0.4086158275604248, + "memory(GiB)": 44.28, + "nll_loss": 0.2584584057331085, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07990145683288574, + "rewards/margins": 4.2991790771484375, + "rewards/rejected": -4.219277381896973, + "step": 177, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.44765796919207795, + "grad_norm": 4.515077590942383, + "learning_rate": 0.00019467625886290167, + "logits/chosen": -0.5108579993247986, + "logits/rejected": -0.6950021386146545, + "logps/chosen": -14.55732250213623, + "logps/rejected": -50.92564392089844, + "loss": 0.9052462577819824, + "memory(GiB)": 44.28, + "nll_loss": 0.6600362062454224, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.2508185803890228, + "rewards/margins": 3.1151885986328125, + "rewards/rejected": -3.366007089614868, + "step": 178, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.4501729016032694, + "grad_norm": 4.742927074432373, + "learning_rate": 0.0001945864701157239, + "logits/chosen": -0.5691394805908203, + "logits/rejected": -0.6816927790641785, + "logps/chosen": -4.945348262786865, + "logps/rejected": -41.207801818847656, + "loss": 0.6003872752189636, + "memory(GiB)": 44.28, + "nll_loss": 0.3502579927444458, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07804224640130997, + "rewards/margins": 2.8702170848846436, + "rewards/rejected": -2.792174816131592, + "step": 179, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.4526878340144609, + "grad_norm": 2.001896619796753, + "learning_rate": 0.00019449595156952827, + "logits/chosen": -0.5839424133300781, + "logits/rejected": -0.7141891717910767, + "logps/chosen": -7.824261665344238, + "logps/rejected": -42.13900375366211, + "loss": 0.5872877240180969, + "memory(GiB)": 44.28, + "nll_loss": 0.3686111569404602, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15194831788539886, + "rewards/margins": 2.942096710205078, + "rewards/rejected": -2.7901487350463867, + "step": 180, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.4552027664256523, + "grad_norm": 2.7639639377593994, + "learning_rate": 0.00019440470392272696, + "logits/chosen": -0.5639482736587524, + "logits/rejected": -0.682327151298523, + "logps/chosen": -4.094886302947998, + "logps/rejected": -37.27549362182617, + "loss": 0.5251520872116089, + "memory(GiB)": 44.28, + "nll_loss": 0.3160794675350189, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16162550449371338, + "rewards/margins": 2.7334065437316895, + "rewards/rejected": -2.5717809200286865, + "step": 181, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.45771769883684377, + "grad_norm": 3.3179478645324707, + "learning_rate": 0.00019431272787935773, + "logits/chosen": -0.4620465636253357, + "logits/rejected": -0.7069180011749268, + "logps/chosen": -10.247895240783691, + "logps/rejected": -44.85274887084961, + "loss": 0.895711362361908, + "memory(GiB)": 44.28, + "nll_loss": 0.6755833625793457, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08936743438243866, + "rewards/margins": 2.546542167663574, + "rewards/rejected": -2.457174777984619, + "step": 182, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.4602326312480352, + "grad_norm": 1.921773910522461, + "learning_rate": 0.00019422002414907837, + "logits/chosen": -0.5177928805351257, + "logits/rejected": -0.6913548707962036, + "logps/chosen": -5.233502388000488, + "logps/rejected": -40.59012985229492, + "loss": 0.5152122974395752, + "memory(GiB)": 44.28, + "nll_loss": 0.34400492906570435, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14941944181919098, + "rewards/margins": 2.640808582305908, + "rewards/rejected": -2.491389513015747, + "step": 183, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.46274756365922665, + "grad_norm": 4.714654922485352, + "learning_rate": 0.00019412659344716124, + "logits/chosen": -0.5423455834388733, + "logits/rejected": -0.6348575949668884, + "logps/chosen": -10.46619987487793, + "logps/rejected": -40.417510986328125, + "loss": 0.974522054195404, + "memory(GiB)": 44.28, + "nll_loss": 0.6639810800552368, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.15414148569107056, + "rewards/margins": 2.581968307495117, + "rewards/rejected": -2.736109972000122, + "step": 184, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.4652624960704181, + "grad_norm": 1.0332216024398804, + "learning_rate": 0.00019403243649448787, + "logits/chosen": -0.4484131336212158, + "logits/rejected": -0.7438564896583557, + "logps/chosen": -2.097586154937744, + "logps/rejected": -52.63762664794922, + "loss": 0.2517527937889099, + "memory(GiB)": 44.28, + "nll_loss": 0.15005721151828766, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27270805835723877, + "rewards/margins": 3.6738533973693848, + "rewards/rejected": -3.4011456966400146, + "step": 185, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.4677774284816096, + "grad_norm": 6.869028091430664, + "learning_rate": 0.00019393755401754322, + "logits/chosen": -0.495494544506073, + "logits/rejected": -0.7087106108665466, + "logps/chosen": -5.248052597045898, + "logps/rejected": -53.633941650390625, + "loss": 0.7538958787918091, + "memory(GiB)": 44.28, + "nll_loss": 0.489072322845459, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09345913678407669, + "rewards/margins": 3.6751837730407715, + "rewards/rejected": -3.7686429023742676, + "step": 186, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.470292360892801, + "grad_norm": 8.480405807495117, + "learning_rate": 0.00019384194674841026, + "logits/chosen": -0.5885705947875977, + "logits/rejected": -0.771264910697937, + "logps/chosen": -8.139009475708008, + "logps/rejected": -56.5067253112793, + "loss": 0.9823139905929565, + "memory(GiB)": 44.28, + "nll_loss": 0.8092753291130066, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3364828824996948, + "rewards/margins": 4.044948577880859, + "rewards/rejected": -4.381431579589844, + "step": 187, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.47280729330399246, + "grad_norm": 5.539621829986572, + "learning_rate": 0.0001937456154247641, + "logits/chosen": -0.5992149114608765, + "logits/rejected": -0.7545720934867859, + "logps/chosen": -5.675577640533447, + "logps/rejected": -53.787532806396484, + "loss": 0.4309059977531433, + "memory(GiB)": 44.28, + "nll_loss": 0.3199475407600403, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08290509879589081, + "rewards/margins": 3.8106563091278076, + "rewards/rejected": -3.893561601638794, + "step": 188, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.4753222257151839, + "grad_norm": 7.750495910644531, + "learning_rate": 0.00019364856078986652, + "logits/chosen": -0.5513782501220703, + "logits/rejected": -0.6628525853157043, + "logps/chosen": -13.800762176513672, + "logps/rejected": -49.375457763671875, + "loss": 1.059139370918274, + "memory(GiB)": 44.28, + "nll_loss": 0.7687480449676514, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11855825036764145, + "rewards/margins": 3.4539475440979004, + "rewards/rejected": -3.5725057125091553, + "step": 189, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.47783715812637534, + "grad_norm": 2.178455352783203, + "learning_rate": 0.0001935507835925601, + "logits/chosen": -0.5001492500305176, + "logits/rejected": -0.683919370174408, + "logps/chosen": -5.1807756423950195, + "logps/rejected": -48.025428771972656, + "loss": 0.4980285167694092, + "memory(GiB)": 44.28, + "nll_loss": 0.3027910590171814, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.057751405984163284, + "rewards/margins": 3.4277446269989014, + "rewards/rejected": -3.3699934482574463, + "step": 190, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.4803520905375668, + "grad_norm": 2.17415714263916, + "learning_rate": 0.00019345228458726252, + "logits/chosen": -0.4294896423816681, + "logits/rejected": -0.624671459197998, + "logps/chosen": -6.253881454467773, + "logps/rejected": -50.155250549316406, + "loss": 0.5238272547721863, + "memory(GiB)": 44.28, + "nll_loss": 0.3330422341823578, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.024177849292755127, + "rewards/margins": 2.913661003112793, + "rewards/rejected": -2.8894832134246826, + "step": 191, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.4828670229487583, + "grad_norm": 5.977149486541748, + "learning_rate": 0.00019335306453396064, + "logits/chosen": -0.33437275886535645, + "logits/rejected": -0.5742294788360596, + "logps/chosen": -4.437854290008545, + "logps/rejected": -45.52449035644531, + "loss": 0.5229467153549194, + "memory(GiB)": 44.28, + "nll_loss": 0.2966506779193878, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.040689948946237564, + "rewards/margins": 2.79740571975708, + "rewards/rejected": -2.7567155361175537, + "step": 192, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.4853819553599497, + "grad_norm": 4.404195308685303, + "learning_rate": 0.00019325312419820473, + "logits/chosen": -0.40588510036468506, + "logits/rejected": -0.6035714149475098, + "logps/chosen": -4.6151909828186035, + "logps/rejected": -44.15699005126953, + "loss": 0.5418132543563843, + "memory(GiB)": 44.28, + "nll_loss": 0.315884530544281, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.029081931337714195, + "rewards/margins": 2.7166473865509033, + "rewards/rejected": -2.687565565109253, + "step": 193, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.48789688777114115, + "grad_norm": 2.59771466255188, + "learning_rate": 0.00019315246435110256, + "logits/chosen": -0.4158392548561096, + "logits/rejected": -0.6061286330223083, + "logps/chosen": -6.033608436584473, + "logps/rejected": -42.764007568359375, + "loss": 0.5932649970054626, + "memory(GiB)": 44.28, + "nll_loss": 0.43631434440612793, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.22790978848934174, + "rewards/margins": 2.7686610221862793, + "rewards/rejected": -2.5407509803771973, + "step": 194, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.4904118201823326, + "grad_norm": 2.5979843139648438, + "learning_rate": 0.00019305108576931336, + "logits/chosen": -0.4368806779384613, + "logits/rejected": -0.6110289692878723, + "logps/chosen": -6.914710998535156, + "logps/rejected": -49.386474609375, + "loss": 0.6119182705879211, + "memory(GiB)": 44.28, + "nll_loss": 0.38600677251815796, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.016620682552456856, + "rewards/margins": 3.258530616760254, + "rewards/rejected": -3.241909980773926, + "step": 195, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.49292675259352403, + "grad_norm": 1.838233232498169, + "learning_rate": 0.0001929489892350419, + "logits/chosen": -0.4425433874130249, + "logits/rejected": -0.643796443939209, + "logps/chosen": -3.6048429012298584, + "logps/rejected": -54.97930908203125, + "loss": 0.386331170797348, + "memory(GiB)": 44.28, + "nll_loss": 0.28649255633354187, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14747180044651031, + "rewards/margins": 4.032291889190674, + "rewards/rejected": -3.884819984436035, + "step": 196, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.49544168500471547, + "grad_norm": 2.9112889766693115, + "learning_rate": 0.00019284617553603242, + "logits/chosen": -0.48377659916877747, + "logits/rejected": -0.605985164642334, + "logps/chosen": -4.890591144561768, + "logps/rejected": -54.24449920654297, + "loss": 0.5024012327194214, + "memory(GiB)": 44.28, + "nll_loss": 0.34084275364875793, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.23494963347911835, + "rewards/margins": 3.875343084335327, + "rewards/rejected": -3.6403934955596924, + "step": 197, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.49795661741590697, + "grad_norm": 3.7803006172180176, + "learning_rate": 0.0001927426454655627, + "logits/chosen": -0.3935869038105011, + "logits/rejected": -0.5888193845748901, + "logps/chosen": -7.011892318725586, + "logps/rejected": -54.100929260253906, + "loss": 0.5712710618972778, + "memory(GiB)": 44.28, + "nll_loss": 0.3695356249809265, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.02423219382762909, + "rewards/margins": 3.535330057144165, + "rewards/rejected": -3.5110979080200195, + "step": 198, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.5004715498270984, + "grad_norm": 5.7016282081604, + "learning_rate": 0.0001926383998224377, + "logits/chosen": -0.3959541618824005, + "logits/rejected": -0.6700530648231506, + "logps/chosen": -2.51404070854187, + "logps/rejected": -55.97195816040039, + "loss": 0.3349680006504059, + "memory(GiB)": 46.04, + "nll_loss": 0.20782868564128876, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11337391287088394, + "rewards/margins": 3.802442789077759, + "rewards/rejected": -3.6890690326690674, + "step": 199, + "train_speed(iter/s)": 0.011241 + }, + { + "epoch": 0.5029864822382899, + "grad_norm": 1.2177132368087769, + "learning_rate": 0.0001925334394109835, + "logits/chosen": -0.39053067564964294, + "logits/rejected": -0.6353622674942017, + "logps/chosen": -4.960577964782715, + "logps/rejected": -55.41823196411133, + "loss": 0.4196793735027313, + "memory(GiB)": 46.04, + "nll_loss": 0.2920268774032593, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06429460644721985, + "rewards/margins": 3.601248025894165, + "rewards/rejected": -3.5369529724121094, + "step": 200, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.5055014146494813, + "grad_norm": 7.681524753570557, + "learning_rate": 0.00019242776504104118, + "logits/chosen": -0.3979511260986328, + "logits/rejected": -0.5972259044647217, + "logps/chosen": -6.038961887359619, + "logps/rejected": -54.249908447265625, + "loss": 0.6535472869873047, + "memory(GiB)": 46.04, + "nll_loss": 0.41866153478622437, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.054757650941610336, + "rewards/margins": 3.7828688621520996, + "rewards/rejected": -3.7281110286712646, + "step": 201, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.5080163470606728, + "grad_norm": 2.600893497467041, + "learning_rate": 0.00019232137752796044, + "logits/chosen": -0.4759564995765686, + "logits/rejected": -0.6580658555030823, + "logps/chosen": -4.586948394775391, + "logps/rejected": -51.12422180175781, + "loss": 0.6128251552581787, + "memory(GiB)": 46.04, + "nll_loss": 0.46890994906425476, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11943411082029343, + "rewards/margins": 3.681659460067749, + "rewards/rejected": -3.562225580215454, + "step": 202, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.5105312794718642, + "grad_norm": 2.17677640914917, + "learning_rate": 0.00019221427769259333, + "logits/chosen": -0.39213788509368896, + "logits/rejected": -0.6274294853210449, + "logps/chosen": -4.02351713180542, + "logps/rejected": -54.13360595703125, + "loss": 0.4597245752811432, + "memory(GiB)": 46.04, + "nll_loss": 0.31017711758613586, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19347746670246124, + "rewards/margins": 3.6046862602233887, + "rewards/rejected": -3.4112088680267334, + "step": 203, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.5130462118830557, + "grad_norm": 3.405174732208252, + "learning_rate": 0.00019210646636128807, + "logits/chosen": -0.4578840434551239, + "logits/rejected": -0.6036797165870667, + "logps/chosen": -4.427443027496338, + "logps/rejected": -44.289703369140625, + "loss": 0.4890953004360199, + "memory(GiB)": 46.04, + "nll_loss": 0.2529718279838562, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10575352609157562, + "rewards/margins": 3.2759764194488525, + "rewards/rejected": -3.170222759246826, + "step": 204, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.515561144294247, + "grad_norm": 1.9355807304382324, + "learning_rate": 0.00019199794436588243, + "logits/chosen": -0.3543737232685089, + "logits/rejected": -0.6088857650756836, + "logps/chosen": -6.951679229736328, + "logps/rejected": -68.91035461425781, + "loss": 0.39467012882232666, + "memory(GiB)": 46.04, + "nll_loss": 0.27390897274017334, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13660939037799835, + "rewards/margins": 4.888247013092041, + "rewards/rejected": -4.751636981964111, + "step": 205, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.5180760767054385, + "grad_norm": 12.973567962646484, + "learning_rate": 0.00019188871254369752, + "logits/chosen": -0.4342585802078247, + "logits/rejected": -0.5918402671813965, + "logps/chosen": -5.572432518005371, + "logps/rejected": -47.04520034790039, + "loss": 0.8162675499916077, + "memory(GiB)": 46.04, + "nll_loss": 0.5575422048568726, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.07569149881601334, + "rewards/margins": 2.895029067993164, + "rewards/rejected": -2.9707207679748535, + "step": 206, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.52059100911663, + "grad_norm": 3.5247535705566406, + "learning_rate": 0.00019177877173753128, + "logits/chosen": -0.48035189509391785, + "logits/rejected": -0.6423895359039307, + "logps/chosen": -6.628141403198242, + "logps/rejected": -52.741973876953125, + "loss": 0.6659956574440002, + "memory(GiB)": 46.04, + "nll_loss": 0.4173967242240906, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.11046299338340759, + "rewards/margins": 3.8508124351501465, + "rewards/rejected": -3.961275577545166, + "step": 207, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5231059415278214, + "grad_norm": 9.15176773071289, + "learning_rate": 0.00019166812279565178, + "logits/chosen": -0.4394856095314026, + "logits/rejected": -0.5828301906585693, + "logps/chosen": -6.419562816619873, + "logps/rejected": -65.78706359863281, + "loss": 0.7201980948448181, + "memory(GiB)": 46.04, + "nll_loss": 0.5739071369171143, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.05124050751328468, + "rewards/margins": 5.137848377227783, + "rewards/rejected": -5.086607933044434, + "step": 208, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5256208739390129, + "grad_norm": 1.9403810501098633, + "learning_rate": 0.000191556766571791, + "logits/chosen": -0.36848342418670654, + "logits/rejected": -0.6423362493515015, + "logps/chosen": -4.248262882232666, + "logps/rejected": -82.75714111328125, + "loss": 0.3508817255496979, + "memory(GiB)": 46.04, + "nll_loss": 0.274202436208725, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14326420426368713, + "rewards/margins": 6.368471622467041, + "rewards/rejected": -6.225207328796387, + "step": 209, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5281358063502043, + "grad_norm": 2.350217342376709, + "learning_rate": 0.000191444703925138, + "logits/chosen": -0.36294057965278625, + "logits/rejected": -0.681117057800293, + "logps/chosen": -3.7879486083984375, + "logps/rejected": -73.4488525390625, + "loss": 0.4432718753814697, + "memory(GiB)": 46.04, + "nll_loss": 0.3073051869869232, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17865654826164246, + "rewards/margins": 5.326575756072998, + "rewards/rejected": -5.14792013168335, + "step": 210, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5306507387613958, + "grad_norm": 9.633001327514648, + "learning_rate": 0.00019133193572033242, + "logits/chosen": -0.35440313816070557, + "logits/rejected": -0.647504448890686, + "logps/chosen": -5.407248020172119, + "logps/rejected": -69.90013122558594, + "loss": 0.4669504165649414, + "memory(GiB)": 46.04, + "nll_loss": 0.26951655745506287, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.040601931512355804, + "rewards/margins": 5.3051629066467285, + "rewards/rejected": -5.345764636993408, + "step": 211, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5331656711725873, + "grad_norm": 2.539501428604126, + "learning_rate": 0.00019121846282745778, + "logits/chosen": -0.4186283349990845, + "logits/rejected": -0.5546895265579224, + "logps/chosen": -5.969156742095947, + "logps/rejected": -55.70306396484375, + "loss": 0.5105364918708801, + "memory(GiB)": 46.04, + "nll_loss": 0.33588486909866333, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05188417807221413, + "rewards/margins": 4.2251996994018555, + "rewards/rejected": -4.173315525054932, + "step": 212, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5356806035837787, + "grad_norm": 2.609890937805176, + "learning_rate": 0.00019110428612203464, + "logits/chosen": -0.40281373262405396, + "logits/rejected": -0.6546099185943604, + "logps/chosen": -6.447667121887207, + "logps/rejected": -89.42594909667969, + "loss": 0.43404707312583923, + "memory(GiB)": 46.04, + "nll_loss": 0.29497838020324707, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05883106589317322, + "rewards/margins": 6.6620965003967285, + "rewards/rejected": -6.603265285491943, + "step": 213, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5381955359949702, + "grad_norm": 9.001029968261719, + "learning_rate": 0.00019098940648501405, + "logits/chosen": -0.4067293107509613, + "logits/rejected": -0.6849221587181091, + "logps/chosen": -3.5787479877471924, + "logps/rejected": -69.67017364501953, + "loss": 0.4548971951007843, + "memory(GiB)": 46.04, + "nll_loss": 0.29569321870803833, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.01640375703573227, + "rewards/margins": 5.399847507476807, + "rewards/rejected": -5.416250705718994, + "step": 214, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5407104684061615, + "grad_norm": 3.073113203048706, + "learning_rate": 0.0001908738248027706, + "logits/chosen": -0.38208895921707153, + "logits/rejected": -0.6967729926109314, + "logps/chosen": -4.078890323638916, + "logps/rejected": -85.77256774902344, + "loss": 0.37352272868156433, + "memory(GiB)": 46.04, + "nll_loss": 0.27824363112449646, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0630294606089592, + "rewards/margins": 6.849693298339844, + "rewards/rejected": -6.78666353225708, + "step": 215, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.543225400817353, + "grad_norm": 4.24471378326416, + "learning_rate": 0.00019075754196709572, + "logits/chosen": -0.4263435900211334, + "logits/rejected": -0.7725025415420532, + "logps/chosen": -2.7350358963012695, + "logps/rejected": -105.30278015136719, + "loss": 0.3268907964229584, + "memory(GiB)": 46.04, + "nll_loss": 0.2297362983226776, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09944246709346771, + "rewards/margins": 8.845938682556152, + "rewards/rejected": -8.746496200561523, + "step": 216, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5457403332285444, + "grad_norm": 7.5312819480896, + "learning_rate": 0.00019064055887519062, + "logits/chosen": -0.48908063769340515, + "logits/rejected": -0.6851401329040527, + "logps/chosen": -8.316049575805664, + "logps/rejected": -87.41441345214844, + "loss": 0.7208706140518188, + "memory(GiB)": 46.04, + "nll_loss": 0.5370450615882874, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.054982058703899384, + "rewards/margins": 7.039226055145264, + "rewards/rejected": -7.094207763671875, + "step": 217, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5482552656397359, + "grad_norm": 5.015770435333252, + "learning_rate": 0.00019052287642965952, + "logits/chosen": -0.4340421259403229, + "logits/rejected": -0.6953417062759399, + "logps/chosen": -6.858056545257568, + "logps/rejected": -93.66557312011719, + "loss": 0.5943813323974609, + "memory(GiB)": 46.04, + "nll_loss": 0.40210384130477905, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.02029874362051487, + "rewards/margins": 7.397557258605957, + "rewards/rejected": -7.417856693267822, + "step": 218, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5507701980509274, + "grad_norm": 25.73984718322754, + "learning_rate": 0.00019040449553850257, + "logits/chosen": -0.4239513576030731, + "logits/rejected": -0.6732109785079956, + "logps/chosen": -8.325424194335938, + "logps/rejected": -87.74398040771484, + "loss": 0.8946905732154846, + "memory(GiB)": 46.04, + "nll_loss": 0.648015558719635, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.2404731959104538, + "rewards/margins": 6.910739898681641, + "rewards/rejected": -7.151212215423584, + "step": 219, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5532851304621188, + "grad_norm": 2.769166946411133, + "learning_rate": 0.000190285417115109, + "logits/chosen": -0.49040669202804565, + "logits/rejected": -0.7523952126502991, + "logps/chosen": -7.0566301345825195, + "logps/rejected": -82.0855941772461, + "loss": 0.5741197466850281, + "memory(GiB)": 46.04, + "nll_loss": 0.4936599135398865, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05626031011343002, + "rewards/margins": 6.2684006690979, + "rewards/rejected": -6.2121405601501465, + "step": 220, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5558000628733103, + "grad_norm": 4.775945663452148, + "learning_rate": 0.00019016564207824992, + "logits/chosen": -0.5227105021476746, + "logits/rejected": -0.6636734008789062, + "logps/chosen": -4.836442947387695, + "logps/rejected": -49.289161682128906, + "loss": 0.5425068736076355, + "memory(GiB)": 46.04, + "nll_loss": 0.35901886224746704, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08150728791952133, + "rewards/margins": 3.6998212337493896, + "rewards/rejected": -3.618313789367676, + "step": 221, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.5583149952845017, + "grad_norm": 4.910019874572754, + "learning_rate": 0.00019004517135207127, + "logits/chosen": -0.3941330313682556, + "logits/rejected": -0.660327136516571, + "logps/chosen": -4.9822516441345215, + "logps/rejected": -77.26637268066406, + "loss": 0.5757743120193481, + "memory(GiB)": 46.04, + "nll_loss": 0.39564475417137146, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17502854764461517, + "rewards/margins": 5.9140496253967285, + "rewards/rejected": -5.739021301269531, + "step": 222, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.5608299276956932, + "grad_norm": 6.759812355041504, + "learning_rate": 0.00018992400586608676, + "logits/chosen": -0.4000013470649719, + "logits/rejected": -0.7170141935348511, + "logps/chosen": -2.594712972640991, + "logps/rejected": -67.61477661132812, + "loss": 0.4701526165008545, + "memory(GiB)": 46.04, + "nll_loss": 0.26969754695892334, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.03438286855816841, + "rewards/margins": 4.8134918212890625, + "rewards/rejected": -4.847874641418457, + "step": 223, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.5633448601068847, + "grad_norm": 2.779904842376709, + "learning_rate": 0.00018980214655517057, + "logits/chosen": -0.3656763732433319, + "logits/rejected": -0.6763076186180115, + "logps/chosen": -2.6532158851623535, + "logps/rejected": -62.15465545654297, + "loss": 0.3870552182197571, + "memory(GiB)": 46.04, + "nll_loss": 0.2073136270046234, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13753379881381989, + "rewards/margins": 4.43159818649292, + "rewards/rejected": -4.294064521789551, + "step": 224, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.565859792518076, + "grad_norm": 4.610898971557617, + "learning_rate": 0.00018967959435955026, + "logits/chosen": -0.41923901438713074, + "logits/rejected": -0.7322454452514648, + "logps/chosen": -4.079246997833252, + "logps/rejected": -85.366943359375, + "loss": 0.5197367668151855, + "memory(GiB)": 46.04, + "nll_loss": 0.3901809751987457, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0011233258992433548, + "rewards/margins": 6.633144378662109, + "rewards/rejected": -6.634267807006836, + "step": 225, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.5683747249292675, + "grad_norm": 1.7400152683258057, + "learning_rate": 0.0001895563502247995, + "logits/chosen": -0.4450499713420868, + "logits/rejected": -0.6796970367431641, + "logps/chosen": -3.0244576930999756, + "logps/rejected": -73.87388610839844, + "loss": 0.3383142650127411, + "memory(GiB)": 46.04, + "nll_loss": 0.21261034905910492, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16421900689601898, + "rewards/margins": 5.867359161376953, + "rewards/rejected": -5.703140735626221, + "step": 226, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5708896573404589, + "grad_norm": 2.3019213676452637, + "learning_rate": 0.00018943241510183063, + "logits/chosen": -0.458271324634552, + "logits/rejected": -0.6559279561042786, + "logps/chosen": -6.137403964996338, + "logps/rejected": -69.26449584960938, + "loss": 0.43954962491989136, + "memory(GiB)": 46.04, + "nll_loss": 0.3364819884300232, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18200799822807312, + "rewards/margins": 5.23583459854126, + "rewards/rejected": -5.053826332092285, + "step": 227, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5734045897516504, + "grad_norm": 4.305068492889404, + "learning_rate": 0.00018930778994688757, + "logits/chosen": -0.4849119186401367, + "logits/rejected": -0.6896030902862549, + "logps/chosen": -6.080837726593018, + "logps/rejected": -76.35552978515625, + "loss": 0.6719568967819214, + "memory(GiB)": 46.04, + "nll_loss": 0.47549009323120117, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.02206701785326004, + "rewards/margins": 5.978794097900391, + "rewards/rejected": -5.956727027893066, + "step": 228, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5759195221628419, + "grad_norm": 2.6933469772338867, + "learning_rate": 0.00018918247572153823, + "logits/chosen": -0.461727499961853, + "logits/rejected": -0.6680722236633301, + "logps/chosen": -4.37862491607666, + "logps/rejected": -86.59188842773438, + "loss": 0.42717570066452026, + "memory(GiB)": 46.04, + "nll_loss": 0.3303125500679016, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08164448291063309, + "rewards/margins": 7.104611873626709, + "rewards/rejected": -7.022968292236328, + "step": 229, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5784344545740333, + "grad_norm": 8.504460334777832, + "learning_rate": 0.0001890564733926672, + "logits/chosen": -0.34246841073036194, + "logits/rejected": -0.5939763188362122, + "logps/chosen": -11.11518383026123, + "logps/rejected": -101.42696380615234, + "loss": 0.7018269300460815, + "memory(GiB)": 46.04, + "nll_loss": 0.4599987864494324, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.15412060916423798, + "rewards/margins": 7.550798416137695, + "rewards/rejected": -7.704918384552002, + "step": 230, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5809493869852248, + "grad_norm": 3.522885322570801, + "learning_rate": 0.00018892978393246818, + "logits/chosen": -0.5426961183547974, + "logits/rejected": -0.6135550737380981, + "logps/chosen": -10.763383865356445, + "logps/rejected": -72.09516143798828, + "loss": 0.5679591298103333, + "memory(GiB)": 46.04, + "nll_loss": 0.39400285482406616, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07710488885641098, + "rewards/margins": 5.834630012512207, + "rewards/rejected": -5.757525444030762, + "step": 231, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5834643193964162, + "grad_norm": 2.6743204593658447, + "learning_rate": 0.00018880240831843666, + "logits/chosen": -0.42806094884872437, + "logits/rejected": -0.6858870983123779, + "logps/chosen": -7.369032859802246, + "logps/rejected": -94.46809387207031, + "loss": 0.49837395548820496, + "memory(GiB)": 46.04, + "nll_loss": 0.3631903827190399, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1583317369222641, + "rewards/margins": 7.130619525909424, + "rewards/rejected": -6.972287654876709, + "step": 232, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5859792518076077, + "grad_norm": 2.545430898666382, + "learning_rate": 0.00018867434753336223, + "logits/chosen": -0.5526742935180664, + "logits/rejected": -0.6764613389968872, + "logps/chosen": -6.2565155029296875, + "logps/rejected": -53.53122329711914, + "loss": 0.6385827660560608, + "memory(GiB)": 46.04, + "nll_loss": 0.47814565896987915, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18877720832824707, + "rewards/margins": 4.254955768585205, + "rewards/rejected": -4.066178321838379, + "step": 233, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5884941842187991, + "grad_norm": 5.874550819396973, + "learning_rate": 0.000188545602565321, + "logits/chosen": -0.5431405901908875, + "logits/rejected": -0.6591349840164185, + "logps/chosen": -7.041271686553955, + "logps/rejected": -51.830989837646484, + "loss": 0.511455774307251, + "memory(GiB)": 46.04, + "nll_loss": 0.3468693494796753, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17714644968509674, + "rewards/margins": 3.9428811073303223, + "rewards/rejected": -3.7657346725463867, + "step": 234, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5910091166299906, + "grad_norm": 11.230829238891602, + "learning_rate": 0.0001884161744076681, + "logits/chosen": -0.4081313908100128, + "logits/rejected": -0.6242074370384216, + "logps/chosen": -5.126794815063477, + "logps/rejected": -49.272377014160156, + "loss": 0.6269602179527283, + "memory(GiB)": 46.04, + "nll_loss": 0.440512478351593, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.023164164274930954, + "rewards/margins": 3.228619337081909, + "rewards/rejected": -3.2054548263549805, + "step": 235, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.5935240490411821, + "grad_norm": 2.906224489212036, + "learning_rate": 0.00018828606405902986, + "logits/chosen": -0.4151853919029236, + "logits/rejected": -0.6895049810409546, + "logps/chosen": -3.3425395488739014, + "logps/rejected": -60.14630126953125, + "loss": 0.40797102451324463, + "memory(GiB)": 46.04, + "nll_loss": 0.3100608289241791, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0746859535574913, + "rewards/margins": 4.381271839141846, + "rewards/rejected": -4.306585788726807, + "step": 236, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.5960389814523734, + "grad_norm": 5.449255466461182, + "learning_rate": 0.00018815527252329624, + "logits/chosen": -0.5026019811630249, + "logits/rejected": -0.6662022471427917, + "logps/chosen": -6.956883430480957, + "logps/rejected": -59.189815521240234, + "loss": 0.6439992785453796, + "memory(GiB)": 46.04, + "nll_loss": 0.448159396648407, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.038928791880607605, + "rewards/margins": 4.4153032302856445, + "rewards/rejected": -4.376374244689941, + "step": 237, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.5985539138635649, + "grad_norm": 1.9680973291397095, + "learning_rate": 0.00018802380080961296, + "logits/chosen": -0.4111896753311157, + "logits/rejected": -0.6969754695892334, + "logps/chosen": -2.109022617340088, + "logps/rejected": -70.76811981201172, + "loss": 0.30665695667266846, + "memory(GiB)": 46.04, + "nll_loss": 0.19306617975234985, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1144958883523941, + "rewards/margins": 5.321486473083496, + "rewards/rejected": -5.206991195678711, + "step": 238, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.6010688462747563, + "grad_norm": 2.106550931930542, + "learning_rate": 0.00018789164993237382, + "logits/chosen": -0.5053385496139526, + "logits/rejected": -0.6760715842247009, + "logps/chosen": -3.813554048538208, + "logps/rejected": -71.93482971191406, + "loss": 0.3176461160182953, + "memory(GiB)": 46.04, + "nll_loss": 0.18995390832424164, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16428788006305695, + "rewards/margins": 5.706036567687988, + "rewards/rejected": -5.541748523712158, + "step": 239, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.6035837786859478, + "grad_norm": 17.008098602294922, + "learning_rate": 0.0001877588209112128, + "logits/chosen": -0.4033944010734558, + "logits/rejected": -0.6984363794326782, + "logps/chosen": -5.072837829589844, + "logps/rejected": -76.01246643066406, + "loss": 0.8396996259689331, + "memory(GiB)": 46.04, + "nll_loss": 0.458006352186203, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.022617299109697342, + "rewards/margins": 5.918478965759277, + "rewards/rejected": -5.941096305847168, + "step": 240, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.6060987110971393, + "grad_norm": 1.8717386722564697, + "learning_rate": 0.00018762531477099626, + "logits/chosen": -0.3073655366897583, + "logits/rejected": -0.6664487719535828, + "logps/chosen": -5.499012470245361, + "logps/rejected": -105.34025573730469, + "loss": 0.46345487236976624, + "memory(GiB)": 46.04, + "nll_loss": 0.321692556142807, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.015820708125829697, + "rewards/margins": 8.114753723144531, + "rewards/rejected": -8.098933219909668, + "step": 241, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.6086136435083307, + "grad_norm": 12.112175941467285, + "learning_rate": 0.00018749113254181498, + "logits/chosen": -0.5718834400177002, + "logits/rejected": -0.7824399471282959, + "logps/chosen": -3.723027229309082, + "logps/rejected": -83.3447265625, + "loss": 0.4468189775943756, + "memory(GiB)": 46.04, + "nll_loss": 0.3446814715862274, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.03424655646085739, + "rewards/margins": 7.22880744934082, + "rewards/rejected": -7.263053894042969, + "step": 242, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6111285759195222, + "grad_norm": 3.188166618347168, + "learning_rate": 0.0001873562752589762, + "logits/chosen": -0.47086432576179504, + "logits/rejected": -0.6838089823722839, + "logps/chosen": -8.102127075195312, + "logps/rejected": -79.43113708496094, + "loss": 0.47154104709625244, + "memory(GiB)": 46.04, + "nll_loss": 0.3822014331817627, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09235017001628876, + "rewards/margins": 6.291683197021484, + "rewards/rejected": -6.1993327140808105, + "step": 243, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6136435083307136, + "grad_norm": 4.067774772644043, + "learning_rate": 0.00018722074396299566, + "logits/chosen": -0.46753576397895813, + "logits/rejected": -0.7191226482391357, + "logps/chosen": -9.372048377990723, + "logps/rejected": -93.54489135742188, + "loss": 0.4576195180416107, + "memory(GiB)": 46.04, + "nll_loss": 0.31360840797424316, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07592315226793289, + "rewards/margins": 7.556896209716797, + "rewards/rejected": -7.480972766876221, + "step": 244, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6161584407419051, + "grad_norm": 2.5217037200927734, + "learning_rate": 0.00018708453969958958, + "logits/chosen": -0.5039016008377075, + "logits/rejected": -0.6719211935997009, + "logps/chosen": -6.228890419006348, + "logps/rejected": -58.18281936645508, + "loss": 0.7915486097335815, + "memory(GiB)": 46.04, + "nll_loss": 0.5135347247123718, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.017953312024474144, + "rewards/margins": 4.20432186126709, + "rewards/rejected": -4.222275733947754, + "step": 245, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6186733731530966, + "grad_norm": 1.6694215536117554, + "learning_rate": 0.00018694766351966667, + "logits/chosen": -0.4057798981666565, + "logits/rejected": -0.6351858973503113, + "logps/chosen": -6.20676326751709, + "logps/rejected": -72.40664672851562, + "loss": 0.4737001359462738, + "memory(GiB)": 46.04, + "nll_loss": 0.35722869634628296, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.26643669605255127, + "rewards/margins": 5.429479598999023, + "rewards/rejected": -5.163043022155762, + "step": 246, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.621188305564288, + "grad_norm": 4.144296169281006, + "learning_rate": 0.00018681011647931974, + "logits/chosen": -0.34827110171318054, + "logits/rejected": -0.6173001527786255, + "logps/chosen": -6.582627773284912, + "logps/rejected": -78.4413070678711, + "loss": 0.6141136288642883, + "memory(GiB)": 46.04, + "nll_loss": 0.5033036470413208, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.004567645490169525, + "rewards/margins": 5.899599552154541, + "rewards/rejected": -5.895031929016113, + "step": 247, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6237032379754794, + "grad_norm": 13.872243881225586, + "learning_rate": 0.00018667189963981793, + "logits/chosen": -0.387536883354187, + "logits/rejected": -0.6405035257339478, + "logps/chosen": -3.9526758193969727, + "logps/rejected": -55.78480529785156, + "loss": 0.5534759163856506, + "memory(GiB)": 46.04, + "nll_loss": 0.38310396671295166, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1983298808336258, + "rewards/margins": 3.9616622924804688, + "rewards/rejected": -3.7633323669433594, + "step": 248, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6262181703866708, + "grad_norm": 4.9104180335998535, + "learning_rate": 0.00018653301406759825, + "logits/chosen": -0.43706703186035156, + "logits/rejected": -0.7033647298812866, + "logps/chosen": -4.780174732208252, + "logps/rejected": -63.24848937988281, + "loss": 0.4667983949184418, + "memory(GiB)": 46.04, + "nll_loss": 0.3596409559249878, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10462813079357147, + "rewards/margins": 4.852310657501221, + "rewards/rejected": -4.747682094573975, + "step": 249, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.6287331027978623, + "grad_norm": 2.1640334129333496, + "learning_rate": 0.00018639346083425748, + "logits/chosen": -0.46963977813720703, + "logits/rejected": -0.6406612396240234, + "logps/chosen": -7.166382789611816, + "logps/rejected": -57.39030838012695, + "loss": 0.6180220246315002, + "memory(GiB)": 46.04, + "nll_loss": 0.49737781286239624, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22726891934871674, + "rewards/margins": 4.274770736694336, + "rewards/rejected": -4.047501564025879, + "step": 250, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.6312480352090537, + "grad_norm": 1.6145262718200684, + "learning_rate": 0.00018625324101654377, + "logits/chosen": -0.30196619033813477, + "logits/rejected": -0.6030948162078857, + "logps/chosen": -4.5810980796813965, + "logps/rejected": -69.94066619873047, + "loss": 0.44845062494277954, + "memory(GiB)": 33.43, + "nll_loss": 0.3242243528366089, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13189926743507385, + "rewards/margins": 5.211311340332031, + "rewards/rejected": -5.07941198348999, + "step": 251, + "train_speed(iter/s)": 2.643562 + }, + { + "epoch": 0.6337629676202452, + "grad_norm": 2.0983619689941406, + "learning_rate": 0.00018611235569634854, + "logits/chosen": -0.33113306760787964, + "logits/rejected": -0.5730663537979126, + "logps/chosen": -5.2111029624938965, + "logps/rejected": -66.04705047607422, + "loss": 0.39135444164276123, + "memory(GiB)": 33.43, + "nll_loss": 0.282539963722229, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20458640158176422, + "rewards/margins": 5.00150203704834, + "rewards/rejected": -4.7969160079956055, + "step": 252, + "train_speed(iter/s)": 1.370537 + }, + { + "epoch": 0.6362779000314367, + "grad_norm": 2.1315553188323975, + "learning_rate": 0.00018597080596069793, + "logits/chosen": -0.45159927010536194, + "logits/rejected": -0.6239475011825562, + "logps/chosen": -8.911016464233398, + "logps/rejected": -51.39215850830078, + "loss": 0.6090635061264038, + "memory(GiB)": 33.43, + "nll_loss": 0.46375155448913574, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16291305422782898, + "rewards/margins": 3.745710611343384, + "rewards/rejected": -3.5827975273132324, + "step": 253, + "train_speed(iter/s)": 0.928331 + }, + { + "epoch": 0.6387928324426281, + "grad_norm": 1.5824214220046997, + "learning_rate": 0.00018582859290174452, + "logits/chosen": -0.4049888849258423, + "logits/rejected": -0.5815865993499756, + "logps/chosen": -4.045827388763428, + "logps/rejected": -46.12110137939453, + "loss": 0.4411042332649231, + "memory(GiB)": 34.43, + "nll_loss": 0.27207979559898376, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.25326088070869446, + "rewards/margins": 3.262639045715332, + "rewards/rejected": -3.009377956390381, + "step": 254, + "train_speed(iter/s)": 0.701344 + }, + { + "epoch": 0.6413077648538196, + "grad_norm": 9.893558502197266, + "learning_rate": 0.00018568571761675893, + "logits/chosen": -0.4062230885028839, + "logits/rejected": -0.5557867288589478, + "logps/chosen": -5.695141792297363, + "logps/rejected": -50.45267105102539, + "loss": 0.6555861234664917, + "memory(GiB)": 34.43, + "nll_loss": 0.4070875644683838, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.018757404759526253, + "rewards/margins": 3.3658759593963623, + "rewards/rejected": -3.347118854522705, + "step": 255, + "train_speed(iter/s)": 0.56489 + }, + { + "epoch": 0.643822697265011, + "grad_norm": 2.4083399772644043, + "learning_rate": 0.00018554218120812123, + "logits/chosen": -0.33443182706832886, + "logits/rejected": -0.5299184322357178, + "logps/chosen": -10.211224555969238, + "logps/rejected": -49.86205291748047, + "loss": 0.7328925132751465, + "memory(GiB)": 34.43, + "nll_loss": 0.535182774066925, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15930601954460144, + "rewards/margins": 3.196995735168457, + "rewards/rejected": -3.037689685821533, + "step": 256, + "train_speed(iter/s)": 0.473313 + }, + { + "epoch": 0.6463376296762025, + "grad_norm": 3.05432391166687, + "learning_rate": 0.00018539798478331253, + "logits/chosen": -0.36439672112464905, + "logits/rejected": -0.5713220834732056, + "logps/chosen": -5.492653846740723, + "logps/rejected": -63.25914001464844, + "loss": 0.4198300540447235, + "memory(GiB)": 34.43, + "nll_loss": 0.3039914071559906, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1935369074344635, + "rewards/margins": 4.819316387176514, + "rewards/rejected": -4.62578010559082, + "step": 257, + "train_speed(iter/s)": 0.40816 + }, + { + "epoch": 0.648852562087394, + "grad_norm": 4.973176956176758, + "learning_rate": 0.00018525312945490648, + "logits/chosen": -0.4139503240585327, + "logits/rejected": -0.49163496494293213, + "logps/chosen": -7.645513534545898, + "logps/rejected": -42.70791244506836, + "loss": 0.7363718748092651, + "memory(GiB)": 34.43, + "nll_loss": 0.4982314109802246, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08269016444683075, + "rewards/margins": 3.138368606567383, + "rewards/rejected": -3.055678606033325, + "step": 258, + "train_speed(iter/s)": 0.359112 + }, + { + "epoch": 0.6513674944985853, + "grad_norm": 1.6409480571746826, + "learning_rate": 0.0001851076163405605, + "logits/chosen": -0.3667711913585663, + "logits/rejected": -0.5486889481544495, + "logps/chosen": -5.454978942871094, + "logps/rejected": -57.34523010253906, + "loss": 0.5182607769966125, + "memory(GiB)": 35.19, + "nll_loss": 0.3399607241153717, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06281061470508575, + "rewards/margins": 4.247139930725098, + "rewards/rejected": -4.184329032897949, + "step": 259, + "train_speed(iter/s)": 0.320666 + }, + { + "epoch": 0.6538824269097768, + "grad_norm": 2.9779512882232666, + "learning_rate": 0.0001849614465630074, + "logits/chosen": -0.43861109018325806, + "logits/rejected": -0.6130017042160034, + "logps/chosen": -4.781963348388672, + "logps/rejected": -56.94814682006836, + "loss": 0.49893221259117126, + "memory(GiB)": 35.19, + "nll_loss": 0.33870410919189453, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12630243599414825, + "rewards/margins": 4.560263633728027, + "rewards/rejected": -4.433961391448975, + "step": 260, + "train_speed(iter/s)": 0.290134 + }, + { + "epoch": 0.6563973593209682, + "grad_norm": 2.126897096633911, + "learning_rate": 0.00018481462125004647, + "logits/chosen": -0.395521879196167, + "logits/rejected": -0.6116307973861694, + "logps/chosen": -7.144432544708252, + "logps/rejected": -73.966552734375, + "loss": 0.5626826882362366, + "memory(GiB)": 35.19, + "nll_loss": 0.3062968850135803, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0832599550485611, + "rewards/margins": 5.646542549133301, + "rewards/rejected": -5.72980260848999, + "step": 261, + "train_speed(iter/s)": 0.264936 + }, + { + "epoch": 0.6589122917321597, + "grad_norm": 5.1192545890808105, + "learning_rate": 0.00018466714153453503, + "logits/chosen": -0.2840866148471832, + "logits/rejected": -0.6146043539047241, + "logps/chosen": -3.4402737617492676, + "logps/rejected": -87.25662231445312, + "loss": 0.4280086159706116, + "memory(GiB)": 35.19, + "nll_loss": 0.30285900831222534, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11021072417497635, + "rewards/margins": 6.3835039138793945, + "rewards/rejected": -6.273293495178223, + "step": 262, + "train_speed(iter/s)": 0.243811 + }, + { + "epoch": 0.6614272241433512, + "grad_norm": 3.2079365253448486, + "learning_rate": 0.0001845190085543795, + "logits/chosen": -0.449633926153183, + "logits/rejected": -0.6547152996063232, + "logps/chosen": -10.382014274597168, + "logps/rejected": -83.1773681640625, + "loss": 0.5558671951293945, + "memory(GiB)": 35.19, + "nll_loss": 0.43276214599609375, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.21494942903518677, + "rewards/margins": 6.166694641113281, + "rewards/rejected": -6.381643772125244, + "step": 263, + "train_speed(iter/s)": 0.226046 + }, + { + "epoch": 0.6639421565545426, + "grad_norm": 2.844895601272583, + "learning_rate": 0.00018437022345252664, + "logits/chosen": -0.442143052816391, + "logits/rejected": -0.6931208372116089, + "logps/chosen": -3.599593162536621, + "logps/rejected": -87.85449981689453, + "loss": 0.37555554509162903, + "memory(GiB)": 35.19, + "nll_loss": 0.27443069219589233, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.02881481871008873, + "rewards/margins": 7.2416534423828125, + "rewards/rejected": -7.212838649749756, + "step": 264, + "train_speed(iter/s)": 0.210855 + }, + { + "epoch": 0.6664570889657341, + "grad_norm": 2.5585174560546875, + "learning_rate": 0.0001842207873769548, + "logits/chosen": -0.42873817682266235, + "logits/rejected": -0.6308642029762268, + "logps/chosen": -5.981243133544922, + "logps/rejected": -70.72628784179688, + "loss": 0.6464019417762756, + "memory(GiB)": 35.19, + "nll_loss": 0.46387210488319397, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.002297995612025261, + "rewards/margins": 5.15910530090332, + "rewards/rejected": -5.161403179168701, + "step": 265, + "train_speed(iter/s)": 0.197608 + }, + { + "epoch": 0.6689720213769255, + "grad_norm": 6.840214729309082, + "learning_rate": 0.00018407070148066513, + "logits/chosen": -0.34393343329429626, + "logits/rejected": -0.5655381679534912, + "logps/chosen": -6.317379474639893, + "logps/rejected": -80.00755310058594, + "loss": 0.6975788474082947, + "memory(GiB)": 35.19, + "nll_loss": 0.5568116307258606, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.04052425175905228, + "rewards/margins": 6.243714332580566, + "rewards/rejected": -6.284238338470459, + "step": 266, + "train_speed(iter/s)": 0.185936 + }, + { + "epoch": 0.671486953788117, + "grad_norm": 1.300536870956421, + "learning_rate": 0.00018391996692167242, + "logits/chosen": -0.44441860914230347, + "logits/rejected": -0.6562116742134094, + "logps/chosen": -3.928229570388794, + "logps/rejected": -69.29019165039062, + "loss": 0.3208352327346802, + "memory(GiB)": 35.19, + "nll_loss": 0.218485489487648, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3024291396141052, + "rewards/margins": 5.891707420349121, + "rewards/rejected": -5.589277744293213, + "step": 267, + "train_speed(iter/s)": 0.175708 + }, + { + "epoch": 0.6740018861993083, + "grad_norm": 2.1066205501556396, + "learning_rate": 0.00018376858486299647, + "logits/chosen": -0.3569512367248535, + "logits/rejected": -0.6223514080047607, + "logps/chosen": -5.05648136138916, + "logps/rejected": -77.00200653076172, + "loss": 0.447488397359848, + "memory(GiB)": 35.19, + "nll_loss": 0.2895119786262512, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07016298919916153, + "rewards/margins": 5.940699100494385, + "rewards/rejected": -5.870535850524902, + "step": 268, + "train_speed(iter/s)": 0.166588 + }, + { + "epoch": 0.6765168186104998, + "grad_norm": 6.73745584487915, + "learning_rate": 0.00018361655647265295, + "logits/chosen": -0.432262122631073, + "logits/rejected": -0.6704272031784058, + "logps/chosen": -4.047597885131836, + "logps/rejected": -66.35338592529297, + "loss": 0.4859369993209839, + "memory(GiB)": 35.19, + "nll_loss": 0.3608669936656952, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07137257605791092, + "rewards/margins": 5.09867000579834, + "rewards/rejected": -5.027297019958496, + "step": 269, + "train_speed(iter/s)": 0.158465 + }, + { + "epoch": 0.6790317510216913, + "grad_norm": 4.508405685424805, + "learning_rate": 0.00018346388292364435, + "logits/chosen": -0.36208435893058777, + "logits/rejected": -0.6471053957939148, + "logps/chosen": -2.5504043102264404, + "logps/rejected": -72.08878326416016, + "loss": 0.26533976197242737, + "memory(GiB)": 35.19, + "nll_loss": 0.15827052295207977, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1948476880788803, + "rewards/margins": 5.411668300628662, + "rewards/rejected": -5.216820240020752, + "step": 270, + "train_speed(iter/s)": 0.151104 + }, + { + "epoch": 0.6815466834328827, + "grad_norm": 2.18328595161438, + "learning_rate": 0.00018331056539395112, + "logits/chosen": -0.49844443798065186, + "logits/rejected": -0.651973307132721, + "logps/chosen": -4.165656566619873, + "logps/rejected": -53.685787200927734, + "loss": 0.5224581360816956, + "memory(GiB)": 35.19, + "nll_loss": 0.3570694625377655, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08425498008728027, + "rewards/margins": 4.271368980407715, + "rewards/rejected": -4.1871137619018555, + "step": 271, + "train_speed(iter/s)": 0.1445 + }, + { + "epoch": 0.6840616158440742, + "grad_norm": 1.904048204421997, + "learning_rate": 0.00018315660506652232, + "logits/chosen": -0.4680135250091553, + "logits/rejected": -0.6710811853408813, + "logps/chosen": -4.441302299499512, + "logps/rejected": -72.74127197265625, + "loss": 0.34467679262161255, + "memory(GiB)": 35.19, + "nll_loss": 0.23408779501914978, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18954063951969147, + "rewards/margins": 6.041881561279297, + "rewards/rejected": -5.852341175079346, + "step": 272, + "train_speed(iter/s)": 0.138445 + }, + { + "epoch": 0.6865765482552656, + "grad_norm": 2.8482308387756348, + "learning_rate": 0.00018300200312926674, + "logits/chosen": -0.4003257751464844, + "logits/rejected": -0.6571944355964661, + "logps/chosen": -4.5086669921875, + "logps/rejected": -66.03219604492188, + "loss": 0.6187813878059387, + "memory(GiB)": 35.19, + "nll_loss": 0.41691598296165466, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07326355576515198, + "rewards/margins": 4.743034362792969, + "rewards/rejected": -4.669771194458008, + "step": 273, + "train_speed(iter/s)": 0.132932 + }, + { + "epoch": 0.6890914806664571, + "grad_norm": 7.494622707366943, + "learning_rate": 0.00018284676077504362, + "logits/chosen": -0.40674543380737305, + "logits/rejected": -0.5925787687301636, + "logps/chosen": -9.069441795349121, + "logps/rejected": -53.76180648803711, + "loss": 0.8244871497154236, + "memory(GiB)": 35.19, + "nll_loss": 0.5905988216400146, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0009557940065860748, + "rewards/margins": 3.9635586738586426, + "rewards/rejected": -3.96451473236084, + "step": 274, + "train_speed(iter/s)": 0.127852 + }, + { + "epoch": 0.6916064130776486, + "grad_norm": 12.676557540893555, + "learning_rate": 0.00018269087920165332, + "logits/chosen": -0.3970157504081726, + "logits/rejected": -0.6446709632873535, + "logps/chosen": -7.009572982788086, + "logps/rejected": -75.48789978027344, + "loss": 0.5546004772186279, + "memory(GiB)": 38.68, + "nll_loss": 0.40932291746139526, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11061696708202362, + "rewards/margins": 5.533331871032715, + "rewards/rejected": -5.422715187072754, + "step": 275, + "train_speed(iter/s)": 0.12316 + }, + { + "epoch": 0.69412134548884, + "grad_norm": 3.2866766452789307, + "learning_rate": 0.00018253435961182845, + "logits/chosen": -0.5122930407524109, + "logits/rejected": -0.7138373255729675, + "logps/chosen": -5.048257827758789, + "logps/rejected": -90.43961334228516, + "loss": 0.4567756950855255, + "memory(GiB)": 38.68, + "nll_loss": 0.3689834177494049, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17359167337417603, + "rewards/margins": 7.448518753051758, + "rewards/rejected": -7.274927139282227, + "step": 276, + "train_speed(iter/s)": 0.11888 + }, + { + "epoch": 0.6966362779000315, + "grad_norm": 1.6649048328399658, + "learning_rate": 0.00018237720321322409, + "logits/chosen": -0.4259472191333771, + "logits/rejected": -0.6217616200447083, + "logps/chosen": -5.190006256103516, + "logps/rejected": -60.61420440673828, + "loss": 0.48118671774864197, + "memory(GiB)": 38.68, + "nll_loss": 0.34742891788482666, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3943961560726166, + "rewards/margins": 4.985601902008057, + "rewards/rejected": -4.591205596923828, + "step": 277, + "train_speed(iter/s)": 0.114903 + }, + { + "epoch": 0.6991512103112228, + "grad_norm": 1.641586184501648, + "learning_rate": 0.00018221941121840892, + "logits/chosen": -0.5152907967567444, + "logits/rejected": -0.7797664403915405, + "logps/chosen": -3.807891845703125, + "logps/rejected": -89.89514923095703, + "loss": 0.4311053454875946, + "memory(GiB)": 38.68, + "nll_loss": 0.3506782352924347, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16795168817043304, + "rewards/margins": 7.562755584716797, + "rewards/rejected": -7.394804000854492, + "step": 278, + "train_speed(iter/s)": 0.111233 + }, + { + "epoch": 0.7016661427224143, + "grad_norm": 2.954766273498535, + "learning_rate": 0.00018206098484485563, + "logits/chosen": -0.47647833824157715, + "logits/rejected": -0.7569775581359863, + "logps/chosen": -3.6604084968566895, + "logps/rejected": -93.30502319335938, + "loss": 0.33102554082870483, + "memory(GiB)": 38.68, + "nll_loss": 0.23846475780010223, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.057397253811359406, + "rewards/margins": 7.980731964111328, + "rewards/rejected": -7.92333459854126, + "step": 279, + "train_speed(iter/s)": 0.107796 + }, + { + "epoch": 0.7041810751336058, + "grad_norm": 1.0509345531463623, + "learning_rate": 0.00018190192531493152, + "logits/chosen": -0.4389861226081848, + "logits/rejected": -0.7513998746871948, + "logps/chosen": -3.3642971515655518, + "logps/rejected": -104.84876251220703, + "loss": 0.2865274250507355, + "memory(GiB)": 38.68, + "nll_loss": 0.1890382468700409, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1356939822435379, + "rewards/margins": 8.732723236083984, + "rewards/rejected": -8.597027778625488, + "step": 280, + "train_speed(iter/s)": 0.104576 + }, + { + "epoch": 0.7066960075447972, + "grad_norm": 2.062405824661255, + "learning_rate": 0.00018174223385588917, + "logits/chosen": -0.4419255256652832, + "logits/rejected": -0.789115309715271, + "logps/chosen": -6.24101448059082, + "logps/rejected": -107.06963348388672, + "loss": 0.38868069648742676, + "memory(GiB)": 38.68, + "nll_loss": 0.27302783727645874, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04043314605951309, + "rewards/margins": 8.512565612792969, + "rewards/rejected": -8.472132682800293, + "step": 281, + "train_speed(iter/s)": 0.101563 + }, + { + "epoch": 0.7092109399559887, + "grad_norm": 1.5434448719024658, + "learning_rate": 0.00018158191169985696, + "logits/chosen": -0.5877774953842163, + "logits/rejected": -0.7937146425247192, + "logps/chosen": -3.5352978706359863, + "logps/rejected": -82.489990234375, + "loss": 0.28792405128479004, + "memory(GiB)": 38.68, + "nll_loss": 0.21994276344776154, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.152220219373703, + "rewards/margins": 7.123733043670654, + "rewards/rejected": -6.971513271331787, + "step": 282, + "train_speed(iter/s)": 0.098762 + }, + { + "epoch": 0.7117258723671801, + "grad_norm": 1.7215518951416016, + "learning_rate": 0.00018142096008382942, + "logits/chosen": -0.5603517889976501, + "logits/rejected": -0.8750638961791992, + "logps/chosen": -3.4122605323791504, + "logps/rejected": -98.63863372802734, + "loss": 0.4363364279270172, + "memory(GiB)": 38.68, + "nll_loss": 0.33546215295791626, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1424112468957901, + "rewards/margins": 8.030111312866211, + "rewards/rejected": -7.887700080871582, + "step": 283, + "train_speed(iter/s)": 0.096121 + }, + { + "epoch": 0.7142408047783716, + "grad_norm": 6.417537212371826, + "learning_rate": 0.00018125938024965793, + "logits/chosen": -0.543463408946991, + "logits/rejected": -0.7929783463478088, + "logps/chosen": -5.197461128234863, + "logps/rejected": -97.00271606445312, + "loss": 0.6537326574325562, + "memory(GiB)": 38.68, + "nll_loss": 0.46845054626464844, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07486869394779205, + "rewards/margins": 8.32656478881836, + "rewards/rejected": -8.251694679260254, + "step": 284, + "train_speed(iter/s)": 0.093639 + }, + { + "epoch": 0.716755737189563, + "grad_norm": 2.125540018081665, + "learning_rate": 0.0001810971734440408, + "logits/chosen": -0.6539860367774963, + "logits/rejected": -0.853480875492096, + "logps/chosen": -5.839959621429443, + "logps/rejected": -91.55633544921875, + "loss": 0.4674933850765228, + "memory(GiB)": 38.68, + "nll_loss": 0.3285233974456787, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0453268326818943, + "rewards/margins": 7.891117572784424, + "rewards/rejected": -7.845790863037109, + "step": 285, + "train_speed(iter/s)": 0.091303 + }, + { + "epoch": 0.7192706696007545, + "grad_norm": 2.496291160583496, + "learning_rate": 0.0001809343409185141, + "logits/chosen": -0.533951997756958, + "logits/rejected": -0.8393864631652832, + "logps/chosen": -5.252377510070801, + "logps/rejected": -108.10926055908203, + "loss": 0.4806031584739685, + "memory(GiB)": 38.68, + "nll_loss": 0.35059598088264465, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.26575902104377747, + "rewards/margins": 9.291682243347168, + "rewards/rejected": -9.025922775268555, + "step": 286, + "train_speed(iter/s)": 0.089086 + }, + { + "epoch": 0.721785602011946, + "grad_norm": 12.633203506469727, + "learning_rate": 0.00018077088392944157, + "logits/chosen": -0.5087528228759766, + "logits/rejected": -0.7988476753234863, + "logps/chosen": -4.803990840911865, + "logps/rejected": -134.75717163085938, + "loss": 0.5371387600898743, + "memory(GiB)": 38.68, + "nll_loss": 0.4104423522949219, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09430995583534241, + "rewards/margins": 11.716846466064453, + "rewards/rejected": -11.622535705566406, + "step": 287, + "train_speed(iter/s)": 0.086965 + }, + { + "epoch": 0.7243005344231374, + "grad_norm": 6.384928226470947, + "learning_rate": 0.00018060680373800517, + "logits/chosen": -0.634199857711792, + "logits/rejected": -0.8538622260093689, + "logps/chosen": -7.401331901550293, + "logps/rejected": -99.94730377197266, + "loss": 0.9957684874534607, + "memory(GiB)": 38.68, + "nll_loss": 0.7036086916923523, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.18307247757911682, + "rewards/margins": 8.665613174438477, + "rewards/rejected": -8.848685264587402, + "step": 288, + "train_speed(iter/s)": 0.084995 + }, + { + "epoch": 0.7268154668343288, + "grad_norm": 1.3055838346481323, + "learning_rate": 0.00018044210161019536, + "logits/chosen": -0.6348901391029358, + "logits/rejected": -0.8540142774581909, + "logps/chosen": -3.750216484069824, + "logps/rejected": -116.36519622802734, + "loss": 0.43587327003479004, + "memory(GiB)": 38.68, + "nll_loss": 0.3083232641220093, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27759620547294617, + "rewards/margins": 10.4413480758667, + "rewards/rejected": -10.163751602172852, + "step": 289, + "train_speed(iter/s)": 0.083116 + }, + { + "epoch": 0.7293303992455202, + "grad_norm": 6.336847305297852, + "learning_rate": 0.0001802767788168011, + "logits/chosen": -0.49168598651885986, + "logits/rejected": -0.7385163307189941, + "logps/chosen": -5.862331867218018, + "logps/rejected": -90.92340850830078, + "loss": 0.5076504349708557, + "memory(GiB)": 42.19, + "nll_loss": 0.40712910890579224, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.007690219208598137, + "rewards/margins": 7.30800724029541, + "rewards/rejected": -7.31569766998291, + "step": 290, + "train_speed(iter/s)": 0.081314 + }, + { + "epoch": 0.7318453316567117, + "grad_norm": 1.5834612846374512, + "learning_rate": 0.0001801108366334004, + "logits/chosen": -0.49157392978668213, + "logits/rejected": -0.8517095446586609, + "logps/chosen": -2.9823904037475586, + "logps/rejected": -128.86550903320312, + "loss": 0.28464004397392273, + "memory(GiB)": 42.19, + "nll_loss": 0.19038352370262146, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11139468103647232, + "rewards/margins": 11.453866004943848, + "rewards/rejected": -11.3424711227417, + "step": 291, + "train_speed(iter/s)": 0.079608 + }, + { + "epoch": 0.7343602640679032, + "grad_norm": 2.694648265838623, + "learning_rate": 0.00017994427634035015, + "logits/chosen": -0.4209265410900116, + "logits/rejected": -0.7504950761795044, + "logps/chosen": -6.3449931144714355, + "logps/rejected": -102.324951171875, + "loss": 0.48612844944000244, + "memory(GiB)": 42.19, + "nll_loss": 0.439787358045578, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.05951198935508728, + "rewards/margins": 8.676319122314453, + "rewards/rejected": -8.735831260681152, + "step": 292, + "train_speed(iter/s)": 0.077977 + }, + { + "epoch": 0.7368751964790946, + "grad_norm": 12.563709259033203, + "learning_rate": 0.00017977709922277634, + "logits/chosen": -0.546952486038208, + "logits/rejected": -0.8098918199539185, + "logps/chosen": -3.903282880783081, + "logps/rejected": -106.09379577636719, + "loss": 0.4184201657772064, + "memory(GiB)": 42.19, + "nll_loss": 0.2776867151260376, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11045068502426147, + "rewards/margins": 9.161507606506348, + "rewards/rejected": -9.051055908203125, + "step": 293, + "train_speed(iter/s)": 0.076428 + }, + { + "epoch": 0.7393901288902861, + "grad_norm": 10.21599292755127, + "learning_rate": 0.00017960930657056438, + "logits/chosen": -0.4862319827079773, + "logits/rejected": -0.8446335196495056, + "logps/chosen": -4.003853797912598, + "logps/rejected": -124.26484680175781, + "loss": 0.5325912833213806, + "memory(GiB)": 42.19, + "nll_loss": 0.4379286766052246, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.017376184463500977, + "rewards/margins": 10.38001823425293, + "rewards/rejected": -10.397393226623535, + "step": 294, + "train_speed(iter/s)": 0.074951 + }, + { + "epoch": 0.7419050613014775, + "grad_norm": 2.1142091751098633, + "learning_rate": 0.00017944089967834875, + "logits/chosen": -0.6149781942367554, + "logits/rejected": -0.8646234273910522, + "logps/chosen": -4.966188907623291, + "logps/rejected": -105.03376007080078, + "loss": 0.3664693236351013, + "memory(GiB)": 42.19, + "nll_loss": 0.29172563552856445, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12611237168312073, + "rewards/margins": 8.863950729370117, + "rewards/rejected": -8.737838745117188, + "step": 295, + "train_speed(iter/s)": 0.073543 + }, + { + "epoch": 0.744419993712669, + "grad_norm": 2.3182928562164307, + "learning_rate": 0.00017927187984550334, + "logits/chosen": -0.4714081883430481, + "logits/rejected": -0.8076429963111877, + "logps/chosen": -5.210522174835205, + "logps/rejected": -94.38253784179688, + "loss": 0.40223944187164307, + "memory(GiB)": 42.19, + "nll_loss": 0.3296588659286499, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.05410341918468475, + "rewards/margins": 7.804743766784668, + "rewards/rejected": -7.750640392303467, + "step": 296, + "train_speed(iter/s)": 0.072191 + }, + { + "epoch": 0.7469349261238605, + "grad_norm": 1.6627568006515503, + "learning_rate": 0.00017910224837613118, + "logits/chosen": -0.5375725030899048, + "logits/rejected": -0.7311803698539734, + "logps/chosen": -5.645050048828125, + "logps/rejected": -86.46006774902344, + "loss": 0.4087020456790924, + "memory(GiB)": 45.72, + "nll_loss": 0.2872757613658905, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12224967032670975, + "rewards/margins": 6.998920440673828, + "rewards/rejected": -6.876670837402344, + "step": 297, + "train_speed(iter/s)": 0.070893 + }, + { + "epoch": 0.7494498585350519, + "grad_norm": 7.537210941314697, + "learning_rate": 0.0001789320065790546, + "logits/chosen": -0.5413732528686523, + "logits/rejected": -0.7798230051994324, + "logps/chosen": -5.977290630340576, + "logps/rejected": -84.39535522460938, + "loss": 0.49224522709846497, + "memory(GiB)": 45.72, + "nll_loss": 0.3515035808086395, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.29318666458129883, + "rewards/margins": 6.832826614379883, + "rewards/rejected": -6.539640426635742, + "step": 298, + "train_speed(iter/s)": 0.06965 + }, + { + "epoch": 0.7519647909462434, + "grad_norm": 13.154047012329102, + "learning_rate": 0.00017876115576780498, + "logits/chosen": -0.56256502866745, + "logits/rejected": -0.7628785371780396, + "logps/chosen": -8.302791595458984, + "logps/rejected": -71.12387084960938, + "loss": 0.7186374068260193, + "memory(GiB)": 45.72, + "nll_loss": 0.5491286516189575, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04678814485669136, + "rewards/margins": 5.721759796142578, + "rewards/rejected": -5.768548488616943, + "step": 299, + "train_speed(iter/s)": 0.068463 + }, + { + "epoch": 0.7544797233574347, + "grad_norm": 2.1101605892181396, + "learning_rate": 0.0001785896972606126, + "logits/chosen": -0.6424190998077393, + "logits/rejected": -0.8492656946182251, + "logps/chosen": -6.954460144042969, + "logps/rejected": -72.85733795166016, + "loss": 0.5062767267227173, + "memory(GiB)": 45.72, + "nll_loss": 0.4268747568130493, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18685287237167358, + "rewards/margins": 6.121685028076172, + "rewards/rejected": -5.934832572937012, + "step": 300, + "train_speed(iter/s)": 0.067324 + }, + { + "epoch": 0.7544797233574347, + "eval_logits/chosen": -0.6104318499565125, + "eval_logits/rejected": -0.8290396332740784, + "eval_logps/chosen": -5.665504455566406, + "eval_logps/rejected": -75.69713592529297, + "eval_loss": 0.5146161317825317, + "eval_nll_loss": 0.36346134543418884, + "eval_rewards/accuracies": 0.953125, + "eval_rewards/chosen": 0.1080789864063263, + "eval_rewards/margins": 6.1169891357421875, + "eval_rewards/rejected": -6.008910179138184, + "eval_runtime": 138.6744, + "eval_samples_per_second": 0.923, + "eval_steps_per_second": 0.462, + "step": 300 + }, + { + "epoch": 0.7569946557686262, + "grad_norm": 2.742777109146118, + "learning_rate": 0.00017841763238039663, + "logits/chosen": -0.5258588194847107, + "logits/rejected": -0.8002297878265381, + "logps/chosen": -9.598472595214844, + "logps/rejected": -73.01333618164062, + "loss": 0.6272799968719482, + "memory(GiB)": 45.72, + "nll_loss": 0.45022809505462646, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14143598079681396, + "rewards/margins": 5.63870096206665, + "rewards/rejected": -5.497264862060547, + "step": 301, + "train_speed(iter/s)": 0.064032 + }, + { + "epoch": 0.7595095881798176, + "grad_norm": 4.982257843017578, + "learning_rate": 0.00017824496245475477, + "logits/chosen": -0.5467951893806458, + "logits/rejected": -0.8088864088058472, + "logps/chosen": -5.489375114440918, + "logps/rejected": -68.20091247558594, + "loss": 0.6761376261711121, + "memory(GiB)": 45.72, + "nll_loss": 0.46992918848991394, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.03441940248012543, + "rewards/margins": 5.287006378173828, + "rewards/rejected": -5.252586841583252, + "step": 302, + "train_speed(iter/s)": 0.063055 + }, + { + "epoch": 0.7620245205910091, + "grad_norm": 2.363013982772827, + "learning_rate": 0.00017807168881595303, + "logits/chosen": -0.486921101808548, + "logits/rejected": -0.7870367765426636, + "logps/chosen": -3.2118942737579346, + "logps/rejected": -80.78832244873047, + "loss": 0.3174203336238861, + "memory(GiB)": 45.72, + "nll_loss": 0.22700631618499756, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0014025792479515076, + "rewards/margins": 6.200619697570801, + "rewards/rejected": -6.199216842651367, + "step": 303, + "train_speed(iter/s)": 0.062112 + }, + { + "epoch": 0.7645394530022006, + "grad_norm": 2.5794389247894287, + "learning_rate": 0.00017789781280091557, + "logits/chosen": -0.5253547430038452, + "logits/rejected": -0.7778689861297607, + "logps/chosen": -2.740814685821533, + "logps/rejected": -63.18760299682617, + "loss": 0.43329575657844543, + "memory(GiB)": 45.72, + "nll_loss": 0.25420501828193665, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11472588777542114, + "rewards/margins": 4.892645359039307, + "rewards/rejected": -4.777919292449951, + "step": 304, + "train_speed(iter/s)": 0.061202 + }, + { + "epoch": 0.767054385413392, + "grad_norm": 3.204162836074829, + "learning_rate": 0.0001777233357512142, + "logits/chosen": -0.48726359009742737, + "logits/rejected": -0.7712681293487549, + "logps/chosen": -4.6791911125183105, + "logps/rejected": -73.8591079711914, + "loss": 0.49168315529823303, + "memory(GiB)": 45.72, + "nll_loss": 0.3878230154514313, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12156429141759872, + "rewards/margins": 5.675488471984863, + "rewards/rejected": -5.553924083709717, + "step": 305, + "train_speed(iter/s)": 0.060326 + }, + { + "epoch": 0.7695693178245835, + "grad_norm": 2.685102939605713, + "learning_rate": 0.00017754825901305813, + "logits/chosen": -0.49732205271720886, + "logits/rejected": -0.6546094417572021, + "logps/chosen": -9.81102180480957, + "logps/rejected": -60.67332077026367, + "loss": 0.6829510927200317, + "memory(GiB)": 45.72, + "nll_loss": 0.5321569442749023, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.23814226686954498, + "rewards/margins": 4.872373104095459, + "rewards/rejected": -4.634230613708496, + "step": 306, + "train_speed(iter/s)": 0.059474 + }, + { + "epoch": 0.7720842502357749, + "grad_norm": 2.191077947616577, + "learning_rate": 0.00017737258393728364, + "logits/chosen": -0.3749886453151703, + "logits/rejected": -0.7177920341491699, + "logps/chosen": -3.679948329925537, + "logps/rejected": -76.73285675048828, + "loss": 0.44855624437332153, + "memory(GiB)": 45.72, + "nll_loss": 0.31453874707221985, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11833418905735016, + "rewards/margins": 5.410902976989746, + "rewards/rejected": -5.292569160461426, + "step": 307, + "train_speed(iter/s)": 0.058652 + }, + { + "epoch": 0.7745991826469664, + "grad_norm": 5.087850570678711, + "learning_rate": 0.00017719631187934352, + "logits/chosen": -0.4312075078487396, + "logits/rejected": -0.7199861407279968, + "logps/chosen": -5.141556739807129, + "logps/rejected": -84.90442657470703, + "loss": 0.49092957377433777, + "memory(GiB)": 45.72, + "nll_loss": 0.3564959764480591, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06924839317798615, + "rewards/margins": 6.585083961486816, + "rewards/rejected": -6.515835762023926, + "step": 308, + "train_speed(iter/s)": 0.057859 + }, + { + "epoch": 0.7771141150581579, + "grad_norm": 1.931964635848999, + "learning_rate": 0.00017701944419929672, + "logits/chosen": -0.5209879279136658, + "logits/rejected": -0.7744486331939697, + "logps/chosen": -4.660381317138672, + "logps/rejected": -75.40290069580078, + "loss": 0.380489319562912, + "memory(GiB)": 45.72, + "nll_loss": 0.2925787568092346, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07492721825838089, + "rewards/margins": 5.759749412536621, + "rewards/rejected": -5.684822082519531, + "step": 309, + "train_speed(iter/s)": 0.057092 + }, + { + "epoch": 0.7796290474693492, + "grad_norm": 1.4231215715408325, + "learning_rate": 0.00017684198226179784, + "logits/chosen": -0.5286337733268738, + "logits/rejected": -0.7741042375564575, + "logps/chosen": -5.322835922241211, + "logps/rejected": -94.39818572998047, + "loss": 0.32215720415115356, + "memory(GiB)": 45.72, + "nll_loss": 0.23559962213039398, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03720661997795105, + "rewards/margins": 7.8387603759765625, + "rewards/rejected": -7.8015546798706055, + "step": 310, + "train_speed(iter/s)": 0.056351 + }, + { + "epoch": 0.7821439798805407, + "grad_norm": 3.5972278118133545, + "learning_rate": 0.00017666392743608656, + "logits/chosen": -0.587047815322876, + "logits/rejected": -0.6886476874351501, + "logps/chosen": -11.391901969909668, + "logps/rejected": -63.259220123291016, + "loss": 0.631456732749939, + "memory(GiB)": 45.72, + "nll_loss": 0.4477965831756592, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2087097316980362, + "rewards/margins": 5.190671920776367, + "rewards/rejected": -4.981962203979492, + "step": 311, + "train_speed(iter/s)": 0.055631 + }, + { + "epoch": 0.7846589122917321, + "grad_norm": 6.570708751678467, + "learning_rate": 0.00017648528109597705, + "logits/chosen": -0.5264805555343628, + "logits/rejected": -0.6204366683959961, + "logps/chosen": -5.628443241119385, + "logps/rejected": -55.631587982177734, + "loss": 0.44488024711608887, + "memory(GiB)": 45.72, + "nll_loss": 0.29311853647232056, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15758532285690308, + "rewards/margins": 4.487579345703125, + "rewards/rejected": -4.329994201660156, + "step": 312, + "train_speed(iter/s)": 0.054937 + }, + { + "epoch": 0.7871738447029236, + "grad_norm": 2.2891812324523926, + "learning_rate": 0.00017630604461984746, + "logits/chosen": -0.49812430143356323, + "logits/rejected": -0.7942063808441162, + "logps/chosen": -4.372743606567383, + "logps/rejected": -94.14138793945312, + "loss": 0.36223098635673523, + "memory(GiB)": 45.72, + "nll_loss": 0.30161458253860474, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10362353175878525, + "rewards/margins": 7.83615255355835, + "rewards/rejected": -7.7325286865234375, + "step": 313, + "train_speed(iter/s)": 0.054263 + }, + { + "epoch": 0.789688777114115, + "grad_norm": 4.033056259155273, + "learning_rate": 0.00017612621939062928, + "logits/chosen": -0.5049434304237366, + "logits/rejected": -0.7435040473937988, + "logps/chosen": -3.835451364517212, + "logps/rejected": -84.56083679199219, + "loss": 0.43754124641418457, + "memory(GiB)": 45.72, + "nll_loss": 0.2978503704071045, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17277862131595612, + "rewards/margins": 6.71329402923584, + "rewards/rejected": -6.540514945983887, + "step": 314, + "train_speed(iter/s)": 0.053607 + }, + { + "epoch": 0.7922037095253065, + "grad_norm": 4.677379608154297, + "learning_rate": 0.00017594580679579656, + "logits/chosen": -0.4425162076950073, + "logits/rejected": -0.7090457677841187, + "logps/chosen": -4.391209602355957, + "logps/rejected": -78.58367919921875, + "loss": 0.5179426670074463, + "memory(GiB)": 45.72, + "nll_loss": 0.3796064257621765, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.003046363592147827, + "rewards/margins": 6.267105579376221, + "rewards/rejected": -6.264059066772461, + "step": 315, + "train_speed(iter/s)": 0.052969 + }, + { + "epoch": 0.794718641936498, + "grad_norm": 2.2975409030914307, + "learning_rate": 0.00017576480822735532, + "logits/chosen": -0.552763819694519, + "logits/rejected": -0.7386415004730225, + "logps/chosen": -6.633852005004883, + "logps/rejected": -75.99591827392578, + "loss": 0.5104320049285889, + "memory(GiB)": 45.72, + "nll_loss": 0.4509289562702179, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13848751783370972, + "rewards/margins": 6.377573013305664, + "rewards/rejected": -6.239084720611572, + "step": 316, + "train_speed(iter/s)": 0.052355 + }, + { + "epoch": 0.7972335743476894, + "grad_norm": 1.692866325378418, + "learning_rate": 0.00017558322508183275, + "logits/chosen": -0.5113404393196106, + "logits/rejected": -0.7546023726463318, + "logps/chosen": -8.723966598510742, + "logps/rejected": -68.92810821533203, + "loss": 0.3866414725780487, + "memory(GiB)": 45.72, + "nll_loss": 0.2663732171058655, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12023188173770905, + "rewards/margins": 5.404682159423828, + "rewards/rejected": -5.284450054168701, + "step": 317, + "train_speed(iter/s)": 0.051754 + }, + { + "epoch": 0.7997485067588809, + "grad_norm": 1.7934551239013672, + "learning_rate": 0.00017540105876026648, + "logits/chosen": -0.5510843396186829, + "logits/rejected": -0.7684234976768494, + "logps/chosen": -4.503855228424072, + "logps/rejected": -73.94355010986328, + "loss": 0.49109137058258057, + "memory(GiB)": 45.72, + "nll_loss": 0.33945515751838684, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08985169976949692, + "rewards/margins": 6.044112682342529, + "rewards/rejected": -5.9542622566223145, + "step": 318, + "train_speed(iter/s)": 0.051176 + }, + { + "epoch": 0.8022634391700723, + "grad_norm": 2.696263074874878, + "learning_rate": 0.00017521831066819363, + "logits/chosen": -0.6849238872528076, + "logits/rejected": -0.7512338757514954, + "logps/chosen": -7.439488410949707, + "logps/rejected": -54.92814636230469, + "loss": 0.5880566835403442, + "memory(GiB)": 45.72, + "nll_loss": 0.32204046845436096, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08035062998533249, + "rewards/margins": 4.1917901039123535, + "rewards/rejected": -4.2721405029296875, + "step": 319, + "train_speed(iter/s)": 0.050611 + }, + { + "epoch": 0.8047783715812638, + "grad_norm": 8.225646018981934, + "learning_rate": 0.00017503498221564025, + "logits/chosen": -0.5244278311729431, + "logits/rejected": -0.8192154169082642, + "logps/chosen": -5.370640754699707, + "logps/rejected": -69.76851654052734, + "loss": 0.5900783538818359, + "memory(GiB)": 46.31, + "nll_loss": 0.43174654245376587, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.15494805574417114, + "rewards/margins": 4.877842903137207, + "rewards/rejected": -4.722894668579102, + "step": 320, + "train_speed(iter/s)": 0.050058 + }, + { + "epoch": 0.8072933039924552, + "grad_norm": 3.1263160705566406, + "learning_rate": 0.00017485107481711012, + "logits/chosen": -0.6532421112060547, + "logits/rejected": -0.8408050537109375, + "logps/chosen": -5.802052974700928, + "logps/rejected": -69.45040893554688, + "loss": 0.4921962022781372, + "memory(GiB)": 46.31, + "nll_loss": 0.3598018288612366, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.016502581536769867, + "rewards/margins": 5.566046714782715, + "rewards/rejected": -5.582549571990967, + "step": 321, + "train_speed(iter/s)": 0.049526 + }, + { + "epoch": 0.8098082364036466, + "grad_norm": 1.3748646974563599, + "learning_rate": 0.00017466658989157406, + "logits/chosen": -0.5933704376220703, + "logits/rejected": -0.8203169107437134, + "logps/chosen": -4.698756694793701, + "logps/rejected": -86.18940734863281, + "loss": 0.39138850569725037, + "memory(GiB)": 46.31, + "nll_loss": 0.3173418641090393, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07632508128881454, + "rewards/margins": 7.136272430419922, + "rewards/rejected": -7.0599470138549805, + "step": 322, + "train_speed(iter/s)": 0.049007 + }, + { + "epoch": 0.8123231688148381, + "grad_norm": 1.2321858406066895, + "learning_rate": 0.00017448152886245888, + "logits/chosen": -0.6055665016174316, + "logits/rejected": -0.835323691368103, + "logps/chosen": -4.2663726806640625, + "logps/rejected": -92.81077575683594, + "loss": 0.31942421197891235, + "memory(GiB)": 46.31, + "nll_loss": 0.23250146210193634, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.29466480016708374, + "rewards/margins": 7.823517799377441, + "rewards/rejected": -7.528852939605713, + "step": 323, + "train_speed(iter/s)": 0.048501 + }, + { + "epoch": 0.8148381012260295, + "grad_norm": 9.860624313354492, + "learning_rate": 0.00017429589315763638, + "logits/chosen": -0.5517026782035828, + "logits/rejected": -0.9087996482849121, + "logps/chosen": -5.479660987854004, + "logps/rejected": -96.31484985351562, + "loss": 0.9468085169792175, + "memory(GiB)": 46.31, + "nll_loss": 0.8088601231575012, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.13317811489105225, + "rewards/margins": 7.745667457580566, + "rewards/rejected": -7.87884521484375, + "step": 324, + "train_speed(iter/s)": 0.048008 + }, + { + "epoch": 0.817353033637221, + "grad_norm": 1.578026294708252, + "learning_rate": 0.00017410968420941244, + "logits/chosen": -0.4647430181503296, + "logits/rejected": -0.863081157207489, + "logps/chosen": -5.37056827545166, + "logps/rejected": -94.24394226074219, + "loss": 0.4825628399848938, + "memory(GiB)": 46.31, + "nll_loss": 0.4036305248737335, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21735483407974243, + "rewards/margins": 7.361626148223877, + "rewards/rejected": -7.144271373748779, + "step": 325, + "train_speed(iter/s)": 0.047523 + }, + { + "epoch": 0.8198679660484125, + "grad_norm": 5.969099044799805, + "learning_rate": 0.00017392290345451592, + "logits/chosen": -0.5345301628112793, + "logits/rejected": -0.820349395275116, + "logps/chosen": -6.029956817626953, + "logps/rejected": -104.83753204345703, + "loss": 0.5511329174041748, + "memory(GiB)": 46.31, + "nll_loss": 0.4094994068145752, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2000085562467575, + "rewards/margins": 8.80609130859375, + "rewards/rejected": -8.606082916259766, + "step": 326, + "train_speed(iter/s)": 0.047054 + }, + { + "epoch": 0.8223828984596039, + "grad_norm": 7.578153133392334, + "learning_rate": 0.0001737355523340875, + "logits/chosen": -0.7083917856216431, + "logits/rejected": -0.9140143990516663, + "logps/chosen": -5.282914161682129, + "logps/rejected": -89.25215148925781, + "loss": 0.6085916757583618, + "memory(GiB)": 46.31, + "nll_loss": 0.4778170585632324, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04538874328136444, + "rewards/margins": 7.695877552032471, + "rewards/rejected": -7.650489807128906, + "step": 327, + "train_speed(iter/s)": 0.046602 + }, + { + "epoch": 0.8248978308707954, + "grad_norm": 7.923031330108643, + "learning_rate": 0.0001735476322936686, + "logits/chosen": -0.48410487174987793, + "logits/rejected": -0.8404121398925781, + "logps/chosen": -5.619724273681641, + "logps/rejected": -111.05279541015625, + "loss": 0.5005356073379517, + "memory(GiB)": 46.31, + "nll_loss": 0.415988028049469, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.04838261753320694, + "rewards/margins": 9.071845054626465, + "rewards/rejected": -9.023462295532227, + "step": 328, + "train_speed(iter/s)": 0.046157 + }, + { + "epoch": 0.8274127632819868, + "grad_norm": 1.6029303073883057, + "learning_rate": 0.00017335914478319028, + "logits/chosen": -0.5107839107513428, + "logits/rejected": -0.811547577381134, + "logps/chosen": -5.087898254394531, + "logps/rejected": -90.02484130859375, + "loss": 0.4403790235519409, + "memory(GiB)": 46.31, + "nll_loss": 0.3421851396560669, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1459052413702011, + "rewards/margins": 7.192948341369629, + "rewards/rejected": -7.047042369842529, + "step": 329, + "train_speed(iter/s)": 0.045725 + }, + { + "epoch": 0.8299276956931783, + "grad_norm": 1.4880335330963135, + "learning_rate": 0.00017317009125696208, + "logits/chosen": -0.5659950375556946, + "logits/rejected": -0.8397157192230225, + "logps/chosen": -4.4726243019104, + "logps/rejected": -92.63803100585938, + "loss": 0.2614552974700928, + "memory(GiB)": 46.31, + "nll_loss": 0.21865251660346985, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20230340957641602, + "rewards/margins": 7.986957550048828, + "rewards/rejected": -7.784653663635254, + "step": 330, + "train_speed(iter/s)": 0.045303 + }, + { + "epoch": 0.8324426281043696, + "grad_norm": 4.477383136749268, + "learning_rate": 0.00017298047317366061, + "logits/chosen": -0.5597605109214783, + "logits/rejected": -0.8359659910202026, + "logps/chosen": -3.7963671684265137, + "logps/rejected": -79.54478454589844, + "loss": 0.386127769947052, + "memory(GiB)": 46.31, + "nll_loss": 0.30003517866134644, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13904371857643127, + "rewards/margins": 6.464727401733398, + "rewards/rejected": -6.32568359375, + "step": 331, + "train_speed(iter/s)": 0.044893 + }, + { + "epoch": 0.8349575605155611, + "grad_norm": 1.3735220432281494, + "learning_rate": 0.00017279029199631857, + "logits/chosen": -0.4946771264076233, + "logits/rejected": -0.8467792272567749, + "logps/chosen": -2.6571059226989746, + "logps/rejected": -94.11196899414062, + "loss": 0.2998129427433014, + "memory(GiB)": 46.31, + "nll_loss": 0.22292877733707428, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12367352843284607, + "rewards/margins": 8.038744926452637, + "rewards/rejected": -7.915071487426758, + "step": 332, + "train_speed(iter/s)": 0.044491 + }, + { + "epoch": 0.8374724929267526, + "grad_norm": 2.6437320709228516, + "learning_rate": 0.0001725995491923131, + "logits/chosen": -0.552052915096283, + "logits/rejected": -0.754858136177063, + "logps/chosen": -11.185404777526855, + "logps/rejected": -86.47320556640625, + "loss": 0.8312444090843201, + "memory(GiB)": 46.31, + "nll_loss": 0.6818321943283081, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.01666903868317604, + "rewards/margins": 6.93008279800415, + "rewards/rejected": -6.946751594543457, + "step": 333, + "train_speed(iter/s)": 0.044095 + }, + { + "epoch": 0.839987425337944, + "grad_norm": 8.506746292114258, + "learning_rate": 0.0001724082462333549, + "logits/chosen": -0.5991488099098206, + "logits/rejected": -0.8054255843162537, + "logps/chosen": -5.842095375061035, + "logps/rejected": -69.10340118408203, + "loss": 0.6470210552215576, + "memory(GiB)": 46.31, + "nll_loss": 0.4332461953163147, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.05430920422077179, + "rewards/margins": 5.4711198806762695, + "rewards/rejected": -5.5254292488098145, + "step": 334, + "train_speed(iter/s)": 0.043713 + }, + { + "epoch": 0.8425023577491355, + "grad_norm": 4.649921894073486, + "learning_rate": 0.00017221638459547655, + "logits/chosen": -0.6327688694000244, + "logits/rejected": -0.8601678609848022, + "logps/chosen": -10.89456558227539, + "logps/rejected": -83.64013671875, + "loss": 0.9836386442184448, + "memory(GiB)": 46.31, + "nll_loss": 0.8277321457862854, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3010229766368866, + "rewards/margins": 6.816217422485352, + "rewards/rejected": -7.1172404289245605, + "step": 335, + "train_speed(iter/s)": 0.043343 + }, + { + "epoch": 0.8450172901603269, + "grad_norm": 3.5049939155578613, + "learning_rate": 0.00017202396575902116, + "logits/chosen": -0.5969817042350769, + "logits/rejected": -0.7890011072158813, + "logps/chosen": -8.715362548828125, + "logps/rejected": -87.10135650634766, + "loss": 0.6040844321250916, + "memory(GiB)": 46.31, + "nll_loss": 0.48410138487815857, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.02549554966390133, + "rewards/margins": 6.946871280670166, + "rewards/rejected": -6.9213762283325195, + "step": 336, + "train_speed(iter/s)": 0.042976 + }, + { + "epoch": 0.8475322225715184, + "grad_norm": 2.5609958171844482, + "learning_rate": 0.00017183099120863116, + "logits/chosen": -0.5624786615371704, + "logits/rejected": -0.8555223941802979, + "logps/chosen": -2.920971393585205, + "logps/rejected": -87.21141815185547, + "loss": 0.25303125381469727, + "memory(GiB)": 46.31, + "nll_loss": 0.17635448276996613, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1297520101070404, + "rewards/margins": 7.165301322937012, + "rewards/rejected": -7.035548210144043, + "step": 337, + "train_speed(iter/s)": 0.04262 + }, + { + "epoch": 0.8500471549827099, + "grad_norm": 2.219430923461914, + "learning_rate": 0.0001716374624332365, + "logits/chosen": -0.4873535931110382, + "logits/rejected": -0.865464985370636, + "logps/chosen": -4.004513740539551, + "logps/rejected": -99.60807800292969, + "loss": 0.3732931613922119, + "memory(GiB)": 46.31, + "nll_loss": 0.32243484258651733, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09392180293798447, + "rewards/margins": 7.731295108795166, + "rewards/rejected": -7.637372970581055, + "step": 338, + "train_speed(iter/s)": 0.042269 + }, + { + "epoch": 0.8525620873939013, + "grad_norm": 1.4633954763412476, + "learning_rate": 0.0001714433809260435, + "logits/chosen": -0.5742372274398804, + "logits/rejected": -0.7338902950286865, + "logps/chosen": -13.275593757629395, + "logps/rejected": -84.15313720703125, + "loss": 0.6252297759056091, + "memory(GiB)": 46.31, + "nll_loss": 0.5572872757911682, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1977962702512741, + "rewards/margins": 6.996209621429443, + "rewards/rejected": -6.798413276672363, + "step": 339, + "train_speed(iter/s)": 0.041927 + }, + { + "epoch": 0.8550770198050928, + "grad_norm": 3.0565924644470215, + "learning_rate": 0.0001712487481845231, + "logits/chosen": -0.3683091104030609, + "logits/rejected": -0.7499415874481201, + "logps/chosen": -6.7348151206970215, + "logps/rejected": -91.9858169555664, + "loss": 0.37550103664398193, + "memory(GiB)": 46.31, + "nll_loss": 0.2465668022632599, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.00661234837025404, + "rewards/margins": 7.401021957397461, + "rewards/rejected": -7.40763521194458, + "step": 340, + "train_speed(iter/s)": 0.041588 + }, + { + "epoch": 0.8575919522162841, + "grad_norm": 1.3918770551681519, + "learning_rate": 0.00017105356571039945, + "logits/chosen": -0.49031901359558105, + "logits/rejected": -0.8110065460205078, + "logps/chosen": -7.980224132537842, + "logps/rejected": -82.05560302734375, + "loss": 0.458918035030365, + "memory(GiB)": 46.31, + "nll_loss": 0.3925769627094269, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.05228738859295845, + "rewards/margins": 6.4176025390625, + "rewards/rejected": -6.365314960479736, + "step": 341, + "train_speed(iter/s)": 0.04126 + }, + { + "epoch": 0.8601068846274756, + "grad_norm": 2.4676175117492676, + "learning_rate": 0.00017085783500963824, + "logits/chosen": -0.4095371961593628, + "logits/rejected": -0.7287023067474365, + "logps/chosen": -3.0237834453582764, + "logps/rejected": -64.21556091308594, + "loss": 0.30038806796073914, + "memory(GiB)": 46.31, + "nll_loss": 0.17780858278274536, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22278741002082825, + "rewards/margins": 4.871509075164795, + "rewards/rejected": -4.648721218109131, + "step": 342, + "train_speed(iter/s)": 0.040936 + }, + { + "epoch": 0.8626218170386671, + "grad_norm": 2.6251206398010254, + "learning_rate": 0.00017066155759243507, + "logits/chosen": -0.559488832950592, + "logits/rejected": -0.7672823667526245, + "logps/chosen": -4.220951557159424, + "logps/rejected": -62.22422790527344, + "loss": 0.30714038014411926, + "memory(GiB)": 46.31, + "nll_loss": 0.201259046792984, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.024884473532438278, + "rewards/margins": 4.834282398223877, + "rewards/rejected": -4.809398174285889, + "step": 343, + "train_speed(iter/s)": 0.040623 + }, + { + "epoch": 0.8651367494498585, + "grad_norm": 1.955743670463562, + "learning_rate": 0.00017046473497320385, + "logits/chosen": -0.5384873151779175, + "logits/rejected": -0.7534351944923401, + "logps/chosen": -4.600622653961182, + "logps/rejected": -62.08992004394531, + "loss": 0.4026404619216919, + "memory(GiB)": 46.31, + "nll_loss": 0.30260390043258667, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04929278418421745, + "rewards/margins": 4.71509313583374, + "rewards/rejected": -4.665800094604492, + "step": 344, + "train_speed(iter/s)": 0.040318 + }, + { + "epoch": 0.86765168186105, + "grad_norm": 1.3359214067459106, + "learning_rate": 0.0001702673686705651, + "logits/chosen": -0.5102373361587524, + "logits/rejected": -0.7203519940376282, + "logps/chosen": -3.7551567554473877, + "logps/rejected": -58.683082580566406, + "loss": 0.26333218812942505, + "memory(GiB)": 46.31, + "nll_loss": 0.1936265230178833, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08393731713294983, + "rewards/margins": 4.568361282348633, + "rewards/rejected": -4.484423637390137, + "step": 345, + "train_speed(iter/s)": 0.040019 + }, + { + "epoch": 0.8701666142722414, + "grad_norm": 1.2940682172775269, + "learning_rate": 0.00017006946020733425, + "logits/chosen": -0.423111230134964, + "logits/rejected": -0.6618233919143677, + "logps/chosen": -3.599187135696411, + "logps/rejected": -57.84706115722656, + "loss": 0.23464316129684448, + "memory(GiB)": 46.31, + "nll_loss": 0.1839112639427185, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3290537893772125, + "rewards/margins": 4.494791507720947, + "rewards/rejected": -4.165737152099609, + "step": 346, + "train_speed(iter/s)": 0.039723 + }, + { + "epoch": 0.8726815466834329, + "grad_norm": 4.882867336273193, + "learning_rate": 0.00016987101111050982, + "logits/chosen": -0.487949937582016, + "logits/rejected": -0.680131733417511, + "logps/chosen": -7.074028015136719, + "logps/rejected": -57.329803466796875, + "loss": 0.5204004645347595, + "memory(GiB)": 46.31, + "nll_loss": 0.4155931770801544, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21489042043685913, + "rewards/margins": 4.179245471954346, + "rewards/rejected": -3.9643547534942627, + "step": 347, + "train_speed(iter/s)": 0.039433 + }, + { + "epoch": 0.8751964790946243, + "grad_norm": 5.844240665435791, + "learning_rate": 0.00016967202291126173, + "logits/chosen": -0.371315062046051, + "logits/rejected": -0.7444156408309937, + "logps/chosen": -5.045580863952637, + "logps/rejected": -67.66508483886719, + "loss": 0.6576581001281738, + "memory(GiB)": 46.31, + "nll_loss": 0.48166701197624207, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07985140383243561, + "rewards/margins": 4.563851833343506, + "rewards/rejected": -4.484000205993652, + "step": 348, + "train_speed(iter/s)": 0.039147 + }, + { + "epoch": 0.8777114115058158, + "grad_norm": 7.42000675201416, + "learning_rate": 0.00016947249714491938, + "logits/chosen": -0.36127984523773193, + "logits/rejected": -0.7726977467536926, + "logps/chosen": -4.132587432861328, + "logps/rejected": -69.11751556396484, + "loss": 0.4309765100479126, + "memory(GiB)": 46.31, + "nll_loss": 0.3314419388771057, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12238576263189316, + "rewards/margins": 4.787508010864258, + "rewards/rejected": -4.6651225090026855, + "step": 349, + "train_speed(iter/s)": 0.038867 + }, + { + "epoch": 0.8802263439170073, + "grad_norm": 8.671714782714844, + "learning_rate": 0.00016927243535095997, + "logits/chosen": -0.584995448589325, + "logits/rejected": -0.7704389691352844, + "logps/chosen": -5.863958358764648, + "logps/rejected": -54.833553314208984, + "loss": 0.6147282123565674, + "memory(GiB)": 46.31, + "nll_loss": 0.5134801864624023, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10420051962137222, + "rewards/margins": 4.378936767578125, + "rewards/rejected": -4.274736404418945, + "step": 350, + "train_speed(iter/s)": 0.038598 + }, + { + "epoch": 0.8827412763281987, + "grad_norm": 1.384485125541687, + "learning_rate": 0.0001690718390729964, + "logits/chosen": -0.49590209126472473, + "logits/rejected": -0.7934409379959106, + "logps/chosen": -3.4274821281433105, + "logps/rejected": -75.05296325683594, + "loss": 0.28905826807022095, + "memory(GiB)": 46.31, + "nll_loss": 0.2423771321773529, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10859310626983643, + "rewards/margins": 5.8243184089660645, + "rewards/rejected": -5.715725898742676, + "step": 351, + "train_speed(iter/s)": 0.038264 + }, + { + "epoch": 0.8852562087393901, + "grad_norm": 2.392730474472046, + "learning_rate": 0.00016887070985876555, + "logits/chosen": -0.6525416970252991, + "logits/rejected": -0.8886513710021973, + "logps/chosen": -3.8878211975097656, + "logps/rejected": -70.61896514892578, + "loss": 0.3180730640888214, + "memory(GiB)": 46.31, + "nll_loss": 0.24728332459926605, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.021800898015499115, + "rewards/margins": 5.6876444816589355, + "rewards/rejected": -5.709445476531982, + "step": 352, + "train_speed(iter/s)": 0.038006 + }, + { + "epoch": 0.8877711411505815, + "grad_norm": 1.8687230348587036, + "learning_rate": 0.00016866904926011623, + "logits/chosen": -0.5100048780441284, + "logits/rejected": -0.8430832028388977, + "logps/chosen": -7.690830230712891, + "logps/rejected": -79.55768585205078, + "loss": 0.5146079063415527, + "memory(GiB)": 46.31, + "nll_loss": 0.480059415102005, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15592242777347565, + "rewards/margins": 6.150962829589844, + "rewards/rejected": -5.995040416717529, + "step": 353, + "train_speed(iter/s)": 0.03775 + }, + { + "epoch": 0.890286073561773, + "grad_norm": 2.0833466053009033, + "learning_rate": 0.00016846685883299732, + "logits/chosen": -0.6130125522613525, + "logits/rejected": -0.8856174945831299, + "logps/chosen": -2.691479206085205, + "logps/rejected": -68.62126922607422, + "loss": 0.29438382387161255, + "memory(GiB)": 46.31, + "nll_loss": 0.21310341358184814, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21031589806079865, + "rewards/margins": 5.494964599609375, + "rewards/rejected": -5.284648418426514, + "step": 354, + "train_speed(iter/s)": 0.037501 + }, + { + "epoch": 0.8928010059729645, + "grad_norm": 5.909437656402588, + "learning_rate": 0.0001682641401374456, + "logits/chosen": -0.5725633502006531, + "logits/rejected": -0.8215086460113525, + "logps/chosen": -7.275115966796875, + "logps/rejected": -68.49534606933594, + "loss": 0.683579683303833, + "memory(GiB)": 46.31, + "nll_loss": 0.408993124961853, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.1348603069782257, + "rewards/margins": 5.070530891418457, + "rewards/rejected": -5.205390930175781, + "step": 355, + "train_speed(iter/s)": 0.037256 + }, + { + "epoch": 0.8953159383841559, + "grad_norm": 12.008193016052246, + "learning_rate": 0.00016806089473757384, + "logits/chosen": -0.48063433170318604, + "logits/rejected": -0.8200632929801941, + "logps/chosen": -5.245819091796875, + "logps/rejected": -75.48336791992188, + "loss": 0.5572218894958496, + "memory(GiB)": 46.31, + "nll_loss": 0.4593680500984192, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.042567022144794464, + "rewards/margins": 5.983489036560059, + "rewards/rejected": -5.940921783447266, + "step": 356, + "train_speed(iter/s)": 0.037015 + }, + { + "epoch": 0.8978308707953474, + "grad_norm": 2.3024280071258545, + "learning_rate": 0.00016785712420155864, + "logits/chosen": -0.5618032813072205, + "logits/rejected": -0.7750505208969116, + "logps/chosen": -5.727665424346924, + "logps/rejected": -66.33029174804688, + "loss": 0.49546992778778076, + "memory(GiB)": 46.31, + "nll_loss": 0.3907400071620941, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2677232623100281, + "rewards/margins": 5.57326078414917, + "rewards/rejected": -5.305538177490234, + "step": 357, + "train_speed(iter/s)": 0.036778 + }, + { + "epoch": 0.9003458032065388, + "grad_norm": 1.5032073259353638, + "learning_rate": 0.00016765283010162845, + "logits/chosen": -0.6311084628105164, + "logits/rejected": -0.884809136390686, + "logps/chosen": -6.822357177734375, + "logps/rejected": -86.34951782226562, + "loss": 0.3546079099178314, + "memory(GiB)": 46.31, + "nll_loss": 0.32874923944473267, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21881002187728882, + "rewards/margins": 7.21632719039917, + "rewards/rejected": -6.997516632080078, + "step": 358, + "train_speed(iter/s)": 0.036547 + }, + { + "epoch": 0.9028607356177303, + "grad_norm": 12.216839790344238, + "learning_rate": 0.0001674480140140514, + "logits/chosen": -0.563357949256897, + "logits/rejected": -0.8381999731063843, + "logps/chosen": -5.951011657714844, + "logps/rejected": -73.21198272705078, + "loss": 0.6865818500518799, + "memory(GiB)": 46.31, + "nll_loss": 0.4365982413291931, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.010548931546509266, + "rewards/margins": 5.462331771850586, + "rewards/rejected": -5.472881317138672, + "step": 359, + "train_speed(iter/s)": 0.036319 + }, + { + "epoch": 0.9053756680289218, + "grad_norm": 2.2943708896636963, + "learning_rate": 0.000167242677519123, + "logits/chosen": -0.5497739315032959, + "logits/rejected": -0.7586756348609924, + "logps/chosen": -6.904477596282959, + "logps/rejected": -63.757484436035156, + "loss": 0.524222195148468, + "memory(GiB)": 46.31, + "nll_loss": 0.4234248697757721, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.012442603707313538, + "rewards/margins": 4.878024578094482, + "rewards/rejected": -4.865581512451172, + "step": 360, + "train_speed(iter/s)": 0.036096 + }, + { + "epoch": 0.9078906004401132, + "grad_norm": 1.8752505779266357, + "learning_rate": 0.00016703682220115416, + "logits/chosen": -0.49279505014419556, + "logits/rejected": -0.6874585747718811, + "logps/chosen": -6.828492641448975, + "logps/rejected": -56.00688552856445, + "loss": 0.5722380876541138, + "memory(GiB)": 46.31, + "nll_loss": 0.4202978312969208, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16803106665611267, + "rewards/margins": 4.519139289855957, + "rewards/rejected": -4.351108074188232, + "step": 361, + "train_speed(iter/s)": 0.035876 + }, + { + "epoch": 0.9104055328513047, + "grad_norm": 13.851007461547852, + "learning_rate": 0.00016683044964845874, + "logits/chosen": -0.3259517550468445, + "logits/rejected": -0.7759101986885071, + "logps/chosen": -2.291940689086914, + "logps/rejected": -82.96823120117188, + "loss": 0.3387893736362457, + "memory(GiB)": 46.31, + "nll_loss": 0.23984995484352112, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13379287719726562, + "rewards/margins": 6.447347640991211, + "rewards/rejected": -6.313554763793945, + "step": 362, + "train_speed(iter/s)": 0.035659 + }, + { + "epoch": 0.912920465262496, + "grad_norm": 3.0961484909057617, + "learning_rate": 0.00016662356145334158, + "logits/chosen": -0.4107140004634857, + "logits/rejected": -0.7844898104667664, + "logps/chosen": -6.643762111663818, + "logps/rejected": -85.53607940673828, + "loss": 0.7083590030670166, + "memory(GiB)": 46.31, + "nll_loss": 0.5615524053573608, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14561021327972412, + "rewards/margins": 6.219028472900391, + "rewards/rejected": -6.0734171867370605, + "step": 363, + "train_speed(iter/s)": 0.035445 + }, + { + "epoch": 0.9154353976736875, + "grad_norm": 3.2900326251983643, + "learning_rate": 0.0001664161592120859, + "logits/chosen": -0.48369789123535156, + "logits/rejected": -0.8355358839035034, + "logps/chosen": -4.989360332489014, + "logps/rejected": -76.567138671875, + "loss": 0.44374826550483704, + "memory(GiB)": 46.31, + "nll_loss": 0.3595760762691498, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.027300171554088593, + "rewards/margins": 6.1481733322143555, + "rewards/rejected": -6.12087345123291, + "step": 364, + "train_speed(iter/s)": 0.035237 + }, + { + "epoch": 0.9179503300848789, + "grad_norm": 2.7764828205108643, + "learning_rate": 0.00016620824452494128, + "logits/chosen": -0.3900059163570404, + "logits/rejected": -0.7216352224349976, + "logps/chosen": -2.781705617904663, + "logps/rejected": -72.1066665649414, + "loss": 0.32665613293647766, + "memory(GiB)": 46.31, + "nll_loss": 0.22233696281909943, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2311152219772339, + "rewards/margins": 5.6492919921875, + "rewards/rejected": -5.418176651000977, + "step": 365, + "train_speed(iter/s)": 0.035032 + }, + { + "epoch": 0.9204652624960704, + "grad_norm": 3.265141010284424, + "learning_rate": 0.00016599981899611104, + "logits/chosen": -0.4622117280960083, + "logits/rejected": -0.7529523372650146, + "logps/chosen": -7.999266624450684, + "logps/rejected": -65.14107513427734, + "loss": 0.578423261642456, + "memory(GiB)": 46.31, + "nll_loss": 0.44822537899017334, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.09772372990846634, + "rewards/margins": 4.932915210723877, + "rewards/rejected": -5.030638694763184, + "step": 366, + "train_speed(iter/s)": 0.034831 + }, + { + "epoch": 0.9229801949072619, + "grad_norm": 7.041696071624756, + "learning_rate": 0.00016579088423374004, + "logits/chosen": -0.5291659235954285, + "logits/rejected": -0.7164372205734253, + "logps/chosen": -8.906322479248047, + "logps/rejected": -56.788475036621094, + "loss": 0.7593228816986084, + "memory(GiB)": 46.31, + "nll_loss": 0.5370157957077026, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.056440941989421844, + "rewards/margins": 4.266091346740723, + "rewards/rejected": -4.3225321769714355, + "step": 367, + "train_speed(iter/s)": 0.034633 + }, + { + "epoch": 0.9254951273184533, + "grad_norm": 5.255563735961914, + "learning_rate": 0.00016558144184990227, + "logits/chosen": -0.37939125299453735, + "logits/rejected": -0.6499576568603516, + "logps/chosen": -9.906354904174805, + "logps/rejected": -63.14579772949219, + "loss": 1.1975581645965576, + "memory(GiB)": 46.31, + "nll_loss": 0.7969497442245483, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.17077556252479553, + "rewards/margins": 4.492245197296143, + "rewards/rejected": -4.663021087646484, + "step": 368, + "train_speed(iter/s)": 0.034438 + }, + { + "epoch": 0.9280100597296448, + "grad_norm": 3.5527825355529785, + "learning_rate": 0.0001653714934605883, + "logits/chosen": -0.39540186524391174, + "logits/rejected": -0.6938212513923645, + "logps/chosen": -4.2778849601745605, + "logps/rejected": -69.90755462646484, + "loss": 0.43115851283073425, + "memory(GiB)": 46.31, + "nll_loss": 0.30130866169929504, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0013723522424697876, + "rewards/margins": 5.105025291442871, + "rewards/rejected": -5.1036529541015625, + "step": 369, + "train_speed(iter/s)": 0.034247 + }, + { + "epoch": 0.9305249921408362, + "grad_norm": 1.7090513706207275, + "learning_rate": 0.0001651610406856929, + "logits/chosen": -0.3463304936885834, + "logits/rejected": -0.579744815826416, + "logps/chosen": -5.828442573547363, + "logps/rejected": -69.42247772216797, + "loss": 0.40808120369911194, + "memory(GiB)": 46.31, + "nll_loss": 0.30207115411758423, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17322836816310883, + "rewards/margins": 5.431868553161621, + "rewards/rejected": -5.258640289306641, + "step": 370, + "train_speed(iter/s)": 0.034058 + }, + { + "epoch": 0.9330399245520277, + "grad_norm": 2.4113481044769287, + "learning_rate": 0.00016495008514900254, + "logits/chosen": -0.33101192116737366, + "logits/rejected": -0.6629313826560974, + "logps/chosen": -3.4581689834594727, + "logps/rejected": -70.76332092285156, + "loss": 0.3193123936653137, + "memory(GiB)": 46.31, + "nll_loss": 0.2055172175168991, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16898323595523834, + "rewards/margins": 5.381399154663086, + "rewards/rejected": -5.2124152183532715, + "step": 371, + "train_speed(iter/s)": 0.033871 + }, + { + "epoch": 0.9355548569632192, + "grad_norm": 1.4116032123565674, + "learning_rate": 0.00016473862847818277, + "logits/chosen": -0.3042815625667572, + "logits/rejected": -0.6890904903411865, + "logps/chosen": -2.9157466888427734, + "logps/rejected": -70.07747650146484, + "loss": 0.35020875930786133, + "memory(GiB)": 46.31, + "nll_loss": 0.2551298141479492, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16930048167705536, + "rewards/margins": 5.438758850097656, + "rewards/rejected": -5.269458293914795, + "step": 372, + "train_speed(iter/s)": 0.033688 + }, + { + "epoch": 0.9380697893744105, + "grad_norm": 3.803670883178711, + "learning_rate": 0.0001645266723047658, + "logits/chosen": -0.43881455063819885, + "logits/rejected": -0.7362341284751892, + "logps/chosen": -3.6323812007904053, + "logps/rejected": -70.09701538085938, + "loss": 0.39496979117393494, + "memory(GiB)": 46.31, + "nll_loss": 0.3047463893890381, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07599923759698868, + "rewards/margins": 5.4243903160095215, + "rewards/rejected": -5.348391532897949, + "step": 373, + "train_speed(iter/s)": 0.03351 + }, + { + "epoch": 0.940584721785602, + "grad_norm": 2.404557466506958, + "learning_rate": 0.0001643142182641379, + "logits/chosen": -0.34889113903045654, + "logits/rejected": -0.6839251518249512, + "logps/chosen": -4.086870193481445, + "logps/rejected": -61.729209899902344, + "loss": 0.3285399377346039, + "memory(GiB)": 46.31, + "nll_loss": 0.24733781814575195, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1459178924560547, + "rewards/margins": 4.617620944976807, + "rewards/rejected": -4.47170352935791, + "step": 374, + "train_speed(iter/s)": 0.033332 + }, + { + "epoch": 0.9430996541967934, + "grad_norm": 1.1544145345687866, + "learning_rate": 0.0001641012679955265, + "logits/chosen": -0.24821627140045166, + "logits/rejected": -0.6281561255455017, + "logps/chosen": -4.0372772216796875, + "logps/rejected": -79.80203247070312, + "loss": 0.2588978707790375, + "memory(GiB)": 46.31, + "nll_loss": 0.21967677772045135, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12974770367145538, + "rewards/margins": 5.533021926879883, + "rewards/rejected": -5.403275012969971, + "step": 375, + "train_speed(iter/s)": 0.033157 + }, + { + "epoch": 0.9456145866079849, + "grad_norm": 0.9758641123771667, + "learning_rate": 0.00016388782314198804, + "logits/chosen": -0.3430945873260498, + "logits/rejected": -0.5772558450698853, + "logps/chosen": -4.472623825073242, + "logps/rejected": -63.992557525634766, + "loss": 0.3319728672504425, + "memory(GiB)": 46.31, + "nll_loss": 0.27475422620773315, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24630330502986908, + "rewards/margins": 5.2393107414245605, + "rewards/rejected": -4.993007183074951, + "step": 376, + "train_speed(iter/s)": 0.032986 + }, + { + "epoch": 0.9481295190191764, + "grad_norm": 2.5297164916992188, + "learning_rate": 0.00016367388535039483, + "logits/chosen": -0.4488414525985718, + "logits/rejected": -0.5838022232055664, + "logps/chosen": -7.7367448806762695, + "logps/rejected": -54.747467041015625, + "loss": 0.8268335461616516, + "memory(GiB)": 46.31, + "nll_loss": 0.5897634029388428, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0750073790550232, + "rewards/margins": 4.319672107696533, + "rewards/rejected": -4.244664669036865, + "step": 377, + "train_speed(iter/s)": 0.032818 + }, + { + "epoch": 0.9506444514303678, + "grad_norm": 1.5900825262069702, + "learning_rate": 0.00016345945627142263, + "logits/chosen": -0.3362869322299957, + "logits/rejected": -0.6326735019683838, + "logps/chosen": -4.714737415313721, + "logps/rejected": -64.11145782470703, + "loss": 0.3902578055858612, + "memory(GiB)": 46.31, + "nll_loss": 0.33638864755630493, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2739555835723877, + "rewards/margins": 5.219550132751465, + "rewards/rejected": -4.945594787597656, + "step": 378, + "train_speed(iter/s)": 0.032653 + }, + { + "epoch": 0.9531593838415593, + "grad_norm": 4.194026470184326, + "learning_rate": 0.00016324453755953773, + "logits/chosen": -0.16701598465442657, + "logits/rejected": -0.524956464767456, + "logps/chosen": -7.983905792236328, + "logps/rejected": -79.0844955444336, + "loss": 0.5492078065872192, + "memory(GiB)": 46.31, + "nll_loss": 0.4571991264820099, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04178844019770622, + "rewards/margins": 5.678005218505859, + "rewards/rejected": -5.636216640472412, + "step": 379, + "train_speed(iter/s)": 0.032486 + }, + { + "epoch": 0.9556743162527507, + "grad_norm": 3.1762311458587646, + "learning_rate": 0.00016302913087298438, + "logits/chosen": -0.23892173171043396, + "logits/rejected": -0.5944012403488159, + "logps/chosen": -2.855877161026001, + "logps/rejected": -71.64038848876953, + "loss": 0.2882783114910126, + "memory(GiB)": 46.31, + "nll_loss": 0.20760102570056915, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15581874549388885, + "rewards/margins": 5.530270099639893, + "rewards/rejected": -5.374452114105225, + "step": 380, + "train_speed(iter/s)": 0.032324 + }, + { + "epoch": 0.9581892486639422, + "grad_norm": 1.3025074005126953, + "learning_rate": 0.0001628132378737718, + "logits/chosen": -0.255632609128952, + "logits/rejected": -0.5729305744171143, + "logps/chosen": -7.531216144561768, + "logps/rejected": -76.65748596191406, + "loss": 0.35401973128318787, + "memory(GiB)": 46.31, + "nll_loss": 0.28915196657180786, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17935439944267273, + "rewards/margins": 5.804968357086182, + "rewards/rejected": -5.625614166259766, + "step": 381, + "train_speed(iter/s)": 0.032164 + }, + { + "epoch": 0.9607041810751336, + "grad_norm": 1.362042784690857, + "learning_rate": 0.00016259686022766147, + "logits/chosen": -0.3349270224571228, + "logits/rejected": -0.5813159346580505, + "logps/chosen": -5.685837268829346, + "logps/rejected": -82.27192687988281, + "loss": 0.3539260923862457, + "memory(GiB)": 46.31, + "nll_loss": 0.2879295349121094, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15435966849327087, + "rewards/margins": 6.62058162689209, + "rewards/rejected": -6.466221809387207, + "step": 382, + "train_speed(iter/s)": 0.032007 + }, + { + "epoch": 0.963219113486325, + "grad_norm": 2.0811262130737305, + "learning_rate": 0.00016237999960415426, + "logits/chosen": -0.41777318716049194, + "logits/rejected": -0.6865772008895874, + "logps/chosen": -4.136772155761719, + "logps/rejected": -66.67522430419922, + "loss": 0.37008586525917053, + "memory(GiB)": 46.31, + "nll_loss": 0.23609820008277893, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.061387136578559875, + "rewards/margins": 5.56961727142334, + "rewards/rejected": -5.508229732513428, + "step": 383, + "train_speed(iter/s)": 0.031855 + }, + { + "epoch": 0.9657340458975165, + "grad_norm": 2.0822818279266357, + "learning_rate": 0.00016216265767647755, + "logits/chosen": -0.34635311365127563, + "logits/rejected": -0.6642223596572876, + "logps/chosen": -7.652328014373779, + "logps/rejected": -82.85174560546875, + "loss": 0.4690261483192444, + "memory(GiB)": 46.31, + "nll_loss": 0.37369227409362793, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1277095228433609, + "rewards/margins": 6.65843391418457, + "rewards/rejected": -6.530723571777344, + "step": 384, + "train_speed(iter/s)": 0.031703 + }, + { + "epoch": 0.9682489783087079, + "grad_norm": 2.0401089191436768, + "learning_rate": 0.0001619448361215723, + "logits/chosen": -0.44850289821624756, + "logits/rejected": -0.7342371940612793, + "logps/chosen": -3.3725736141204834, + "logps/rejected": -70.59913635253906, + "loss": 0.3117908835411072, + "memory(GiB)": 46.31, + "nll_loss": 0.24734090268611908, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2703946530818939, + "rewards/margins": 5.976531982421875, + "rewards/rejected": -5.706137657165527, + "step": 385, + "train_speed(iter/s)": 0.031555 + }, + { + "epoch": 0.9707639107198994, + "grad_norm": 3.712329387664795, + "learning_rate": 0.0001617265366200801, + "logits/chosen": -0.5690063834190369, + "logits/rejected": -0.7474982738494873, + "logps/chosen": -7.040001392364502, + "logps/rejected": -58.06250762939453, + "loss": 0.7071576118469238, + "memory(GiB)": 46.31, + "nll_loss": 0.5602840781211853, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.24300126731395721, + "rewards/margins": 4.731121063232422, + "rewards/rejected": -4.488119602203369, + "step": 386, + "train_speed(iter/s)": 0.031409 + }, + { + "epoch": 0.9732788431310908, + "grad_norm": 4.140589714050293, + "learning_rate": 0.0001615077608563302, + "logits/chosen": -0.4431472420692444, + "logits/rejected": -0.6148325800895691, + "logps/chosen": -6.866641521453857, + "logps/rejected": -60.58491516113281, + "loss": 0.716019868850708, + "memory(GiB)": 46.31, + "nll_loss": 0.53892582654953, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04589424282312393, + "rewards/margins": 4.8714165687561035, + "rewards/rejected": -4.825522422790527, + "step": 387, + "train_speed(iter/s)": 0.031265 + }, + { + "epoch": 0.9757937755422823, + "grad_norm": 1.4952077865600586, + "learning_rate": 0.00016128851051832663, + "logits/chosen": -0.3862327039241791, + "logits/rejected": -0.6277360320091248, + "logps/chosen": -4.6577043533325195, + "logps/rejected": -65.08265686035156, + "loss": 0.39572662115097046, + "memory(GiB)": 46.31, + "nll_loss": 0.32492324709892273, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1440601646900177, + "rewards/margins": 4.995415687561035, + "rewards/rejected": -4.85135555267334, + "step": 388, + "train_speed(iter/s)": 0.031122 + }, + { + "epoch": 0.9783087079534738, + "grad_norm": 6.760348320007324, + "learning_rate": 0.000161068787297735, + "logits/chosen": -0.36251139640808105, + "logits/rejected": -0.6239615082740784, + "logps/chosen": -7.945605754852295, + "logps/rejected": -73.32543182373047, + "loss": 0.7213991284370422, + "memory(GiB)": 46.31, + "nll_loss": 0.4911874234676361, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.02407783269882202, + "rewards/margins": 5.578208923339844, + "rewards/rejected": -5.602287292480469, + "step": 389, + "train_speed(iter/s)": 0.030981 + }, + { + "epoch": 0.9808236403646652, + "grad_norm": 1.4369536638259888, + "learning_rate": 0.00016084859288986955, + "logits/chosen": -0.4005224108695984, + "logits/rejected": -0.7330536246299744, + "logps/chosen": -2.2685210704803467, + "logps/rejected": -75.97575378417969, + "loss": 0.22442395985126495, + "memory(GiB)": 46.31, + "nll_loss": 0.15800324082374573, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1866656392812729, + "rewards/margins": 6.141209602355957, + "rewards/rejected": -5.9545440673828125, + "step": 390, + "train_speed(iter/s)": 0.030843 + }, + { + "epoch": 0.9833385727758567, + "grad_norm": 2.5951297283172607, + "learning_rate": 0.0001606279289936801, + "logits/chosen": -0.3820829689502716, + "logits/rejected": -0.6506544351577759, + "logps/chosen": -5.631180763244629, + "logps/rejected": -72.71686553955078, + "loss": 0.4367509186267853, + "memory(GiB)": 46.31, + "nll_loss": 0.30333346128463745, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2744603753089905, + "rewards/margins": 5.615926742553711, + "rewards/rejected": -5.341466426849365, + "step": 391, + "train_speed(iter/s)": 0.030705 + }, + { + "epoch": 0.9858535051870481, + "grad_norm": 1.5032093524932861, + "learning_rate": 0.00016040679731173883, + "logits/chosen": -0.4364391565322876, + "logits/rejected": -0.8331149220466614, + "logps/chosen": -4.418792247772217, + "logps/rejected": -87.72761535644531, + "loss": 0.33695462346076965, + "memory(GiB)": 46.31, + "nll_loss": 0.2598365545272827, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15108640491962433, + "rewards/margins": 7.023785591125488, + "rewards/rejected": -6.872698783874512, + "step": 392, + "train_speed(iter/s)": 0.03057 + }, + { + "epoch": 0.9883684375982396, + "grad_norm": 1.427937388420105, + "learning_rate": 0.0001601851995502272, + "logits/chosen": -0.43706899881362915, + "logits/rejected": -0.6605555415153503, + "logps/chosen": -6.906405925750732, + "logps/rejected": -74.19134521484375, + "loss": 0.44790229201316833, + "memory(GiB)": 46.31, + "nll_loss": 0.36099445819854736, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21629741787910461, + "rewards/margins": 6.02875280380249, + "rewards/rejected": -5.812455177307129, + "step": 393, + "train_speed(iter/s)": 0.030436 + }, + { + "epoch": 0.9908833700094309, + "grad_norm": 1.9399830102920532, + "learning_rate": 0.00015996313741892287, + "logits/chosen": -0.26576218008995056, + "logits/rejected": -0.6935127377510071, + "logps/chosen": -3.865288257598877, + "logps/rejected": -82.85759735107422, + "loss": 0.4476841688156128, + "memory(GiB)": 46.31, + "nll_loss": 0.32863834500312805, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09060452878475189, + "rewards/margins": 6.340301990509033, + "rewards/rejected": -6.249697685241699, + "step": 394, + "train_speed(iter/s)": 0.030303 + }, + { + "epoch": 0.9933983024206224, + "grad_norm": 1.766074299812317, + "learning_rate": 0.00015974061263118633, + "logits/chosen": -0.45828965306282043, + "logits/rejected": -0.7260013222694397, + "logps/chosen": -5.735683917999268, + "logps/rejected": -84.8984375, + "loss": 0.4351236820220947, + "memory(GiB)": 46.31, + "nll_loss": 0.34114885330200195, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15323616564273834, + "rewards/margins": 6.745055198669434, + "rewards/rejected": -6.591818809509277, + "step": 395, + "train_speed(iter/s)": 0.030174 + }, + { + "epoch": 0.9959132348318139, + "grad_norm": 1.4137344360351562, + "learning_rate": 0.00015951762690394786, + "logits/chosen": -0.36978721618652344, + "logits/rejected": -0.6695666313171387, + "logps/chosen": -6.405080318450928, + "logps/rejected": -82.90550231933594, + "loss": 0.3862062990665436, + "memory(GiB)": 46.31, + "nll_loss": 0.30257418751716614, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23101738095283508, + "rewards/margins": 6.621737480163574, + "rewards/rejected": -6.390720367431641, + "step": 396, + "train_speed(iter/s)": 0.030046 + }, + { + "epoch": 0.9984281672430053, + "grad_norm": 4.094442844390869, + "learning_rate": 0.00015929418195769417, + "logits/chosen": -0.48969727754592896, + "logits/rejected": -0.6828620433807373, + "logps/chosen": -6.582212448120117, + "logps/rejected": -70.81021881103516, + "loss": 0.4160895049571991, + "memory(GiB)": 46.31, + "nll_loss": 0.31492117047309875, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13509458303451538, + "rewards/margins": 5.814253330230713, + "rewards/rejected": -5.679159164428711, + "step": 397, + "train_speed(iter/s)": 0.02992 + }, + { + "epoch": 1.0025149324111915, + "grad_norm": 1.9179918766021729, + "learning_rate": 0.0001590702795164551, + "logits/chosen": -0.35988548398017883, + "logits/rejected": -0.6789807081222534, + "logps/chosen": -4.162905693054199, + "logps/rejected": -76.96558380126953, + "loss": 0.7711372971534729, + "memory(GiB)": 46.31, + "nll_loss": 0.3491760492324829, + "rewards/accuracies": 0.9807692170143127, + "rewards/chosen": 0.2266516387462616, + "rewards/margins": 6.186110496520996, + "rewards/rejected": -5.959458351135254, + "step": 398, + "train_speed(iter/s)": 0.029672 + }, + { + "epoch": 1.005029864822383, + "grad_norm": 4.1499152183532715, + "learning_rate": 0.00015884592130779054, + "logits/chosen": -0.39633291959762573, + "logits/rejected": -0.7770199775695801, + "logps/chosen": -3.2545783519744873, + "logps/rejected": -85.13819122314453, + "loss": 0.3328564167022705, + "memory(GiB)": 46.31, + "nll_loss": 0.2181829959154129, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2071080356836319, + "rewards/margins": 6.915965557098389, + "rewards/rejected": -6.708857536315918, + "step": 399, + "train_speed(iter/s)": 0.029551 + }, + { + "epoch": 1.0075447972335743, + "grad_norm": 1.9317526817321777, + "learning_rate": 0.00015862110906277675, + "logits/chosen": -0.3625487983226776, + "logits/rejected": -0.7721232175827026, + "logps/chosen": -2.4324581623077393, + "logps/rejected": -94.80921173095703, + "loss": 0.1829817295074463, + "memory(GiB)": 46.31, + "nll_loss": 0.13283208012580872, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14578577876091003, + "rewards/margins": 7.785459041595459, + "rewards/rejected": -7.639672756195068, + "step": 400, + "train_speed(iter/s)": 0.029431 + } + ], + "logging_steps": 1, + "max_steps": 1191, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1869819546986086e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}