{ "best_global_step": 300, "best_metric": 0.51461613, "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt10_noidk__pari0.5_0602_p2prompt/v3-20250603-112113/checkpoint-300", "epoch": 1.0075447972335743, "eval_steps": 300, "global_step": 400, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.002514932411191449, "grad_norm": 7.572820663452148, "learning_rate": 3.3333333333333333e-06, "logits/chosen": -0.5129972100257874, "logits/rejected": -0.6397510766983032, "logps/chosen": -8.847090721130371, "logps/rejected": -18.28310203552246, "loss": 1.2403231859207153, "memory(GiB)": 30.59, "nll_loss": 0.5471758246421814, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1, "train_speed(iter/s)": 0.010604 }, { "epoch": 0.005029864822382898, "grad_norm": 7.455193042755127, "learning_rate": 6.666666666666667e-06, "logits/chosen": -0.5209798812866211, "logits/rejected": -0.6239324808120728, "logps/chosen": -4.766471862792969, "logps/rejected": -15.138076782226562, "loss": 1.065555453300476, "memory(GiB)": 33.8, "nll_loss": 0.37240836024284363, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 2, "train_speed(iter/s)": 0.01094 }, { "epoch": 0.007544797233574348, "grad_norm": 10.782440185546875, "learning_rate": 1e-05, "logits/chosen": -0.4890531897544861, "logits/rejected": -0.6535542011260986, "logps/chosen": -4.760848522186279, "logps/rejected": -17.92791175842285, "loss": 1.0910710096359253, "memory(GiB)": 33.8, "nll_loss": 0.3938094973564148, "rewards/accuracies": 0.4375, "rewards/chosen": -0.0034015527926385403, "rewards/margins": -0.008013413287699223, "rewards/rejected": 0.004611859563738108, "step": 3, "train_speed(iter/s)": 0.011055 }, { "epoch": 0.010059729644765796, "grad_norm": 13.736228942871094, "learning_rate": 1.3333333333333333e-05, "logits/chosen": -0.4501972496509552, "logits/rejected": -0.6398910880088806, "logps/chosen": -3.6690049171447754, "logps/rejected": -21.246660232543945, "loss": 1.0828931331634521, "memory(GiB)": 37.26, "nll_loss": 0.39956799149513245, "rewards/accuracies": 0.75, "rewards/chosen": 0.006963790860027075, "rewards/margins": 0.019915670156478882, "rewards/rejected": -0.012951879762113094, "step": 4, "train_speed(iter/s)": 0.011095 }, { "epoch": 0.012574662055957246, "grad_norm": 8.89594554901123, "learning_rate": 1.6666666666666667e-05, "logits/chosen": -0.41603705286979675, "logits/rejected": -0.5863696336746216, "logps/chosen": -8.551430702209473, "logps/rejected": -19.92246437072754, "loss": 1.1882262229919434, "memory(GiB)": 37.26, "nll_loss": 0.5108650922775269, "rewards/accuracies": 0.75, "rewards/chosen": 0.014113119803369045, "rewards/margins": 0.03240397199988365, "rewards/rejected": -0.018290851265192032, "step": 5, "train_speed(iter/s)": 0.011125 }, { "epoch": 0.015089594467148696, "grad_norm": 5.3424577713012695, "learning_rate": 2e-05, "logits/chosen": -0.4721744656562805, "logits/rejected": -0.5799455642700195, "logps/chosen": -9.535165786743164, "logps/rejected": -19.043678283691406, "loss": 1.188051462173462, "memory(GiB)": 37.26, "nll_loss": 0.524101972579956, "rewards/accuracies": 0.6875, "rewards/chosen": 0.010117664933204651, "rewards/margins": 0.0616740845143795, "rewards/rejected": -0.05155642703175545, "step": 6, "train_speed(iter/s)": 0.011145 }, { "epoch": 0.017604526878340146, "grad_norm": 3.5404231548309326, "learning_rate": 2.3333333333333336e-05, "logits/chosen": -0.4064752459526062, "logits/rejected": -0.5387755036354065, "logps/chosen": -9.426046371459961, "logps/rejected": -24.705936431884766, "loss": 1.1507773399353027, "memory(GiB)": 40.75, "nll_loss": 0.5126925706863403, "rewards/accuracies": 0.65625, "rewards/chosen": -0.0260818712413311, "rewards/margins": 0.1276150643825531, "rewards/rejected": -0.1536969393491745, "step": 7, "train_speed(iter/s)": 0.011146 }, { "epoch": 0.020119459289531592, "grad_norm": 4.785594463348389, "learning_rate": 2.6666666666666667e-05, "logits/chosen": -0.40693986415863037, "logits/rejected": -0.5652169585227966, "logps/chosen": -5.92374849319458, "logps/rejected": -20.89351463317871, "loss": 1.02293860912323, "memory(GiB)": 40.75, "nll_loss": 0.39562517404556274, "rewards/accuracies": 0.71875, "rewards/chosen": -0.05045217275619507, "rewards/margins": 0.1817830502986908, "rewards/rejected": -0.23223522305488586, "step": 8, "train_speed(iter/s)": 0.011156 }, { "epoch": 0.022634391700723042, "grad_norm": 4.752041816711426, "learning_rate": 3e-05, "logits/chosen": -0.3253605365753174, "logits/rejected": -0.5100574493408203, "logps/chosen": -5.970067024230957, "logps/rejected": -27.08715057373047, "loss": 0.8663768768310547, "memory(GiB)": 40.75, "nll_loss": 0.36089974641799927, "rewards/accuracies": 0.875, "rewards/chosen": 0.04699152708053589, "rewards/margins": 0.49801987409591675, "rewards/rejected": -0.45102834701538086, "step": 9, "train_speed(iter/s)": 0.011163 }, { "epoch": 0.025149324111914492, "grad_norm": 5.771737575531006, "learning_rate": 3.3333333333333335e-05, "logits/chosen": -0.4344920516014099, "logits/rejected": -0.5643672347068787, "logps/chosen": -6.490581035614014, "logps/rejected": -19.547592163085938, "loss": 1.0153567790985107, "memory(GiB)": 40.75, "nll_loss": 0.429390549659729, "rewards/accuracies": 0.6875, "rewards/chosen": -0.19264209270477295, "rewards/margins": 0.43160852789878845, "rewards/rejected": -0.6242506504058838, "step": 10, "train_speed(iter/s)": 0.011177 }, { "epoch": 0.027664256523105942, "grad_norm": 5.376907825469971, "learning_rate": 3.6666666666666666e-05, "logits/chosen": -0.4512503147125244, "logits/rejected": -0.6003657579421997, "logps/chosen": -6.500467300415039, "logps/rejected": -27.486557006835938, "loss": 0.9551109075546265, "memory(GiB)": 40.75, "nll_loss": 0.40139278769493103, "rewards/accuracies": 0.75, "rewards/chosen": -0.12439044564962387, "rewards/margins": 0.5762982368469238, "rewards/rejected": -0.7006886601448059, "step": 11, "train_speed(iter/s)": 0.011187 }, { "epoch": 0.030179188934297392, "grad_norm": 3.0979809761047363, "learning_rate": 4e-05, "logits/chosen": -0.4187595844268799, "logits/rejected": -0.5968605279922485, "logps/chosen": -4.248518466949463, "logps/rejected": -26.590652465820312, "loss": 0.7605504393577576, "memory(GiB)": 40.75, "nll_loss": 0.2799570858478546, "rewards/accuracies": 0.8125, "rewards/chosen": -0.14420613646507263, "rewards/margins": 0.6546018123626709, "rewards/rejected": -0.7988079190254211, "step": 12, "train_speed(iter/s)": 0.011196 }, { "epoch": 0.03269412134548884, "grad_norm": 9.02223014831543, "learning_rate": 4.3333333333333334e-05, "logits/chosen": -0.40893471240997314, "logits/rejected": -0.5400319695472717, "logps/chosen": -7.627485275268555, "logps/rejected": -21.307247161865234, "loss": 1.2244255542755127, "memory(GiB)": 40.75, "nll_loss": 0.6750409007072449, "rewards/accuracies": 0.71875, "rewards/chosen": -0.2602284550666809, "rewards/margins": 0.49883735179901123, "rewards/rejected": -0.7590658664703369, "step": 13, "train_speed(iter/s)": 0.011204 }, { "epoch": 0.03520905375668029, "grad_norm": 5.483772277832031, "learning_rate": 4.666666666666667e-05, "logits/chosen": -0.5219076871871948, "logits/rejected": -0.593062698841095, "logps/chosen": -8.136625289916992, "logps/rejected": -17.90158462524414, "loss": 1.2523046731948853, "memory(GiB)": 40.75, "nll_loss": 0.6594831347465515, "rewards/accuracies": 0.65625, "rewards/chosen": -0.14057493209838867, "rewards/margins": 0.4059276878833771, "rewards/rejected": -0.5465026497840881, "step": 14, "train_speed(iter/s)": 0.011215 }, { "epoch": 0.03772398616787174, "grad_norm": 5.3392133712768555, "learning_rate": 5e-05, "logits/chosen": -0.40230512619018555, "logits/rejected": -0.587448000907898, "logps/chosen": -3.3861958980560303, "logps/rejected": -22.72595977783203, "loss": 0.8488077521324158, "memory(GiB)": 40.75, "nll_loss": 0.4007309377193451, "rewards/accuracies": 0.8125, "rewards/chosen": -0.02581140398979187, "rewards/margins": 0.6975324153900146, "rewards/rejected": -0.7233438491821289, "step": 15, "train_speed(iter/s)": 0.011221 }, { "epoch": 0.040238918579063185, "grad_norm": 3.3262414932250977, "learning_rate": 5.333333333333333e-05, "logits/chosen": -0.342201292514801, "logits/rejected": -0.575752854347229, "logps/chosen": -3.893523693084717, "logps/rejected": -28.40308952331543, "loss": 0.8233633637428284, "memory(GiB)": 40.75, "nll_loss": 0.3312762379646301, "rewards/accuracies": 0.84375, "rewards/chosen": 0.03883545473217964, "rewards/margins": 0.5127238035202026, "rewards/rejected": -0.4738883376121521, "step": 16, "train_speed(iter/s)": 0.011223 }, { "epoch": 0.04275385099025464, "grad_norm": 2.462789297103882, "learning_rate": 5.666666666666667e-05, "logits/chosen": -0.49846047163009644, "logits/rejected": -0.6083415746688843, "logps/chosen": -1.8747515678405762, "logps/rejected": -16.662185668945312, "loss": 0.7689712047576904, "memory(GiB)": 40.75, "nll_loss": 0.238669753074646, "rewards/accuracies": 0.875, "rewards/chosen": 0.041000571101903915, "rewards/margins": 0.40174293518066406, "rewards/rejected": -0.36074236035346985, "step": 17, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.045268783401446085, "grad_norm": 4.245744228363037, "learning_rate": 6e-05, "logits/chosen": -0.4577159881591797, "logits/rejected": -0.5291006565093994, "logps/chosen": -8.686376571655273, "logps/rejected": -17.176931381225586, "loss": 1.2110098600387573, "memory(GiB)": 40.75, "nll_loss": 0.5645009875297546, "rewards/accuracies": 0.625, "rewards/chosen": -0.049589186906814575, "rewards/margins": 0.15553997457027435, "rewards/rejected": -0.20512914657592773, "step": 18, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.04778371581263754, "grad_norm": 3.633544683456421, "learning_rate": 6.333333333333333e-05, "logits/chosen": -0.4796344041824341, "logits/rejected": -0.5480799674987793, "logps/chosen": -10.090361595153809, "logps/rejected": -17.440088272094727, "loss": 1.1647558212280273, "memory(GiB)": 40.75, "nll_loss": 0.5513034462928772, "rewards/accuracies": 0.65625, "rewards/chosen": -0.004148788750171661, "rewards/margins": 0.22297149896621704, "rewards/rejected": -0.2271203100681305, "step": 19, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.050298648223828984, "grad_norm": 3.275191307067871, "learning_rate": 6.666666666666667e-05, "logits/chosen": -0.46574562788009644, "logits/rejected": -0.5685222744941711, "logps/chosen": -5.4855055809021, "logps/rejected": -15.27071762084961, "loss": 1.0493377447128296, "memory(GiB)": 40.75, "nll_loss": 0.40973347425460815, "rewards/accuracies": 0.6875, "rewards/chosen": 0.02530045434832573, "rewards/margins": 0.1835480034351349, "rewards/rejected": -0.15824756026268005, "step": 20, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.05281358063502043, "grad_norm": 2.346269130706787, "learning_rate": 7e-05, "logits/chosen": -0.5085228085517883, "logits/rejected": -0.6115283966064453, "logps/chosen": -8.632377624511719, "logps/rejected": -18.418798446655273, "loss": 1.1822807788848877, "memory(GiB)": 40.75, "nll_loss": 0.6134262084960938, "rewards/accuracies": 0.78125, "rewards/chosen": 0.07014060765504837, "rewards/margins": 0.30156341195106506, "rewards/rejected": -0.2314227819442749, "step": 21, "train_speed(iter/s)": 0.011245 }, { "epoch": 0.055328513046211884, "grad_norm": 3.8027472496032715, "learning_rate": 7.333333333333333e-05, "logits/chosen": -0.4623318314552307, "logits/rejected": -0.5286604762077332, "logps/chosen": -4.330241680145264, "logps/rejected": -15.440506935119629, "loss": 0.9398723840713501, "memory(GiB)": 40.75, "nll_loss": 0.32075199484825134, "rewards/accuracies": 0.71875, "rewards/chosen": -0.009936274029314518, "rewards/margins": 0.20203867554664612, "rewards/rejected": -0.21197494864463806, "step": 22, "train_speed(iter/s)": 0.011248 }, { "epoch": 0.05784344545740333, "grad_norm": 2.3595120906829834, "learning_rate": 7.666666666666667e-05, "logits/chosen": -0.49782079458236694, "logits/rejected": -0.5459281206130981, "logps/chosen": -5.645054817199707, "logps/rejected": -15.544858932495117, "loss": 0.9763747453689575, "memory(GiB)": 40.75, "nll_loss": 0.3531709313392639, "rewards/accuracies": 0.71875, "rewards/chosen": 0.011355478316545486, "rewards/margins": 0.17947493493556976, "rewards/rejected": -0.16811946034431458, "step": 23, "train_speed(iter/s)": 0.011252 }, { "epoch": 0.060358377868594784, "grad_norm": 2.5567357540130615, "learning_rate": 8e-05, "logits/chosen": -0.52140212059021, "logits/rejected": -0.5767900943756104, "logps/chosen": -9.158100128173828, "logps/rejected": -18.047792434692383, "loss": 1.1764006614685059, "memory(GiB)": 40.75, "nll_loss": 0.6046724915504456, "rewards/accuracies": 0.71875, "rewards/chosen": 0.03944522514939308, "rewards/margins": 0.2968653440475464, "rewards/rejected": -0.2574201226234436, "step": 24, "train_speed(iter/s)": 0.011255 }, { "epoch": 0.06287331027978624, "grad_norm": 3.3667352199554443, "learning_rate": 8.333333333333334e-05, "logits/chosen": -0.4414227604866028, "logits/rejected": -0.5774338245391846, "logps/chosen": -5.001893997192383, "logps/rejected": -18.086332321166992, "loss": 1.0027401447296143, "memory(GiB)": 40.75, "nll_loss": 0.40850552916526794, "rewards/accuracies": 0.71875, "rewards/chosen": 0.01537637785077095, "rewards/margins": 0.2611372172832489, "rewards/rejected": -0.24576085805892944, "step": 25, "train_speed(iter/s)": 0.011255 }, { "epoch": 0.06538824269097768, "grad_norm": 2.4368274211883545, "learning_rate": 8.666666666666667e-05, "logits/chosen": -0.33819591999053955, "logits/rejected": -0.5409271121025085, "logps/chosen": -11.725374221801758, "logps/rejected": -26.261959075927734, "loss": 1.0086052417755127, "memory(GiB)": 40.75, "nll_loss": 0.5157173275947571, "rewards/accuracies": 0.84375, "rewards/chosen": 0.07034377753734589, "rewards/margins": 0.5667710900306702, "rewards/rejected": -0.49642735719680786, "step": 26, "train_speed(iter/s)": 0.011254 }, { "epoch": 0.06790317510216913, "grad_norm": 4.366164207458496, "learning_rate": 9e-05, "logits/chosen": -0.3777789771556854, "logits/rejected": -0.5889321565628052, "logps/chosen": -5.34947395324707, "logps/rejected": -27.725887298583984, "loss": 0.8724238872528076, "memory(GiB)": 40.75, "nll_loss": 0.39934495091438293, "rewards/accuracies": 0.875, "rewards/chosen": -0.022416604682803154, "rewards/margins": 0.6145756244659424, "rewards/rejected": -0.6369922161102295, "step": 27, "train_speed(iter/s)": 0.011254 }, { "epoch": 0.07041810751336058, "grad_norm": 3.8030436038970947, "learning_rate": 9.333333333333334e-05, "logits/chosen": -0.43299004435539246, "logits/rejected": -0.5541425347328186, "logps/chosen": -8.235193252563477, "logps/rejected": -26.827539443969727, "loss": 0.874066174030304, "memory(GiB)": 40.75, "nll_loss": 0.45102763175964355, "rewards/accuracies": 0.875, "rewards/chosen": -0.07945333421230316, "rewards/margins": 0.8036540746688843, "rewards/rejected": -0.8831074237823486, "step": 28, "train_speed(iter/s)": 0.011253 }, { "epoch": 0.07293303992455202, "grad_norm": 3.0242788791656494, "learning_rate": 9.666666666666667e-05, "logits/chosen": -0.37702634930610657, "logits/rejected": -0.5286097526550293, "logps/chosen": -6.507118225097656, "logps/rejected": -25.659378051757812, "loss": 0.9697012901306152, "memory(GiB)": 40.75, "nll_loss": 0.4811092019081116, "rewards/accuracies": 0.75, "rewards/chosen": -0.04141651839017868, "rewards/margins": 0.5634151101112366, "rewards/rejected": -0.6048315763473511, "step": 29, "train_speed(iter/s)": 0.011253 }, { "epoch": 0.07544797233574348, "grad_norm": 2.460881233215332, "learning_rate": 0.0001, "logits/chosen": -0.371891051530838, "logits/rejected": -0.45985037088394165, "logps/chosen": -6.48842716217041, "logps/rejected": -18.697708129882812, "loss": 0.9460075497627258, "memory(GiB)": 40.75, "nll_loss": 0.3780769407749176, "rewards/accuracies": 0.78125, "rewards/chosen": 0.0511014498770237, "rewards/margins": 0.3350966274738312, "rewards/rejected": -0.2839951515197754, "step": 30, "train_speed(iter/s)": 0.011254 }, { "epoch": 0.07796290474693493, "grad_norm": 3.6821117401123047, "learning_rate": 0.00010333333333333334, "logits/chosen": -0.39938536286354065, "logits/rejected": -0.49178823828697205, "logps/chosen": -6.468450546264648, "logps/rejected": -18.927385330200195, "loss": 1.019504189491272, "memory(GiB)": 40.75, "nll_loss": 0.5059184432029724, "rewards/accuracies": 0.8125, "rewards/chosen": 0.0709109753370285, "rewards/margins": 0.5467783808708191, "rewards/rejected": -0.4758673906326294, "step": 31, "train_speed(iter/s)": 0.011256 }, { "epoch": 0.08047783715812637, "grad_norm": 2.8029754161834717, "learning_rate": 0.00010666666666666667, "logits/chosen": -0.38790467381477356, "logits/rejected": -0.4925430417060852, "logps/chosen": -5.981967926025391, "logps/rejected": -28.314722061157227, "loss": 0.9189813733100891, "memory(GiB)": 40.75, "nll_loss": 0.3903328776359558, "rewards/accuracies": 0.75, "rewards/chosen": 0.08847282826900482, "rewards/margins": 0.501548171043396, "rewards/rejected": -0.41307532787323, "step": 32, "train_speed(iter/s)": 0.011256 }, { "epoch": 0.08299276956931782, "grad_norm": 3.7524354457855225, "learning_rate": 0.00011000000000000002, "logits/chosen": -0.3702802360057831, "logits/rejected": -0.4830480217933655, "logps/chosen": -6.820221424102783, "logps/rejected": -18.442974090576172, "loss": 0.9390460252761841, "memory(GiB)": 40.75, "nll_loss": 0.4203004837036133, "rewards/accuracies": 0.90625, "rewards/chosen": 0.05472969263792038, "rewards/margins": 0.4696621000766754, "rewards/rejected": -0.41493239998817444, "step": 33, "train_speed(iter/s)": 0.011257 }, { "epoch": 0.08550770198050928, "grad_norm": 2.968419075012207, "learning_rate": 0.00011333333333333334, "logits/chosen": -0.35177576541900635, "logits/rejected": -0.45535844564437866, "logps/chosen": -4.456936836242676, "logps/rejected": -22.066486358642578, "loss": 0.8263772130012512, "memory(GiB)": 40.75, "nll_loss": 0.35278084874153137, "rewards/accuracies": 0.84375, "rewards/chosen": -0.009354054927825928, "rewards/margins": 0.7917211651802063, "rewards/rejected": -0.8010752201080322, "step": 34, "train_speed(iter/s)": 0.011259 }, { "epoch": 0.08802263439170073, "grad_norm": 2.9763481616973877, "learning_rate": 0.00011666666666666668, "logits/chosen": -0.2393473982810974, "logits/rejected": -0.4172413945198059, "logps/chosen": -4.113461017608643, "logps/rejected": -30.265304565429688, "loss": 0.8524877429008484, "memory(GiB)": 40.75, "nll_loss": 0.471011221408844, "rewards/accuracies": 0.8125, "rewards/chosen": 0.05692119151353836, "rewards/margins": 1.161907434463501, "rewards/rejected": -1.1049861907958984, "step": 35, "train_speed(iter/s)": 0.011258 }, { "epoch": 0.09053756680289217, "grad_norm": 2.4537270069122314, "learning_rate": 0.00012, "logits/chosen": -0.276807963848114, "logits/rejected": -0.41199547052383423, "logps/chosen": -7.090043544769287, "logps/rejected": -29.689010620117188, "loss": 0.7714070081710815, "memory(GiB)": 40.75, "nll_loss": 0.3793071210384369, "rewards/accuracies": 0.84375, "rewards/chosen": 0.036474838852882385, "rewards/margins": 1.0788801908493042, "rewards/rejected": -1.0424052476882935, "step": 36, "train_speed(iter/s)": 0.011257 }, { "epoch": 0.09305249921408362, "grad_norm": 3.29215145111084, "learning_rate": 0.00012333333333333334, "logits/chosen": -0.3102665841579437, "logits/rejected": -0.4546661078929901, "logps/chosen": -5.750801086425781, "logps/rejected": -31.45724868774414, "loss": 0.7538890838623047, "memory(GiB)": 40.75, "nll_loss": 0.4176124334335327, "rewards/accuracies": 0.875, "rewards/chosen": -0.1214592307806015, "rewards/margins": 1.2956981658935547, "rewards/rejected": -1.4171574115753174, "step": 37, "train_speed(iter/s)": 0.011259 }, { "epoch": 0.09556743162527508, "grad_norm": 3.8826475143432617, "learning_rate": 0.00012666666666666666, "logits/chosen": -0.37785372138023376, "logits/rejected": -0.45433154702186584, "logps/chosen": -8.914945602416992, "logps/rejected": -22.28145408630371, "loss": 0.9718711972236633, "memory(GiB)": 40.75, "nll_loss": 0.542230486869812, "rewards/accuracies": 0.8125, "rewards/chosen": -0.05876921862363815, "rewards/margins": 0.9656041860580444, "rewards/rejected": -1.0243734121322632, "step": 38, "train_speed(iter/s)": 0.01126 }, { "epoch": 0.09808236403646652, "grad_norm": 10.463335990905762, "learning_rate": 0.00013000000000000002, "logits/chosen": -0.40348339080810547, "logits/rejected": -0.4396521747112274, "logps/chosen": -9.546710014343262, "logps/rejected": -24.998611450195312, "loss": 1.1912176609039307, "memory(GiB)": 40.75, "nll_loss": 0.6404471397399902, "rewards/accuracies": 0.65625, "rewards/chosen": -0.18508824706077576, "rewards/margins": 0.8297311663627625, "rewards/rejected": -1.0148195028305054, "step": 39, "train_speed(iter/s)": 0.01126 }, { "epoch": 0.10059729644765797, "grad_norm": 3.4194676876068115, "learning_rate": 0.00013333333333333334, "logits/chosen": -0.36826181411743164, "logits/rejected": -0.5072036981582642, "logps/chosen": -6.227029800415039, "logps/rejected": -28.42563247680664, "loss": 0.7728609442710876, "memory(GiB)": 40.75, "nll_loss": 0.3573758006095886, "rewards/accuracies": 0.8125, "rewards/chosen": 0.021011125296354294, "rewards/margins": 1.0436646938323975, "rewards/rejected": -1.0226534605026245, "step": 40, "train_speed(iter/s)": 0.011261 }, { "epoch": 0.10311222885884942, "grad_norm": 4.902315139770508, "learning_rate": 0.00013666666666666666, "logits/chosen": -0.3183041512966156, "logits/rejected": -0.48007485270500183, "logps/chosen": -9.719247817993164, "logps/rejected": -22.464027404785156, "loss": 1.06601881980896, "memory(GiB)": 40.75, "nll_loss": 0.5293514728546143, "rewards/accuracies": 0.71875, "rewards/chosen": 0.04249940067529678, "rewards/margins": 0.45808011293411255, "rewards/rejected": -0.4155806601047516, "step": 41, "train_speed(iter/s)": 0.01126 }, { "epoch": 0.10562716127004086, "grad_norm": 5.45733118057251, "learning_rate": 0.00014, "logits/chosen": -0.3853970468044281, "logits/rejected": -0.4931572675704956, "logps/chosen": -8.472291946411133, "logps/rejected": -24.512069702148438, "loss": 0.943773627281189, "memory(GiB)": 40.75, "nll_loss": 0.4035080075263977, "rewards/accuracies": 0.75, "rewards/chosen": -0.07248181104660034, "rewards/margins": 0.5888606309890747, "rewards/rejected": -0.661342442035675, "step": 42, "train_speed(iter/s)": 0.011261 }, { "epoch": 0.10814209368123232, "grad_norm": 3.7701737880706787, "learning_rate": 0.00014333333333333334, "logits/chosen": -0.24171334505081177, "logits/rejected": -0.46347737312316895, "logps/chosen": -3.810516119003296, "logps/rejected": -28.03646469116211, "loss": 0.7043237090110779, "memory(GiB)": 40.75, "nll_loss": 0.33659031987190247, "rewards/accuracies": 0.90625, "rewards/chosen": 0.10358152538537979, "rewards/margins": 1.084468960762024, "rewards/rejected": -0.9808874726295471, "step": 43, "train_speed(iter/s)": 0.01126 }, { "epoch": 0.11065702609242377, "grad_norm": 5.19516658782959, "learning_rate": 0.00014666666666666666, "logits/chosen": -0.3297630548477173, "logits/rejected": -0.4781489074230194, "logps/chosen": -8.038228988647461, "logps/rejected": -33.8675651550293, "loss": 0.8617865443229675, "memory(GiB)": 40.75, "nll_loss": 0.5089200139045715, "rewards/accuracies": 0.8125, "rewards/chosen": -0.21755161881446838, "rewards/margins": 1.4295934438705444, "rewards/rejected": -1.6471449136734009, "step": 44, "train_speed(iter/s)": 0.01126 }, { "epoch": 0.11317195850361522, "grad_norm": 4.779160499572754, "learning_rate": 0.00015000000000000001, "logits/chosen": -0.49006256461143494, "logits/rejected": -0.5443361401557922, "logps/chosen": -9.670989990234375, "logps/rejected": -27.758705139160156, "loss": 0.9880771636962891, "memory(GiB)": 40.75, "nll_loss": 0.5687355995178223, "rewards/accuracies": 0.75, "rewards/chosen": -0.21813175082206726, "rewards/margins": 1.2758864164352417, "rewards/rejected": -1.4940180778503418, "step": 45, "train_speed(iter/s)": 0.011261 }, { "epoch": 0.11568689091480666, "grad_norm": 4.207818031311035, "learning_rate": 0.00015333333333333334, "logits/chosen": -0.3747759461402893, "logits/rejected": -0.5024475455284119, "logps/chosen": -6.3994646072387695, "logps/rejected": -32.430179595947266, "loss": 0.9543445706367493, "memory(GiB)": 40.75, "nll_loss": 0.5410714745521545, "rewards/accuracies": 0.78125, "rewards/chosen": -0.1253054440021515, "rewards/margins": 1.216636300086975, "rewards/rejected": -1.3419415950775146, "step": 46, "train_speed(iter/s)": 0.011262 }, { "epoch": 0.11820182332599811, "grad_norm": 2.187269926071167, "learning_rate": 0.00015666666666666666, "logits/chosen": -0.34373798966407776, "logits/rejected": -0.5492506623268127, "logps/chosen": -5.171298027038574, "logps/rejected": -34.377967834472656, "loss": 0.5840844511985779, "memory(GiB)": 40.75, "nll_loss": 0.2608174979686737, "rewards/accuracies": 0.875, "rewards/chosen": 0.04855790734291077, "rewards/margins": 1.4605152606964111, "rewards/rejected": -1.4119572639465332, "step": 47, "train_speed(iter/s)": 0.011263 }, { "epoch": 0.12071675573718957, "grad_norm": 2.492182970046997, "learning_rate": 0.00016, "logits/chosen": -0.3911299705505371, "logits/rejected": -0.5427611470222473, "logps/chosen": -5.3708953857421875, "logps/rejected": -30.956531524658203, "loss": 0.7430465221405029, "memory(GiB)": 40.75, "nll_loss": 0.3517543077468872, "rewards/accuracies": 0.84375, "rewards/chosen": -0.04663639888167381, "rewards/margins": 1.1671051979064941, "rewards/rejected": -1.2137415409088135, "step": 48, "train_speed(iter/s)": 0.011264 }, { "epoch": 0.12323168814838101, "grad_norm": 2.454068422317505, "learning_rate": 0.00016333333333333334, "logits/chosen": -0.30930572748184204, "logits/rejected": -0.5388137698173523, "logps/chosen": -4.821499824523926, "logps/rejected": -33.31132125854492, "loss": 0.6669176816940308, "memory(GiB)": 40.75, "nll_loss": 0.34143078327178955, "rewards/accuracies": 0.90625, "rewards/chosen": 0.09115171432495117, "rewards/margins": 1.3478808403015137, "rewards/rejected": -1.2567291259765625, "step": 49, "train_speed(iter/s)": 0.011264 }, { "epoch": 0.12574662055957248, "grad_norm": 2.7763614654541016, "learning_rate": 0.0001666666666666667, "logits/chosen": -0.22302384674549103, "logits/rejected": -0.476222962141037, "logps/chosen": -6.5321455001831055, "logps/rejected": -33.70460891723633, "loss": 0.8022267818450928, "memory(GiB)": 40.75, "nll_loss": 0.3984905481338501, "rewards/accuracies": 0.8125, "rewards/chosen": 0.011320816352963448, "rewards/margins": 1.2645773887634277, "rewards/rejected": -1.2532566785812378, "step": 50, "train_speed(iter/s)": 0.011263 }, { "epoch": 0.12826155297076391, "grad_norm": 3.5810701847076416, "learning_rate": 0.00017, "logits/chosen": -0.2979552447795868, "logits/rejected": -0.4668833017349243, "logps/chosen": -7.627035140991211, "logps/rejected": -35.17571258544922, "loss": 0.8107961416244507, "memory(GiB)": 40.75, "nll_loss": 0.4284512996673584, "rewards/accuracies": 0.875, "rewards/chosen": -0.06179499253630638, "rewards/margins": 1.205071210861206, "rewards/rejected": -1.2668663263320923, "step": 51, "train_speed(iter/s)": 0.011221 }, { "epoch": 0.13077648538195535, "grad_norm": 2.9454948902130127, "learning_rate": 0.00017333333333333334, "logits/chosen": -0.2851395010948181, "logits/rejected": -0.4914030134677887, "logps/chosen": -6.73934268951416, "logps/rejected": -44.49215316772461, "loss": 0.7192223072052002, "memory(GiB)": 40.75, "nll_loss": 0.3857932686805725, "rewards/accuracies": 0.875, "rewards/chosen": -0.06120564043521881, "rewards/margins": 2.360339403152466, "rewards/rejected": -2.4215452671051025, "step": 52, "train_speed(iter/s)": 0.011221 }, { "epoch": 0.13329141779314682, "grad_norm": 2.907097339630127, "learning_rate": 0.00017666666666666666, "logits/chosen": -0.29824766516685486, "logits/rejected": -0.5210357904434204, "logps/chosen": -8.840283393859863, "logps/rejected": -34.15322494506836, "loss": 0.9132320284843445, "memory(GiB)": 40.75, "nll_loss": 0.5277525782585144, "rewards/accuracies": 0.78125, "rewards/chosen": -0.013686132617294788, "rewards/margins": 1.6544121503829956, "rewards/rejected": -1.6680983304977417, "step": 53, "train_speed(iter/s)": 0.011222 }, { "epoch": 0.13580635020433826, "grad_norm": 3.2579548358917236, "learning_rate": 0.00018, "logits/chosen": -0.3622821867465973, "logits/rejected": -0.49751946330070496, "logps/chosen": -9.77501106262207, "logps/rejected": -34.95624542236328, "loss": 0.8283704519271851, "memory(GiB)": 40.75, "nll_loss": 0.478503942489624, "rewards/accuracies": 0.875, "rewards/chosen": 0.04811567813158035, "rewards/margins": 1.6710915565490723, "rewards/rejected": -1.6229758262634277, "step": 54, "train_speed(iter/s)": 0.011222 }, { "epoch": 0.1383212826155297, "grad_norm": 6.844382286071777, "learning_rate": 0.00018333333333333334, "logits/chosen": -0.34049201011657715, "logits/rejected": -0.5361011028289795, "logps/chosen": -4.566088676452637, "logps/rejected": -41.054141998291016, "loss": 0.7934082746505737, "memory(GiB)": 40.75, "nll_loss": 0.4864325225353241, "rewards/accuracies": 0.84375, "rewards/chosen": -0.022840898483991623, "rewards/margins": 2.574941873550415, "rewards/rejected": -2.597783088684082, "step": 55, "train_speed(iter/s)": 0.011224 }, { "epoch": 0.14083621502672117, "grad_norm": 5.409123420715332, "learning_rate": 0.0001866666666666667, "logits/chosen": -0.3723406195640564, "logits/rejected": -0.4861374497413635, "logps/chosen": -11.017836570739746, "logps/rejected": -34.19987487792969, "loss": 1.1561821699142456, "memory(GiB)": 40.75, "nll_loss": 0.7329697608947754, "rewards/accuracies": 0.78125, "rewards/chosen": -0.1480638086795807, "rewards/margins": 2.0217599868774414, "rewards/rejected": -2.16982364654541, "step": 56, "train_speed(iter/s)": 0.011225 }, { "epoch": 0.1433511474379126, "grad_norm": 7.563216686248779, "learning_rate": 0.00019, "logits/chosen": -0.32556983828544617, "logits/rejected": -0.5365452170372009, "logps/chosen": -5.449405193328857, "logps/rejected": -46.6372184753418, "loss": 0.8106161952018738, "memory(GiB)": 40.75, "nll_loss": 0.5291056632995605, "rewards/accuracies": 0.8125, "rewards/chosen": -0.11195089668035507, "rewards/margins": 2.898953676223755, "rewards/rejected": -3.010904312133789, "step": 57, "train_speed(iter/s)": 0.011226 }, { "epoch": 0.14586607984910405, "grad_norm": 9.61186408996582, "learning_rate": 0.00019333333333333333, "logits/chosen": -0.4079486131668091, "logits/rejected": -0.5931285619735718, "logps/chosen": -3.3833391666412354, "logps/rejected": -41.95534896850586, "loss": 0.559334397315979, "memory(GiB)": 40.75, "nll_loss": 0.30327463150024414, "rewards/accuracies": 0.875, "rewards/chosen": 0.04168446362018585, "rewards/margins": 2.6080174446105957, "rewards/rejected": -2.566333055496216, "step": 58, "train_speed(iter/s)": 0.011229 }, { "epoch": 0.14838101226029551, "grad_norm": 3.3261170387268066, "learning_rate": 0.00019666666666666666, "logits/chosen": -0.4187636971473694, "logits/rejected": -0.4730520248413086, "logps/chosen": -8.343189239501953, "logps/rejected": -25.519798278808594, "loss": 1.058987021446228, "memory(GiB)": 40.75, "nll_loss": 0.5353435277938843, "rewards/accuracies": 0.6875, "rewards/chosen": -0.0768323540687561, "rewards/margins": 1.170108437538147, "rewards/rejected": -1.2469407320022583, "step": 59, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.15089594467148695, "grad_norm": 3.7475709915161133, "learning_rate": 0.0002, "logits/chosen": -0.31574761867523193, "logits/rejected": -0.4878937005996704, "logps/chosen": -6.4234299659729, "logps/rejected": -30.35470199584961, "loss": 0.7999802827835083, "memory(GiB)": 40.75, "nll_loss": 0.3874775171279907, "rewards/accuracies": 0.78125, "rewards/chosen": 0.019132789224386215, "rewards/margins": 1.3630902767181396, "rewards/rejected": -1.3439574241638184, "step": 60, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.1534108770826784, "grad_norm": 2.8922085762023926, "learning_rate": 0.0001999996142159566, "logits/chosen": -0.35342201590538025, "logits/rejected": -0.4222828149795532, "logps/chosen": -7.674136638641357, "logps/rejected": -26.560787200927734, "loss": 0.7506588101387024, "memory(GiB)": 40.75, "nll_loss": 0.3475904166698456, "rewards/accuracies": 0.75, "rewards/chosen": 0.04072082042694092, "rewards/margins": 1.29744291305542, "rewards/rejected": -1.2567219734191895, "step": 61, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.15592580949386986, "grad_norm": 2.781667947769165, "learning_rate": 0.0001999984568668029, "logits/chosen": -0.33968424797058105, "logits/rejected": -0.45871520042419434, "logps/chosen": -3.5645596981048584, "logps/rejected": -33.30908203125, "loss": 0.6515002250671387, "memory(GiB)": 40.75, "nll_loss": 0.28380468487739563, "rewards/accuracies": 0.875, "rewards/chosen": 0.10595045238733292, "rewards/margins": 1.6635351181030273, "rewards/rejected": -1.5575847625732422, "step": 62, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.1584407419050613, "grad_norm": 3.0299599170684814, "learning_rate": 0.00019999652796146876, "logits/chosen": -0.3050180971622467, "logits/rejected": -0.4909503161907196, "logps/chosen": -7.468794822692871, "logps/rejected": -40.68026351928711, "loss": 0.7424418330192566, "memory(GiB)": 40.75, "nll_loss": 0.41286778450012207, "rewards/accuracies": 0.78125, "rewards/chosen": 0.06561660766601562, "rewards/margins": 1.9888997077941895, "rewards/rejected": -1.9232829809188843, "step": 63, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.16095567431625274, "grad_norm": 2.0761804580688477, "learning_rate": 0.0001999938275148369, "logits/chosen": -0.33227968215942383, "logits/rejected": -0.45812830328941345, "logps/chosen": -4.157924652099609, "logps/rejected": -40.23651123046875, "loss": 0.40325966477394104, "memory(GiB)": 40.75, "nll_loss": 0.19691286981105804, "rewards/accuracies": 0.9375, "rewards/chosen": -0.10368794947862625, "rewards/margins": 2.3141493797302246, "rewards/rejected": -2.417837619781494, "step": 64, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.1634706067274442, "grad_norm": 3.9792428016662598, "learning_rate": 0.00019999035554774314, "logits/chosen": -0.3644486367702484, "logits/rejected": -0.5156970024108887, "logps/chosen": -5.496757984161377, "logps/rejected": -43.13388442993164, "loss": 0.759586751461029, "memory(GiB)": 40.75, "nll_loss": 0.5011360049247742, "rewards/accuracies": 0.90625, "rewards/chosen": -0.13431251049041748, "rewards/margins": 2.4990766048431396, "rewards/rejected": -2.6333892345428467, "step": 65, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.16598553913863565, "grad_norm": 3.7544219493865967, "learning_rate": 0.00019998611208697607, "logits/chosen": -0.3731898367404938, "logits/rejected": -0.5181675553321838, "logps/chosen": -4.577354431152344, "logps/rejected": -36.74962615966797, "loss": 0.7680931091308594, "memory(GiB)": 40.75, "nll_loss": 0.4070277214050293, "rewards/accuracies": 0.75, "rewards/chosen": -0.09029161185026169, "rewards/margins": 1.9024454355239868, "rewards/rejected": -1.992737054824829, "step": 66, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.16850047154982709, "grad_norm": 2.9526925086975098, "learning_rate": 0.00019998109716527686, "logits/chosen": -0.247542604804039, "logits/rejected": -0.45712605118751526, "logps/chosen": -3.374553918838501, "logps/rejected": -43.25963592529297, "loss": 0.49172505736351013, "memory(GiB)": 40.75, "nll_loss": 0.22025981545448303, "rewards/accuracies": 0.75, "rewards/chosen": 0.04232815280556679, "rewards/margins": 2.575462579727173, "rewards/rejected": -2.5331344604492188, "step": 67, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.17101540396101855, "grad_norm": 5.039420127868652, "learning_rate": 0.00019997531082133904, "logits/chosen": -0.3098013997077942, "logits/rejected": -0.42491626739501953, "logps/chosen": -7.849802494049072, "logps/rejected": -31.89962387084961, "loss": 1.0089820623397827, "memory(GiB)": 40.75, "nll_loss": 0.5222876071929932, "rewards/accuracies": 0.8125, "rewards/chosen": -0.09253531694412231, "rewards/margins": 1.4570719003677368, "rewards/rejected": -1.549607276916504, "step": 68, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.17353033637221, "grad_norm": 2.4941506385803223, "learning_rate": 0.00019996875309980826, "logits/chosen": -0.33015602827072144, "logits/rejected": -0.4958527684211731, "logps/chosen": -4.380959510803223, "logps/rejected": -36.05854415893555, "loss": 0.6548417806625366, "memory(GiB)": 40.75, "nll_loss": 0.38906389474868774, "rewards/accuracies": 0.875, "rewards/chosen": 0.07272060960531235, "rewards/margins": 2.0416646003723145, "rewards/rejected": -1.9689440727233887, "step": 69, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.17604526878340146, "grad_norm": 3.3756778240203857, "learning_rate": 0.0001999614240512817, "logits/chosen": -0.2773034870624542, "logits/rejected": -0.48985666036605835, "logps/chosen": -7.3368024826049805, "logps/rejected": -28.274978637695312, "loss": 0.7877169251441956, "memory(GiB)": 40.75, "nll_loss": 0.4065099060535431, "rewards/accuracies": 0.8125, "rewards/chosen": 0.1428125947713852, "rewards/margins": 1.3781116008758545, "rewards/rejected": -1.2352991104125977, "step": 70, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.1785602011945929, "grad_norm": 3.289449453353882, "learning_rate": 0.00019995332373230808, "logits/chosen": -0.2643814980983734, "logits/rejected": -0.441372275352478, "logps/chosen": -4.96966552734375, "logps/rejected": -32.28032684326172, "loss": 0.6629552841186523, "memory(GiB)": 40.75, "nll_loss": 0.2761652171611786, "rewards/accuracies": 0.84375, "rewards/chosen": 0.028238017112016678, "rewards/margins": 1.3064823150634766, "rewards/rejected": -1.2782442569732666, "step": 71, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.18107513360578434, "grad_norm": 2.8338823318481445, "learning_rate": 0.00019994445220538677, "logits/chosen": -0.2861837148666382, "logits/rejected": -0.39666813611984253, "logps/chosen": -6.460289001464844, "logps/rejected": -33.8844108581543, "loss": 0.7865106463432312, "memory(GiB)": 40.75, "nll_loss": 0.4185709059238434, "rewards/accuracies": 0.84375, "rewards/chosen": -0.03227805718779564, "rewards/margins": 1.8171049356460571, "rewards/rejected": -1.849382996559143, "step": 72, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.1835900660169758, "grad_norm": 4.286563396453857, "learning_rate": 0.0001999348095389677, "logits/chosen": -0.2642534673213959, "logits/rejected": -0.4921380281448364, "logps/chosen": -6.74280309677124, "logps/rejected": -48.935733795166016, "loss": 0.6765459775924683, "memory(GiB)": 40.75, "nll_loss": 0.4257548451423645, "rewards/accuracies": 0.96875, "rewards/chosen": -0.023126238957047462, "rewards/margins": 2.3492531776428223, "rewards/rejected": -2.3723795413970947, "step": 73, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.18610499842816725, "grad_norm": 6.242228984832764, "learning_rate": 0.0001999243958074506, "logits/chosen": -0.24496644735336304, "logits/rejected": -0.4604867100715637, "logps/chosen": -14.664331436157227, "logps/rejected": -44.660789489746094, "loss": 1.2609174251556396, "memory(GiB)": 40.75, "nll_loss": 0.827994704246521, "rewards/accuracies": 0.6875, "rewards/chosen": -0.4426795244216919, "rewards/margins": 1.524437427520752, "rewards/rejected": -1.9671170711517334, "step": 74, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.18861993083935868, "grad_norm": 2.4951188564300537, "learning_rate": 0.00019991321109118448, "logits/chosen": -0.40645161271095276, "logits/rejected": -0.4706539511680603, "logps/chosen": -6.498587131500244, "logps/rejected": -31.508909225463867, "loss": 0.6895300149917603, "memory(GiB)": 40.75, "nll_loss": 0.3541663587093353, "rewards/accuracies": 0.90625, "rewards/chosen": 0.004417119547724724, "rewards/margins": 1.655824899673462, "rewards/rejected": -1.6514074802398682, "step": 75, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.19113486325055015, "grad_norm": 2.4047465324401855, "learning_rate": 0.00019990125547646704, "logits/chosen": -0.33654719591140747, "logits/rejected": -0.49985551834106445, "logps/chosen": -3.1749448776245117, "logps/rejected": -31.598142623901367, "loss": 0.6832078695297241, "memory(GiB)": 40.75, "nll_loss": 0.28699803352355957, "rewards/accuracies": 0.8125, "rewards/chosen": 0.018617400899529457, "rewards/margins": 1.5892831087112427, "rewards/rejected": -1.570665717124939, "step": 76, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.1936497956617416, "grad_norm": 2.1775262355804443, "learning_rate": 0.00019988852905554404, "logits/chosen": -0.2977695167064667, "logits/rejected": -0.4677082896232605, "logps/chosen": -5.7894110679626465, "logps/rejected": -33.92140579223633, "loss": 0.6954871416091919, "memory(GiB)": 40.75, "nll_loss": 0.35726743936538696, "rewards/accuracies": 0.9375, "rewards/chosen": 0.11432527005672455, "rewards/margins": 1.4882932901382446, "rewards/rejected": -1.3739678859710693, "step": 77, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.19616472807293303, "grad_norm": 2.386960983276367, "learning_rate": 0.0001998750319266084, "logits/chosen": -0.34930819272994995, "logits/rejected": -0.4996439814567566, "logps/chosen": -5.451239585876465, "logps/rejected": -30.024625778198242, "loss": 0.8510737419128418, "memory(GiB)": 40.75, "nll_loss": 0.38498157262802124, "rewards/accuracies": 0.71875, "rewards/chosen": 0.08021941781044006, "rewards/margins": 1.2762808799743652, "rewards/rejected": -1.1960614919662476, "step": 78, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.1986796604841245, "grad_norm": 2.540059804916382, "learning_rate": 0.00019986076419379974, "logits/chosen": -0.2711866497993469, "logits/rejected": -0.46172043681144714, "logps/chosen": -7.032684803009033, "logps/rejected": -37.4212760925293, "loss": 0.8995335698127747, "memory(GiB)": 40.75, "nll_loss": 0.5719486474990845, "rewards/accuracies": 0.90625, "rewards/chosen": 0.1289057433605194, "rewards/margins": 1.6372534036636353, "rewards/rejected": -1.508347749710083, "step": 79, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.20119459289531594, "grad_norm": 2.888524293899536, "learning_rate": 0.00019984572596720324, "logits/chosen": -0.4125524163246155, "logits/rejected": -0.550200343132019, "logps/chosen": -4.954836845397949, "logps/rejected": -31.84545135498047, "loss": 0.7266713976860046, "memory(GiB)": 40.75, "nll_loss": 0.4038025140762329, "rewards/accuracies": 0.875, "rewards/chosen": -0.018991030752658844, "rewards/margins": 1.6552242040634155, "rewards/rejected": -1.6742154359817505, "step": 80, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.20370952530650738, "grad_norm": 2.54023814201355, "learning_rate": 0.00019982991736284915, "logits/chosen": -0.36888328194618225, "logits/rejected": -0.48596253991127014, "logps/chosen": -6.946263313293457, "logps/rejected": -37.912750244140625, "loss": 0.6610869765281677, "memory(GiB)": 40.75, "nll_loss": 0.3929502069950104, "rewards/accuracies": 0.90625, "rewards/chosen": 0.005873316898941994, "rewards/margins": 2.2654542922973633, "rewards/rejected": -2.2595808506011963, "step": 81, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.20622445771769884, "grad_norm": 5.155144214630127, "learning_rate": 0.00019981333850271158, "logits/chosen": -0.37043288350105286, "logits/rejected": -0.5056601166725159, "logps/chosen": -7.0625691413879395, "logps/rejected": -45.226646423339844, "loss": 0.6925163269042969, "memory(GiB)": 40.75, "nll_loss": 0.4076215326786041, "rewards/accuracies": 0.8125, "rewards/chosen": -0.03391120955348015, "rewards/margins": 3.1254899501800537, "rewards/rejected": -3.1594011783599854, "step": 82, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.20873939012889028, "grad_norm": 4.161493301391602, "learning_rate": 0.0001997959895147077, "logits/chosen": -0.3119804859161377, "logits/rejected": -0.519420862197876, "logps/chosen": -5.138388156890869, "logps/rejected": -51.65058517456055, "loss": 0.6154710054397583, "memory(GiB)": 40.75, "nll_loss": 0.3612300753593445, "rewards/accuracies": 0.9375, "rewards/chosen": 0.08062062412500381, "rewards/margins": 3.3395729064941406, "rewards/rejected": -3.2589523792266846, "step": 83, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.21125432254008172, "grad_norm": 3.640066146850586, "learning_rate": 0.0001997778705326968, "logits/chosen": -0.3015586733818054, "logits/rejected": -0.561132550239563, "logps/chosen": -2.76611328125, "logps/rejected": -47.526405334472656, "loss": 0.5920863151550293, "memory(GiB)": 40.75, "nll_loss": 0.3293038606643677, "rewards/accuracies": 0.84375, "rewards/chosen": 0.07201344519853592, "rewards/margins": 2.9245572090148926, "rewards/rejected": -2.852543592453003, "step": 84, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.2137692549512732, "grad_norm": 9.056829452514648, "learning_rate": 0.00019975898169647915, "logits/chosen": -0.3201826810836792, "logits/rejected": -0.49801909923553467, "logps/chosen": -8.43231201171875, "logps/rejected": -44.97834014892578, "loss": 0.7947379350662231, "memory(GiB)": 40.75, "nll_loss": 0.45788073539733887, "rewards/accuracies": 0.84375, "rewards/chosen": -0.03715405985713005, "rewards/margins": 2.6956162452697754, "rewards/rejected": -2.7327704429626465, "step": 85, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.21628418736246463, "grad_norm": 3.5543746948242188, "learning_rate": 0.000199739323151795, "logits/chosen": -0.2616378962993622, "logits/rejected": -0.5049853920936584, "logps/chosen": -5.9475860595703125, "logps/rejected": -39.59865951538086, "loss": 0.7357087135314941, "memory(GiB)": 40.75, "nll_loss": 0.3480994701385498, "rewards/accuracies": 0.78125, "rewards/chosen": -0.013387706130743027, "rewards/margins": 2.0890746116638184, "rewards/rejected": -2.1024622917175293, "step": 86, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.21879911977365607, "grad_norm": 2.5816776752471924, "learning_rate": 0.00019971889505032334, "logits/chosen": -0.3568105101585388, "logits/rejected": -0.5804364681243896, "logps/chosen": -3.9897284507751465, "logps/rejected": -40.31490707397461, "loss": 0.6060020923614502, "memory(GiB)": 40.75, "nll_loss": 0.3237951099872589, "rewards/accuracies": 0.875, "rewards/chosen": -0.0022782403975725174, "rewards/margins": 2.383437156677246, "rewards/rejected": -2.3857154846191406, "step": 87, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.22131405218484754, "grad_norm": 3.132185935974121, "learning_rate": 0.00019969769754968098, "logits/chosen": -0.28933870792388916, "logits/rejected": -0.5249110460281372, "logps/chosen": -5.762962818145752, "logps/rejected": -34.77198791503906, "loss": 0.6889637112617493, "memory(GiB)": 40.75, "nll_loss": 0.3635081648826599, "rewards/accuracies": 0.84375, "rewards/chosen": 0.13572651147842407, "rewards/margins": 1.7912075519561768, "rewards/rejected": -1.655480980873108, "step": 88, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.22382898459603898, "grad_norm": 2.8023223876953125, "learning_rate": 0.00019967573081342103, "logits/chosen": -0.3481064736843109, "logits/rejected": -0.5555439591407776, "logps/chosen": -5.762640476226807, "logps/rejected": -35.07036209106445, "loss": 0.6983572244644165, "memory(GiB)": 40.75, "nll_loss": 0.3852231502532959, "rewards/accuracies": 0.9375, "rewards/chosen": 0.04139009863138199, "rewards/margins": 2.027146816253662, "rewards/rejected": -1.985756754875183, "step": 89, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.22634391700723044, "grad_norm": 8.448691368103027, "learning_rate": 0.00019965299501103177, "logits/chosen": -0.28187096118927, "logits/rejected": -0.5138179063796997, "logps/chosen": -11.35129165649414, "logps/rejected": -54.19221496582031, "loss": 0.9737562537193298, "memory(GiB)": 40.75, "nll_loss": 0.6502060890197754, "rewards/accuracies": 0.8125, "rewards/chosen": -0.1801879107952118, "rewards/margins": 3.0160610675811768, "rewards/rejected": -3.196249008178711, "step": 90, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.22885884941842188, "grad_norm": 4.455811977386475, "learning_rate": 0.00019962949031793542, "logits/chosen": -0.388353168964386, "logits/rejected": -0.5416743755340576, "logps/chosen": -7.5033183097839355, "logps/rejected": -38.96351623535156, "loss": 0.8328152298927307, "memory(GiB)": 40.75, "nll_loss": 0.5262293219566345, "rewards/accuracies": 0.84375, "rewards/chosen": -0.18585588037967682, "rewards/margins": 2.3831686973571777, "rewards/rejected": -2.5690245628356934, "step": 91, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.23137378182961332, "grad_norm": 5.176366806030273, "learning_rate": 0.00019960521691548674, "logits/chosen": -0.30729976296424866, "logits/rejected": -0.5507268905639648, "logps/chosen": -6.70347785949707, "logps/rejected": -47.423580169677734, "loss": 0.7757164835929871, "memory(GiB)": 40.75, "nll_loss": 0.5047218203544617, "rewards/accuracies": 0.90625, "rewards/chosen": -0.212627112865448, "rewards/margins": 2.656092643737793, "rewards/rejected": -2.8687198162078857, "step": 92, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.2338887142408048, "grad_norm": 2.7338004112243652, "learning_rate": 0.0001995801749909715, "logits/chosen": -0.310178279876709, "logits/rejected": -0.5231863856315613, "logps/chosen": -6.638127326965332, "logps/rejected": -34.64179992675781, "loss": 0.6857970952987671, "memory(GiB)": 42.51, "nll_loss": 0.341688871383667, "rewards/accuracies": 0.75, "rewards/chosen": 0.04894203692674637, "rewards/margins": 1.8471993207931519, "rewards/rejected": -1.7982572317123413, "step": 93, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.23640364665199623, "grad_norm": 2.074855327606201, "learning_rate": 0.00019955436473760525, "logits/chosen": -0.2659550607204437, "logits/rejected": -0.5499891042709351, "logps/chosen": -4.5521063804626465, "logps/rejected": -44.503509521484375, "loss": 0.668745219707489, "memory(GiB)": 42.51, "nll_loss": 0.3918987512588501, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1607407182455063, "rewards/margins": 2.3078651428222656, "rewards/rejected": -2.1471242904663086, "step": 94, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.23891857906318767, "grad_norm": 2.6431190967559814, "learning_rate": 0.0001995277863545316, "logits/chosen": -0.38493573665618896, "logits/rejected": -0.5655022859573364, "logps/chosen": -3.839203119277954, "logps/rejected": -27.24703025817871, "loss": 0.5949357151985168, "memory(GiB)": 42.51, "nll_loss": 0.2755538821220398, "rewards/accuracies": 0.90625, "rewards/chosen": 0.11298017203807831, "rewards/margins": 1.4701435565948486, "rewards/rejected": -1.357163429260254, "step": 95, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.24143351147437914, "grad_norm": 2.935624837875366, "learning_rate": 0.00019950044004682092, "logits/chosen": -0.357890248298645, "logits/rejected": -0.5321043729782104, "logps/chosen": -9.981462478637695, "logps/rejected": -34.8548583984375, "loss": 0.9003065228462219, "memory(GiB)": 42.51, "nll_loss": 0.5299741625785828, "rewards/accuracies": 0.8125, "rewards/chosen": 0.04542495310306549, "rewards/margins": 1.7066924571990967, "rewards/rejected": -1.6612673997879028, "step": 96, "train_speed(iter/s)": 0.011243 }, { "epoch": 0.24394844388557058, "grad_norm": 2.2122833728790283, "learning_rate": 0.0001994723260254686, "logits/chosen": -0.2592640519142151, "logits/rejected": -0.4961639940738678, "logps/chosen": -4.603950500488281, "logps/rejected": -41.2968864440918, "loss": 0.5464479923248291, "memory(GiB)": 42.51, "nll_loss": 0.3016549050807953, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1378301978111267, "rewards/margins": 2.3867669105529785, "rewards/rejected": -2.248936653137207, "step": 97, "train_speed(iter/s)": 0.011244 }, { "epoch": 0.24646337629676202, "grad_norm": 3.110325574874878, "learning_rate": 0.0001994434445073934, "logits/chosen": -0.25290989875793457, "logits/rejected": -0.5424689650535583, "logps/chosen": -3.2259504795074463, "logps/rejected": -50.58758544921875, "loss": 0.48420143127441406, "memory(GiB)": 42.51, "nll_loss": 0.24583308398723602, "rewards/accuracies": 0.90625, "rewards/chosen": 0.06779447197914124, "rewards/margins": 2.959743022918701, "rewards/rejected": -2.891948699951172, "step": 98, "train_speed(iter/s)": 0.011244 }, { "epoch": 0.24897830870795348, "grad_norm": 5.068027019500732, "learning_rate": 0.00019941379571543596, "logits/chosen": -0.2531135678291321, "logits/rejected": -0.4778846800327301, "logps/chosen": -4.735211372375488, "logps/rejected": -50.30940246582031, "loss": 0.6116160750389099, "memory(GiB)": 42.51, "nll_loss": 0.39248543977737427, "rewards/accuracies": 0.90625, "rewards/chosen": -0.009140911512076855, "rewards/margins": 3.1750411987304688, "rewards/rejected": -3.1841821670532227, "step": 99, "train_speed(iter/s)": 0.011244 }, { "epoch": 0.25149324111914495, "grad_norm": 5.654895305633545, "learning_rate": 0.00019938337987835688, "logits/chosen": -0.29766467213630676, "logits/rejected": -0.39502421021461487, "logps/chosen": -9.370654106140137, "logps/rejected": -37.111106872558594, "loss": 0.9719223976135254, "memory(GiB)": 42.51, "nll_loss": 0.5046005249023438, "rewards/accuracies": 0.75, "rewards/chosen": -0.21612250804901123, "rewards/margins": 2.0566141605377197, "rewards/rejected": -2.2727365493774414, "step": 100, "train_speed(iter/s)": 0.011244 }, { "epoch": 0.2540081735303364, "grad_norm": 2.3178176879882812, "learning_rate": 0.00019935219723083502, "logits/chosen": -0.3190368115901947, "logits/rejected": -0.4808640480041504, "logps/chosen": -4.635265350341797, "logps/rejected": -46.21360397338867, "loss": 0.5798269510269165, "memory(GiB)": 42.51, "nll_loss": 0.3243585228919983, "rewards/accuracies": 0.90625, "rewards/chosen": 0.07786446064710617, "rewards/margins": 3.2863645553588867, "rewards/rejected": -3.2085001468658447, "step": 101, "train_speed(iter/s)": 0.011225 }, { "epoch": 0.25652310594152783, "grad_norm": 1.5522849559783936, "learning_rate": 0.0001993202480134658, "logits/chosen": -0.3053315281867981, "logits/rejected": -0.5489925742149353, "logps/chosen": -2.3068742752075195, "logps/rejected": -59.077171325683594, "loss": 0.29376542568206787, "memory(GiB)": 42.51, "nll_loss": 0.15781843662261963, "rewards/accuracies": 0.96875, "rewards/chosen": 0.14335167407989502, "rewards/margins": 4.1030802726745605, "rewards/rejected": -3.959728240966797, "step": 102, "train_speed(iter/s)": 0.011225 }, { "epoch": 0.25903803835271927, "grad_norm": 3.088001012802124, "learning_rate": 0.0001992875324727591, "logits/chosen": -0.29225340485572815, "logits/rejected": -0.4629225730895996, "logps/chosen": -7.737708568572998, "logps/rejected": -56.16609573364258, "loss": 0.657672107219696, "memory(GiB)": 42.51, "nll_loss": 0.4136083722114563, "rewards/accuracies": 0.9375, "rewards/chosen": -0.15128585696220398, "rewards/margins": 3.662168025970459, "rewards/rejected": -3.8134536743164062, "step": 103, "train_speed(iter/s)": 0.011226 }, { "epoch": 0.2615529707639107, "grad_norm": 16.196247100830078, "learning_rate": 0.00019925405086113768, "logits/chosen": -0.24154868721961975, "logits/rejected": -0.42765212059020996, "logps/chosen": -7.00141716003418, "logps/rejected": -44.41450500488281, "loss": 0.9225707054138184, "memory(GiB)": 42.51, "nll_loss": 0.5817552804946899, "rewards/accuracies": 0.875, "rewards/chosen": -0.18904252350330353, "rewards/margins": 2.805342674255371, "rewards/rejected": -2.994385004043579, "step": 104, "train_speed(iter/s)": 0.011227 }, { "epoch": 0.26406790317510215, "grad_norm": 4.839797496795654, "learning_rate": 0.0001992198034369349, "logits/chosen": -0.21534261107444763, "logits/rejected": -0.4093911051750183, "logps/chosen": -6.268553733825684, "logps/rejected": -44.56496047973633, "loss": 0.7437994480133057, "memory(GiB)": 42.51, "nll_loss": 0.4600142240524292, "rewards/accuracies": 0.90625, "rewards/chosen": 0.12164817750453949, "rewards/margins": 2.817305088043213, "rewards/rejected": -2.695657253265381, "step": 105, "train_speed(iter/s)": 0.011227 }, { "epoch": 0.26658283558629364, "grad_norm": 6.186526298522949, "learning_rate": 0.000199184790464393, "logits/chosen": -0.28391483426094055, "logits/rejected": -0.47835806012153625, "logps/chosen": -4.414430618286133, "logps/rejected": -35.84981155395508, "loss": 0.7718313336372375, "memory(GiB)": 42.51, "nll_loss": 0.451696515083313, "rewards/accuracies": 0.8125, "rewards/chosen": 0.061221636831760406, "rewards/margins": 2.390685558319092, "rewards/rejected": -2.3294641971588135, "step": 106, "train_speed(iter/s)": 0.011228 }, { "epoch": 0.2690977679974851, "grad_norm": 4.50436544418335, "learning_rate": 0.00019914901221366086, "logits/chosen": -0.23250971734523773, "logits/rejected": -0.4437968134880066, "logps/chosen": -6.316987037658691, "logps/rejected": -49.236106872558594, "loss": 0.7839150428771973, "memory(GiB)": 42.51, "nll_loss": 0.46380385756492615, "rewards/accuracies": 0.78125, "rewards/chosen": -0.052675511687994, "rewards/margins": 3.1932520866394043, "rewards/rejected": -3.245927572250366, "step": 107, "train_speed(iter/s)": 0.011229 }, { "epoch": 0.2716127004086765, "grad_norm": 3.6184003353118896, "learning_rate": 0.0001991124689607921, "logits/chosen": -0.19258703291416168, "logits/rejected": -0.43321287631988525, "logps/chosen": -7.002651691436768, "logps/rejected": -41.406925201416016, "loss": 0.8718461990356445, "memory(GiB)": 42.51, "nll_loss": 0.5701597929000854, "rewards/accuracies": 0.90625, "rewards/chosen": -0.0768214762210846, "rewards/margins": 2.482198715209961, "rewards/rejected": -2.559020519256592, "step": 108, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.27412763281986796, "grad_norm": 10.216064453125, "learning_rate": 0.00019907516098774275, "logits/chosen": -0.3110716640949249, "logits/rejected": -0.4275364577770233, "logps/chosen": -5.352696418762207, "logps/rejected": -44.723453521728516, "loss": 0.8070493340492249, "memory(GiB)": 42.51, "nll_loss": 0.5113809704780579, "rewards/accuracies": 0.90625, "rewards/chosen": 0.021823018789291382, "rewards/margins": 3.496793508529663, "rewards/rejected": -3.474970579147339, "step": 109, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.2766425652310594, "grad_norm": 4.2641143798828125, "learning_rate": 0.00019903708858236925, "logits/chosen": -0.17137043178081512, "logits/rejected": -0.3559553623199463, "logps/chosen": -8.645333290100098, "logps/rejected": -56.14238357543945, "loss": 0.8187519311904907, "memory(GiB)": 42.51, "nll_loss": 0.4967080354690552, "rewards/accuracies": 0.84375, "rewards/chosen": -0.22593286633491516, "rewards/margins": 3.5679526329040527, "rewards/rejected": -3.7938857078552246, "step": 110, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.27915749764225084, "grad_norm": 5.008543014526367, "learning_rate": 0.00019899825203842613, "logits/chosen": -0.22317875921726227, "logits/rejected": -0.4507110118865967, "logps/chosen": -4.583744525909424, "logps/rejected": -52.95720672607422, "loss": 0.6659380793571472, "memory(GiB)": 42.51, "nll_loss": 0.42531025409698486, "rewards/accuracies": 0.875, "rewards/chosen": 0.08066091686487198, "rewards/margins": 3.507352828979492, "rewards/rejected": -3.426692247390747, "step": 111, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.28167243005344234, "grad_norm": 3.2912778854370117, "learning_rate": 0.00019895865165556377, "logits/chosen": -0.20796722173690796, "logits/rejected": -0.4548439085483551, "logps/chosen": -6.193486213684082, "logps/rejected": -50.00212860107422, "loss": 0.724488377571106, "memory(GiB)": 42.51, "nll_loss": 0.36660081148147583, "rewards/accuracies": 0.78125, "rewards/chosen": -0.09731753170490265, "rewards/margins": 2.9351813793182373, "rewards/rejected": -3.032498598098755, "step": 112, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.2841873624646338, "grad_norm": 2.3630876541137695, "learning_rate": 0.00019891828773932604, "logits/chosen": -0.29762569069862366, "logits/rejected": -0.496326208114624, "logps/chosen": -5.607400417327881, "logps/rejected": -45.21501922607422, "loss": 0.5872230529785156, "memory(GiB)": 42.51, "nll_loss": 0.36216530203819275, "rewards/accuracies": 0.9375, "rewards/chosen": 0.2503179609775543, "rewards/margins": 3.1686577796936035, "rewards/rejected": -2.918339729309082, "step": 113, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.2867022948758252, "grad_norm": 2.2401249408721924, "learning_rate": 0.0001988771606011481, "logits/chosen": -0.2846427857875824, "logits/rejected": -0.46442368626594543, "logps/chosen": -7.414566993713379, "logps/rejected": -38.23175048828125, "loss": 0.8090199828147888, "memory(GiB)": 42.51, "nll_loss": 0.5426386594772339, "rewards/accuracies": 0.96875, "rewards/chosen": 0.0931171178817749, "rewards/margins": 2.2654707431793213, "rewards/rejected": -2.172353744506836, "step": 114, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.28921722728701665, "grad_norm": 2.3249471187591553, "learning_rate": 0.0001988352705583538, "logits/chosen": -0.36136990785598755, "logits/rejected": -0.49599409103393555, "logps/chosen": -6.019715785980225, "logps/rejected": -31.482872009277344, "loss": 0.716136634349823, "memory(GiB)": 42.51, "nll_loss": 0.38180771470069885, "rewards/accuracies": 0.875, "rewards/chosen": 0.07284583151340485, "rewards/margins": 1.6758012771606445, "rewards/rejected": -1.6029551029205322, "step": 115, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.2917321596982081, "grad_norm": 2.0250649452209473, "learning_rate": 0.0001987926179341533, "logits/chosen": -0.3277466893196106, "logits/rejected": -0.49884647130966187, "logps/chosen": -6.582662105560303, "logps/rejected": -34.93992614746094, "loss": 0.6409764289855957, "memory(GiB)": 42.51, "nll_loss": 0.3818390965461731, "rewards/accuracies": 0.9375, "rewards/chosen": 0.219668447971344, "rewards/margins": 2.2215864658355713, "rewards/rejected": -2.001918315887451, "step": 116, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.29424709210939953, "grad_norm": 3.080130100250244, "learning_rate": 0.00019874920305764068, "logits/chosen": -0.3647570312023163, "logits/rejected": -0.5425549149513245, "logps/chosen": -4.569772720336914, "logps/rejected": -29.92070198059082, "loss": 0.733748197555542, "memory(GiB)": 42.51, "nll_loss": 0.3840240240097046, "rewards/accuracies": 0.84375, "rewards/chosen": 0.09753906726837158, "rewards/margins": 1.5259737968444824, "rewards/rejected": -1.4284348487854004, "step": 117, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.29676202452059103, "grad_norm": 3.2526772022247314, "learning_rate": 0.00019870502626379127, "logits/chosen": -0.39825165271759033, "logits/rejected": -0.5752891898155212, "logps/chosen": -6.833606243133545, "logps/rejected": -35.04240798950195, "loss": 0.7230013012886047, "memory(GiB)": 42.51, "nll_loss": 0.4058716297149658, "rewards/accuracies": 0.90625, "rewards/chosen": -0.1110297441482544, "rewards/margins": 1.915820837020874, "rewards/rejected": -2.026850461959839, "step": 118, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.29927695693178247, "grad_norm": 2.977874994277954, "learning_rate": 0.00019866008789345917, "logits/chosen": -0.4147549271583557, "logits/rejected": -0.5979161262512207, "logps/chosen": -5.960415363311768, "logps/rejected": -40.34906768798828, "loss": 0.6719368696212769, "memory(GiB)": 42.51, "nll_loss": 0.38052472472190857, "rewards/accuracies": 0.9375, "rewards/chosen": 0.015904245898127556, "rewards/margins": 2.2843177318573, "rewards/rejected": -2.2684133052825928, "step": 119, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.3017918893429739, "grad_norm": 3.058821439743042, "learning_rate": 0.00019861438829337438, "logits/chosen": -0.3485436737537384, "logits/rejected": -0.5699384808540344, "logps/chosen": -8.048928260803223, "logps/rejected": -55.26435852050781, "loss": 0.8104152083396912, "memory(GiB)": 42.51, "nll_loss": 0.5313911437988281, "rewards/accuracies": 0.84375, "rewards/chosen": 0.08313308656215668, "rewards/margins": 3.3404150009155273, "rewards/rejected": -3.257282257080078, "step": 120, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.30430682175416535, "grad_norm": 2.809389591217041, "learning_rate": 0.00019856792781614054, "logits/chosen": -0.40515586733818054, "logits/rejected": -0.6652223467826843, "logps/chosen": -2.280519962310791, "logps/rejected": -50.222084045410156, "loss": 0.45217883586883545, "memory(GiB)": 42.51, "nll_loss": 0.26132771372795105, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1198212057352066, "rewards/margins": 3.28947114944458, "rewards/rejected": -3.169650077819824, "step": 121, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.3068217541653568, "grad_norm": 3.070807456970215, "learning_rate": 0.00019852070682023176, "logits/chosen": -0.4808189272880554, "logits/rejected": -0.6974934339523315, "logps/chosen": -7.208076477050781, "logps/rejected": -52.298465728759766, "loss": 0.5899146795272827, "memory(GiB)": 42.51, "nll_loss": 0.42121508717536926, "rewards/accuracies": 0.9375, "rewards/chosen": 0.16679850220680237, "rewards/margins": 3.6304702758789062, "rewards/rejected": -3.4636716842651367, "step": 122, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.3093366865765483, "grad_norm": 2.8824384212493896, "learning_rate": 0.00019847272566999026, "logits/chosen": -0.49758437275886536, "logits/rejected": -0.6554095149040222, "logps/chosen": -10.397936820983887, "logps/rejected": -42.242393493652344, "loss": 0.8943163156509399, "memory(GiB)": 42.51, "nll_loss": 0.5859109163284302, "rewards/accuracies": 0.90625, "rewards/chosen": 0.0896286740899086, "rewards/margins": 2.782667875289917, "rewards/rejected": -2.6930387020111084, "step": 123, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.3118516189877397, "grad_norm": 2.103641986846924, "learning_rate": 0.0001984239847356233, "logits/chosen": -0.5333049297332764, "logits/rejected": -0.6946154832839966, "logps/chosen": -4.652249336242676, "logps/rejected": -53.446041107177734, "loss": 0.5551345944404602, "memory(GiB)": 42.51, "nll_loss": 0.3173026144504547, "rewards/accuracies": 0.84375, "rewards/chosen": 0.0959150567650795, "rewards/margins": 4.159012794494629, "rewards/rejected": -4.063097953796387, "step": 124, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.31436655139893116, "grad_norm": 2.719147205352783, "learning_rate": 0.00019837448439320027, "logits/chosen": -0.5486172437667847, "logits/rejected": -0.7822794318199158, "logps/chosen": -3.360109567642212, "logps/rejected": -77.09004211425781, "loss": 0.4081324338912964, "memory(GiB)": 42.51, "nll_loss": 0.27365225553512573, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09313642978668213, "rewards/margins": 6.103594779968262, "rewards/rejected": -6.010458469390869, "step": 125, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.3168814838101226, "grad_norm": 3.821580410003662, "learning_rate": 0.0001983242250246501, "logits/chosen": -0.49823230504989624, "logits/rejected": -0.746193528175354, "logps/chosen": -8.326932907104492, "logps/rejected": -85.3308334350586, "loss": 0.6026039719581604, "memory(GiB)": 42.51, "nll_loss": 0.48026546835899353, "rewards/accuracies": 0.9375, "rewards/chosen": -0.15297873318195343, "rewards/margins": 6.628488063812256, "rewards/rejected": -6.781466484069824, "step": 126, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.31939641622131404, "grad_norm": 2.334726333618164, "learning_rate": 0.00019827320701775806, "logits/chosen": -0.4417359530925751, "logits/rejected": -0.6943372488021851, "logps/chosen": -5.183182716369629, "logps/rejected": -74.23169708251953, "loss": 0.5311453938484192, "memory(GiB)": 42.51, "nll_loss": 0.31170862913131714, "rewards/accuracies": 0.9375, "rewards/chosen": 0.06631641089916229, "rewards/margins": 5.296952724456787, "rewards/rejected": -5.2306365966796875, "step": 127, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.3219113486325055, "grad_norm": 3.0242960453033447, "learning_rate": 0.0001982214307661627, "logits/chosen": -0.39977923035621643, "logits/rejected": -0.6834134459495544, "logps/chosen": -4.503589153289795, "logps/rejected": -70.66866302490234, "loss": 0.5889554619789124, "memory(GiB)": 42.51, "nll_loss": 0.3487299084663391, "rewards/accuracies": 0.90625, "rewards/chosen": 0.18435406684875488, "rewards/margins": 5.4617156982421875, "rewards/rejected": -5.277361869812012, "step": 128, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.324426281043697, "grad_norm": 2.191619396209717, "learning_rate": 0.00019816889666935317, "logits/chosen": -0.3248615860939026, "logits/rejected": -0.6416115164756775, "logps/chosen": -6.371191024780273, "logps/rejected": -76.59197998046875, "loss": 0.4878350794315338, "memory(GiB)": 42.51, "nll_loss": 0.33031994104385376, "rewards/accuracies": 0.96875, "rewards/chosen": 0.13981086015701294, "rewards/margins": 5.163226127624512, "rewards/rejected": -5.0234150886535645, "step": 129, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.3269412134548884, "grad_norm": 3.211867570877075, "learning_rate": 0.00019811560513266572, "logits/chosen": -0.3845720887184143, "logits/rejected": -0.6389954090118408, "logps/chosen": -5.841503620147705, "logps/rejected": -71.1533203125, "loss": 0.586833655834198, "memory(GiB)": 42.51, "nll_loss": 0.3950338065624237, "rewards/accuracies": 0.9375, "rewards/chosen": 0.09367645531892776, "rewards/margins": 5.478381156921387, "rewards/rejected": -5.384705066680908, "step": 130, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.32945614586607985, "grad_norm": 9.350093841552734, "learning_rate": 0.00019806155656728084, "logits/chosen": -0.2960500419139862, "logits/rejected": -0.6573159098625183, "logps/chosen": -9.058235168457031, "logps/rejected": -103.11417388916016, "loss": 0.9421167373657227, "memory(GiB)": 42.51, "nll_loss": 0.7972053289413452, "rewards/accuracies": 0.90625, "rewards/chosen": -0.26350751519203186, "rewards/margins": 7.675902366638184, "rewards/rejected": -7.939410209655762, "step": 131, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.3319710782772713, "grad_norm": 5.248542308807373, "learning_rate": 0.00019800675139022006, "logits/chosen": -0.47517600655555725, "logits/rejected": -0.6538941264152527, "logps/chosen": -9.417102813720703, "logps/rejected": -74.27867126464844, "loss": 0.6414040327072144, "memory(GiB)": 42.51, "nll_loss": 0.3988567888736725, "rewards/accuracies": 0.84375, "rewards/chosen": -0.300504595041275, "rewards/margins": 5.568276405334473, "rewards/rejected": -5.868781089782715, "step": 132, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.33448601068846273, "grad_norm": 3.997988224029541, "learning_rate": 0.00019795119002434262, "logits/chosen": -0.47632429003715515, "logits/rejected": -0.6377424001693726, "logps/chosen": -6.212442874908447, "logps/rejected": -72.35940551757812, "loss": 0.6233670711517334, "memory(GiB)": 42.51, "nll_loss": 0.36861366033554077, "rewards/accuracies": 0.9375, "rewards/chosen": 0.04314713925123215, "rewards/margins": 5.591425895690918, "rewards/rejected": -5.548279285430908, "step": 133, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.33700094309965417, "grad_norm": 2.701449394226074, "learning_rate": 0.00019789487289834228, "logits/chosen": -0.35887065529823303, "logits/rejected": -0.5933378338813782, "logps/chosen": -3.4826135635375977, "logps/rejected": -62.30265808105469, "loss": 0.4813486635684967, "memory(GiB)": 42.51, "nll_loss": 0.28860440850257874, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1144554391503334, "rewards/margins": 4.846682548522949, "rewards/rejected": -4.732227325439453, "step": 134, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.33951587551084567, "grad_norm": 1.554006814956665, "learning_rate": 0.000197837800446744, "logits/chosen": -0.26935291290283203, "logits/rejected": -0.6075947284698486, "logps/chosen": -5.505460262298584, "logps/rejected": -90.5344467163086, "loss": 0.36852002143859863, "memory(GiB)": 42.51, "nll_loss": 0.22335653007030487, "rewards/accuracies": 0.96875, "rewards/chosen": -0.009416868910193443, "rewards/margins": 6.831361770629883, "rewards/rejected": -6.840778350830078, "step": 135, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.3420308079220371, "grad_norm": 2.480741024017334, "learning_rate": 0.00019777997310990063, "logits/chosen": -0.3000340461730957, "logits/rejected": -0.6057184338569641, "logps/chosen": -3.0569756031036377, "logps/rejected": -74.86396789550781, "loss": 0.4303661286830902, "memory(GiB)": 42.51, "nll_loss": 0.29833856225013733, "rewards/accuracies": 1.0, "rewards/chosen": 0.10127638280391693, "rewards/margins": 5.511883735656738, "rewards/rejected": -5.410606861114502, "step": 136, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.34454574033322855, "grad_norm": 4.56061315536499, "learning_rate": 0.00019772139133398942, "logits/chosen": -0.3569238781929016, "logits/rejected": -0.5552278161048889, "logps/chosen": -6.037662506103516, "logps/rejected": -72.8807373046875, "loss": 0.5785601139068604, "memory(GiB)": 42.51, "nll_loss": 0.3691541850566864, "rewards/accuracies": 0.90625, "rewards/chosen": -0.07385054975748062, "rewards/margins": 5.239901542663574, "rewards/rejected": -5.313753128051758, "step": 137, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.34706067274442, "grad_norm": 3.128384828567505, "learning_rate": 0.00019766205557100868, "logits/chosen": -0.45258647203445435, "logits/rejected": -0.6256503462791443, "logps/chosen": -6.615728855133057, "logps/rejected": -54.988853454589844, "loss": 0.6647377014160156, "memory(GiB)": 42.51, "nll_loss": 0.4122207760810852, "rewards/accuracies": 0.90625, "rewards/chosen": 0.1877025067806244, "rewards/margins": 4.346957206726074, "rewards/rejected": -4.159254550933838, "step": 138, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.3495756051556114, "grad_norm": 3.3702149391174316, "learning_rate": 0.00019760196627877422, "logits/chosen": -0.22564947605133057, "logits/rejected": -0.6019137501716614, "logps/chosen": -2.935713052749634, "logps/rejected": -91.22589111328125, "loss": 0.4306371212005615, "memory(GiB)": 42.51, "nll_loss": 0.2682075500488281, "rewards/accuracies": 0.90625, "rewards/chosen": 0.09053624421358109, "rewards/margins": 6.857407569885254, "rewards/rejected": -6.766871452331543, "step": 139, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.3520905375668029, "grad_norm": 17.25383949279785, "learning_rate": 0.0001975411239209158, "logits/chosen": -0.4722020626068115, "logits/rejected": -0.5809731483459473, "logps/chosen": -11.125019073486328, "logps/rejected": -47.38395690917969, "loss": 1.216600775718689, "memory(GiB)": 42.51, "nll_loss": 0.792304515838623, "rewards/accuracies": 0.75, "rewards/chosen": -0.23399552702903748, "rewards/margins": 3.3990795612335205, "rewards/rejected": -3.63307523727417, "step": 140, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.35460546997799436, "grad_norm": 3.727832794189453, "learning_rate": 0.0001974795289668737, "logits/chosen": -0.35420382022857666, "logits/rejected": -0.718546986579895, "logps/chosen": -7.968466758728027, "logps/rejected": -91.39598846435547, "loss": 0.7644503712654114, "memory(GiB)": 42.51, "nll_loss": 0.6175839900970459, "rewards/accuracies": 0.96875, "rewards/chosen": -0.3464764654636383, "rewards/margins": 6.540220737457275, "rewards/rejected": -6.886697292327881, "step": 141, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.3571204023891858, "grad_norm": 13.208379745483398, "learning_rate": 0.00019741718189189485, "logits/chosen": -0.41622060537338257, "logits/rejected": -0.6492832899093628, "logps/chosen": -6.71783447265625, "logps/rejected": -69.75479888916016, "loss": 0.8267927169799805, "memory(GiB)": 42.51, "nll_loss": 0.5884179472923279, "rewards/accuracies": 0.96875, "rewards/chosen": -0.1379324495792389, "rewards/margins": 4.956573963165283, "rewards/rejected": -5.094506740570068, "step": 142, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.35963533480037724, "grad_norm": 4.856459617614746, "learning_rate": 0.00019735408317702948, "logits/chosen": -0.4186263382434845, "logits/rejected": -0.6451130509376526, "logps/chosen": -4.069480895996094, "logps/rejected": -62.28482437133789, "loss": 0.4998188018798828, "memory(GiB)": 42.51, "nll_loss": 0.3359423875808716, "rewards/accuracies": 0.96875, "rewards/chosen": 0.012309092096984386, "rewards/margins": 4.688631057739258, "rewards/rejected": -4.676321983337402, "step": 143, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.3621502672115687, "grad_norm": 4.043496608734131, "learning_rate": 0.0001972902333091271, "logits/chosen": -0.3931187391281128, "logits/rejected": -0.5637274384498596, "logps/chosen": -8.15110969543457, "logps/rejected": -48.92616271972656, "loss": 0.7109068632125854, "memory(GiB)": 42.51, "nll_loss": 0.4194295108318329, "rewards/accuracies": 0.9375, "rewards/chosen": 0.008020445704460144, "rewards/margins": 3.4844448566436768, "rewards/rejected": -3.476424217224121, "step": 144, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.3646651996227601, "grad_norm": 3.2291014194488525, "learning_rate": 0.00019722563278083287, "logits/chosen": -0.3712625503540039, "logits/rejected": -0.6469380855560303, "logps/chosen": -2.269747734069824, "logps/rejected": -41.929664611816406, "loss": 0.4003128111362457, "memory(GiB)": 42.51, "nll_loss": 0.182529017329216, "rewards/accuracies": 0.9375, "rewards/chosen": 0.18503372371196747, "rewards/margins": 2.92374849319458, "rewards/rejected": -2.7387146949768066, "step": 145, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.3671801320339516, "grad_norm": 4.1039204597473145, "learning_rate": 0.00019716028209058387, "logits/chosen": -0.44887417554855347, "logits/rejected": -0.6388370990753174, "logps/chosen": -5.760115623474121, "logps/rejected": -43.50143051147461, "loss": 0.7498366236686707, "memory(GiB)": 42.51, "nll_loss": 0.48517200350761414, "rewards/accuracies": 0.9375, "rewards/chosen": -0.02230704203248024, "rewards/margins": 2.708374500274658, "rewards/rejected": -2.7306814193725586, "step": 146, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.36969506444514305, "grad_norm": 3.1436734199523926, "learning_rate": 0.0001970941817426052, "logits/chosen": -0.31916379928588867, "logits/rejected": -0.577805757522583, "logps/chosen": -3.4299936294555664, "logps/rejected": -47.918155670166016, "loss": 0.4161657989025116, "memory(GiB)": 42.51, "nll_loss": 0.21494346857070923, "rewards/accuracies": 0.96875, "rewards/chosen": 0.14184261858463287, "rewards/margins": 2.971557855606079, "rewards/rejected": -2.8297150135040283, "step": 147, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.3722099968563345, "grad_norm": 2.7959487438201904, "learning_rate": 0.00019702733224690605, "logits/chosen": -0.43002602458000183, "logits/rejected": -0.6337836980819702, "logps/chosen": -8.148699760437012, "logps/rejected": -42.15251922607422, "loss": 0.7381757497787476, "memory(GiB)": 42.51, "nll_loss": 0.4981796443462372, "rewards/accuracies": 0.875, "rewards/chosen": -0.013383438810706139, "rewards/margins": 2.8139827251434326, "rewards/rejected": -2.8273658752441406, "step": 148, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.37472492926752593, "grad_norm": 1.9313372373580933, "learning_rate": 0.00019695973411927578, "logits/chosen": -0.500542402267456, "logits/rejected": -0.7104104161262512, "logps/chosen": -2.929166316986084, "logps/rejected": -54.1365966796875, "loss": 0.42066073417663574, "memory(GiB)": 42.51, "nll_loss": 0.22536346316337585, "rewards/accuracies": 0.90625, "rewards/chosen": 0.06520625203847885, "rewards/margins": 3.698242664337158, "rewards/rejected": -3.6330366134643555, "step": 149, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.37723986167871737, "grad_norm": 1.9907225370407104, "learning_rate": 0.00019689138788127995, "logits/chosen": -0.49561041593551636, "logits/rejected": -0.6665260195732117, "logps/chosen": -5.222489356994629, "logps/rejected": -42.92105484008789, "loss": 0.6262026429176331, "memory(GiB)": 42.51, "nll_loss": 0.385796457529068, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09229743480682373, "rewards/margins": 3.072531223297119, "rewards/rejected": -2.980233669281006, "step": 150, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.3797547940899088, "grad_norm": 3.5840723514556885, "learning_rate": 0.00019682229406025635, "logits/chosen": -0.4681642949581146, "logits/rejected": -0.6473852396011353, "logps/chosen": -8.059581756591797, "logps/rejected": -54.22500228881836, "loss": 0.6054436564445496, "memory(GiB)": 42.51, "nll_loss": 0.38610902428627014, "rewards/accuracies": 0.90625, "rewards/chosen": 0.097575843334198, "rewards/margins": 3.8078057765960693, "rewards/rejected": -3.7102296352386475, "step": 151, "train_speed(iter/s)": 0.011228 }, { "epoch": 0.3822697265011003, "grad_norm": 4.087256908416748, "learning_rate": 0.00019675245318931083, "logits/chosen": -0.5369412899017334, "logits/rejected": -0.749391496181488, "logps/chosen": -9.42272663116455, "logps/rejected": -59.544307708740234, "loss": 0.6138618588447571, "memory(GiB)": 42.51, "nll_loss": 0.4435786008834839, "rewards/accuracies": 0.96875, "rewards/chosen": -0.10584679991006851, "rewards/margins": 4.319212436676025, "rewards/rejected": -4.425059795379639, "step": 152, "train_speed(iter/s)": 0.011228 }, { "epoch": 0.38478465891229174, "grad_norm": 7.815120697021484, "learning_rate": 0.0001966818658073133, "logits/chosen": -0.6010515689849854, "logits/rejected": -0.7820409536361694, "logps/chosen": -7.579458713531494, "logps/rejected": -62.63882064819336, "loss": 0.709304928779602, "memory(GiB)": 42.51, "nll_loss": 0.5141942501068115, "rewards/accuracies": 0.9375, "rewards/chosen": -0.13567981123924255, "rewards/margins": 4.736479759216309, "rewards/rejected": -4.872159481048584, "step": 153, "train_speed(iter/s)": 0.011229 }, { "epoch": 0.3872995913234832, "grad_norm": 7.4138031005859375, "learning_rate": 0.0001966105324588934, "logits/chosen": -0.5685268640518188, "logits/rejected": -0.7353744506835938, "logps/chosen": -11.372655868530273, "logps/rejected": -61.317073822021484, "loss": 0.8202418684959412, "memory(GiB)": 42.51, "nll_loss": 0.583480715751648, "rewards/accuracies": 0.875, "rewards/chosen": 0.05895308405160904, "rewards/margins": 4.791215419769287, "rewards/rejected": -4.73226261138916, "step": 154, "train_speed(iter/s)": 0.011229 }, { "epoch": 0.3898145237346746, "grad_norm": 2.7024075984954834, "learning_rate": 0.00019653845369443657, "logits/chosen": -0.4999295175075531, "logits/rejected": -0.7417422533035278, "logps/chosen": -6.620236873626709, "logps/rejected": -70.78112030029297, "loss": 0.6438719034194946, "memory(GiB)": 42.51, "nll_loss": 0.5055179595947266, "rewards/accuracies": 0.96875, "rewards/chosen": 0.060695331543684006, "rewards/margins": 5.345253944396973, "rewards/rejected": -5.284558296203613, "step": 155, "train_speed(iter/s)": 0.011229 }, { "epoch": 0.39232945614586606, "grad_norm": 4.160597801208496, "learning_rate": 0.00019646563007007952, "logits/chosen": -0.5322343707084656, "logits/rejected": -0.7421768307685852, "logps/chosen": -6.158726215362549, "logps/rejected": -47.234554290771484, "loss": 0.6948550343513489, "memory(GiB)": 42.51, "nll_loss": 0.41276055574417114, "rewards/accuracies": 0.9375, "rewards/chosen": -0.09865140169858932, "rewards/margins": 2.9533112049102783, "rewards/rejected": -3.0519626140594482, "step": 156, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.3948443885570575, "grad_norm": 4.034154891967773, "learning_rate": 0.00019639206214770608, "logits/chosen": -0.5310461521148682, "logits/rejected": -0.7443493008613586, "logps/chosen": -3.5962212085723877, "logps/rejected": -53.00592803955078, "loss": 0.48415228724479675, "memory(GiB)": 42.51, "nll_loss": 0.2999834716320038, "rewards/accuracies": 0.875, "rewards/chosen": 0.020860865712165833, "rewards/margins": 3.5892655849456787, "rewards/rejected": -3.5684046745300293, "step": 157, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.397359320968249, "grad_norm": 9.343074798583984, "learning_rate": 0.00019631775049494285, "logits/chosen": -0.44315972924232483, "logits/rejected": -0.7157086730003357, "logps/chosen": -4.082563400268555, "logps/rejected": -64.51760864257812, "loss": 0.5230289697647095, "memory(GiB)": 42.51, "nll_loss": 0.2998603284358978, "rewards/accuracies": 0.9375, "rewards/chosen": -0.021998610347509384, "rewards/margins": 3.9378349781036377, "rewards/rejected": -3.959833860397339, "step": 158, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.39987425337944044, "grad_norm": 2.403951644897461, "learning_rate": 0.00019624269568515486, "logits/chosen": -0.4427807629108429, "logits/rejected": -0.674246072769165, "logps/chosen": -5.679157257080078, "logps/rejected": -46.368919372558594, "loss": 0.5779350996017456, "memory(GiB)": 42.51, "nll_loss": 0.39234140515327454, "rewards/accuracies": 0.9375, "rewards/chosen": 0.11242523044347763, "rewards/margins": 2.7115559577941895, "rewards/rejected": -2.599130630493164, "step": 159, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.4023891857906319, "grad_norm": 3.0051136016845703, "learning_rate": 0.00019616689829744105, "logits/chosen": -0.595456600189209, "logits/rejected": -0.7742252945899963, "logps/chosen": -6.239538669586182, "logps/rejected": -36.2790641784668, "loss": 0.6849649548530579, "memory(GiB)": 42.51, "nll_loss": 0.43636244535446167, "rewards/accuracies": 0.9375, "rewards/chosen": 0.12961390614509583, "rewards/margins": 2.334453821182251, "rewards/rejected": -2.2048397064208984, "step": 160, "train_speed(iter/s)": 0.01123 }, { "epoch": 0.4049041182018233, "grad_norm": 4.199156761169434, "learning_rate": 0.0001960903589166299, "logits/chosen": -0.6130431890487671, "logits/rejected": -0.6931115388870239, "logps/chosen": -8.752334594726562, "logps/rejected": -42.808692932128906, "loss": 0.6933425664901733, "memory(GiB)": 42.51, "nll_loss": 0.39625656604766846, "rewards/accuracies": 0.8125, "rewards/chosen": 0.2133105993270874, "rewards/margins": 3.1959619522094727, "rewards/rejected": -2.9826512336730957, "step": 161, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.40741905061301475, "grad_norm": 4.59311056137085, "learning_rate": 0.00019601307813327482, "logits/chosen": -0.5155867338180542, "logits/rejected": -0.6933278441429138, "logps/chosen": -3.984743595123291, "logps/rejected": -44.87555694580078, "loss": 0.6113356351852417, "memory(GiB)": 42.51, "nll_loss": 0.3314781188964844, "rewards/accuracies": 0.84375, "rewards/chosen": 0.04041612148284912, "rewards/margins": 2.863827705383301, "rewards/rejected": -2.8234119415283203, "step": 162, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.40993398302420625, "grad_norm": 2.7300822734832764, "learning_rate": 0.00019593505654364965, "logits/chosen": -0.4362912178039551, "logits/rejected": -0.6915664076805115, "logps/chosen": -3.529961585998535, "logps/rejected": -61.31735610961914, "loss": 0.4943440556526184, "memory(GiB)": 42.51, "nll_loss": 0.3457809090614319, "rewards/accuracies": 0.96875, "rewards/chosen": 0.07190480828285217, "rewards/margins": 4.230862617492676, "rewards/rejected": -4.158957481384277, "step": 163, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.4124489154353977, "grad_norm": 1.3495978116989136, "learning_rate": 0.00019585629474974415, "logits/chosen": -0.3829024136066437, "logits/rejected": -0.7348803877830505, "logps/chosen": -0.20720741152763367, "logps/rejected": -75.26912689208984, "loss": 0.09561214596033096, "memory(GiB)": 42.51, "nll_loss": 0.0268473532050848, "rewards/accuracies": 1.0, "rewards/chosen": 0.11308196187019348, "rewards/margins": 5.472672462463379, "rewards/rejected": -5.359590530395508, "step": 164, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.41496384784658913, "grad_norm": 2.3953280448913574, "learning_rate": 0.0001957767933592591, "logits/chosen": -0.49378660321235657, "logits/rejected": -0.6871320009231567, "logps/chosen": -5.856287002563477, "logps/rejected": -55.68489456176758, "loss": 0.5843676924705505, "memory(GiB)": 42.51, "nll_loss": 0.3655776381492615, "rewards/accuracies": 0.90625, "rewards/chosen": -0.0030907923355698586, "rewards/margins": 4.014834403991699, "rewards/rejected": -4.017925262451172, "step": 165, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.41747878025778057, "grad_norm": 5.787476062774658, "learning_rate": 0.00019569655298560195, "logits/chosen": -0.4845959544181824, "logits/rejected": -0.725760281085968, "logps/chosen": -3.9740681648254395, "logps/rejected": -63.2627067565918, "loss": 0.4704618752002716, "memory(GiB)": 42.51, "nll_loss": 0.34831976890563965, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09146178513765335, "rewards/margins": 4.952096939086914, "rewards/rejected": -4.860634803771973, "step": 166, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.419993712668972, "grad_norm": 2.719466209411621, "learning_rate": 0.0001956155742478817, "logits/chosen": -0.4632588326931, "logits/rejected": -0.7149248123168945, "logps/chosen": -2.24023175239563, "logps/rejected": -70.20541381835938, "loss": 0.28046005964279175, "memory(GiB)": 42.51, "nll_loss": 0.18644680082798004, "rewards/accuracies": 0.96875, "rewards/chosen": 0.19088204205036163, "rewards/margins": 5.480358123779297, "rewards/rejected": -5.289475917816162, "step": 167, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.42250864508016345, "grad_norm": 3.770395517349243, "learning_rate": 0.00019553385777090458, "logits/chosen": -0.43829765915870667, "logits/rejected": -0.7132550477981567, "logps/chosen": -9.330952644348145, "logps/rejected": -72.52558135986328, "loss": 0.5571638941764832, "memory(GiB)": 44.28, "nll_loss": 0.46076467633247375, "rewards/accuracies": 1.0, "rewards/chosen": 0.10896007716655731, "rewards/margins": 5.002399444580078, "rewards/rejected": -4.893439292907715, "step": 168, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.42502357749135494, "grad_norm": 13.122285842895508, "learning_rate": 0.00019545140418516873, "logits/chosen": -0.628214418888092, "logits/rejected": -0.7334375977516174, "logps/chosen": -7.368585586547852, "logps/rejected": -48.79350662231445, "loss": 0.7424007654190063, "memory(GiB)": 44.28, "nll_loss": 0.5702238082885742, "rewards/accuracies": 0.90625, "rewards/chosen": 0.07907923310995102, "rewards/margins": 3.957150459289551, "rewards/rejected": -3.8780713081359863, "step": 169, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.4275385099025464, "grad_norm": 2.2032506465911865, "learning_rate": 0.0001953682141268598, "logits/chosen": -0.4527589678764343, "logits/rejected": -0.6895320415496826, "logps/chosen": -3.53696870803833, "logps/rejected": -64.63774871826172, "loss": 0.35415133833885193, "memory(GiB)": 44.28, "nll_loss": 0.22634199261665344, "rewards/accuracies": 1.0, "rewards/chosen": 0.032016728073358536, "rewards/margins": 4.734713077545166, "rewards/rejected": -4.702696800231934, "step": 170, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.4300534423137378, "grad_norm": 2.4580419063568115, "learning_rate": 0.00019528428823784567, "logits/chosen": -0.37069255113601685, "logits/rejected": -0.7563280463218689, "logps/chosen": -2.2511513233184814, "logps/rejected": -86.21305847167969, "loss": 0.22136372327804565, "memory(GiB)": 44.28, "nll_loss": 0.18806159496307373, "rewards/accuracies": 1.0, "rewards/chosen": 0.13967791199684143, "rewards/margins": 6.765411853790283, "rewards/rejected": -6.625734329223633, "step": 171, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.43256837472492926, "grad_norm": 2.1516947746276855, "learning_rate": 0.0001951996271656717, "logits/chosen": -0.5217421650886536, "logits/rejected": -0.7590201497077942, "logps/chosen": -4.64290714263916, "logps/rejected": -64.5576171875, "loss": 0.4696881175041199, "memory(GiB)": 44.28, "nll_loss": 0.3352178931236267, "rewards/accuracies": 0.96875, "rewards/chosen": 0.0438573844730854, "rewards/margins": 4.907254695892334, "rewards/rejected": -4.863397598266602, "step": 172, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.4350833071361207, "grad_norm": 3.411616563796997, "learning_rate": 0.00019511423156355577, "logits/chosen": -0.4631859362125397, "logits/rejected": -0.7260483503341675, "logps/chosen": -5.667875289916992, "logps/rejected": -75.57254028320312, "loss": 0.35108137130737305, "memory(GiB)": 44.28, "nll_loss": 0.30264440178871155, "rewards/accuracies": 1.0, "rewards/chosen": 0.08381501585245132, "rewards/margins": 5.755500793457031, "rewards/rejected": -5.671685695648193, "step": 173, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.43759823954731214, "grad_norm": 4.192603588104248, "learning_rate": 0.00019502810209038303, "logits/chosen": -0.5316020250320435, "logits/rejected": -0.6840960383415222, "logps/chosen": -10.37562084197998, "logps/rejected": -67.83643341064453, "loss": 1.085424542427063, "memory(GiB)": 44.28, "nll_loss": 0.7863109111785889, "rewards/accuracies": 0.9375, "rewards/chosen": -0.0769924521446228, "rewards/margins": 4.868200778961182, "rewards/rejected": -4.945193767547607, "step": 174, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.44011317195850364, "grad_norm": 4.43406343460083, "learning_rate": 0.00019494123941070108, "logits/chosen": -0.6808348894119263, "logits/rejected": -0.7925645112991333, "logps/chosen": -7.7486252784729, "logps/rejected": -48.33315658569336, "loss": 0.9540278911590576, "memory(GiB)": 44.28, "nll_loss": 0.7564924955368042, "rewards/accuracies": 0.96875, "rewards/chosen": -0.1413968950510025, "rewards/margins": 3.6723074913024902, "rewards/rejected": -3.813704013824463, "step": 175, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.4426281043696951, "grad_norm": 6.438490867614746, "learning_rate": 0.00019485364419471454, "logits/chosen": -0.6449403762817383, "logits/rejected": -0.7584658861160278, "logps/chosen": -6.1986799240112305, "logps/rejected": -49.09370422363281, "loss": 0.6880564093589783, "memory(GiB)": 44.28, "nll_loss": 0.38199642300605774, "rewards/accuracies": 0.8125, "rewards/chosen": -0.20061062276363373, "rewards/margins": 3.613797426223755, "rewards/rejected": -3.814408302307129, "step": 176, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.4451430367808865, "grad_norm": 2.3246142864227295, "learning_rate": 0.00019476531711828027, "logits/chosen": -0.524186372756958, "logits/rejected": -0.6847387552261353, "logps/chosen": -6.170748710632324, "logps/rejected": -58.475975036621094, "loss": 0.4086158275604248, "memory(GiB)": 44.28, "nll_loss": 0.2584584057331085, "rewards/accuracies": 0.96875, "rewards/chosen": 0.07990145683288574, "rewards/margins": 4.2991790771484375, "rewards/rejected": -4.219277381896973, "step": 177, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.44765796919207795, "grad_norm": 4.515077590942383, "learning_rate": 0.00019467625886290167, "logits/chosen": -0.5108579993247986, "logits/rejected": -0.6950021386146545, "logps/chosen": -14.55732250213623, "logps/rejected": -50.92564392089844, "loss": 0.9052462577819824, "memory(GiB)": 44.28, "nll_loss": 0.6600362062454224, "rewards/accuracies": 0.84375, "rewards/chosen": -0.2508185803890228, "rewards/margins": 3.1151885986328125, "rewards/rejected": -3.366007089614868, "step": 178, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.4501729016032694, "grad_norm": 4.742927074432373, "learning_rate": 0.0001945864701157239, "logits/chosen": -0.5691394805908203, "logits/rejected": -0.6816927790641785, "logps/chosen": -4.945348262786865, "logps/rejected": -41.207801818847656, "loss": 0.6003872752189636, "memory(GiB)": 44.28, "nll_loss": 0.3502579927444458, "rewards/accuracies": 0.90625, "rewards/chosen": 0.07804224640130997, "rewards/margins": 2.8702170848846436, "rewards/rejected": -2.792174816131592, "step": 179, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.4526878340144609, "grad_norm": 2.001896619796753, "learning_rate": 0.00019449595156952827, "logits/chosen": -0.5839424133300781, "logits/rejected": -0.7141891717910767, "logps/chosen": -7.824261665344238, "logps/rejected": -42.13900375366211, "loss": 0.5872877240180969, "memory(GiB)": 44.28, "nll_loss": 0.3686111569404602, "rewards/accuracies": 0.9375, "rewards/chosen": 0.15194831788539886, "rewards/margins": 2.942096710205078, "rewards/rejected": -2.7901487350463867, "step": 180, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.4552027664256523, "grad_norm": 2.7639639377593994, "learning_rate": 0.00019440470392272696, "logits/chosen": -0.5639482736587524, "logits/rejected": -0.682327151298523, "logps/chosen": -4.094886302947998, "logps/rejected": -37.27549362182617, "loss": 0.5251520872116089, "memory(GiB)": 44.28, "nll_loss": 0.3160794675350189, "rewards/accuracies": 0.90625, "rewards/chosen": 0.16162550449371338, "rewards/margins": 2.7334065437316895, "rewards/rejected": -2.5717809200286865, "step": 181, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.45771769883684377, "grad_norm": 3.3179478645324707, "learning_rate": 0.00019431272787935773, "logits/chosen": -0.4620465636253357, "logits/rejected": -0.7069180011749268, "logps/chosen": -10.247895240783691, "logps/rejected": -44.85274887084961, "loss": 0.895711362361908, "memory(GiB)": 44.28, "nll_loss": 0.6755833625793457, "rewards/accuracies": 0.96875, "rewards/chosen": 0.08936743438243866, "rewards/margins": 2.546542167663574, "rewards/rejected": -2.457174777984619, "step": 182, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.4602326312480352, "grad_norm": 1.921773910522461, "learning_rate": 0.00019422002414907837, "logits/chosen": -0.5177928805351257, "logits/rejected": -0.6913548707962036, "logps/chosen": -5.233502388000488, "logps/rejected": -40.59012985229492, "loss": 0.5152122974395752, "memory(GiB)": 44.28, "nll_loss": 0.34400492906570435, "rewards/accuracies": 0.96875, "rewards/chosen": 0.14941944181919098, "rewards/margins": 2.640808582305908, "rewards/rejected": -2.491389513015747, "step": 183, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.46274756365922665, "grad_norm": 4.714654922485352, "learning_rate": 0.00019412659344716124, "logits/chosen": -0.5423455834388733, "logits/rejected": -0.6348575949668884, "logps/chosen": -10.46619987487793, "logps/rejected": -40.417510986328125, "loss": 0.974522054195404, "memory(GiB)": 44.28, "nll_loss": 0.6639810800552368, "rewards/accuracies": 0.875, "rewards/chosen": -0.15414148569107056, "rewards/margins": 2.581968307495117, "rewards/rejected": -2.736109972000122, "step": 184, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.4652624960704181, "grad_norm": 1.0332216024398804, "learning_rate": 0.00019403243649448787, "logits/chosen": -0.4484131336212158, "logits/rejected": -0.7438564896583557, "logps/chosen": -2.097586154937744, "logps/rejected": -52.63762664794922, "loss": 0.2517527937889099, "memory(GiB)": 44.28, "nll_loss": 0.15005721151828766, "rewards/accuracies": 0.96875, "rewards/chosen": 0.27270805835723877, "rewards/margins": 3.6738533973693848, "rewards/rejected": -3.4011456966400146, "step": 185, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.4677774284816096, "grad_norm": 6.869028091430664, "learning_rate": 0.00019393755401754322, "logits/chosen": -0.495494544506073, "logits/rejected": -0.7087106108665466, "logps/chosen": -5.248052597045898, "logps/rejected": -53.633941650390625, "loss": 0.7538958787918091, "memory(GiB)": 44.28, "nll_loss": 0.489072322845459, "rewards/accuracies": 0.875, "rewards/chosen": -0.09345913678407669, "rewards/margins": 3.6751837730407715, "rewards/rejected": -3.7686429023742676, "step": 186, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.470292360892801, "grad_norm": 8.480405807495117, "learning_rate": 0.00019384194674841026, "logits/chosen": -0.5885705947875977, "logits/rejected": -0.771264910697937, "logps/chosen": -8.139009475708008, "logps/rejected": -56.5067253112793, "loss": 0.9823139905929565, "memory(GiB)": 44.28, "nll_loss": 0.8092753291130066, "rewards/accuracies": 0.9375, "rewards/chosen": -0.3364828824996948, "rewards/margins": 4.044948577880859, "rewards/rejected": -4.381431579589844, "step": 187, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.47280729330399246, "grad_norm": 5.539621829986572, "learning_rate": 0.0001937456154247641, "logits/chosen": -0.5992149114608765, "logits/rejected": -0.7545720934867859, "logps/chosen": -5.675577640533447, "logps/rejected": -53.787532806396484, "loss": 0.4309059977531433, "memory(GiB)": 44.28, "nll_loss": 0.3199475407600403, "rewards/accuracies": 1.0, "rewards/chosen": -0.08290509879589081, "rewards/margins": 3.8106563091278076, "rewards/rejected": -3.893561601638794, "step": 188, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.4753222257151839, "grad_norm": 7.750495910644531, "learning_rate": 0.00019364856078986652, "logits/chosen": -0.5513782501220703, "logits/rejected": -0.6628525853157043, "logps/chosen": -13.800762176513672, "logps/rejected": -49.375457763671875, "loss": 1.059139370918274, "memory(GiB)": 44.28, "nll_loss": 0.7687480449676514, "rewards/accuracies": 0.875, "rewards/chosen": -0.11855825036764145, "rewards/margins": 3.4539475440979004, "rewards/rejected": -3.5725057125091553, "step": 189, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.47783715812637534, "grad_norm": 2.178455352783203, "learning_rate": 0.0001935507835925601, "logits/chosen": -0.5001492500305176, "logits/rejected": -0.683919370174408, "logps/chosen": -5.1807756423950195, "logps/rejected": -48.025428771972656, "loss": 0.4980285167694092, "memory(GiB)": 44.28, "nll_loss": 0.3027910590171814, "rewards/accuracies": 0.875, "rewards/chosen": 0.057751405984163284, "rewards/margins": 3.4277446269989014, "rewards/rejected": -3.3699934482574463, "step": 190, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.4803520905375668, "grad_norm": 2.17415714263916, "learning_rate": 0.00019345228458726252, "logits/chosen": -0.4294896423816681, "logits/rejected": -0.624671459197998, "logps/chosen": -6.253881454467773, "logps/rejected": -50.155250549316406, "loss": 0.5238272547721863, "memory(GiB)": 44.28, "nll_loss": 0.3330422341823578, "rewards/accuracies": 0.9375, "rewards/chosen": 0.024177849292755127, "rewards/margins": 2.913661003112793, "rewards/rejected": -2.8894832134246826, "step": 191, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.4828670229487583, "grad_norm": 5.977149486541748, "learning_rate": 0.00019335306453396064, "logits/chosen": -0.33437275886535645, "logits/rejected": -0.5742294788360596, "logps/chosen": -4.437854290008545, "logps/rejected": -45.52449035644531, "loss": 0.5229467153549194, "memory(GiB)": 44.28, "nll_loss": 0.2966506779193878, "rewards/accuracies": 0.875, "rewards/chosen": 0.040689948946237564, "rewards/margins": 2.79740571975708, "rewards/rejected": -2.7567155361175537, "step": 192, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.4853819553599497, "grad_norm": 4.404195308685303, "learning_rate": 0.00019325312419820473, "logits/chosen": -0.40588510036468506, "logits/rejected": -0.6035714149475098, "logps/chosen": -4.6151909828186035, "logps/rejected": -44.15699005126953, "loss": 0.5418132543563843, "memory(GiB)": 44.28, "nll_loss": 0.315884530544281, "rewards/accuracies": 0.96875, "rewards/chosen": 0.029081931337714195, "rewards/margins": 2.7166473865509033, "rewards/rejected": -2.687565565109253, "step": 193, "train_speed(iter/s)": 0.01124 }, { "epoch": 0.48789688777114115, "grad_norm": 2.59771466255188, "learning_rate": 0.00019315246435110256, "logits/chosen": -0.4158392548561096, "logits/rejected": -0.6061286330223083, "logps/chosen": -6.033608436584473, "logps/rejected": -42.764007568359375, "loss": 0.5932649970054626, "memory(GiB)": 44.28, "nll_loss": 0.43631434440612793, "rewards/accuracies": 0.9375, "rewards/chosen": 0.22790978848934174, "rewards/margins": 2.7686610221862793, "rewards/rejected": -2.5407509803771973, "step": 194, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.4904118201823326, "grad_norm": 2.5979843139648438, "learning_rate": 0.00019305108576931336, "logits/chosen": -0.4368806779384613, "logits/rejected": -0.6110289692878723, "logps/chosen": -6.914710998535156, "logps/rejected": -49.386474609375, "loss": 0.6119182705879211, "memory(GiB)": 44.28, "nll_loss": 0.38600677251815796, "rewards/accuracies": 0.90625, "rewards/chosen": 0.016620682552456856, "rewards/margins": 3.258530616760254, "rewards/rejected": -3.241909980773926, "step": 195, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.49292675259352403, "grad_norm": 1.838233232498169, "learning_rate": 0.0001929489892350419, "logits/chosen": -0.4425433874130249, "logits/rejected": -0.643796443939209, "logps/chosen": -3.6048429012298584, "logps/rejected": -54.97930908203125, "loss": 0.386331170797348, "memory(GiB)": 44.28, "nll_loss": 0.28649255633354187, "rewards/accuracies": 1.0, "rewards/chosen": 0.14747180044651031, "rewards/margins": 4.032291889190674, "rewards/rejected": -3.884819984436035, "step": 196, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.49544168500471547, "grad_norm": 2.9112889766693115, "learning_rate": 0.00019284617553603242, "logits/chosen": -0.48377659916877747, "logits/rejected": -0.605985164642334, "logps/chosen": -4.890591144561768, "logps/rejected": -54.24449920654297, "loss": 0.5024012327194214, "memory(GiB)": 44.28, "nll_loss": 0.34084275364875793, "rewards/accuracies": 0.9375, "rewards/chosen": 0.23494963347911835, "rewards/margins": 3.875343084335327, "rewards/rejected": -3.6403934955596924, "step": 197, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.49795661741590697, "grad_norm": 3.7803006172180176, "learning_rate": 0.0001927426454655627, "logits/chosen": -0.3935869038105011, "logits/rejected": -0.5888193845748901, "logps/chosen": -7.011892318725586, "logps/rejected": -54.100929260253906, "loss": 0.5712710618972778, "memory(GiB)": 44.28, "nll_loss": 0.3695356249809265, "rewards/accuracies": 0.9375, "rewards/chosen": 0.02423219382762909, "rewards/margins": 3.535330057144165, "rewards/rejected": -3.5110979080200195, "step": 198, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.5004715498270984, "grad_norm": 5.7016282081604, "learning_rate": 0.0001926383998224377, "logits/chosen": -0.3959541618824005, "logits/rejected": -0.6700530648231506, "logps/chosen": -2.51404070854187, "logps/rejected": -55.97195816040039, "loss": 0.3349680006504059, "memory(GiB)": 46.04, "nll_loss": 0.20782868564128876, "rewards/accuracies": 0.9375, "rewards/chosen": 0.11337391287088394, "rewards/margins": 3.802442789077759, "rewards/rejected": -3.6890690326690674, "step": 199, "train_speed(iter/s)": 0.011241 }, { "epoch": 0.5029864822382899, "grad_norm": 1.2177132368087769, "learning_rate": 0.0001925334394109835, "logits/chosen": -0.39053067564964294, "logits/rejected": -0.6353622674942017, "logps/chosen": -4.960577964782715, "logps/rejected": -55.41823196411133, "loss": 0.4196793735027313, "memory(GiB)": 46.04, "nll_loss": 0.2920268774032593, "rewards/accuracies": 0.96875, "rewards/chosen": 0.06429460644721985, "rewards/margins": 3.601248025894165, "rewards/rejected": -3.5369529724121094, "step": 200, "train_speed(iter/s)": 0.011242 }, { "epoch": 0.5055014146494813, "grad_norm": 7.681524753570557, "learning_rate": 0.00019242776504104118, "logits/chosen": -0.3979511260986328, "logits/rejected": -0.5972259044647217, "logps/chosen": -6.038961887359619, "logps/rejected": -54.249908447265625, "loss": 0.6535472869873047, "memory(GiB)": 46.04, "nll_loss": 0.41866153478622437, "rewards/accuracies": 0.84375, "rewards/chosen": 0.054757650941610336, "rewards/margins": 3.7828688621520996, "rewards/rejected": -3.7281110286712646, "step": 201, "train_speed(iter/s)": 0.011231 }, { "epoch": 0.5080163470606728, "grad_norm": 2.600893497467041, "learning_rate": 0.00019232137752796044, "logits/chosen": -0.4759564995765686, "logits/rejected": -0.6580658555030823, "logps/chosen": -4.586948394775391, "logps/rejected": -51.12422180175781, "loss": 0.6128251552581787, "memory(GiB)": 46.04, "nll_loss": 0.46890994906425476, "rewards/accuracies": 0.96875, "rewards/chosen": 0.11943411082029343, "rewards/margins": 3.681659460067749, "rewards/rejected": -3.562225580215454, "step": 202, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.5105312794718642, "grad_norm": 2.17677640914917, "learning_rate": 0.00019221427769259333, "logits/chosen": -0.39213788509368896, "logits/rejected": -0.6274294853210449, "logps/chosen": -4.02351713180542, "logps/rejected": -54.13360595703125, "loss": 0.4597245752811432, "memory(GiB)": 46.04, "nll_loss": 0.31017711758613586, "rewards/accuracies": 0.96875, "rewards/chosen": 0.19347746670246124, "rewards/margins": 3.6046862602233887, "rewards/rejected": -3.4112088680267334, "step": 203, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.5130462118830557, "grad_norm": 3.405174732208252, "learning_rate": 0.00019210646636128807, "logits/chosen": -0.4578840434551239, "logits/rejected": -0.6036797165870667, "logps/chosen": -4.427443027496338, "logps/rejected": -44.289703369140625, "loss": 0.4890953004360199, "memory(GiB)": 46.04, "nll_loss": 0.2529718279838562, "rewards/accuracies": 0.90625, "rewards/chosen": 0.10575352609157562, "rewards/margins": 3.2759764194488525, "rewards/rejected": -3.170222759246826, "step": 204, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.515561144294247, "grad_norm": 1.9355807304382324, "learning_rate": 0.00019199794436588243, "logits/chosen": -0.3543737232685089, "logits/rejected": -0.6088857650756836, "logps/chosen": -6.951679229736328, "logps/rejected": -68.91035461425781, "loss": 0.39467012882232666, "memory(GiB)": 46.04, "nll_loss": 0.27390897274017334, "rewards/accuracies": 0.96875, "rewards/chosen": 0.13660939037799835, "rewards/margins": 4.888247013092041, "rewards/rejected": -4.751636981964111, "step": 205, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.5180760767054385, "grad_norm": 12.973567962646484, "learning_rate": 0.00019188871254369752, "logits/chosen": -0.4342585802078247, "logits/rejected": -0.5918402671813965, "logps/chosen": -5.572432518005371, "logps/rejected": -47.04520034790039, "loss": 0.8162675499916077, "memory(GiB)": 46.04, "nll_loss": 0.5575422048568726, "rewards/accuracies": 0.90625, "rewards/chosen": -0.07569149881601334, "rewards/margins": 2.895029067993164, "rewards/rejected": -2.9707207679748535, "step": 206, "train_speed(iter/s)": 0.011232 }, { "epoch": 0.52059100911663, "grad_norm": 3.5247535705566406, "learning_rate": 0.00019177877173753128, "logits/chosen": -0.48035189509391785, "logits/rejected": -0.6423895359039307, "logps/chosen": -6.628141403198242, "logps/rejected": -52.741973876953125, "loss": 0.6659956574440002, "memory(GiB)": 46.04, "nll_loss": 0.4173967242240906, "rewards/accuracies": 0.84375, "rewards/chosen": -0.11046299338340759, "rewards/margins": 3.8508124351501465, "rewards/rejected": -3.961275577545166, "step": 207, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5231059415278214, "grad_norm": 9.15176773071289, "learning_rate": 0.00019166812279565178, "logits/chosen": -0.4394856095314026, "logits/rejected": -0.5828301906585693, "logps/chosen": -6.419562816619873, "logps/rejected": -65.78706359863281, "loss": 0.7201980948448181, "memory(GiB)": 46.04, "nll_loss": 0.5739071369171143, "rewards/accuracies": 0.9375, "rewards/chosen": 0.05124050751328468, "rewards/margins": 5.137848377227783, "rewards/rejected": -5.086607933044434, "step": 208, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5256208739390129, "grad_norm": 1.9403810501098633, "learning_rate": 0.000191556766571791, "logits/chosen": -0.36848342418670654, "logits/rejected": -0.6423362493515015, "logps/chosen": -4.248262882232666, "logps/rejected": -82.75714111328125, "loss": 0.3508817255496979, "memory(GiB)": 46.04, "nll_loss": 0.274202436208725, "rewards/accuracies": 1.0, "rewards/chosen": 0.14326420426368713, "rewards/margins": 6.368471622467041, "rewards/rejected": -6.225207328796387, "step": 209, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5281358063502043, "grad_norm": 2.350217342376709, "learning_rate": 0.000191444703925138, "logits/chosen": -0.36294057965278625, "logits/rejected": -0.681117057800293, "logps/chosen": -3.7879486083984375, "logps/rejected": -73.4488525390625, "loss": 0.4432718753814697, "memory(GiB)": 46.04, "nll_loss": 0.3073051869869232, "rewards/accuracies": 0.96875, "rewards/chosen": 0.17865654826164246, "rewards/margins": 5.326575756072998, "rewards/rejected": -5.14792013168335, "step": 210, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5306507387613958, "grad_norm": 9.633001327514648, "learning_rate": 0.00019133193572033242, "logits/chosen": -0.35440313816070557, "logits/rejected": -0.647504448890686, "logps/chosen": -5.407248020172119, "logps/rejected": -69.90013122558594, "loss": 0.4669504165649414, "memory(GiB)": 46.04, "nll_loss": 0.26951655745506287, "rewards/accuracies": 0.875, "rewards/chosen": -0.040601931512355804, "rewards/margins": 5.3051629066467285, "rewards/rejected": -5.345764636993408, "step": 211, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5331656711725873, "grad_norm": 2.539501428604126, "learning_rate": 0.00019121846282745778, "logits/chosen": -0.4186283349990845, "logits/rejected": -0.5546895265579224, "logps/chosen": -5.969156742095947, "logps/rejected": -55.70306396484375, "loss": 0.5105364918708801, "memory(GiB)": 46.04, "nll_loss": 0.33588486909866333, "rewards/accuracies": 0.96875, "rewards/chosen": 0.05188417807221413, "rewards/margins": 4.2251996994018555, "rewards/rejected": -4.173315525054932, "step": 212, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5356806035837787, "grad_norm": 2.609890937805176, "learning_rate": 0.00019110428612203464, "logits/chosen": -0.40281373262405396, "logits/rejected": -0.6546099185943604, "logps/chosen": -6.447667121887207, "logps/rejected": -89.42594909667969, "loss": 0.43404707312583923, "memory(GiB)": 46.04, "nll_loss": 0.29497838020324707, "rewards/accuracies": 0.96875, "rewards/chosen": 0.05883106589317322, "rewards/margins": 6.6620965003967285, "rewards/rejected": -6.603265285491943, "step": 213, "train_speed(iter/s)": 0.011233 }, { "epoch": 0.5381955359949702, "grad_norm": 9.001029968261719, "learning_rate": 0.00019098940648501405, "logits/chosen": -0.4067293107509613, "logits/rejected": -0.6849221587181091, "logps/chosen": -3.5787479877471924, "logps/rejected": -69.67017364501953, "loss": 0.4548971951007843, "memory(GiB)": 46.04, "nll_loss": 0.29569321870803833, "rewards/accuracies": 0.9375, "rewards/chosen": -0.01640375703573227, "rewards/margins": 5.399847507476807, "rewards/rejected": -5.416250705718994, "step": 214, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5407104684061615, "grad_norm": 3.073113203048706, "learning_rate": 0.0001908738248027706, "logits/chosen": -0.38208895921707153, "logits/rejected": -0.6967729926109314, "logps/chosen": -4.078890323638916, "logps/rejected": -85.77256774902344, "loss": 0.37352272868156433, "memory(GiB)": 46.04, "nll_loss": 0.27824363112449646, "rewards/accuracies": 0.96875, "rewards/chosen": 0.0630294606089592, "rewards/margins": 6.849693298339844, "rewards/rejected": -6.78666353225708, "step": 215, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.543225400817353, "grad_norm": 4.24471378326416, "learning_rate": 0.00019075754196709572, "logits/chosen": -0.4263435900211334, "logits/rejected": -0.7725025415420532, "logps/chosen": -2.7350358963012695, "logps/rejected": -105.30278015136719, "loss": 0.3268907964229584, "memory(GiB)": 46.04, "nll_loss": 0.2297362983226776, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09944246709346771, "rewards/margins": 8.845938682556152, "rewards/rejected": -8.746496200561523, "step": 216, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5457403332285444, "grad_norm": 7.5312819480896, "learning_rate": 0.00019064055887519062, "logits/chosen": -0.48908063769340515, "logits/rejected": -0.6851401329040527, "logps/chosen": -8.316049575805664, "logps/rejected": -87.41441345214844, "loss": 0.7208706140518188, "memory(GiB)": 46.04, "nll_loss": 0.5370450615882874, "rewards/accuracies": 0.9375, "rewards/chosen": -0.054982058703899384, "rewards/margins": 7.039226055145264, "rewards/rejected": -7.094207763671875, "step": 217, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5482552656397359, "grad_norm": 5.015770435333252, "learning_rate": 0.00019052287642965952, "logits/chosen": -0.4340421259403229, "logits/rejected": -0.6953417062759399, "logps/chosen": -6.858056545257568, "logps/rejected": -93.66557312011719, "loss": 0.5943813323974609, "memory(GiB)": 46.04, "nll_loss": 0.40210384130477905, "rewards/accuracies": 0.875, "rewards/chosen": -0.02029874362051487, "rewards/margins": 7.397557258605957, "rewards/rejected": -7.417856693267822, "step": 218, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5507701980509274, "grad_norm": 25.73984718322754, "learning_rate": 0.00019040449553850257, "logits/chosen": -0.4239513576030731, "logits/rejected": -0.6732109785079956, "logps/chosen": -8.325424194335938, "logps/rejected": -87.74398040771484, "loss": 0.8946905732154846, "memory(GiB)": 46.04, "nll_loss": 0.648015558719635, "rewards/accuracies": 0.9375, "rewards/chosen": -0.2404731959104538, "rewards/margins": 6.910739898681641, "rewards/rejected": -7.151212215423584, "step": 219, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5532851304621188, "grad_norm": 2.769166946411133, "learning_rate": 0.000190285417115109, "logits/chosen": -0.49040669202804565, "logits/rejected": -0.7523952126502991, "logps/chosen": -7.0566301345825195, "logps/rejected": -82.0855941772461, "loss": 0.5741197466850281, "memory(GiB)": 46.04, "nll_loss": 0.4936599135398865, "rewards/accuracies": 0.96875, "rewards/chosen": 0.05626031011343002, "rewards/margins": 6.2684006690979, "rewards/rejected": -6.2121405601501465, "step": 220, "train_speed(iter/s)": 0.011234 }, { "epoch": 0.5558000628733103, "grad_norm": 4.775945663452148, "learning_rate": 0.00019016564207824992, "logits/chosen": -0.5227105021476746, "logits/rejected": -0.6636734008789062, "logps/chosen": -4.836442947387695, "logps/rejected": -49.289161682128906, "loss": 0.5425068736076355, "memory(GiB)": 46.04, "nll_loss": 0.35901886224746704, "rewards/accuracies": 0.96875, "rewards/chosen": 0.08150728791952133, "rewards/margins": 3.6998212337493896, "rewards/rejected": -3.618313789367676, "step": 221, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.5583149952845017, "grad_norm": 4.910019874572754, "learning_rate": 0.00019004517135207127, "logits/chosen": -0.3941330313682556, "logits/rejected": -0.660327136516571, "logps/chosen": -4.9822516441345215, "logps/rejected": -77.26637268066406, "loss": 0.5757743120193481, "memory(GiB)": 46.04, "nll_loss": 0.39564475417137146, "rewards/accuracies": 0.96875, "rewards/chosen": 0.17502854764461517, "rewards/margins": 5.9140496253967285, "rewards/rejected": -5.739021301269531, "step": 222, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.5608299276956932, "grad_norm": 6.759812355041504, "learning_rate": 0.00018992400586608676, "logits/chosen": -0.4000013470649719, "logits/rejected": -0.7170141935348511, "logps/chosen": -2.594712972640991, "logps/rejected": -67.61477661132812, "loss": 0.4701526165008545, "memory(GiB)": 46.04, "nll_loss": 0.26969754695892334, "rewards/accuracies": 0.875, "rewards/chosen": -0.03438286855816841, "rewards/margins": 4.8134918212890625, "rewards/rejected": -4.847874641418457, "step": 223, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.5633448601068847, "grad_norm": 2.779904842376709, "learning_rate": 0.00018980214655517057, "logits/chosen": -0.3656763732433319, "logits/rejected": -0.6763076186180115, "logps/chosen": -2.6532158851623535, "logps/rejected": -62.15465545654297, "loss": 0.3870552182197571, "memory(GiB)": 46.04, "nll_loss": 0.2073136270046234, "rewards/accuracies": 0.90625, "rewards/chosen": 0.13753379881381989, "rewards/margins": 4.43159818649292, "rewards/rejected": -4.294064521789551, "step": 224, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.565859792518076, "grad_norm": 4.610898971557617, "learning_rate": 0.00018967959435955026, "logits/chosen": -0.41923901438713074, "logits/rejected": -0.7322454452514648, "logps/chosen": -4.079246997833252, "logps/rejected": -85.366943359375, "loss": 0.5197367668151855, "memory(GiB)": 46.04, "nll_loss": 0.3901809751987457, "rewards/accuracies": 0.90625, "rewards/chosen": -0.0011233258992433548, "rewards/margins": 6.633144378662109, "rewards/rejected": -6.634267807006836, "step": 225, "train_speed(iter/s)": 0.011235 }, { "epoch": 0.5683747249292675, "grad_norm": 1.7400152683258057, "learning_rate": 0.0001895563502247995, "logits/chosen": -0.4450499713420868, "logits/rejected": -0.6796970367431641, "logps/chosen": -3.0244576930999756, "logps/rejected": -73.87388610839844, "loss": 0.3383142650127411, "memory(GiB)": 46.04, "nll_loss": 0.21261034905910492, "rewards/accuracies": 1.0, "rewards/chosen": 0.16421900689601898, "rewards/margins": 5.867359161376953, "rewards/rejected": -5.703140735626221, "step": 226, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5708896573404589, "grad_norm": 2.3019213676452637, "learning_rate": 0.00018943241510183063, "logits/chosen": -0.458271324634552, "logits/rejected": -0.6559279561042786, "logps/chosen": -6.137403964996338, "logps/rejected": -69.26449584960938, "loss": 0.43954962491989136, "memory(GiB)": 46.04, "nll_loss": 0.3364819884300232, "rewards/accuracies": 0.96875, "rewards/chosen": 0.18200799822807312, "rewards/margins": 5.23583459854126, "rewards/rejected": -5.053826332092285, "step": 227, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5734045897516504, "grad_norm": 4.305068492889404, "learning_rate": 0.00018930778994688757, "logits/chosen": -0.4849119186401367, "logits/rejected": -0.6896030902862549, "logps/chosen": -6.080837726593018, "logps/rejected": -76.35552978515625, "loss": 0.6719568967819214, "memory(GiB)": 46.04, "nll_loss": 0.47549009323120117, "rewards/accuracies": 0.9375, "rewards/chosen": 0.02206701785326004, "rewards/margins": 5.978794097900391, "rewards/rejected": -5.956727027893066, "step": 228, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5759195221628419, "grad_norm": 2.6933469772338867, "learning_rate": 0.00018918247572153823, "logits/chosen": -0.461727499961853, "logits/rejected": -0.6680722236633301, "logps/chosen": -4.37862491607666, "logps/rejected": -86.59188842773438, "loss": 0.42717570066452026, "memory(GiB)": 46.04, "nll_loss": 0.3303125500679016, "rewards/accuracies": 0.9375, "rewards/chosen": 0.08164448291063309, "rewards/margins": 7.104611873626709, "rewards/rejected": -7.022968292236328, "step": 229, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5784344545740333, "grad_norm": 8.504460334777832, "learning_rate": 0.0001890564733926672, "logits/chosen": -0.34246841073036194, "logits/rejected": -0.5939763188362122, "logps/chosen": -11.11518383026123, "logps/rejected": -101.42696380615234, "loss": 0.7018269300460815, "memory(GiB)": 46.04, "nll_loss": 0.4599987864494324, "rewards/accuracies": 0.84375, "rewards/chosen": -0.15412060916423798, "rewards/margins": 7.550798416137695, "rewards/rejected": -7.704918384552002, "step": 230, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5809493869852248, "grad_norm": 3.522885322570801, "learning_rate": 0.00018892978393246818, "logits/chosen": -0.5426961183547974, "logits/rejected": -0.6135550737380981, "logps/chosen": -10.763383865356445, "logps/rejected": -72.09516143798828, "loss": 0.5679591298103333, "memory(GiB)": 46.04, "nll_loss": 0.39400285482406616, "rewards/accuracies": 0.9375, "rewards/chosen": 0.07710488885641098, "rewards/margins": 5.834630012512207, "rewards/rejected": -5.757525444030762, "step": 231, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5834643193964162, "grad_norm": 2.6743204593658447, "learning_rate": 0.00018880240831843666, "logits/chosen": -0.42806094884872437, "logits/rejected": -0.6858870983123779, "logps/chosen": -7.369032859802246, "logps/rejected": -94.46809387207031, "loss": 0.49837395548820496, "memory(GiB)": 46.04, "nll_loss": 0.3631903827190399, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1583317369222641, "rewards/margins": 7.130619525909424, "rewards/rejected": -6.972287654876709, "step": 232, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5859792518076077, "grad_norm": 2.545430898666382, "learning_rate": 0.00018867434753336223, "logits/chosen": -0.5526742935180664, "logits/rejected": -0.6764613389968872, "logps/chosen": -6.2565155029296875, "logps/rejected": -53.53122329711914, "loss": 0.6385827660560608, "memory(GiB)": 46.04, "nll_loss": 0.47814565896987915, "rewards/accuracies": 0.9375, "rewards/chosen": 0.18877720832824707, "rewards/margins": 4.254955768585205, "rewards/rejected": -4.066178321838379, "step": 233, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5884941842187991, "grad_norm": 5.874550819396973, "learning_rate": 0.000188545602565321, "logits/chosen": -0.5431405901908875, "logits/rejected": -0.6591349840164185, "logps/chosen": -7.041271686553955, "logps/rejected": -51.830989837646484, "loss": 0.511455774307251, "memory(GiB)": 46.04, "nll_loss": 0.3468693494796753, "rewards/accuracies": 0.96875, "rewards/chosen": 0.17714644968509674, "rewards/margins": 3.9428811073303223, "rewards/rejected": -3.7657346725463867, "step": 234, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5910091166299906, "grad_norm": 11.230829238891602, "learning_rate": 0.0001884161744076681, "logits/chosen": -0.4081313908100128, "logits/rejected": -0.6242074370384216, "logps/chosen": -5.126794815063477, "logps/rejected": -49.272377014160156, "loss": 0.6269602179527283, "memory(GiB)": 46.04, "nll_loss": 0.440512478351593, "rewards/accuracies": 0.90625, "rewards/chosen": 0.023164164274930954, "rewards/margins": 3.228619337081909, "rewards/rejected": -3.2054548263549805, "step": 235, "train_speed(iter/s)": 0.011236 }, { "epoch": 0.5935240490411821, "grad_norm": 2.906224489212036, "learning_rate": 0.00018828606405902986, "logits/chosen": -0.4151853919029236, "logits/rejected": -0.6895049810409546, "logps/chosen": -3.3425395488739014, "logps/rejected": -60.14630126953125, "loss": 0.40797102451324463, "memory(GiB)": 46.04, "nll_loss": 0.3100608289241791, "rewards/accuracies": 1.0, "rewards/chosen": 0.0746859535574913, "rewards/margins": 4.381271839141846, "rewards/rejected": -4.306585788726807, "step": 236, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.5960389814523734, "grad_norm": 5.449255466461182, "learning_rate": 0.00018815527252329624, "logits/chosen": -0.5026019811630249, "logits/rejected": -0.6662022471427917, "logps/chosen": -6.956883430480957, "logps/rejected": -59.189815521240234, "loss": 0.6439992785453796, "memory(GiB)": 46.04, "nll_loss": 0.448159396648407, "rewards/accuracies": 0.875, "rewards/chosen": 0.038928791880607605, "rewards/margins": 4.4153032302856445, "rewards/rejected": -4.376374244689941, "step": 237, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.5985539138635649, "grad_norm": 1.9680973291397095, "learning_rate": 0.00018802380080961296, "logits/chosen": -0.4111896753311157, "logits/rejected": -0.6969754695892334, "logps/chosen": -2.109022617340088, "logps/rejected": -70.76811981201172, "loss": 0.30665695667266846, "memory(GiB)": 46.04, "nll_loss": 0.19306617975234985, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1144958883523941, "rewards/margins": 5.321486473083496, "rewards/rejected": -5.206991195678711, "step": 238, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.6010688462747563, "grad_norm": 2.106550931930542, "learning_rate": 0.00018789164993237382, "logits/chosen": -0.5053385496139526, "logits/rejected": -0.6760715842247009, "logps/chosen": -3.813554048538208, "logps/rejected": -71.93482971191406, "loss": 0.3176461160182953, "memory(GiB)": 46.04, "nll_loss": 0.18995390832424164, "rewards/accuracies": 0.9375, "rewards/chosen": 0.16428788006305695, "rewards/margins": 5.706036567687988, "rewards/rejected": -5.541748523712158, "step": 239, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.6035837786859478, "grad_norm": 17.008098602294922, "learning_rate": 0.0001877588209112128, "logits/chosen": -0.4033944010734558, "logits/rejected": -0.6984363794326782, "logps/chosen": -5.072837829589844, "logps/rejected": -76.01246643066406, "loss": 0.8396996259689331, "memory(GiB)": 46.04, "nll_loss": 0.458006352186203, "rewards/accuracies": 0.875, "rewards/chosen": -0.022617299109697342, "rewards/margins": 5.918478965759277, "rewards/rejected": -5.941096305847168, "step": 240, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.6060987110971393, "grad_norm": 1.8717386722564697, "learning_rate": 0.00018762531477099626, "logits/chosen": -0.3073655366897583, "logits/rejected": -0.6664487719535828, "logps/chosen": -5.499012470245361, "logps/rejected": -105.34025573730469, "loss": 0.46345487236976624, "memory(GiB)": 46.04, "nll_loss": 0.321692556142807, "rewards/accuracies": 0.96875, "rewards/chosen": 0.015820708125829697, "rewards/margins": 8.114753723144531, "rewards/rejected": -8.098933219909668, "step": 241, "train_speed(iter/s)": 0.011237 }, { "epoch": 0.6086136435083307, "grad_norm": 12.112175941467285, "learning_rate": 0.00018749113254181498, "logits/chosen": -0.5718834400177002, "logits/rejected": -0.7824399471282959, "logps/chosen": -3.723027229309082, "logps/rejected": -83.3447265625, "loss": 0.4468189775943756, "memory(GiB)": 46.04, "nll_loss": 0.3446814715862274, "rewards/accuracies": 0.96875, "rewards/chosen": -0.03424655646085739, "rewards/margins": 7.22880744934082, "rewards/rejected": -7.263053894042969, "step": 242, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6111285759195222, "grad_norm": 3.188166618347168, "learning_rate": 0.0001873562752589762, "logits/chosen": -0.47086432576179504, "logits/rejected": -0.6838089823722839, "logps/chosen": -8.102127075195312, "logps/rejected": -79.43113708496094, "loss": 0.47154104709625244, "memory(GiB)": 46.04, "nll_loss": 0.3822014331817627, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09235017001628876, "rewards/margins": 6.291683197021484, "rewards/rejected": -6.1993327140808105, "step": 243, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6136435083307136, "grad_norm": 4.067774772644043, "learning_rate": 0.00018722074396299566, "logits/chosen": -0.46753576397895813, "logits/rejected": -0.7191226482391357, "logps/chosen": -9.372048377990723, "logps/rejected": -93.54489135742188, "loss": 0.4576195180416107, "memory(GiB)": 46.04, "nll_loss": 0.31360840797424316, "rewards/accuracies": 0.875, "rewards/chosen": 0.07592315226793289, "rewards/margins": 7.556896209716797, "rewards/rejected": -7.480972766876221, "step": 244, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6161584407419051, "grad_norm": 2.5217037200927734, "learning_rate": 0.00018708453969958958, "logits/chosen": -0.5039016008377075, "logits/rejected": -0.6719211935997009, "logps/chosen": -6.228890419006348, "logps/rejected": -58.18281936645508, "loss": 0.7915486097335815, "memory(GiB)": 46.04, "nll_loss": 0.5135347247123718, "rewards/accuracies": 0.9375, "rewards/chosen": -0.017953312024474144, "rewards/margins": 4.20432186126709, "rewards/rejected": -4.222275733947754, "step": 245, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6186733731530966, "grad_norm": 1.6694215536117554, "learning_rate": 0.00018694766351966667, "logits/chosen": -0.4057798981666565, "logits/rejected": -0.6351858973503113, "logps/chosen": -6.20676326751709, "logps/rejected": -72.40664672851562, "loss": 0.4737001359462738, "memory(GiB)": 46.04, "nll_loss": 0.35722869634628296, "rewards/accuracies": 0.96875, "rewards/chosen": 0.26643669605255127, "rewards/margins": 5.429479598999023, "rewards/rejected": -5.163043022155762, "step": 246, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.621188305564288, "grad_norm": 4.144296169281006, "learning_rate": 0.00018681011647931974, "logits/chosen": -0.34827110171318054, "logits/rejected": -0.6173001527786255, "logps/chosen": -6.582627773284912, "logps/rejected": -78.4413070678711, "loss": 0.6141136288642883, "memory(GiB)": 46.04, "nll_loss": 0.5033036470413208, "rewards/accuracies": 1.0, "rewards/chosen": 0.004567645490169525, "rewards/margins": 5.899599552154541, "rewards/rejected": -5.895031929016113, "step": 247, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6237032379754794, "grad_norm": 13.872243881225586, "learning_rate": 0.00018667189963981793, "logits/chosen": -0.387536883354187, "logits/rejected": -0.6405035257339478, "logps/chosen": -3.9526758193969727, "logps/rejected": -55.78480529785156, "loss": 0.5534759163856506, "memory(GiB)": 46.04, "nll_loss": 0.38310396671295166, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1983298808336258, "rewards/margins": 3.9616622924804688, "rewards/rejected": -3.7633323669433594, "step": 248, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6262181703866708, "grad_norm": 4.9104180335998535, "learning_rate": 0.00018653301406759825, "logits/chosen": -0.43706703186035156, "logits/rejected": -0.7033647298812866, "logps/chosen": -4.780174732208252, "logps/rejected": -63.24848937988281, "loss": 0.4667983949184418, "memory(GiB)": 46.04, "nll_loss": 0.3596409559249878, "rewards/accuracies": 0.96875, "rewards/chosen": 0.10462813079357147, "rewards/margins": 4.852310657501221, "rewards/rejected": -4.747682094573975, "step": 249, "train_speed(iter/s)": 0.011238 }, { "epoch": 0.6287331027978623, "grad_norm": 2.1640334129333496, "learning_rate": 0.00018639346083425748, "logits/chosen": -0.46963977813720703, "logits/rejected": -0.6406612396240234, "logps/chosen": -7.166382789611816, "logps/rejected": -57.39030838012695, "loss": 0.6180220246315002, "memory(GiB)": 46.04, "nll_loss": 0.49737781286239624, "rewards/accuracies": 0.96875, "rewards/chosen": 0.22726891934871674, "rewards/margins": 4.274770736694336, "rewards/rejected": -4.047501564025879, "step": 250, "train_speed(iter/s)": 0.011239 }, { "epoch": 0.6312480352090537, "grad_norm": 1.6145262718200684, "learning_rate": 0.00018625324101654377, "logits/chosen": -0.30196619033813477, "logits/rejected": -0.6030948162078857, "logps/chosen": -4.5810980796813965, "logps/rejected": -69.94066619873047, "loss": 0.44845062494277954, "memory(GiB)": 33.43, "nll_loss": 0.3242243528366089, "rewards/accuracies": 0.96875, "rewards/chosen": 0.13189926743507385, "rewards/margins": 5.211311340332031, "rewards/rejected": -5.07941198348999, "step": 251, "train_speed(iter/s)": 2.643562 }, { "epoch": 0.6337629676202452, "grad_norm": 2.0983619689941406, "learning_rate": 0.00018611235569634854, "logits/chosen": -0.33113306760787964, "logits/rejected": -0.5730663537979126, "logps/chosen": -5.2111029624938965, "logps/rejected": -66.04705047607422, "loss": 0.39135444164276123, "memory(GiB)": 33.43, "nll_loss": 0.282539963722229, "rewards/accuracies": 1.0, "rewards/chosen": 0.20458640158176422, "rewards/margins": 5.00150203704834, "rewards/rejected": -4.7969160079956055, "step": 252, "train_speed(iter/s)": 1.370537 }, { "epoch": 0.6362779000314367, "grad_norm": 2.1315553188323975, "learning_rate": 0.00018597080596069793, "logits/chosen": -0.45159927010536194, "logits/rejected": -0.6239475011825562, "logps/chosen": -8.911016464233398, "logps/rejected": -51.39215850830078, "loss": 0.6090635061264038, "memory(GiB)": 33.43, "nll_loss": 0.46375155448913574, "rewards/accuracies": 0.96875, "rewards/chosen": 0.16291305422782898, "rewards/margins": 3.745710611343384, "rewards/rejected": -3.5827975273132324, "step": 253, "train_speed(iter/s)": 0.928331 }, { "epoch": 0.6387928324426281, "grad_norm": 1.5824214220046997, "learning_rate": 0.00018582859290174452, "logits/chosen": -0.4049888849258423, "logits/rejected": -0.5815865993499756, "logps/chosen": -4.045827388763428, "logps/rejected": -46.12110137939453, "loss": 0.4411042332649231, "memory(GiB)": 34.43, "nll_loss": 0.27207979559898376, "rewards/accuracies": 0.9375, "rewards/chosen": 0.25326088070869446, "rewards/margins": 3.262639045715332, "rewards/rejected": -3.009377956390381, "step": 254, "train_speed(iter/s)": 0.701344 }, { "epoch": 0.6413077648538196, "grad_norm": 9.893558502197266, "learning_rate": 0.00018568571761675893, "logits/chosen": -0.4062230885028839, "logits/rejected": -0.5557867288589478, "logps/chosen": -5.695141792297363, "logps/rejected": -50.45267105102539, "loss": 0.6555861234664917, "memory(GiB)": 34.43, "nll_loss": 0.4070875644683838, "rewards/accuracies": 0.875, "rewards/chosen": 0.018757404759526253, "rewards/margins": 3.3658759593963623, "rewards/rejected": -3.347118854522705, "step": 255, "train_speed(iter/s)": 0.56489 }, { "epoch": 0.643822697265011, "grad_norm": 2.4083399772644043, "learning_rate": 0.00018554218120812123, "logits/chosen": -0.33443182706832886, "logits/rejected": -0.5299184322357178, "logps/chosen": -10.211224555969238, "logps/rejected": -49.86205291748047, "loss": 0.7328925132751465, "memory(GiB)": 34.43, "nll_loss": 0.535182774066925, "rewards/accuracies": 0.9375, "rewards/chosen": 0.15930601954460144, "rewards/margins": 3.196995735168457, "rewards/rejected": -3.037689685821533, "step": 256, "train_speed(iter/s)": 0.473313 }, { "epoch": 0.6463376296762025, "grad_norm": 3.05432391166687, "learning_rate": 0.00018539798478331253, "logits/chosen": -0.36439672112464905, "logits/rejected": -0.5713220834732056, "logps/chosen": -5.492653846740723, "logps/rejected": -63.25914001464844, "loss": 0.4198300540447235, "memory(GiB)": 34.43, "nll_loss": 0.3039914071559906, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1935369074344635, "rewards/margins": 4.819316387176514, "rewards/rejected": -4.62578010559082, "step": 257, "train_speed(iter/s)": 0.40816 }, { "epoch": 0.648852562087394, "grad_norm": 4.973176956176758, "learning_rate": 0.00018525312945490648, "logits/chosen": -0.4139503240585327, "logits/rejected": -0.49163496494293213, "logps/chosen": -7.645513534545898, "logps/rejected": -42.70791244506836, "loss": 0.7363718748092651, "memory(GiB)": 34.43, "nll_loss": 0.4982314109802246, "rewards/accuracies": 0.9375, "rewards/chosen": 0.08269016444683075, "rewards/margins": 3.138368606567383, "rewards/rejected": -3.055678606033325, "step": 258, "train_speed(iter/s)": 0.359112 }, { "epoch": 0.6513674944985853, "grad_norm": 1.6409480571746826, "learning_rate": 0.0001851076163405605, "logits/chosen": -0.3667711913585663, "logits/rejected": -0.5486889481544495, "logps/chosen": -5.454978942871094, "logps/rejected": -57.34523010253906, "loss": 0.5182607769966125, "memory(GiB)": 35.19, "nll_loss": 0.3399607241153717, "rewards/accuracies": 0.90625, "rewards/chosen": 0.06281061470508575, "rewards/margins": 4.247139930725098, "rewards/rejected": -4.184329032897949, "step": 259, "train_speed(iter/s)": 0.320666 }, { "epoch": 0.6538824269097768, "grad_norm": 2.9779512882232666, "learning_rate": 0.0001849614465630074, "logits/chosen": -0.43861109018325806, "logits/rejected": -0.6130017042160034, "logps/chosen": -4.781963348388672, "logps/rejected": -56.94814682006836, "loss": 0.49893221259117126, "memory(GiB)": 35.19, "nll_loss": 0.33870410919189453, "rewards/accuracies": 0.9375, "rewards/chosen": 0.12630243599414825, "rewards/margins": 4.560263633728027, "rewards/rejected": -4.433961391448975, "step": 260, "train_speed(iter/s)": 0.290134 }, { "epoch": 0.6563973593209682, "grad_norm": 2.126897096633911, "learning_rate": 0.00018481462125004647, "logits/chosen": -0.395521879196167, "logits/rejected": -0.6116307973861694, "logps/chosen": -7.144432544708252, "logps/rejected": -73.966552734375, "loss": 0.5626826882362366, "memory(GiB)": 35.19, "nll_loss": 0.3062968850135803, "rewards/accuracies": 0.875, "rewards/chosen": -0.0832599550485611, "rewards/margins": 5.646542549133301, "rewards/rejected": -5.72980260848999, "step": 261, "train_speed(iter/s)": 0.264936 }, { "epoch": 0.6589122917321597, "grad_norm": 5.1192545890808105, "learning_rate": 0.00018466714153453503, "logits/chosen": -0.2840866148471832, "logits/rejected": -0.6146043539047241, "logps/chosen": -3.4402737617492676, "logps/rejected": -87.25662231445312, "loss": 0.4280086159706116, "memory(GiB)": 35.19, "nll_loss": 0.30285900831222534, "rewards/accuracies": 0.96875, "rewards/chosen": 0.11021072417497635, "rewards/margins": 6.3835039138793945, "rewards/rejected": -6.273293495178223, "step": 262, "train_speed(iter/s)": 0.243811 }, { "epoch": 0.6614272241433512, "grad_norm": 3.2079365253448486, "learning_rate": 0.0001845190085543795, "logits/chosen": -0.449633926153183, "logits/rejected": -0.6547152996063232, "logps/chosen": -10.382014274597168, "logps/rejected": -83.1773681640625, "loss": 0.5558671951293945, "memory(GiB)": 35.19, "nll_loss": 0.43276214599609375, "rewards/accuracies": 0.9375, "rewards/chosen": -0.21494942903518677, "rewards/margins": 6.166694641113281, "rewards/rejected": -6.381643772125244, "step": 263, "train_speed(iter/s)": 0.226046 }, { "epoch": 0.6639421565545426, "grad_norm": 2.844895601272583, "learning_rate": 0.00018437022345252664, "logits/chosen": -0.442143052816391, "logits/rejected": -0.6931208372116089, "logps/chosen": -3.599593162536621, "logps/rejected": -87.85449981689453, "loss": 0.37555554509162903, "memory(GiB)": 35.19, "nll_loss": 0.27443069219589233, "rewards/accuracies": 0.96875, "rewards/chosen": 0.02881481871008873, "rewards/margins": 7.2416534423828125, "rewards/rejected": -7.212838649749756, "step": 264, "train_speed(iter/s)": 0.210855 }, { "epoch": 0.6664570889657341, "grad_norm": 2.5585174560546875, "learning_rate": 0.0001842207873769548, "logits/chosen": -0.42873817682266235, "logits/rejected": -0.6308642029762268, "logps/chosen": -5.981243133544922, "logps/rejected": -70.72628784179688, "loss": 0.6464019417762756, "memory(GiB)": 35.19, "nll_loss": 0.46387210488319397, "rewards/accuracies": 0.875, "rewards/chosen": -0.002297995612025261, "rewards/margins": 5.15910530090332, "rewards/rejected": -5.161403179168701, "step": 265, "train_speed(iter/s)": 0.197608 }, { "epoch": 0.6689720213769255, "grad_norm": 6.840214729309082, "learning_rate": 0.00018407070148066513, "logits/chosen": -0.34393343329429626, "logits/rejected": -0.5655381679534912, "logps/chosen": -6.317379474639893, "logps/rejected": -80.00755310058594, "loss": 0.6975788474082947, "memory(GiB)": 35.19, "nll_loss": 0.5568116307258606, "rewards/accuracies": 1.0, "rewards/chosen": -0.04052425175905228, "rewards/margins": 6.243714332580566, "rewards/rejected": -6.284238338470459, "step": 266, "train_speed(iter/s)": 0.185936 }, { "epoch": 0.671486953788117, "grad_norm": 1.300536870956421, "learning_rate": 0.00018391996692167242, "logits/chosen": -0.44441860914230347, "logits/rejected": -0.6562116742134094, "logps/chosen": -3.928229570388794, "logps/rejected": -69.29019165039062, "loss": 0.3208352327346802, "memory(GiB)": 35.19, "nll_loss": 0.218485489487648, "rewards/accuracies": 0.9375, "rewards/chosen": 0.3024291396141052, "rewards/margins": 5.891707420349121, "rewards/rejected": -5.589277744293213, "step": 267, "train_speed(iter/s)": 0.175708 }, { "epoch": 0.6740018861993083, "grad_norm": 2.1066205501556396, "learning_rate": 0.00018376858486299647, "logits/chosen": -0.3569512367248535, "logits/rejected": -0.6223514080047607, "logps/chosen": -5.05648136138916, "logps/rejected": -77.00200653076172, "loss": 0.447488397359848, "memory(GiB)": 35.19, "nll_loss": 0.2895119786262512, "rewards/accuracies": 0.96875, "rewards/chosen": 0.07016298919916153, "rewards/margins": 5.940699100494385, "rewards/rejected": -5.870535850524902, "step": 268, "train_speed(iter/s)": 0.166588 }, { "epoch": 0.6765168186104998, "grad_norm": 6.73745584487915, "learning_rate": 0.00018361655647265295, "logits/chosen": -0.432262122631073, "logits/rejected": -0.6704272031784058, "logps/chosen": -4.047597885131836, "logps/rejected": -66.35338592529297, "loss": 0.4859369993209839, "memory(GiB)": 35.19, "nll_loss": 0.3608669936656952, "rewards/accuracies": 0.9375, "rewards/chosen": 0.07137257605791092, "rewards/margins": 5.09867000579834, "rewards/rejected": -5.027297019958496, "step": 269, "train_speed(iter/s)": 0.158465 }, { "epoch": 0.6790317510216913, "grad_norm": 4.508405685424805, "learning_rate": 0.00018346388292364435, "logits/chosen": -0.36208435893058777, "logits/rejected": -0.6471053957939148, "logps/chosen": -2.5504043102264404, "logps/rejected": -72.08878326416016, "loss": 0.26533976197242737, "memory(GiB)": 35.19, "nll_loss": 0.15827052295207977, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1948476880788803, "rewards/margins": 5.411668300628662, "rewards/rejected": -5.216820240020752, "step": 270, "train_speed(iter/s)": 0.151104 }, { "epoch": 0.6815466834328827, "grad_norm": 2.18328595161438, "learning_rate": 0.00018331056539395112, "logits/chosen": -0.49844443798065186, "logits/rejected": -0.651973307132721, "logps/chosen": -4.165656566619873, "logps/rejected": -53.685787200927734, "loss": 0.5224581360816956, "memory(GiB)": 35.19, "nll_loss": 0.3570694625377655, "rewards/accuracies": 0.9375, "rewards/chosen": 0.08425498008728027, "rewards/margins": 4.271368980407715, "rewards/rejected": -4.1871137619018555, "step": 271, "train_speed(iter/s)": 0.1445 }, { "epoch": 0.6840616158440742, "grad_norm": 1.904048204421997, "learning_rate": 0.00018315660506652232, "logits/chosen": -0.4680135250091553, "logits/rejected": -0.6710811853408813, "logps/chosen": -4.441302299499512, "logps/rejected": -72.74127197265625, "loss": 0.34467679262161255, "memory(GiB)": 35.19, "nll_loss": 0.23408779501914978, "rewards/accuracies": 1.0, "rewards/chosen": 0.18954063951969147, "rewards/margins": 6.041881561279297, "rewards/rejected": -5.852341175079346, "step": 272, "train_speed(iter/s)": 0.138445 }, { "epoch": 0.6865765482552656, "grad_norm": 2.8482308387756348, "learning_rate": 0.00018300200312926674, "logits/chosen": -0.4003257751464844, "logits/rejected": -0.6571944355964661, "logps/chosen": -4.5086669921875, "logps/rejected": -66.03219604492188, "loss": 0.6187813878059387, "memory(GiB)": 35.19, "nll_loss": 0.41691598296165466, "rewards/accuracies": 0.9375, "rewards/chosen": 0.07326355576515198, "rewards/margins": 4.743034362792969, "rewards/rejected": -4.669771194458008, "step": 273, "train_speed(iter/s)": 0.132932 }, { "epoch": 0.6890914806664571, "grad_norm": 7.494622707366943, "learning_rate": 0.00018284676077504362, "logits/chosen": -0.40674543380737305, "logits/rejected": -0.5925787687301636, "logps/chosen": -9.069441795349121, "logps/rejected": -53.76180648803711, "loss": 0.8244871497154236, "memory(GiB)": 35.19, "nll_loss": 0.5905988216400146, "rewards/accuracies": 0.875, "rewards/chosen": -0.0009557940065860748, "rewards/margins": 3.9635586738586426, "rewards/rejected": -3.96451473236084, "step": 274, "train_speed(iter/s)": 0.127852 }, { "epoch": 0.6916064130776486, "grad_norm": 12.676557540893555, "learning_rate": 0.00018269087920165332, "logits/chosen": -0.3970157504081726, "logits/rejected": -0.6446709632873535, "logps/chosen": -7.009572982788086, "logps/rejected": -75.48789978027344, "loss": 0.5546004772186279, "memory(GiB)": 38.68, "nll_loss": 0.40932291746139526, "rewards/accuracies": 1.0, "rewards/chosen": 0.11061696708202362, "rewards/margins": 5.533331871032715, "rewards/rejected": -5.422715187072754, "step": 275, "train_speed(iter/s)": 0.12316 }, { "epoch": 0.69412134548884, "grad_norm": 3.2866766452789307, "learning_rate": 0.00018253435961182845, "logits/chosen": -0.5122930407524109, "logits/rejected": -0.7138373255729675, "logps/chosen": -5.048257827758789, "logps/rejected": -90.43961334228516, "loss": 0.4567756950855255, "memory(GiB)": 38.68, "nll_loss": 0.3689834177494049, "rewards/accuracies": 1.0, "rewards/chosen": 0.17359167337417603, "rewards/margins": 7.448518753051758, "rewards/rejected": -7.274927139282227, "step": 276, "train_speed(iter/s)": 0.11888 }, { "epoch": 0.6966362779000315, "grad_norm": 1.6649048328399658, "learning_rate": 0.00018237720321322409, "logits/chosen": -0.4259472191333771, "logits/rejected": -0.6217616200447083, "logps/chosen": -5.190006256103516, "logps/rejected": -60.61420440673828, "loss": 0.48118671774864197, "memory(GiB)": 38.68, "nll_loss": 0.34742891788482666, "rewards/accuracies": 0.96875, "rewards/chosen": 0.3943961560726166, "rewards/margins": 4.985601902008057, "rewards/rejected": -4.591205596923828, "step": 277, "train_speed(iter/s)": 0.114903 }, { "epoch": 0.6991512103112228, "grad_norm": 1.641586184501648, "learning_rate": 0.00018221941121840892, "logits/chosen": -0.5152907967567444, "logits/rejected": -0.7797664403915405, "logps/chosen": -3.807891845703125, "logps/rejected": -89.89514923095703, "loss": 0.4311053454875946, "memory(GiB)": 38.68, "nll_loss": 0.3506782352924347, "rewards/accuracies": 0.96875, "rewards/chosen": 0.16795168817043304, "rewards/margins": 7.562755584716797, "rewards/rejected": -7.394804000854492, "step": 278, "train_speed(iter/s)": 0.111233 }, { "epoch": 0.7016661427224143, "grad_norm": 2.954766273498535, "learning_rate": 0.00018206098484485563, "logits/chosen": -0.47647833824157715, "logits/rejected": -0.7569775581359863, "logps/chosen": -3.6604084968566895, "logps/rejected": -93.30502319335938, "loss": 0.33102554082870483, "memory(GiB)": 38.68, "nll_loss": 0.23846475780010223, "rewards/accuracies": 1.0, "rewards/chosen": 0.057397253811359406, "rewards/margins": 7.980731964111328, "rewards/rejected": -7.92333459854126, "step": 279, "train_speed(iter/s)": 0.107796 }, { "epoch": 0.7041810751336058, "grad_norm": 1.0509345531463623, "learning_rate": 0.00018190192531493152, "logits/chosen": -0.4389861226081848, "logits/rejected": -0.7513998746871948, "logps/chosen": -3.3642971515655518, "logps/rejected": -104.84876251220703, "loss": 0.2865274250507355, "memory(GiB)": 38.68, "nll_loss": 0.1890382468700409, "rewards/accuracies": 0.9375, "rewards/chosen": 0.1356939822435379, "rewards/margins": 8.732723236083984, "rewards/rejected": -8.597027778625488, "step": 280, "train_speed(iter/s)": 0.104576 }, { "epoch": 0.7066960075447972, "grad_norm": 2.062405824661255, "learning_rate": 0.00018174223385588917, "logits/chosen": -0.4419255256652832, "logits/rejected": -0.789115309715271, "logps/chosen": -6.24101448059082, "logps/rejected": -107.06963348388672, "loss": 0.38868069648742676, "memory(GiB)": 38.68, "nll_loss": 0.27302783727645874, "rewards/accuracies": 0.9375, "rewards/chosen": 0.04043314605951309, "rewards/margins": 8.512565612792969, "rewards/rejected": -8.472132682800293, "step": 281, "train_speed(iter/s)": 0.101563 }, { "epoch": 0.7092109399559887, "grad_norm": 1.5434448719024658, "learning_rate": 0.00018158191169985696, "logits/chosen": -0.5877774953842163, "logits/rejected": -0.7937146425247192, "logps/chosen": -3.5352978706359863, "logps/rejected": -82.489990234375, "loss": 0.28792405128479004, "memory(GiB)": 38.68, "nll_loss": 0.21994276344776154, "rewards/accuracies": 1.0, "rewards/chosen": 0.152220219373703, "rewards/margins": 7.123733043670654, "rewards/rejected": -6.971513271331787, "step": 282, "train_speed(iter/s)": 0.098762 }, { "epoch": 0.7117258723671801, "grad_norm": 1.7215518951416016, "learning_rate": 0.00018142096008382942, "logits/chosen": -0.5603517889976501, "logits/rejected": -0.8750638961791992, "logps/chosen": -3.4122605323791504, "logps/rejected": -98.63863372802734, "loss": 0.4363364279270172, "memory(GiB)": 38.68, "nll_loss": 0.33546215295791626, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1424112468957901, "rewards/margins": 8.030111312866211, "rewards/rejected": -7.887700080871582, "step": 283, "train_speed(iter/s)": 0.096121 }, { "epoch": 0.7142408047783716, "grad_norm": 6.417537212371826, "learning_rate": 0.00018125938024965793, "logits/chosen": -0.543463408946991, "logits/rejected": -0.7929783463478088, "logps/chosen": -5.197461128234863, "logps/rejected": -97.00271606445312, "loss": 0.6537326574325562, "memory(GiB)": 38.68, "nll_loss": 0.46845054626464844, "rewards/accuracies": 0.875, "rewards/chosen": 0.07486869394779205, "rewards/margins": 8.32656478881836, "rewards/rejected": -8.251694679260254, "step": 284, "train_speed(iter/s)": 0.093639 }, { "epoch": 0.716755737189563, "grad_norm": 2.125540018081665, "learning_rate": 0.0001810971734440408, "logits/chosen": -0.6539860367774963, "logits/rejected": -0.853480875492096, "logps/chosen": -5.839959621429443, "logps/rejected": -91.55633544921875, "loss": 0.4674933850765228, "memory(GiB)": 38.68, "nll_loss": 0.3285233974456787, "rewards/accuracies": 0.9375, "rewards/chosen": 0.0453268326818943, "rewards/margins": 7.891117572784424, "rewards/rejected": -7.845790863037109, "step": 285, "train_speed(iter/s)": 0.091303 }, { "epoch": 0.7192706696007545, "grad_norm": 2.496291160583496, "learning_rate": 0.0001809343409185141, "logits/chosen": -0.533951997756958, "logits/rejected": -0.8393864631652832, "logps/chosen": -5.252377510070801, "logps/rejected": -108.10926055908203, "loss": 0.4806031584739685, "memory(GiB)": 38.68, "nll_loss": 0.35059598088264465, "rewards/accuracies": 0.9375, "rewards/chosen": 0.26575902104377747, "rewards/margins": 9.291682243347168, "rewards/rejected": -9.025922775268555, "step": 286, "train_speed(iter/s)": 0.089086 }, { "epoch": 0.721785602011946, "grad_norm": 12.633203506469727, "learning_rate": 0.00018077088392944157, "logits/chosen": -0.5087528228759766, "logits/rejected": -0.7988476753234863, "logps/chosen": -4.803990840911865, "logps/rejected": -134.75717163085938, "loss": 0.5371387600898743, "memory(GiB)": 38.68, "nll_loss": 0.4104423522949219, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09430995583534241, "rewards/margins": 11.716846466064453, "rewards/rejected": -11.622535705566406, "step": 287, "train_speed(iter/s)": 0.086965 }, { "epoch": 0.7243005344231374, "grad_norm": 6.384928226470947, "learning_rate": 0.00018060680373800517, "logits/chosen": -0.634199857711792, "logits/rejected": -0.8538622260093689, "logps/chosen": -7.401331901550293, "logps/rejected": -99.94730377197266, "loss": 0.9957684874534607, "memory(GiB)": 38.68, "nll_loss": 0.7036086916923523, "rewards/accuracies": 0.9375, "rewards/chosen": -0.18307247757911682, "rewards/margins": 8.665613174438477, "rewards/rejected": -8.848685264587402, "step": 288, "train_speed(iter/s)": 0.084995 }, { "epoch": 0.7268154668343288, "grad_norm": 1.3055838346481323, "learning_rate": 0.00018044210161019536, "logits/chosen": -0.6348901391029358, "logits/rejected": -0.8540142774581909, "logps/chosen": -3.750216484069824, "logps/rejected": -116.36519622802734, "loss": 0.43587327003479004, "memory(GiB)": 38.68, "nll_loss": 0.3083232641220093, "rewards/accuracies": 1.0, "rewards/chosen": 0.27759620547294617, "rewards/margins": 10.4413480758667, "rewards/rejected": -10.163751602172852, "step": 289, "train_speed(iter/s)": 0.083116 }, { "epoch": 0.7293303992455202, "grad_norm": 6.336847305297852, "learning_rate": 0.0001802767788168011, "logits/chosen": -0.49168598651885986, "logits/rejected": -0.7385163307189941, "logps/chosen": -5.862331867218018, "logps/rejected": -90.92340850830078, "loss": 0.5076504349708557, "memory(GiB)": 42.19, "nll_loss": 0.40712910890579224, "rewards/accuracies": 1.0, "rewards/chosen": -0.007690219208598137, "rewards/margins": 7.30800724029541, "rewards/rejected": -7.31569766998291, "step": 290, "train_speed(iter/s)": 0.081314 }, { "epoch": 0.7318453316567117, "grad_norm": 1.5834612846374512, "learning_rate": 0.0001801108366334004, "logits/chosen": -0.49157392978668213, "logits/rejected": -0.8517095446586609, "logps/chosen": -2.9823904037475586, "logps/rejected": -128.86550903320312, "loss": 0.28464004397392273, "memory(GiB)": 42.19, "nll_loss": 0.19038352370262146, "rewards/accuracies": 0.9375, "rewards/chosen": 0.11139468103647232, "rewards/margins": 11.453866004943848, "rewards/rejected": -11.3424711227417, "step": 291, "train_speed(iter/s)": 0.079608 }, { "epoch": 0.7343602640679032, "grad_norm": 2.694648265838623, "learning_rate": 0.00017994427634035015, "logits/chosen": -0.4209265410900116, "logits/rejected": -0.7504950761795044, "logps/chosen": -6.3449931144714355, "logps/rejected": -102.324951171875, "loss": 0.48612844944000244, "memory(GiB)": 42.19, "nll_loss": 0.439787358045578, "rewards/accuracies": 1.0, "rewards/chosen": -0.05951198935508728, "rewards/margins": 8.676319122314453, "rewards/rejected": -8.735831260681152, "step": 292, "train_speed(iter/s)": 0.077977 }, { "epoch": 0.7368751964790946, "grad_norm": 12.563709259033203, "learning_rate": 0.00017977709922277634, "logits/chosen": -0.546952486038208, "logits/rejected": -0.8098918199539185, "logps/chosen": -3.903282880783081, "logps/rejected": -106.09379577636719, "loss": 0.4184201657772064, "memory(GiB)": 42.19, "nll_loss": 0.2776867151260376, "rewards/accuracies": 0.96875, "rewards/chosen": 0.11045068502426147, "rewards/margins": 9.161507606506348, "rewards/rejected": -9.051055908203125, "step": 293, "train_speed(iter/s)": 0.076428 }, { "epoch": 0.7393901288902861, "grad_norm": 10.21599292755127, "learning_rate": 0.00017960930657056438, "logits/chosen": -0.4862319827079773, "logits/rejected": -0.8446335196495056, "logps/chosen": -4.003853797912598, "logps/rejected": -124.26484680175781, "loss": 0.5325912833213806, "memory(GiB)": 42.19, "nll_loss": 0.4379286766052246, "rewards/accuracies": 0.96875, "rewards/chosen": -0.017376184463500977, "rewards/margins": 10.38001823425293, "rewards/rejected": -10.397393226623535, "step": 294, "train_speed(iter/s)": 0.074951 }, { "epoch": 0.7419050613014775, "grad_norm": 2.1142091751098633, "learning_rate": 0.00017944089967834875, "logits/chosen": -0.6149781942367554, "logits/rejected": -0.8646234273910522, "logps/chosen": -4.966188907623291, "logps/rejected": -105.03376007080078, "loss": 0.3664693236351013, "memory(GiB)": 42.19, "nll_loss": 0.29172563552856445, "rewards/accuracies": 1.0, "rewards/chosen": 0.12611237168312073, "rewards/margins": 8.863950729370117, "rewards/rejected": -8.737838745117188, "step": 295, "train_speed(iter/s)": 0.073543 }, { "epoch": 0.744419993712669, "grad_norm": 2.3182928562164307, "learning_rate": 0.00017927187984550334, "logits/chosen": -0.4714081883430481, "logits/rejected": -0.8076429963111877, "logps/chosen": -5.210522174835205, "logps/rejected": -94.38253784179688, "loss": 0.40223944187164307, "memory(GiB)": 42.19, "nll_loss": 0.3296588659286499, "rewards/accuracies": 0.9375, "rewards/chosen": 0.05410341918468475, "rewards/margins": 7.804743766784668, "rewards/rejected": -7.750640392303467, "step": 296, "train_speed(iter/s)": 0.072191 }, { "epoch": 0.7469349261238605, "grad_norm": 1.6627568006515503, "learning_rate": 0.00017910224837613118, "logits/chosen": -0.5375725030899048, "logits/rejected": -0.7311803698539734, "logps/chosen": -5.645050048828125, "logps/rejected": -86.46006774902344, "loss": 0.4087020456790924, "memory(GiB)": 45.72, "nll_loss": 0.2872757613658905, "rewards/accuracies": 0.96875, "rewards/chosen": 0.12224967032670975, "rewards/margins": 6.998920440673828, "rewards/rejected": -6.876670837402344, "step": 297, "train_speed(iter/s)": 0.070893 }, { "epoch": 0.7494498585350519, "grad_norm": 7.537210941314697, "learning_rate": 0.0001789320065790546, "logits/chosen": -0.5413732528686523, "logits/rejected": -0.7798230051994324, "logps/chosen": -5.977290630340576, "logps/rejected": -84.39535522460938, "loss": 0.49224522709846497, "memory(GiB)": 45.72, "nll_loss": 0.3515035808086395, "rewards/accuracies": 0.9375, "rewards/chosen": 0.29318666458129883, "rewards/margins": 6.832826614379883, "rewards/rejected": -6.539640426635742, "step": 298, "train_speed(iter/s)": 0.06965 }, { "epoch": 0.7519647909462434, "grad_norm": 13.154047012329102, "learning_rate": 0.00017876115576780498, "logits/chosen": -0.56256502866745, "logits/rejected": -0.7628785371780396, "logps/chosen": -8.302791595458984, "logps/rejected": -71.12387084960938, "loss": 0.7186374068260193, "memory(GiB)": 45.72, "nll_loss": 0.5491286516189575, "rewards/accuracies": 0.875, "rewards/chosen": -0.04678814485669136, "rewards/margins": 5.721759796142578, "rewards/rejected": -5.768548488616943, "step": 299, "train_speed(iter/s)": 0.068463 }, { "epoch": 0.7544797233574347, "grad_norm": 2.1101605892181396, "learning_rate": 0.0001785896972606126, "logits/chosen": -0.6424190998077393, "logits/rejected": -0.8492656946182251, "logps/chosen": -6.954460144042969, "logps/rejected": -72.85733795166016, "loss": 0.5062767267227173, "memory(GiB)": 45.72, "nll_loss": 0.4268747568130493, "rewards/accuracies": 0.9375, "rewards/chosen": 0.18685287237167358, "rewards/margins": 6.121685028076172, "rewards/rejected": -5.934832572937012, "step": 300, "train_speed(iter/s)": 0.067324 }, { "epoch": 0.7544797233574347, "eval_logits/chosen": -0.6104318499565125, "eval_logits/rejected": -0.8290396332740784, "eval_logps/chosen": -5.665504455566406, "eval_logps/rejected": -75.69713592529297, "eval_loss": 0.5146161317825317, "eval_nll_loss": 0.36346134543418884, "eval_rewards/accuracies": 0.953125, "eval_rewards/chosen": 0.1080789864063263, "eval_rewards/margins": 6.1169891357421875, "eval_rewards/rejected": -6.008910179138184, "eval_runtime": 138.6744, "eval_samples_per_second": 0.923, "eval_steps_per_second": 0.462, "step": 300 }, { "epoch": 0.7569946557686262, "grad_norm": 2.742777109146118, "learning_rate": 0.00017841763238039663, "logits/chosen": -0.5258588194847107, "logits/rejected": -0.8002297878265381, "logps/chosen": -9.598472595214844, "logps/rejected": -73.01333618164062, "loss": 0.6272799968719482, "memory(GiB)": 45.72, "nll_loss": 0.45022809505462646, "rewards/accuracies": 0.90625, "rewards/chosen": 0.14143598079681396, "rewards/margins": 5.63870096206665, "rewards/rejected": -5.497264862060547, "step": 301, "train_speed(iter/s)": 0.064032 }, { "epoch": 0.7595095881798176, "grad_norm": 4.982257843017578, "learning_rate": 0.00017824496245475477, "logits/chosen": -0.5467951893806458, "logits/rejected": -0.8088864088058472, "logps/chosen": -5.489375114440918, "logps/rejected": -68.20091247558594, "loss": 0.6761376261711121, "memory(GiB)": 45.72, "nll_loss": 0.46992918848991394, "rewards/accuracies": 0.90625, "rewards/chosen": 0.03441940248012543, "rewards/margins": 5.287006378173828, "rewards/rejected": -5.252586841583252, "step": 302, "train_speed(iter/s)": 0.063055 }, { "epoch": 0.7620245205910091, "grad_norm": 2.363013982772827, "learning_rate": 0.00017807168881595303, "logits/chosen": -0.486921101808548, "logits/rejected": -0.7870367765426636, "logps/chosen": -3.2118942737579346, "logps/rejected": -80.78832244873047, "loss": 0.3174203336238861, "memory(GiB)": 45.72, "nll_loss": 0.22700631618499756, "rewards/accuracies": 0.96875, "rewards/chosen": 0.0014025792479515076, "rewards/margins": 6.200619697570801, "rewards/rejected": -6.199216842651367, "step": 303, "train_speed(iter/s)": 0.062112 }, { "epoch": 0.7645394530022006, "grad_norm": 2.5794389247894287, "learning_rate": 0.00017789781280091557, "logits/chosen": -0.5253547430038452, "logits/rejected": -0.7778689861297607, "logps/chosen": -2.740814685821533, "logps/rejected": -63.18760299682617, "loss": 0.43329575657844543, "memory(GiB)": 45.72, "nll_loss": 0.25420501828193665, "rewards/accuracies": 0.9375, "rewards/chosen": 0.11472588777542114, "rewards/margins": 4.892645359039307, "rewards/rejected": -4.777919292449951, "step": 304, "train_speed(iter/s)": 0.061202 }, { "epoch": 0.767054385413392, "grad_norm": 3.204162836074829, "learning_rate": 0.0001777233357512142, "logits/chosen": -0.48726359009742737, "logits/rejected": -0.7712681293487549, "logps/chosen": -4.6791911125183105, "logps/rejected": -73.8591079711914, "loss": 0.49168315529823303, "memory(GiB)": 45.72, "nll_loss": 0.3878230154514313, "rewards/accuracies": 1.0, "rewards/chosen": 0.12156429141759872, "rewards/margins": 5.675488471984863, "rewards/rejected": -5.553924083709717, "step": 305, "train_speed(iter/s)": 0.060326 }, { "epoch": 0.7695693178245835, "grad_norm": 2.685102939605713, "learning_rate": 0.00017754825901305813, "logits/chosen": -0.49732205271720886, "logits/rejected": -0.6546094417572021, "logps/chosen": -9.81102180480957, "logps/rejected": -60.67332077026367, "loss": 0.6829510927200317, "memory(GiB)": 45.72, "nll_loss": 0.5321569442749023, "rewards/accuracies": 0.90625, "rewards/chosen": 0.23814226686954498, "rewards/margins": 4.872373104095459, "rewards/rejected": -4.634230613708496, "step": 306, "train_speed(iter/s)": 0.059474 }, { "epoch": 0.7720842502357749, "grad_norm": 2.191077947616577, "learning_rate": 0.00017737258393728364, "logits/chosen": -0.3749886453151703, "logits/rejected": -0.7177920341491699, "logps/chosen": -3.679948329925537, "logps/rejected": -76.73285675048828, "loss": 0.44855624437332153, "memory(GiB)": 45.72, "nll_loss": 0.31453874707221985, "rewards/accuracies": 0.96875, "rewards/chosen": 0.11833418905735016, "rewards/margins": 5.410902976989746, "rewards/rejected": -5.292569160461426, "step": 307, "train_speed(iter/s)": 0.058652 }, { "epoch": 0.7745991826469664, "grad_norm": 5.087850570678711, "learning_rate": 0.00017719631187934352, "logits/chosen": -0.4312075078487396, "logits/rejected": -0.7199861407279968, "logps/chosen": -5.141556739807129, "logps/rejected": -84.90442657470703, "loss": 0.49092957377433777, "memory(GiB)": 45.72, "nll_loss": 0.3564959764480591, "rewards/accuracies": 0.96875, "rewards/chosen": 0.06924839317798615, "rewards/margins": 6.585083961486816, "rewards/rejected": -6.515835762023926, "step": 308, "train_speed(iter/s)": 0.057859 }, { "epoch": 0.7771141150581579, "grad_norm": 1.931964635848999, "learning_rate": 0.00017701944419929672, "logits/chosen": -0.5209879279136658, "logits/rejected": -0.7744486331939697, "logps/chosen": -4.660381317138672, "logps/rejected": -75.40290069580078, "loss": 0.380489319562912, "memory(GiB)": 45.72, "nll_loss": 0.2925787568092346, "rewards/accuracies": 1.0, "rewards/chosen": 0.07492721825838089, "rewards/margins": 5.759749412536621, "rewards/rejected": -5.684822082519531, "step": 309, "train_speed(iter/s)": 0.057092 }, { "epoch": 0.7796290474693492, "grad_norm": 1.4231215715408325, "learning_rate": 0.00017684198226179784, "logits/chosen": -0.5286337733268738, "logits/rejected": -0.7741042375564575, "logps/chosen": -5.322835922241211, "logps/rejected": -94.39818572998047, "loss": 0.32215720415115356, "memory(GiB)": 45.72, "nll_loss": 0.23559962213039398, "rewards/accuracies": 0.9375, "rewards/chosen": 0.03720661997795105, "rewards/margins": 7.8387603759765625, "rewards/rejected": -7.8015546798706055, "step": 310, "train_speed(iter/s)": 0.056351 }, { "epoch": 0.7821439798805407, "grad_norm": 3.5972278118133545, "learning_rate": 0.00017666392743608656, "logits/chosen": -0.587047815322876, "logits/rejected": -0.6886476874351501, "logps/chosen": -11.391901969909668, "logps/rejected": -63.259220123291016, "loss": 0.631456732749939, "memory(GiB)": 45.72, "nll_loss": 0.4477965831756592, "rewards/accuracies": 1.0, "rewards/chosen": 0.2087097316980362, "rewards/margins": 5.190671920776367, "rewards/rejected": -4.981962203979492, "step": 311, "train_speed(iter/s)": 0.055631 }, { "epoch": 0.7846589122917321, "grad_norm": 6.570708751678467, "learning_rate": 0.00017648528109597705, "logits/chosen": -0.5264805555343628, "logits/rejected": -0.6204366683959961, "logps/chosen": -5.628443241119385, "logps/rejected": -55.631587982177734, "loss": 0.44488024711608887, "memory(GiB)": 45.72, "nll_loss": 0.29311853647232056, "rewards/accuracies": 0.96875, "rewards/chosen": 0.15758532285690308, "rewards/margins": 4.487579345703125, "rewards/rejected": -4.329994201660156, "step": 312, "train_speed(iter/s)": 0.054937 }, { "epoch": 0.7871738447029236, "grad_norm": 2.2891812324523926, "learning_rate": 0.00017630604461984746, "logits/chosen": -0.49812430143356323, "logits/rejected": -0.7942063808441162, "logps/chosen": -4.372743606567383, "logps/rejected": -94.14138793945312, "loss": 0.36223098635673523, "memory(GiB)": 45.72, "nll_loss": 0.30161458253860474, "rewards/accuracies": 1.0, "rewards/chosen": 0.10362353175878525, "rewards/margins": 7.83615255355835, "rewards/rejected": -7.7325286865234375, "step": 313, "train_speed(iter/s)": 0.054263 }, { "epoch": 0.789688777114115, "grad_norm": 4.033056259155273, "learning_rate": 0.00017612621939062928, "logits/chosen": -0.5049434304237366, "logits/rejected": -0.7435040473937988, "logps/chosen": -3.835451364517212, "logps/rejected": -84.56083679199219, "loss": 0.43754124641418457, "memory(GiB)": 45.72, "nll_loss": 0.2978503704071045, "rewards/accuracies": 0.9375, "rewards/chosen": 0.17277862131595612, "rewards/margins": 6.71329402923584, "rewards/rejected": -6.540514945983887, "step": 314, "train_speed(iter/s)": 0.053607 }, { "epoch": 0.7922037095253065, "grad_norm": 4.677379608154297, "learning_rate": 0.00017594580679579656, "logits/chosen": -0.4425162076950073, "logits/rejected": -0.7090457677841187, "logps/chosen": -4.391209602355957, "logps/rejected": -78.58367919921875, "loss": 0.5179426670074463, "memory(GiB)": 45.72, "nll_loss": 0.3796064257621765, "rewards/accuracies": 0.96875, "rewards/chosen": 0.003046363592147827, "rewards/margins": 6.267105579376221, "rewards/rejected": -6.264059066772461, "step": 315, "train_speed(iter/s)": 0.052969 }, { "epoch": 0.794718641936498, "grad_norm": 2.2975409030914307, "learning_rate": 0.00017576480822735532, "logits/chosen": -0.552763819694519, "logits/rejected": -0.7386415004730225, "logps/chosen": -6.633852005004883, "logps/rejected": -75.99591827392578, "loss": 0.5104320049285889, "memory(GiB)": 45.72, "nll_loss": 0.4509289562702179, "rewards/accuracies": 1.0, "rewards/chosen": 0.13848751783370972, "rewards/margins": 6.377573013305664, "rewards/rejected": -6.239084720611572, "step": 316, "train_speed(iter/s)": 0.052355 }, { "epoch": 0.7972335743476894, "grad_norm": 1.692866325378418, "learning_rate": 0.00017558322508183275, "logits/chosen": -0.5113404393196106, "logits/rejected": -0.7546023726463318, "logps/chosen": -8.723966598510742, "logps/rejected": -68.92810821533203, "loss": 0.3866414725780487, "memory(GiB)": 45.72, "nll_loss": 0.2663732171058655, "rewards/accuracies": 0.96875, "rewards/chosen": 0.12023188173770905, "rewards/margins": 5.404682159423828, "rewards/rejected": -5.284450054168701, "step": 317, "train_speed(iter/s)": 0.051754 }, { "epoch": 0.7997485067588809, "grad_norm": 1.7934551239013672, "learning_rate": 0.00017540105876026648, "logits/chosen": -0.5510843396186829, "logits/rejected": -0.7684234976768494, "logps/chosen": -4.503855228424072, "logps/rejected": -73.94355010986328, "loss": 0.49109137058258057, "memory(GiB)": 45.72, "nll_loss": 0.33945515751838684, "rewards/accuracies": 0.96875, "rewards/chosen": 0.08985169976949692, "rewards/margins": 6.044112682342529, "rewards/rejected": -5.9542622566223145, "step": 318, "train_speed(iter/s)": 0.051176 }, { "epoch": 0.8022634391700723, "grad_norm": 2.696263074874878, "learning_rate": 0.00017521831066819363, "logits/chosen": -0.6849238872528076, "logits/rejected": -0.7512338757514954, "logps/chosen": -7.439488410949707, "logps/rejected": -54.92814636230469, "loss": 0.5880566835403442, "memory(GiB)": 45.72, "nll_loss": 0.32204046845436096, "rewards/accuracies": 0.875, "rewards/chosen": -0.08035062998533249, "rewards/margins": 4.1917901039123535, "rewards/rejected": -4.2721405029296875, "step": 319, "train_speed(iter/s)": 0.050611 }, { "epoch": 0.8047783715812638, "grad_norm": 8.225646018981934, "learning_rate": 0.00017503498221564025, "logits/chosen": -0.5244278311729431, "logits/rejected": -0.8192154169082642, "logps/chosen": -5.370640754699707, "logps/rejected": -69.76851654052734, "loss": 0.5900783538818359, "memory(GiB)": 46.31, "nll_loss": 0.43174654245376587, "rewards/accuracies": 0.90625, "rewards/chosen": 0.15494805574417114, "rewards/margins": 4.877842903137207, "rewards/rejected": -4.722894668579102, "step": 320, "train_speed(iter/s)": 0.050058 }, { "epoch": 0.8072933039924552, "grad_norm": 3.1263160705566406, "learning_rate": 0.00017485107481711012, "logits/chosen": -0.6532421112060547, "logits/rejected": -0.8408050537109375, "logps/chosen": -5.802052974700928, "logps/rejected": -69.45040893554688, "loss": 0.4921962022781372, "memory(GiB)": 46.31, "nll_loss": 0.3598018288612366, "rewards/accuracies": 0.90625, "rewards/chosen": -0.016502581536769867, "rewards/margins": 5.566046714782715, "rewards/rejected": -5.582549571990967, "step": 321, "train_speed(iter/s)": 0.049526 }, { "epoch": 0.8098082364036466, "grad_norm": 1.3748646974563599, "learning_rate": 0.00017466658989157406, "logits/chosen": -0.5933704376220703, "logits/rejected": -0.8203169107437134, "logps/chosen": -4.698756694793701, "logps/rejected": -86.18940734863281, "loss": 0.39138850569725037, "memory(GiB)": 46.31, "nll_loss": 0.3173418641090393, "rewards/accuracies": 1.0, "rewards/chosen": 0.07632508128881454, "rewards/margins": 7.136272430419922, "rewards/rejected": -7.0599470138549805, "step": 322, "train_speed(iter/s)": 0.049007 }, { "epoch": 0.8123231688148381, "grad_norm": 1.2321858406066895, "learning_rate": 0.00017448152886245888, "logits/chosen": -0.6055665016174316, "logits/rejected": -0.835323691368103, "logps/chosen": -4.2663726806640625, "logps/rejected": -92.81077575683594, "loss": 0.31942421197891235, "memory(GiB)": 46.31, "nll_loss": 0.23250146210193634, "rewards/accuracies": 0.96875, "rewards/chosen": 0.29466480016708374, "rewards/margins": 7.823517799377441, "rewards/rejected": -7.528852939605713, "step": 323, "train_speed(iter/s)": 0.048501 }, { "epoch": 0.8148381012260295, "grad_norm": 9.860624313354492, "learning_rate": 0.00017429589315763638, "logits/chosen": -0.5517026782035828, "logits/rejected": -0.9087996482849121, "logps/chosen": -5.479660987854004, "logps/rejected": -96.31484985351562, "loss": 0.9468085169792175, "memory(GiB)": 46.31, "nll_loss": 0.8088601231575012, "rewards/accuracies": 0.9375, "rewards/chosen": -0.13317811489105225, "rewards/margins": 7.745667457580566, "rewards/rejected": -7.87884521484375, "step": 324, "train_speed(iter/s)": 0.048008 }, { "epoch": 0.817353033637221, "grad_norm": 1.578026294708252, "learning_rate": 0.00017410968420941244, "logits/chosen": -0.4647430181503296, "logits/rejected": -0.863081157207489, "logps/chosen": -5.37056827545166, "logps/rejected": -94.24394226074219, "loss": 0.4825628399848938, "memory(GiB)": 46.31, "nll_loss": 0.4036305248737335, "rewards/accuracies": 1.0, "rewards/chosen": 0.21735483407974243, "rewards/margins": 7.361626148223877, "rewards/rejected": -7.144271373748779, "step": 325, "train_speed(iter/s)": 0.047523 }, { "epoch": 0.8198679660484125, "grad_norm": 5.969099044799805, "learning_rate": 0.00017392290345451592, "logits/chosen": -0.5345301628112793, "logits/rejected": -0.820349395275116, "logps/chosen": -6.029956817626953, "logps/rejected": -104.83753204345703, "loss": 0.5511329174041748, "memory(GiB)": 46.31, "nll_loss": 0.4094994068145752, "rewards/accuracies": 0.96875, "rewards/chosen": 0.2000085562467575, "rewards/margins": 8.80609130859375, "rewards/rejected": -8.606082916259766, "step": 326, "train_speed(iter/s)": 0.047054 }, { "epoch": 0.8223828984596039, "grad_norm": 7.578153133392334, "learning_rate": 0.0001737355523340875, "logits/chosen": -0.7083917856216431, "logits/rejected": -0.9140143990516663, "logps/chosen": -5.282914161682129, "logps/rejected": -89.25215148925781, "loss": 0.6085916757583618, "memory(GiB)": 46.31, "nll_loss": 0.4778170585632324, "rewards/accuracies": 0.96875, "rewards/chosen": 0.04538874328136444, "rewards/margins": 7.695877552032471, "rewards/rejected": -7.650489807128906, "step": 327, "train_speed(iter/s)": 0.046602 }, { "epoch": 0.8248978308707954, "grad_norm": 7.923031330108643, "learning_rate": 0.0001735476322936686, "logits/chosen": -0.48410487174987793, "logits/rejected": -0.8404121398925781, "logps/chosen": -5.619724273681641, "logps/rejected": -111.05279541015625, "loss": 0.5005356073379517, "memory(GiB)": 46.31, "nll_loss": 0.415988028049469, "rewards/accuracies": 1.0, "rewards/chosen": 0.04838261753320694, "rewards/margins": 9.071845054626465, "rewards/rejected": -9.023462295532227, "step": 328, "train_speed(iter/s)": 0.046157 }, { "epoch": 0.8274127632819868, "grad_norm": 1.6029303073883057, "learning_rate": 0.00017335914478319028, "logits/chosen": -0.5107839107513428, "logits/rejected": -0.811547577381134, "logps/chosen": -5.087898254394531, "logps/rejected": -90.02484130859375, "loss": 0.4403790235519409, "memory(GiB)": 46.31, "nll_loss": 0.3421851396560669, "rewards/accuracies": 1.0, "rewards/chosen": 0.1459052413702011, "rewards/margins": 7.192948341369629, "rewards/rejected": -7.047042369842529, "step": 329, "train_speed(iter/s)": 0.045725 }, { "epoch": 0.8299276956931783, "grad_norm": 1.4880335330963135, "learning_rate": 0.00017317009125696208, "logits/chosen": -0.5659950375556946, "logits/rejected": -0.8397157192230225, "logps/chosen": -4.4726243019104, "logps/rejected": -92.63803100585938, "loss": 0.2614552974700928, "memory(GiB)": 46.31, "nll_loss": 0.21865251660346985, "rewards/accuracies": 1.0, "rewards/chosen": 0.20230340957641602, "rewards/margins": 7.986957550048828, "rewards/rejected": -7.784653663635254, "step": 330, "train_speed(iter/s)": 0.045303 }, { "epoch": 0.8324426281043696, "grad_norm": 4.477383136749268, "learning_rate": 0.00017298047317366061, "logits/chosen": -0.5597605109214783, "logits/rejected": -0.8359659910202026, "logps/chosen": -3.7963671684265137, "logps/rejected": -79.54478454589844, "loss": 0.386127769947052, "memory(GiB)": 46.31, "nll_loss": 0.30003517866134644, "rewards/accuracies": 0.96875, "rewards/chosen": 0.13904371857643127, "rewards/margins": 6.464727401733398, "rewards/rejected": -6.32568359375, "step": 331, "train_speed(iter/s)": 0.044893 }, { "epoch": 0.8349575605155611, "grad_norm": 1.3735220432281494, "learning_rate": 0.00017279029199631857, "logits/chosen": -0.4946771264076233, "logits/rejected": -0.8467792272567749, "logps/chosen": -2.6571059226989746, "logps/rejected": -94.11196899414062, "loss": 0.2998129427433014, "memory(GiB)": 46.31, "nll_loss": 0.22292877733707428, "rewards/accuracies": 0.96875, "rewards/chosen": 0.12367352843284607, "rewards/margins": 8.038744926452637, "rewards/rejected": -7.915071487426758, "step": 332, "train_speed(iter/s)": 0.044491 }, { "epoch": 0.8374724929267526, "grad_norm": 2.6437320709228516, "learning_rate": 0.0001725995491923131, "logits/chosen": -0.552052915096283, "logits/rejected": -0.754858136177063, "logps/chosen": -11.185404777526855, "logps/rejected": -86.47320556640625, "loss": 0.8312444090843201, "memory(GiB)": 46.31, "nll_loss": 0.6818321943283081, "rewards/accuracies": 0.96875, "rewards/chosen": -0.01666903868317604, "rewards/margins": 6.93008279800415, "rewards/rejected": -6.946751594543457, "step": 333, "train_speed(iter/s)": 0.044095 }, { "epoch": 0.839987425337944, "grad_norm": 8.506746292114258, "learning_rate": 0.0001724082462333549, "logits/chosen": -0.5991488099098206, "logits/rejected": -0.8054255843162537, "logps/chosen": -5.842095375061035, "logps/rejected": -69.10340118408203, "loss": 0.6470210552215576, "memory(GiB)": 46.31, "nll_loss": 0.4332461953163147, "rewards/accuracies": 0.9375, "rewards/chosen": -0.05430920422077179, "rewards/margins": 5.4711198806762695, "rewards/rejected": -5.5254292488098145, "step": 334, "train_speed(iter/s)": 0.043713 }, { "epoch": 0.8425023577491355, "grad_norm": 4.649921894073486, "learning_rate": 0.00017221638459547655, "logits/chosen": -0.6327688694000244, "logits/rejected": -0.8601678609848022, "logps/chosen": -10.89456558227539, "logps/rejected": -83.64013671875, "loss": 0.9836386442184448, "memory(GiB)": 46.31, "nll_loss": 0.8277321457862854, "rewards/accuracies": 0.9375, "rewards/chosen": -0.3010229766368866, "rewards/margins": 6.816217422485352, "rewards/rejected": -7.1172404289245605, "step": 335, "train_speed(iter/s)": 0.043343 }, { "epoch": 0.8450172901603269, "grad_norm": 3.5049939155578613, "learning_rate": 0.00017202396575902116, "logits/chosen": -0.5969817042350769, "logits/rejected": -0.7890011072158813, "logps/chosen": -8.715362548828125, "logps/rejected": -87.10135650634766, "loss": 0.6040844321250916, "memory(GiB)": 46.31, "nll_loss": 0.48410138487815857, "rewards/accuracies": 0.9375, "rewards/chosen": 0.02549554966390133, "rewards/margins": 6.946871280670166, "rewards/rejected": -6.9213762283325195, "step": 336, "train_speed(iter/s)": 0.042976 }, { "epoch": 0.8475322225715184, "grad_norm": 2.5609958171844482, "learning_rate": 0.00017183099120863116, "logits/chosen": -0.5624786615371704, "logits/rejected": -0.8555223941802979, "logps/chosen": -2.920971393585205, "logps/rejected": -87.21141815185547, "loss": 0.25303125381469727, "memory(GiB)": 46.31, "nll_loss": 0.17635448276996613, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1297520101070404, "rewards/margins": 7.165301322937012, "rewards/rejected": -7.035548210144043, "step": 337, "train_speed(iter/s)": 0.04262 }, { "epoch": 0.8500471549827099, "grad_norm": 2.219430923461914, "learning_rate": 0.0001716374624332365, "logits/chosen": -0.4873535931110382, "logits/rejected": -0.865464985370636, "logps/chosen": -4.004513740539551, "logps/rejected": -99.60807800292969, "loss": 0.3732931613922119, "memory(GiB)": 46.31, "nll_loss": 0.32243484258651733, "rewards/accuracies": 1.0, "rewards/chosen": 0.09392180293798447, "rewards/margins": 7.731295108795166, "rewards/rejected": -7.637372970581055, "step": 338, "train_speed(iter/s)": 0.042269 }, { "epoch": 0.8525620873939013, "grad_norm": 1.4633954763412476, "learning_rate": 0.0001714433809260435, "logits/chosen": -0.5742372274398804, "logits/rejected": -0.7338902950286865, "logps/chosen": -13.275593757629395, "logps/rejected": -84.15313720703125, "loss": 0.6252297759056091, "memory(GiB)": 46.31, "nll_loss": 0.5572872757911682, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1977962702512741, "rewards/margins": 6.996209621429443, "rewards/rejected": -6.798413276672363, "step": 339, "train_speed(iter/s)": 0.041927 }, { "epoch": 0.8550770198050928, "grad_norm": 3.0565924644470215, "learning_rate": 0.0001712487481845231, "logits/chosen": -0.3683091104030609, "logits/rejected": -0.7499415874481201, "logps/chosen": -6.7348151206970215, "logps/rejected": -91.9858169555664, "loss": 0.37550103664398193, "memory(GiB)": 46.31, "nll_loss": 0.2465668022632599, "rewards/accuracies": 0.9375, "rewards/chosen": -0.00661234837025404, "rewards/margins": 7.401021957397461, "rewards/rejected": -7.40763521194458, "step": 340, "train_speed(iter/s)": 0.041588 }, { "epoch": 0.8575919522162841, "grad_norm": 1.3918770551681519, "learning_rate": 0.00017105356571039945, "logits/chosen": -0.49031901359558105, "logits/rejected": -0.8110065460205078, "logps/chosen": -7.980224132537842, "logps/rejected": -82.05560302734375, "loss": 0.458918035030365, "memory(GiB)": 46.31, "nll_loss": 0.3925769627094269, "rewards/accuracies": 1.0, "rewards/chosen": 0.05228738859295845, "rewards/margins": 6.4176025390625, "rewards/rejected": -6.365314960479736, "step": 341, "train_speed(iter/s)": 0.04126 }, { "epoch": 0.8601068846274756, "grad_norm": 2.4676175117492676, "learning_rate": 0.00017085783500963824, "logits/chosen": -0.4095371961593628, "logits/rejected": -0.7287023067474365, "logps/chosen": -3.0237834453582764, "logps/rejected": -64.21556091308594, "loss": 0.30038806796073914, "memory(GiB)": 46.31, "nll_loss": 0.17780858278274536, "rewards/accuracies": 0.96875, "rewards/chosen": 0.22278741002082825, "rewards/margins": 4.871509075164795, "rewards/rejected": -4.648721218109131, "step": 342, "train_speed(iter/s)": 0.040936 }, { "epoch": 0.8626218170386671, "grad_norm": 2.6251206398010254, "learning_rate": 0.00017066155759243507, "logits/chosen": -0.559488832950592, "logits/rejected": -0.7672823667526245, "logps/chosen": -4.220951557159424, "logps/rejected": -62.22422790527344, "loss": 0.30714038014411926, "memory(GiB)": 46.31, "nll_loss": 0.201259046792984, "rewards/accuracies": 0.96875, "rewards/chosen": 0.024884473532438278, "rewards/margins": 4.834282398223877, "rewards/rejected": -4.809398174285889, "step": 343, "train_speed(iter/s)": 0.040623 }, { "epoch": 0.8651367494498585, "grad_norm": 1.955743670463562, "learning_rate": 0.00017046473497320385, "logits/chosen": -0.5384873151779175, "logits/rejected": -0.7534351944923401, "logps/chosen": -4.600622653961182, "logps/rejected": -62.08992004394531, "loss": 0.4026404619216919, "memory(GiB)": 46.31, "nll_loss": 0.30260390043258667, "rewards/accuracies": 0.96875, "rewards/chosen": 0.04929278418421745, "rewards/margins": 4.71509313583374, "rewards/rejected": -4.665800094604492, "step": 344, "train_speed(iter/s)": 0.040318 }, { "epoch": 0.86765168186105, "grad_norm": 1.3359214067459106, "learning_rate": 0.0001702673686705651, "logits/chosen": -0.5102373361587524, "logits/rejected": -0.7203519940376282, "logps/chosen": -3.7551567554473877, "logps/rejected": -58.683082580566406, "loss": 0.26333218812942505, "memory(GiB)": 46.31, "nll_loss": 0.1936265230178833, "rewards/accuracies": 1.0, "rewards/chosen": 0.08393731713294983, "rewards/margins": 4.568361282348633, "rewards/rejected": -4.484423637390137, "step": 345, "train_speed(iter/s)": 0.040019 }, { "epoch": 0.8701666142722414, "grad_norm": 1.2940682172775269, "learning_rate": 0.00017006946020733425, "logits/chosen": -0.423111230134964, "logits/rejected": -0.6618233919143677, "logps/chosen": -3.599187135696411, "logps/rejected": -57.84706115722656, "loss": 0.23464316129684448, "memory(GiB)": 46.31, "nll_loss": 0.1839112639427185, "rewards/accuracies": 1.0, "rewards/chosen": 0.3290537893772125, "rewards/margins": 4.494791507720947, "rewards/rejected": -4.165737152099609, "step": 346, "train_speed(iter/s)": 0.039723 }, { "epoch": 0.8726815466834329, "grad_norm": 4.882867336273193, "learning_rate": 0.00016987101111050982, "logits/chosen": -0.487949937582016, "logits/rejected": -0.680131733417511, "logps/chosen": -7.074028015136719, "logps/rejected": -57.329803466796875, "loss": 0.5204004645347595, "memory(GiB)": 46.31, "nll_loss": 0.4155931770801544, "rewards/accuracies": 0.96875, "rewards/chosen": 0.21489042043685913, "rewards/margins": 4.179245471954346, "rewards/rejected": -3.9643547534942627, "step": 347, "train_speed(iter/s)": 0.039433 }, { "epoch": 0.8751964790946243, "grad_norm": 5.844240665435791, "learning_rate": 0.00016967202291126173, "logits/chosen": -0.371315062046051, "logits/rejected": -0.7444156408309937, "logps/chosen": -5.045580863952637, "logps/rejected": -67.66508483886719, "loss": 0.6576581001281738, "memory(GiB)": 46.31, "nll_loss": 0.48166701197624207, "rewards/accuracies": 0.9375, "rewards/chosen": 0.07985140383243561, "rewards/margins": 4.563851833343506, "rewards/rejected": -4.484000205993652, "step": 348, "train_speed(iter/s)": 0.039147 }, { "epoch": 0.8777114115058158, "grad_norm": 7.42000675201416, "learning_rate": 0.00016947249714491938, "logits/chosen": -0.36127984523773193, "logits/rejected": -0.7726977467536926, "logps/chosen": -4.132587432861328, "logps/rejected": -69.11751556396484, "loss": 0.4309765100479126, "memory(GiB)": 46.31, "nll_loss": 0.3314419388771057, "rewards/accuracies": 0.96875, "rewards/chosen": 0.12238576263189316, "rewards/margins": 4.787508010864258, "rewards/rejected": -4.6651225090026855, "step": 349, "train_speed(iter/s)": 0.038867 }, { "epoch": 0.8802263439170073, "grad_norm": 8.671714782714844, "learning_rate": 0.00016927243535095997, "logits/chosen": -0.584995448589325, "logits/rejected": -0.7704389691352844, "logps/chosen": -5.863958358764648, "logps/rejected": -54.833553314208984, "loss": 0.6147282123565674, "memory(GiB)": 46.31, "nll_loss": 0.5134801864624023, "rewards/accuracies": 0.9375, "rewards/chosen": 0.10420051962137222, "rewards/margins": 4.378936767578125, "rewards/rejected": -4.274736404418945, "step": 350, "train_speed(iter/s)": 0.038598 }, { "epoch": 0.8827412763281987, "grad_norm": 1.384485125541687, "learning_rate": 0.0001690718390729964, "logits/chosen": -0.49590209126472473, "logits/rejected": -0.7934409379959106, "logps/chosen": -3.4274821281433105, "logps/rejected": -75.05296325683594, "loss": 0.28905826807022095, "memory(GiB)": 46.31, "nll_loss": 0.2423771321773529, "rewards/accuracies": 1.0, "rewards/chosen": 0.10859310626983643, "rewards/margins": 5.8243184089660645, "rewards/rejected": -5.715725898742676, "step": 351, "train_speed(iter/s)": 0.038264 }, { "epoch": 0.8852562087393901, "grad_norm": 2.392730474472046, "learning_rate": 0.00016887070985876555, "logits/chosen": -0.6525416970252991, "logits/rejected": -0.8886513710021973, "logps/chosen": -3.8878211975097656, "logps/rejected": -70.61896514892578, "loss": 0.3180730640888214, "memory(GiB)": 46.31, "nll_loss": 0.24728332459926605, "rewards/accuracies": 0.9375, "rewards/chosen": -0.021800898015499115, "rewards/margins": 5.6876444816589355, "rewards/rejected": -5.709445476531982, "step": 352, "train_speed(iter/s)": 0.038006 }, { "epoch": 0.8877711411505815, "grad_norm": 1.8687230348587036, "learning_rate": 0.00016866904926011623, "logits/chosen": -0.5100048780441284, "logits/rejected": -0.8430832028388977, "logps/chosen": -7.690830230712891, "logps/rejected": -79.55768585205078, "loss": 0.5146079063415527, "memory(GiB)": 46.31, "nll_loss": 0.480059415102005, "rewards/accuracies": 1.0, "rewards/chosen": 0.15592242777347565, "rewards/margins": 6.150962829589844, "rewards/rejected": -5.995040416717529, "step": 353, "train_speed(iter/s)": 0.03775 }, { "epoch": 0.890286073561773, "grad_norm": 2.0833466053009033, "learning_rate": 0.00016846685883299732, "logits/chosen": -0.6130125522613525, "logits/rejected": -0.8856174945831299, "logps/chosen": -2.691479206085205, "logps/rejected": -68.62126922607422, "loss": 0.29438382387161255, "memory(GiB)": 46.31, "nll_loss": 0.21310341358184814, "rewards/accuracies": 0.96875, "rewards/chosen": 0.21031589806079865, "rewards/margins": 5.494964599609375, "rewards/rejected": -5.284648418426514, "step": 354, "train_speed(iter/s)": 0.037501 }, { "epoch": 0.8928010059729645, "grad_norm": 5.909437656402588, "learning_rate": 0.0001682641401374456, "logits/chosen": -0.5725633502006531, "logits/rejected": -0.8215086460113525, "logps/chosen": -7.275115966796875, "logps/rejected": -68.49534606933594, "loss": 0.683579683303833, "memory(GiB)": 46.31, "nll_loss": 0.408993124961853, "rewards/accuracies": 0.96875, "rewards/chosen": -0.1348603069782257, "rewards/margins": 5.070530891418457, "rewards/rejected": -5.205390930175781, "step": 355, "train_speed(iter/s)": 0.037256 }, { "epoch": 0.8953159383841559, "grad_norm": 12.008193016052246, "learning_rate": 0.00016806089473757384, "logits/chosen": -0.48063433170318604, "logits/rejected": -0.8200632929801941, "logps/chosen": -5.245819091796875, "logps/rejected": -75.48336791992188, "loss": 0.5572218894958496, "memory(GiB)": 46.31, "nll_loss": 0.4593680500984192, "rewards/accuracies": 0.9375, "rewards/chosen": 0.042567022144794464, "rewards/margins": 5.983489036560059, "rewards/rejected": -5.940921783447266, "step": 356, "train_speed(iter/s)": 0.037015 }, { "epoch": 0.8978308707953474, "grad_norm": 2.3024280071258545, "learning_rate": 0.00016785712420155864, "logits/chosen": -0.5618032813072205, "logits/rejected": -0.7750505208969116, "logps/chosen": -5.727665424346924, "logps/rejected": -66.33029174804688, "loss": 0.49546992778778076, "memory(GiB)": 46.31, "nll_loss": 0.3907400071620941, "rewards/accuracies": 0.96875, "rewards/chosen": 0.2677232623100281, "rewards/margins": 5.57326078414917, "rewards/rejected": -5.305538177490234, "step": 357, "train_speed(iter/s)": 0.036778 }, { "epoch": 0.9003458032065388, "grad_norm": 1.5032073259353638, "learning_rate": 0.00016765283010162845, "logits/chosen": -0.6311084628105164, "logits/rejected": -0.884809136390686, "logps/chosen": -6.822357177734375, "logps/rejected": -86.34951782226562, "loss": 0.3546079099178314, "memory(GiB)": 46.31, "nll_loss": 0.32874923944473267, "rewards/accuracies": 1.0, "rewards/chosen": 0.21881002187728882, "rewards/margins": 7.21632719039917, "rewards/rejected": -6.997516632080078, "step": 358, "train_speed(iter/s)": 0.036547 }, { "epoch": 0.9028607356177303, "grad_norm": 12.216839790344238, "learning_rate": 0.0001674480140140514, "logits/chosen": -0.563357949256897, "logits/rejected": -0.8381999731063843, "logps/chosen": -5.951011657714844, "logps/rejected": -73.21198272705078, "loss": 0.6865818500518799, "memory(GiB)": 46.31, "nll_loss": 0.4365982413291931, "rewards/accuracies": 0.875, "rewards/chosen": -0.010548931546509266, "rewards/margins": 5.462331771850586, "rewards/rejected": -5.472881317138672, "step": 359, "train_speed(iter/s)": 0.036319 }, { "epoch": 0.9053756680289218, "grad_norm": 2.2943708896636963, "learning_rate": 0.000167242677519123, "logits/chosen": -0.5497739315032959, "logits/rejected": -0.7586756348609924, "logps/chosen": -6.904477596282959, "logps/rejected": -63.757484436035156, "loss": 0.524222195148468, "memory(GiB)": 46.31, "nll_loss": 0.4234248697757721, "rewards/accuracies": 0.96875, "rewards/chosen": 0.012442603707313538, "rewards/margins": 4.878024578094482, "rewards/rejected": -4.865581512451172, "step": 360, "train_speed(iter/s)": 0.036096 }, { "epoch": 0.9078906004401132, "grad_norm": 1.8752505779266357, "learning_rate": 0.00016703682220115416, "logits/chosen": -0.49279505014419556, "logits/rejected": -0.6874585747718811, "logps/chosen": -6.828492641448975, "logps/rejected": -56.00688552856445, "loss": 0.5722380876541138, "memory(GiB)": 46.31, "nll_loss": 0.4202978312969208, "rewards/accuracies": 0.90625, "rewards/chosen": 0.16803106665611267, "rewards/margins": 4.519139289855957, "rewards/rejected": -4.351108074188232, "step": 361, "train_speed(iter/s)": 0.035876 }, { "epoch": 0.9104055328513047, "grad_norm": 13.851007461547852, "learning_rate": 0.00016683044964845874, "logits/chosen": -0.3259517550468445, "logits/rejected": -0.7759101986885071, "logps/chosen": -2.291940689086914, "logps/rejected": -82.96823120117188, "loss": 0.3387893736362457, "memory(GiB)": 46.31, "nll_loss": 0.23984995484352112, "rewards/accuracies": 0.9375, "rewards/chosen": 0.13379287719726562, "rewards/margins": 6.447347640991211, "rewards/rejected": -6.313554763793945, "step": 362, "train_speed(iter/s)": 0.035659 }, { "epoch": 0.912920465262496, "grad_norm": 3.0961484909057617, "learning_rate": 0.00016662356145334158, "logits/chosen": -0.4107140004634857, "logits/rejected": -0.7844898104667664, "logps/chosen": -6.643762111663818, "logps/rejected": -85.53607940673828, "loss": 0.7083590030670166, "memory(GiB)": 46.31, "nll_loss": 0.5615524053573608, "rewards/accuracies": 0.9375, "rewards/chosen": 0.14561021327972412, "rewards/margins": 6.219028472900391, "rewards/rejected": -6.0734171867370605, "step": 363, "train_speed(iter/s)": 0.035445 }, { "epoch": 0.9154353976736875, "grad_norm": 3.2900326251983643, "learning_rate": 0.0001664161592120859, "logits/chosen": -0.48369789123535156, "logits/rejected": -0.8355358839035034, "logps/chosen": -4.989360332489014, "logps/rejected": -76.567138671875, "loss": 0.44374826550483704, "memory(GiB)": 46.31, "nll_loss": 0.3595760762691498, "rewards/accuracies": 0.96875, "rewards/chosen": 0.027300171554088593, "rewards/margins": 6.1481733322143555, "rewards/rejected": -6.12087345123291, "step": 364, "train_speed(iter/s)": 0.035237 }, { "epoch": 0.9179503300848789, "grad_norm": 2.7764828205108643, "learning_rate": 0.00016620824452494128, "logits/chosen": -0.3900059163570404, "logits/rejected": -0.7216352224349976, "logps/chosen": -2.781705617904663, "logps/rejected": -72.1066665649414, "loss": 0.32665613293647766, "memory(GiB)": 46.31, "nll_loss": 0.22233696281909943, "rewards/accuracies": 0.96875, "rewards/chosen": 0.2311152219772339, "rewards/margins": 5.6492919921875, "rewards/rejected": -5.418176651000977, "step": 365, "train_speed(iter/s)": 0.035032 }, { "epoch": 0.9204652624960704, "grad_norm": 3.265141010284424, "learning_rate": 0.00016599981899611104, "logits/chosen": -0.4622117280960083, "logits/rejected": -0.7529523372650146, "logps/chosen": -7.999266624450684, "logps/rejected": -65.14107513427734, "loss": 0.578423261642456, "memory(GiB)": 46.31, "nll_loss": 0.44822537899017334, "rewards/accuracies": 0.96875, "rewards/chosen": -0.09772372990846634, "rewards/margins": 4.932915210723877, "rewards/rejected": -5.030638694763184, "step": 366, "train_speed(iter/s)": 0.034831 }, { "epoch": 0.9229801949072619, "grad_norm": 7.041696071624756, "learning_rate": 0.00016579088423374004, "logits/chosen": -0.5291659235954285, "logits/rejected": -0.7164372205734253, "logps/chosen": -8.906322479248047, "logps/rejected": -56.788475036621094, "loss": 0.7593228816986084, "memory(GiB)": 46.31, "nll_loss": 0.5370157957077026, "rewards/accuracies": 0.90625, "rewards/chosen": -0.056440941989421844, "rewards/margins": 4.266091346740723, "rewards/rejected": -4.3225321769714355, "step": 367, "train_speed(iter/s)": 0.034633 }, { "epoch": 0.9254951273184533, "grad_norm": 5.255563735961914, "learning_rate": 0.00016558144184990227, "logits/chosen": -0.37939125299453735, "logits/rejected": -0.6499576568603516, "logps/chosen": -9.906354904174805, "logps/rejected": -63.14579772949219, "loss": 1.1975581645965576, "memory(GiB)": 46.31, "nll_loss": 0.7969497442245483, "rewards/accuracies": 0.84375, "rewards/chosen": -0.17077556252479553, "rewards/margins": 4.492245197296143, "rewards/rejected": -4.663021087646484, "step": 368, "train_speed(iter/s)": 0.034438 }, { "epoch": 0.9280100597296448, "grad_norm": 3.5527825355529785, "learning_rate": 0.0001653714934605883, "logits/chosen": -0.39540186524391174, "logits/rejected": -0.6938212513923645, "logps/chosen": -4.2778849601745605, "logps/rejected": -69.90755462646484, "loss": 0.43115851283073425, "memory(GiB)": 46.31, "nll_loss": 0.30130866169929504, "rewards/accuracies": 0.9375, "rewards/chosen": 0.0013723522424697876, "rewards/margins": 5.105025291442871, "rewards/rejected": -5.1036529541015625, "step": 369, "train_speed(iter/s)": 0.034247 }, { "epoch": 0.9305249921408362, "grad_norm": 1.7090513706207275, "learning_rate": 0.0001651610406856929, "logits/chosen": -0.3463304936885834, "logits/rejected": -0.579744815826416, "logps/chosen": -5.828442573547363, "logps/rejected": -69.42247772216797, "loss": 0.40808120369911194, "memory(GiB)": 46.31, "nll_loss": 0.30207115411758423, "rewards/accuracies": 0.96875, "rewards/chosen": 0.17322836816310883, "rewards/margins": 5.431868553161621, "rewards/rejected": -5.258640289306641, "step": 370, "train_speed(iter/s)": 0.034058 }, { "epoch": 0.9330399245520277, "grad_norm": 2.4113481044769287, "learning_rate": 0.00016495008514900254, "logits/chosen": -0.33101192116737366, "logits/rejected": -0.6629313826560974, "logps/chosen": -3.4581689834594727, "logps/rejected": -70.76332092285156, "loss": 0.3193123936653137, "memory(GiB)": 46.31, "nll_loss": 0.2055172175168991, "rewards/accuracies": 0.96875, "rewards/chosen": 0.16898323595523834, "rewards/margins": 5.381399154663086, "rewards/rejected": -5.2124152183532715, "step": 371, "train_speed(iter/s)": 0.033871 }, { "epoch": 0.9355548569632192, "grad_norm": 1.4116032123565674, "learning_rate": 0.00016473862847818277, "logits/chosen": -0.3042815625667572, "logits/rejected": -0.6890904903411865, "logps/chosen": -2.9157466888427734, "logps/rejected": -70.07747650146484, "loss": 0.35020875930786133, "memory(GiB)": 46.31, "nll_loss": 0.2551298141479492, "rewards/accuracies": 1.0, "rewards/chosen": 0.16930048167705536, "rewards/margins": 5.438758850097656, "rewards/rejected": -5.269458293914795, "step": 372, "train_speed(iter/s)": 0.033688 }, { "epoch": 0.9380697893744105, "grad_norm": 3.803670883178711, "learning_rate": 0.0001645266723047658, "logits/chosen": -0.43881455063819885, "logits/rejected": -0.7362341284751892, "logps/chosen": -3.6323812007904053, "logps/rejected": -70.09701538085938, "loss": 0.39496979117393494, "memory(GiB)": 46.31, "nll_loss": 0.3047463893890381, "rewards/accuracies": 0.96875, "rewards/chosen": 0.07599923759698868, "rewards/margins": 5.4243903160095215, "rewards/rejected": -5.348391532897949, "step": 373, "train_speed(iter/s)": 0.03351 }, { "epoch": 0.940584721785602, "grad_norm": 2.404557466506958, "learning_rate": 0.0001643142182641379, "logits/chosen": -0.34889113903045654, "logits/rejected": -0.6839251518249512, "logps/chosen": -4.086870193481445, "logps/rejected": -61.729209899902344, "loss": 0.3285399377346039, "memory(GiB)": 46.31, "nll_loss": 0.24733781814575195, "rewards/accuracies": 1.0, "rewards/chosen": 0.1459178924560547, "rewards/margins": 4.617620944976807, "rewards/rejected": -4.47170352935791, "step": 374, "train_speed(iter/s)": 0.033332 }, { "epoch": 0.9430996541967934, "grad_norm": 1.1544145345687866, "learning_rate": 0.0001641012679955265, "logits/chosen": -0.24821627140045166, "logits/rejected": -0.6281561255455017, "logps/chosen": -4.0372772216796875, "logps/rejected": -79.80203247070312, "loss": 0.2588978707790375, "memory(GiB)": 46.31, "nll_loss": 0.21967677772045135, "rewards/accuracies": 1.0, "rewards/chosen": 0.12974770367145538, "rewards/margins": 5.533021926879883, "rewards/rejected": -5.403275012969971, "step": 375, "train_speed(iter/s)": 0.033157 }, { "epoch": 0.9456145866079849, "grad_norm": 0.9758641123771667, "learning_rate": 0.00016388782314198804, "logits/chosen": -0.3430945873260498, "logits/rejected": -0.5772558450698853, "logps/chosen": -4.472623825073242, "logps/rejected": -63.992557525634766, "loss": 0.3319728672504425, "memory(GiB)": 46.31, "nll_loss": 0.27475422620773315, "rewards/accuracies": 0.96875, "rewards/chosen": 0.24630330502986908, "rewards/margins": 5.2393107414245605, "rewards/rejected": -4.993007183074951, "step": 376, "train_speed(iter/s)": 0.032986 }, { "epoch": 0.9481295190191764, "grad_norm": 2.5297164916992188, "learning_rate": 0.00016367388535039483, "logits/chosen": -0.4488414525985718, "logits/rejected": -0.5838022232055664, "logps/chosen": -7.7367448806762695, "logps/rejected": -54.747467041015625, "loss": 0.8268335461616516, "memory(GiB)": 46.31, "nll_loss": 0.5897634029388428, "rewards/accuracies": 0.9375, "rewards/chosen": 0.0750073790550232, "rewards/margins": 4.319672107696533, "rewards/rejected": -4.244664669036865, "step": 377, "train_speed(iter/s)": 0.032818 }, { "epoch": 0.9506444514303678, "grad_norm": 1.5900825262069702, "learning_rate": 0.00016345945627142263, "logits/chosen": -0.3362869322299957, "logits/rejected": -0.6326735019683838, "logps/chosen": -4.714737415313721, "logps/rejected": -64.11145782470703, "loss": 0.3902578055858612, "memory(GiB)": 46.31, "nll_loss": 0.33638864755630493, "rewards/accuracies": 1.0, "rewards/chosen": 0.2739555835723877, "rewards/margins": 5.219550132751465, "rewards/rejected": -4.945594787597656, "step": 378, "train_speed(iter/s)": 0.032653 }, { "epoch": 0.9531593838415593, "grad_norm": 4.194026470184326, "learning_rate": 0.00016324453755953773, "logits/chosen": -0.16701598465442657, "logits/rejected": -0.524956464767456, "logps/chosen": -7.983905792236328, "logps/rejected": -79.0844955444336, "loss": 0.5492078065872192, "memory(GiB)": 46.31, "nll_loss": 0.4571991264820099, "rewards/accuracies": 0.96875, "rewards/chosen": 0.04178844019770622, "rewards/margins": 5.678005218505859, "rewards/rejected": -5.636216640472412, "step": 379, "train_speed(iter/s)": 0.032486 }, { "epoch": 0.9556743162527507, "grad_norm": 3.1762311458587646, "learning_rate": 0.00016302913087298438, "logits/chosen": -0.23892173171043396, "logits/rejected": -0.5944012403488159, "logps/chosen": -2.855877161026001, "logps/rejected": -71.64038848876953, "loss": 0.2882783114910126, "memory(GiB)": 46.31, "nll_loss": 0.20760102570056915, "rewards/accuracies": 0.96875, "rewards/chosen": 0.15581874549388885, "rewards/margins": 5.530270099639893, "rewards/rejected": -5.374452114105225, "step": 380, "train_speed(iter/s)": 0.032324 }, { "epoch": 0.9581892486639422, "grad_norm": 1.3025074005126953, "learning_rate": 0.0001628132378737718, "logits/chosen": -0.255632609128952, "logits/rejected": -0.5729305744171143, "logps/chosen": -7.531216144561768, "logps/rejected": -76.65748596191406, "loss": 0.35401973128318787, "memory(GiB)": 46.31, "nll_loss": 0.28915196657180786, "rewards/accuracies": 1.0, "rewards/chosen": 0.17935439944267273, "rewards/margins": 5.804968357086182, "rewards/rejected": -5.625614166259766, "step": 381, "train_speed(iter/s)": 0.032164 }, { "epoch": 0.9607041810751336, "grad_norm": 1.362042784690857, "learning_rate": 0.00016259686022766147, "logits/chosen": -0.3349270224571228, "logits/rejected": -0.5813159346580505, "logps/chosen": -5.685837268829346, "logps/rejected": -82.27192687988281, "loss": 0.3539260923862457, "memory(GiB)": 46.31, "nll_loss": 0.2879295349121094, "rewards/accuracies": 0.96875, "rewards/chosen": 0.15435966849327087, "rewards/margins": 6.62058162689209, "rewards/rejected": -6.466221809387207, "step": 382, "train_speed(iter/s)": 0.032007 }, { "epoch": 0.963219113486325, "grad_norm": 2.0811262130737305, "learning_rate": 0.00016237999960415426, "logits/chosen": -0.41777318716049194, "logits/rejected": -0.6865772008895874, "logps/chosen": -4.136772155761719, "logps/rejected": -66.67522430419922, "loss": 0.37008586525917053, "memory(GiB)": 46.31, "nll_loss": 0.23609820008277893, "rewards/accuracies": 0.9375, "rewards/chosen": 0.061387136578559875, "rewards/margins": 5.56961727142334, "rewards/rejected": -5.508229732513428, "step": 383, "train_speed(iter/s)": 0.031855 }, { "epoch": 0.9657340458975165, "grad_norm": 2.0822818279266357, "learning_rate": 0.00016216265767647755, "logits/chosen": -0.34635311365127563, "logits/rejected": -0.6642223596572876, "logps/chosen": -7.652328014373779, "logps/rejected": -82.85174560546875, "loss": 0.4690261483192444, "memory(GiB)": 46.31, "nll_loss": 0.37369227409362793, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1277095228433609, "rewards/margins": 6.65843391418457, "rewards/rejected": -6.530723571777344, "step": 384, "train_speed(iter/s)": 0.031703 }, { "epoch": 0.9682489783087079, "grad_norm": 2.0401089191436768, "learning_rate": 0.0001619448361215723, "logits/chosen": -0.44850289821624756, "logits/rejected": -0.7342371940612793, "logps/chosen": -3.3725736141204834, "logps/rejected": -70.59913635253906, "loss": 0.3117908835411072, "memory(GiB)": 46.31, "nll_loss": 0.24734090268611908, "rewards/accuracies": 0.96875, "rewards/chosen": 0.2703946530818939, "rewards/margins": 5.976531982421875, "rewards/rejected": -5.706137657165527, "step": 385, "train_speed(iter/s)": 0.031555 }, { "epoch": 0.9707639107198994, "grad_norm": 3.712329387664795, "learning_rate": 0.0001617265366200801, "logits/chosen": -0.5690063834190369, "logits/rejected": -0.7474982738494873, "logps/chosen": -7.040001392364502, "logps/rejected": -58.06250762939453, "loss": 0.7071576118469238, "memory(GiB)": 46.31, "nll_loss": 0.5602840781211853, "rewards/accuracies": 0.875, "rewards/chosen": 0.24300126731395721, "rewards/margins": 4.731121063232422, "rewards/rejected": -4.488119602203369, "step": 386, "train_speed(iter/s)": 0.031409 }, { "epoch": 0.9732788431310908, "grad_norm": 4.140589714050293, "learning_rate": 0.0001615077608563302, "logits/chosen": -0.4431472420692444, "logits/rejected": -0.6148325800895691, "logps/chosen": -6.866641521453857, "logps/rejected": -60.58491516113281, "loss": 0.716019868850708, "memory(GiB)": 46.31, "nll_loss": 0.53892582654953, "rewards/accuracies": 0.9375, "rewards/chosen": 0.04589424282312393, "rewards/margins": 4.8714165687561035, "rewards/rejected": -4.825522422790527, "step": 387, "train_speed(iter/s)": 0.031265 }, { "epoch": 0.9757937755422823, "grad_norm": 1.4952077865600586, "learning_rate": 0.00016128851051832663, "logits/chosen": -0.3862327039241791, "logits/rejected": -0.6277360320091248, "logps/chosen": -4.6577043533325195, "logps/rejected": -65.08265686035156, "loss": 0.39572662115097046, "memory(GiB)": 46.31, "nll_loss": 0.32492324709892273, "rewards/accuracies": 1.0, "rewards/chosen": 0.1440601646900177, "rewards/margins": 4.995415687561035, "rewards/rejected": -4.85135555267334, "step": 388, "train_speed(iter/s)": 0.031122 }, { "epoch": 0.9783087079534738, "grad_norm": 6.760348320007324, "learning_rate": 0.000161068787297735, "logits/chosen": -0.36251139640808105, "logits/rejected": -0.6239615082740784, "logps/chosen": -7.945605754852295, "logps/rejected": -73.32543182373047, "loss": 0.7213991284370422, "memory(GiB)": 46.31, "nll_loss": 0.4911874234676361, "rewards/accuracies": 0.90625, "rewards/chosen": -0.02407783269882202, "rewards/margins": 5.578208923339844, "rewards/rejected": -5.602287292480469, "step": 389, "train_speed(iter/s)": 0.030981 }, { "epoch": 0.9808236403646652, "grad_norm": 1.4369536638259888, "learning_rate": 0.00016084859288986955, "logits/chosen": -0.4005224108695984, "logits/rejected": -0.7330536246299744, "logps/chosen": -2.2685210704803467, "logps/rejected": -75.97575378417969, "loss": 0.22442395985126495, "memory(GiB)": 46.31, "nll_loss": 0.15800324082374573, "rewards/accuracies": 0.96875, "rewards/chosen": 0.1866656392812729, "rewards/margins": 6.141209602355957, "rewards/rejected": -5.9545440673828125, "step": 390, "train_speed(iter/s)": 0.030843 }, { "epoch": 0.9833385727758567, "grad_norm": 2.5951297283172607, "learning_rate": 0.0001606279289936801, "logits/chosen": -0.3820829689502716, "logits/rejected": -0.6506544351577759, "logps/chosen": -5.631180763244629, "logps/rejected": -72.71686553955078, "loss": 0.4367509186267853, "memory(GiB)": 46.31, "nll_loss": 0.30333346128463745, "rewards/accuracies": 0.9375, "rewards/chosen": 0.2744603753089905, "rewards/margins": 5.615926742553711, "rewards/rejected": -5.341466426849365, "step": 391, "train_speed(iter/s)": 0.030705 }, { "epoch": 0.9858535051870481, "grad_norm": 1.5032093524932861, "learning_rate": 0.00016040679731173883, "logits/chosen": -0.4364391565322876, "logits/rejected": -0.8331149220466614, "logps/chosen": -4.418792247772217, "logps/rejected": -87.72761535644531, "loss": 0.33695462346076965, "memory(GiB)": 46.31, "nll_loss": 0.2598365545272827, "rewards/accuracies": 0.96875, "rewards/chosen": 0.15108640491962433, "rewards/margins": 7.023785591125488, "rewards/rejected": -6.872698783874512, "step": 392, "train_speed(iter/s)": 0.03057 }, { "epoch": 0.9883684375982396, "grad_norm": 1.427937388420105, "learning_rate": 0.0001601851995502272, "logits/chosen": -0.43706899881362915, "logits/rejected": -0.6605555415153503, "logps/chosen": -6.906405925750732, "logps/rejected": -74.19134521484375, "loss": 0.44790229201316833, "memory(GiB)": 46.31, "nll_loss": 0.36099445819854736, "rewards/accuracies": 0.96875, "rewards/chosen": 0.21629741787910461, "rewards/margins": 6.02875280380249, "rewards/rejected": -5.812455177307129, "step": 393, "train_speed(iter/s)": 0.030436 }, { "epoch": 0.9908833700094309, "grad_norm": 1.9399830102920532, "learning_rate": 0.00015996313741892287, "logits/chosen": -0.26576218008995056, "logits/rejected": -0.6935127377510071, "logps/chosen": -3.865288257598877, "logps/rejected": -82.85759735107422, "loss": 0.4476841688156128, "memory(GiB)": 46.31, "nll_loss": 0.32863834500312805, "rewards/accuracies": 0.96875, "rewards/chosen": 0.09060452878475189, "rewards/margins": 6.340301990509033, "rewards/rejected": -6.249697685241699, "step": 394, "train_speed(iter/s)": 0.030303 }, { "epoch": 0.9933983024206224, "grad_norm": 1.766074299812317, "learning_rate": 0.00015974061263118633, "logits/chosen": -0.45828965306282043, "logits/rejected": -0.7260013222694397, "logps/chosen": -5.735683917999268, "logps/rejected": -84.8984375, "loss": 0.4351236820220947, "memory(GiB)": 46.31, "nll_loss": 0.34114885330200195, "rewards/accuracies": 0.9375, "rewards/chosen": 0.15323616564273834, "rewards/margins": 6.745055198669434, "rewards/rejected": -6.591818809509277, "step": 395, "train_speed(iter/s)": 0.030174 }, { "epoch": 0.9959132348318139, "grad_norm": 1.4137344360351562, "learning_rate": 0.00015951762690394786, "logits/chosen": -0.36978721618652344, "logits/rejected": -0.6695666313171387, "logps/chosen": -6.405080318450928, "logps/rejected": -82.90550231933594, "loss": 0.3862062990665436, "memory(GiB)": 46.31, "nll_loss": 0.30257418751716614, "rewards/accuracies": 1.0, "rewards/chosen": 0.23101738095283508, "rewards/margins": 6.621737480163574, "rewards/rejected": -6.390720367431641, "step": 396, "train_speed(iter/s)": 0.030046 }, { "epoch": 0.9984281672430053, "grad_norm": 4.094442844390869, "learning_rate": 0.00015929418195769417, "logits/chosen": -0.48969727754592896, "logits/rejected": -0.6828620433807373, "logps/chosen": -6.582212448120117, "logps/rejected": -70.81021881103516, "loss": 0.4160895049571991, "memory(GiB)": 46.31, "nll_loss": 0.31492117047309875, "rewards/accuracies": 0.96875, "rewards/chosen": 0.13509458303451538, "rewards/margins": 5.814253330230713, "rewards/rejected": -5.679159164428711, "step": 397, "train_speed(iter/s)": 0.02992 }, { "epoch": 1.0025149324111915, "grad_norm": 1.9179918766021729, "learning_rate": 0.0001590702795164551, "logits/chosen": -0.35988548398017883, "logits/rejected": -0.6789807081222534, "logps/chosen": -4.162905693054199, "logps/rejected": -76.96558380126953, "loss": 0.7711372971534729, "memory(GiB)": 46.31, "nll_loss": 0.3491760492324829, "rewards/accuracies": 0.9807692170143127, "rewards/chosen": 0.2266516387462616, "rewards/margins": 6.186110496520996, "rewards/rejected": -5.959458351135254, "step": 398, "train_speed(iter/s)": 0.029672 }, { "epoch": 1.005029864822383, "grad_norm": 4.1499152183532715, "learning_rate": 0.00015884592130779054, "logits/chosen": -0.39633291959762573, "logits/rejected": -0.7770199775695801, "logps/chosen": -3.2545783519744873, "logps/rejected": -85.13819122314453, "loss": 0.3328564167022705, "memory(GiB)": 46.31, "nll_loss": 0.2181829959154129, "rewards/accuracies": 0.96875, "rewards/chosen": 0.2071080356836319, "rewards/margins": 6.915965557098389, "rewards/rejected": -6.708857536315918, "step": 399, "train_speed(iter/s)": 0.029551 }, { "epoch": 1.0075447972335743, "grad_norm": 1.9317526817321777, "learning_rate": 0.00015862110906277675, "logits/chosen": -0.3625487983226776, "logits/rejected": -0.7721232175827026, "logps/chosen": -2.4324581623077393, "logps/rejected": -94.80921173095703, "loss": 0.1829817295074463, "memory(GiB)": 46.31, "nll_loss": 0.13283208012580872, "rewards/accuracies": 1.0, "rewards/chosen": 0.14578577876091003, "rewards/margins": 7.785459041595459, "rewards/rejected": -7.639672756195068, "step": 400, "train_speed(iter/s)": 0.029431 } ], "logging_steps": 1, "max_steps": 1191, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 2.1869819546986086e+17, "train_batch_size": 2, "trial_name": null, "trial_params": null }