diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,8151 @@ +{ + "best_global_step": 300, + "best_metric": 0.60372525, + "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt7_noidk__pari0.5_0604/v0-20250604-225329/checkpoint-300", + "epoch": 1.002231520223152, + "eval_steps": 300, + "global_step": 450, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0022315202231520223, + "grad_norm": 11.248553276062012, + "learning_rate": 4.444444444444445e-06, + "logits/chosen": -0.48349103331565857, + "logits/rejected": -0.652603030204773, + "logps/chosen": -8.859519004821777, + "logps/rejected": -16.016435623168945, + "loss": 1.3468515872955322, + "memory(GiB)": 33.34, + "nll_loss": 0.6537042856216431, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1, + "train_speed(iter/s)": 0.010678 + }, + { + "epoch": 0.004463040446304045, + "grad_norm": 8.97808837890625, + "learning_rate": 8.88888888888889e-06, + "logits/chosen": -0.5174899101257324, + "logits/rejected": -0.6262180805206299, + "logps/chosen": -9.287378311157227, + "logps/rejected": -13.981985092163086, + "loss": 1.290852665901184, + "memory(GiB)": 36.2, + "nll_loss": 0.5977055430412292, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2, + "train_speed(iter/s)": 0.010974 + }, + { + "epoch": 0.0066945606694560665, + "grad_norm": 7.36937141418457, + "learning_rate": 1.3333333333333333e-05, + "logits/chosen": -0.42612141370773315, + "logits/rejected": -0.6118584871292114, + "logps/chosen": -6.56147575378418, + "logps/rejected": -16.162681579589844, + "loss": 1.1864153146743774, + "memory(GiB)": 36.2, + "nll_loss": 0.492550790309906, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.0009138870518654585, + "rewards/margins": -0.0012306026183068752, + "rewards/rejected": 0.00031671510078012943, + "step": 3, + "train_speed(iter/s)": 0.011052 + }, + { + "epoch": 0.00892608089260809, + "grad_norm": 7.918132305145264, + "learning_rate": 1.777777777777778e-05, + "logits/chosen": -0.5244420170783997, + "logits/rejected": -0.6519989371299744, + "logps/chosen": -7.461406707763672, + "logps/rejected": -17.006786346435547, + "loss": 1.3234074115753174, + "memory(GiB)": 39.28, + "nll_loss": 0.6342031359672546, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0032776433508843184, + "rewards/margins": 0.008089437149465084, + "rewards/rejected": -0.004811794031411409, + "step": 4, + "train_speed(iter/s)": 0.011094 + }, + { + "epoch": 0.011157601115760111, + "grad_norm": 9.900442123413086, + "learning_rate": 2.2222222222222223e-05, + "logits/chosen": -0.5073186159133911, + "logits/rejected": -0.6049282550811768, + "logps/chosen": -7.057607650756836, + "logps/rejected": -15.16832447052002, + "loss": 1.071779489517212, + "memory(GiB)": 39.28, + "nll_loss": 0.40041351318359375, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.022623565047979355, + "rewards/margins": 0.04467733949422836, + "rewards/rejected": -0.02205377072095871, + "step": 5, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.013389121338912133, + "grad_norm": 5.12174129486084, + "learning_rate": 2.6666666666666667e-05, + "logits/chosen": -0.48164039850234985, + "logits/rejected": -0.5844419002532959, + "logps/chosen": -6.2985005378723145, + "logps/rejected": -16.83810043334961, + "loss": 1.2731884717941284, + "memory(GiB)": 39.28, + "nll_loss": 0.600845992565155, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.010373853147029877, + "rewards/margins": 0.046181634068489075, + "rewards/rejected": -0.0358077809214592, + "step": 6, + "train_speed(iter/s)": 0.011143 + }, + { + "epoch": 0.015620641562064157, + "grad_norm": 4.1249213218688965, + "learning_rate": 3.111111111111111e-05, + "logits/chosen": -0.5566272735595703, + "logits/rejected": -0.6051127910614014, + "logps/chosen": -5.424515724182129, + "logps/rejected": -11.893548011779785, + "loss": 1.1159045696258545, + "memory(GiB)": 39.28, + "nll_loss": 0.46413302421569824, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.009653191082179546, + "rewards/margins": 0.11444520205259323, + "rewards/rejected": -0.10479200631380081, + "step": 7, + "train_speed(iter/s)": 0.011163 + }, + { + "epoch": 0.01785216178521618, + "grad_norm": 3.8928415775299072, + "learning_rate": 3.555555555555556e-05, + "logits/chosen": -0.5597310662269592, + "logits/rejected": -0.6341894268989563, + "logps/chosen": -6.350505352020264, + "logps/rejected": -20.239299774169922, + "loss": 0.8606890439987183, + "memory(GiB)": 39.28, + "nll_loss": 0.3352409899234772, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.018700765445828438, + "rewards/margins": 0.419380784034729, + "rewards/rejected": -0.4380815923213959, + "step": 8, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.0200836820083682, + "grad_norm": 7.264344692230225, + "learning_rate": 4e-05, + "logits/chosen": -0.500182569026947, + "logits/rejected": -0.595187246799469, + "logps/chosen": -16.194358825683594, + "logps/rejected": -20.5479793548584, + "loss": 1.6191558837890625, + "memory(GiB)": 39.28, + "nll_loss": 1.0435733795166016, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.21446219086647034, + "rewards/margins": 0.40107572078704834, + "rewards/rejected": -0.6155378818511963, + "step": 9, + "train_speed(iter/s)": 0.011188 + }, + { + "epoch": 0.022315202231520222, + "grad_norm": 7.542423725128174, + "learning_rate": 4.4444444444444447e-05, + "logits/chosen": -0.41380318999290466, + "logits/rejected": -0.5418319702148438, + "logps/chosen": -10.30124568939209, + "logps/rejected": -26.6314697265625, + "loss": 1.2661710977554321, + "memory(GiB)": 39.28, + "nll_loss": 0.6692220568656921, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.29620301723480225, + "rewards/margins": 0.5043031573295593, + "rewards/rejected": -0.8005062341690063, + "step": 10, + "train_speed(iter/s)": 0.01119 + }, + { + "epoch": 0.024546722454672244, + "grad_norm": 12.055508613586426, + "learning_rate": 4.888888888888889e-05, + "logits/chosen": -0.45050907135009766, + "logits/rejected": -0.5843921899795532, + "logps/chosen": -15.530860900878906, + "logps/rejected": -27.80389976501465, + "loss": 1.6187087297439575, + "memory(GiB)": 39.28, + "nll_loss": 1.0532186031341553, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.47808465361595154, + "rewards/margins": 0.41364309191703796, + "rewards/rejected": -0.8917278051376343, + "step": 11, + "train_speed(iter/s)": 0.011193 + }, + { + "epoch": 0.026778242677824266, + "grad_norm": 6.556560516357422, + "learning_rate": 5.333333333333333e-05, + "logits/chosen": -0.4449861943721771, + "logits/rejected": -0.5602853894233704, + "logps/chosen": -10.715709686279297, + "logps/rejected": -27.810680389404297, + "loss": 1.2998032569885254, + "memory(GiB)": 42.68, + "nll_loss": 0.6746751070022583, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.33532828092575073, + "rewards/margins": 0.39097556471824646, + "rewards/rejected": -0.7263038754463196, + "step": 12, + "train_speed(iter/s)": 0.011193 + }, + { + "epoch": 0.02900976290097629, + "grad_norm": 6.815319061279297, + "learning_rate": 5.7777777777777776e-05, + "logits/chosen": -0.45461201667785645, + "logits/rejected": -0.5696772933006287, + "logps/chosen": -8.947586059570312, + "logps/rejected": -23.452707290649414, + "loss": 1.347914218902588, + "memory(GiB)": 42.68, + "nll_loss": 0.7918389439582825, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14807622134685516, + "rewards/margins": 0.4436675012111664, + "rewards/rejected": -0.5917437076568604, + "step": 13, + "train_speed(iter/s)": 0.011196 + }, + { + "epoch": 0.031241283124128313, + "grad_norm": 4.015015125274658, + "learning_rate": 6.222222222222222e-05, + "logits/chosen": -0.4277312457561493, + "logits/rejected": -0.5162039995193481, + "logps/chosen": -7.859683513641357, + "logps/rejected": -25.14496612548828, + "loss": 1.1709389686584473, + "memory(GiB)": 46.13, + "nll_loss": 0.5424311757087708, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.09304673224687576, + "rewards/margins": 0.27104437351226807, + "rewards/rejected": -0.36409106850624084, + "step": 14, + "train_speed(iter/s)": 0.011191 + }, + { + "epoch": 0.03347280334728033, + "grad_norm": 2.5992424488067627, + "learning_rate": 6.666666666666667e-05, + "logits/chosen": -0.4692430794239044, + "logits/rejected": -0.5111399292945862, + "logps/chosen": -8.574230194091797, + "logps/rejected": -16.565174102783203, + "loss": 1.1323829889297485, + "memory(GiB)": 46.13, + "nll_loss": 0.5113993287086487, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.08413823693990707, + "rewards/margins": 0.19407325983047485, + "rewards/rejected": -0.27821147441864014, + "step": 15, + "train_speed(iter/s)": 0.011195 + }, + { + "epoch": 0.03570432357043236, + "grad_norm": 2.479030132293701, + "learning_rate": 7.111111111111112e-05, + "logits/chosen": -0.38820019364356995, + "logits/rejected": -0.5241715908050537, + "logps/chosen": -4.694685935974121, + "logps/rejected": -19.62047576904297, + "loss": 1.052955985069275, + "memory(GiB)": 46.13, + "nll_loss": 0.47422000765800476, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.06994247436523438, + "rewards/margins": 0.2729727327823639, + "rewards/rejected": -0.2030302733182907, + "step": 16, + "train_speed(iter/s)": 0.011196 + }, + { + "epoch": 0.03793584379358438, + "grad_norm": 2.1875240802764893, + "learning_rate": 7.555555555555556e-05, + "logits/chosen": -0.4483625292778015, + "logits/rejected": -0.5458305478096008, + "logps/chosen": -9.014262199401855, + "logps/rejected": -18.748149871826172, + "loss": 1.2149709463119507, + "memory(GiB)": 46.13, + "nll_loss": 0.6185500621795654, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1215842068195343, + "rewards/margins": 0.24843284487724304, + "rewards/rejected": -0.12684863805770874, + "step": 17, + "train_speed(iter/s)": 0.011197 + }, + { + "epoch": 0.0401673640167364, + "grad_norm": 2.639186382293701, + "learning_rate": 8e-05, + "logits/chosen": -0.4394489824771881, + "logits/rejected": -0.48552829027175903, + "logps/chosen": -9.954276084899902, + "logps/rejected": -14.681390762329102, + "loss": 1.1648714542388916, + "memory(GiB)": 46.13, + "nll_loss": 0.46223676204681396, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.011617322452366352, + "rewards/margins": 0.034979358315467834, + "rewards/rejected": -0.02336202934384346, + "step": 18, + "train_speed(iter/s)": 0.011199 + }, + { + "epoch": 0.042398884239888426, + "grad_norm": 2.351567029953003, + "learning_rate": 8.444444444444444e-05, + "logits/chosen": -0.4882590174674988, + "logits/rejected": -0.5300821661949158, + "logps/chosen": -7.595667362213135, + "logps/rejected": -17.29962921142578, + "loss": 1.1524200439453125, + "memory(GiB)": 46.13, + "nll_loss": 0.5075826644897461, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.06894361972808838, + "rewards/margins": 0.13017994165420532, + "rewards/rejected": -0.061236318200826645, + "step": 19, + "train_speed(iter/s)": 0.011201 + }, + { + "epoch": 0.044630404463040445, + "grad_norm": 2.8206121921539307, + "learning_rate": 8.888888888888889e-05, + "logits/chosen": -0.4782502055168152, + "logits/rejected": -0.5565172433853149, + "logps/chosen": -6.507530212402344, + "logps/rejected": -13.604747772216797, + "loss": 1.089160442352295, + "memory(GiB)": 46.13, + "nll_loss": 0.45874831080436707, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.06730557233095169, + "rewards/margins": 0.18410882353782654, + "rewards/rejected": -0.11680323630571365, + "step": 20, + "train_speed(iter/s)": 0.011206 + }, + { + "epoch": 0.04686192468619247, + "grad_norm": 2.452335834503174, + "learning_rate": 9.333333333333334e-05, + "logits/chosen": -0.4106925129890442, + "logits/rejected": -0.48896345496177673, + "logps/chosen": -7.722554683685303, + "logps/rejected": -22.094852447509766, + "loss": 1.153997540473938, + "memory(GiB)": 46.71, + "nll_loss": 0.5091268420219421, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.060057930648326874, + "rewards/margins": 0.16068722307682037, + "rewards/rejected": -0.1006293073296547, + "step": 21, + "train_speed(iter/s)": 0.011203 + }, + { + "epoch": 0.04909344490934449, + "grad_norm": 2.3324975967407227, + "learning_rate": 9.777777777777778e-05, + "logits/chosen": -0.4366399943828583, + "logits/rejected": -0.4600733816623688, + "logps/chosen": -9.774506568908691, + "logps/rejected": -14.850456237792969, + "loss": 1.185971975326538, + "memory(GiB)": 46.71, + "nll_loss": 0.5413804650306702, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.08716820925474167, + "rewards/margins": 0.17242644727230072, + "rewards/rejected": -0.08525823801755905, + "step": 22, + "train_speed(iter/s)": 0.011203 + }, + { + "epoch": 0.051324965132496514, + "grad_norm": 2.7291500568389893, + "learning_rate": 0.00010222222222222222, + "logits/chosen": -0.4168204665184021, + "logits/rejected": -0.5373830795288086, + "logps/chosen": -7.675040245056152, + "logps/rejected": -20.635356903076172, + "loss": 1.1511971950531006, + "memory(GiB)": 46.71, + "nll_loss": 0.4784548282623291, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.05419651046395302, + "rewards/margins": 0.0959930419921875, + "rewards/rejected": -0.04179652780294418, + "step": 23, + "train_speed(iter/s)": 0.011204 + }, + { + "epoch": 0.05355648535564853, + "grad_norm": 2.6357204914093018, + "learning_rate": 0.00010666666666666667, + "logits/chosen": -0.39612719416618347, + "logits/rejected": -0.4547193944454193, + "logps/chosen": -9.427398681640625, + "logps/rejected": -20.611848831176758, + "loss": 1.0459527969360352, + "memory(GiB)": 46.71, + "nll_loss": 0.42900872230529785, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.0577767938375473, + "rewards/margins": 0.24017955362796783, + "rewards/rejected": -0.18240275979042053, + "step": 24, + "train_speed(iter/s)": 0.011203 + }, + { + "epoch": 0.05578800557880056, + "grad_norm": 2.7458126544952393, + "learning_rate": 0.00011111111111111112, + "logits/chosen": -0.3978196978569031, + "logits/rejected": -0.4898090958595276, + "logps/chosen": -9.801487922668457, + "logps/rejected": -21.7921199798584, + "loss": 1.1577973365783691, + "memory(GiB)": 46.71, + "nll_loss": 0.5382578372955322, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.03896612673997879, + "rewards/margins": 0.24017201364040375, + "rewards/rejected": -0.20120586454868317, + "step": 25, + "train_speed(iter/s)": 0.011205 + }, + { + "epoch": 0.05801952580195258, + "grad_norm": 4.185514450073242, + "learning_rate": 0.00011555555555555555, + "logits/chosen": -0.43978849053382874, + "logits/rejected": -0.5358645915985107, + "logps/chosen": -4.664649963378906, + "logps/rejected": -18.568632125854492, + "loss": 0.9911558032035828, + "memory(GiB)": 46.71, + "nll_loss": 0.39514851570129395, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.01693350076675415, + "rewards/margins": 0.28997230529785156, + "rewards/rejected": -0.3069058060646057, + "step": 26, + "train_speed(iter/s)": 0.011208 + }, + { + "epoch": 0.0602510460251046, + "grad_norm": 2.983151912689209, + "learning_rate": 0.00012, + "logits/chosen": -0.46957868337631226, + "logits/rejected": -0.5086190700531006, + "logps/chosen": -14.16710376739502, + "logps/rejected": -15.19424819946289, + "loss": 1.3119632005691528, + "memory(GiB)": 46.71, + "nll_loss": 0.732680082321167, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.1205766424536705, + "rewards/margins": 0.2763415277004242, + "rewards/rejected": -0.1557648628950119, + "step": 27, + "train_speed(iter/s)": 0.01121 + }, + { + "epoch": 0.06248256624825663, + "grad_norm": 3.7991929054260254, + "learning_rate": 0.00012444444444444444, + "logits/chosen": -0.3287929892539978, + "logits/rejected": -0.46299535036087036, + "logps/chosen": -6.527912616729736, + "logps/rejected": -23.983104705810547, + "loss": 1.0033739805221558, + "memory(GiB)": 46.71, + "nll_loss": 0.44460976123809814, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.07841134071350098, + "rewards/margins": 0.3698306679725647, + "rewards/rejected": -0.29141926765441895, + "step": 28, + "train_speed(iter/s)": 0.011207 + }, + { + "epoch": 0.06471408647140865, + "grad_norm": 3.0063183307647705, + "learning_rate": 0.00012888888888888892, + "logits/chosen": -0.3735123574733734, + "logits/rejected": -0.458845853805542, + "logps/chosen": -7.325934886932373, + "logps/rejected": -21.65030288696289, + "loss": 1.0098869800567627, + "memory(GiB)": 46.71, + "nll_loss": 0.477042555809021, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.025919266045093536, + "rewards/margins": 0.47939103841781616, + "rewards/rejected": -0.4534717798233032, + "step": 29, + "train_speed(iter/s)": 0.011208 + }, + { + "epoch": 0.06694560669456066, + "grad_norm": 2.6957406997680664, + "learning_rate": 0.00013333333333333334, + "logits/chosen": -0.3790513873100281, + "logits/rejected": -0.4815320372581482, + "logps/chosen": -6.8280110359191895, + "logps/rejected": -22.987621307373047, + "loss": 1.0349267721176147, + "memory(GiB)": 46.71, + "nll_loss": 0.5045362114906311, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.09961891919374466, + "rewards/margins": 0.4102502763271332, + "rewards/rejected": -0.3106313645839691, + "step": 30, + "train_speed(iter/s)": 0.011209 + }, + { + "epoch": 0.06917712691771269, + "grad_norm": 2.7142391204833984, + "learning_rate": 0.0001377777777777778, + "logits/chosen": -0.36308056116104126, + "logits/rejected": -0.44216084480285645, + "logps/chosen": -5.850837230682373, + "logps/rejected": -21.365463256835938, + "loss": 0.9850523471832275, + "memory(GiB)": 46.71, + "nll_loss": 0.43448853492736816, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.0431375615298748, + "rewards/margins": 0.3983423113822937, + "rewards/rejected": -0.3552047610282898, + "step": 31, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.07140864714086471, + "grad_norm": 3.897137403488159, + "learning_rate": 0.00014222222222222224, + "logits/chosen": -0.3979527950286865, + "logits/rejected": -0.43778184056282043, + "logps/chosen": -10.285924911499023, + "logps/rejected": -18.7073974609375, + "loss": 1.1233854293823242, + "memory(GiB)": 46.71, + "nll_loss": 0.5153944492340088, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.026699109002947807, + "rewards/margins": 0.23945647478103638, + "rewards/rejected": -0.21275737881660461, + "step": 32, + "train_speed(iter/s)": 0.011213 + }, + { + "epoch": 0.07364016736401674, + "grad_norm": 3.93380069732666, + "learning_rate": 0.00014666666666666666, + "logits/chosen": -0.2758394777774811, + "logits/rejected": -0.37545740604400635, + "logps/chosen": -12.159762382507324, + "logps/rejected": -24.926610946655273, + "loss": 1.2487003803253174, + "memory(GiB)": 46.71, + "nll_loss": 0.6138952970504761, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.09166956692934036, + "rewards/margins": 0.2808037996292114, + "rewards/rejected": -0.3724733591079712, + "step": 33, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.07587168758716877, + "grad_norm": 3.21878981590271, + "learning_rate": 0.0001511111111111111, + "logits/chosen": -0.24420826137065887, + "logits/rejected": -0.38765522837638855, + "logps/chosen": -6.041399955749512, + "logps/rejected": -22.503984451293945, + "loss": 0.9895752668380737, + "memory(GiB)": 46.71, + "nll_loss": 0.44009482860565186, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.010253368876874447, + "rewards/margins": 0.5208209753036499, + "rewards/rejected": -0.5105676054954529, + "step": 34, + "train_speed(iter/s)": 0.01121 + }, + { + "epoch": 0.07810320781032078, + "grad_norm": 5.1504316329956055, + "learning_rate": 0.00015555555555555556, + "logits/chosen": -0.2664889693260193, + "logits/rejected": -0.40212178230285645, + "logps/chosen": -5.9132609367370605, + "logps/rejected": -24.527639389038086, + "loss": 0.8355879187583923, + "memory(GiB)": 46.71, + "nll_loss": 0.4006969928741455, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.03980417177081108, + "rewards/margins": 0.8774286508560181, + "rewards/rejected": -0.9172328114509583, + "step": 35, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.0803347280334728, + "grad_norm": 3.388197660446167, + "learning_rate": 0.00016, + "logits/chosen": -0.25354304909706116, + "logits/rejected": -0.33330878615379333, + "logps/chosen": -9.807186126708984, + "logps/rejected": -22.963045120239258, + "loss": 1.1024540662765503, + "memory(GiB)": 46.71, + "nll_loss": 0.6454060673713684, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.001722879707813263, + "rewards/margins": 0.8297844529151917, + "rewards/rejected": -0.8315073847770691, + "step": 36, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.08256624825662483, + "grad_norm": 2.596273899078369, + "learning_rate": 0.00016444444444444444, + "logits/chosen": -0.21093566715717316, + "logits/rejected": -0.3617200553417206, + "logps/chosen": -2.8380072116851807, + "logps/rejected": -30.42251968383789, + "loss": 0.6882003545761108, + "memory(GiB)": 46.71, + "nll_loss": 0.2877930700778961, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.06296033412218094, + "rewards/margins": 1.0582321882247925, + "rewards/rejected": -0.9952719211578369, + "step": 37, + "train_speed(iter/s)": 0.01121 + }, + { + "epoch": 0.08479776847977685, + "grad_norm": 2.358452796936035, + "learning_rate": 0.00016888888888888889, + "logits/chosen": -0.2989475429058075, + "logits/rejected": -0.37469911575317383, + "logps/chosen": -6.1970534324646, + "logps/rejected": -19.100358963012695, + "loss": 0.8981406092643738, + "memory(GiB)": 46.71, + "nll_loss": 0.3934318423271179, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.002394435927271843, + "rewards/margins": 0.6755657196044922, + "rewards/rejected": -0.6731712818145752, + "step": 38, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.08702928870292886, + "grad_norm": 2.9299051761627197, + "learning_rate": 0.00017333333333333334, + "logits/chosen": -0.2796667516231537, + "logits/rejected": -0.34129300713539124, + "logps/chosen": -8.498173713684082, + "logps/rejected": -26.264965057373047, + "loss": 1.012364387512207, + "memory(GiB)": 46.71, + "nll_loss": 0.586083173751831, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07207094132900238, + "rewards/margins": 1.1341111660003662, + "rewards/rejected": -1.0620403289794922, + "step": 39, + "train_speed(iter/s)": 0.011212 + }, + { + "epoch": 0.08926080892608089, + "grad_norm": 3.547189474105835, + "learning_rate": 0.00017777777777777779, + "logits/chosen": -0.28194305300712585, + "logits/rejected": -0.33060967922210693, + "logps/chosen": -7.982609748840332, + "logps/rejected": -20.187652587890625, + "loss": 0.9403265118598938, + "memory(GiB)": 46.71, + "nll_loss": 0.42783123254776, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.002833280712366104, + "rewards/margins": 0.7690448760986328, + "rewards/rejected": -0.7662116289138794, + "step": 40, + "train_speed(iter/s)": 0.011213 + }, + { + "epoch": 0.09149232914923291, + "grad_norm": 3.4967706203460693, + "learning_rate": 0.00018222222222222224, + "logits/chosen": -0.21270643174648285, + "logits/rejected": -0.3004887104034424, + "logps/chosen": -5.226779937744141, + "logps/rejected": -22.208778381347656, + "loss": 0.8620257377624512, + "memory(GiB)": 46.71, + "nll_loss": 0.3914262056350708, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.018529795110225677, + "rewards/margins": 0.9286664128303528, + "rewards/rejected": -0.9101365804672241, + "step": 41, + "train_speed(iter/s)": 0.011213 + }, + { + "epoch": 0.09372384937238494, + "grad_norm": 6.100057601928711, + "learning_rate": 0.0001866666666666667, + "logits/chosen": -0.29974937438964844, + "logits/rejected": -0.36353808641433716, + "logps/chosen": -10.446137428283691, + "logps/rejected": -22.973119735717773, + "loss": 1.2477580308914185, + "memory(GiB)": 46.71, + "nll_loss": 0.6882976293563843, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.2526606023311615, + "rewards/margins": 0.5461662411689758, + "rewards/rejected": -0.7988268136978149, + "step": 42, + "train_speed(iter/s)": 0.011214 + }, + { + "epoch": 0.09595536959553697, + "grad_norm": 4.625721454620361, + "learning_rate": 0.00019111111111111114, + "logits/chosen": -0.21875600516796112, + "logits/rejected": -0.3691503405570984, + "logps/chosen": -8.029109001159668, + "logps/rejected": -33.986671447753906, + "loss": 0.9927695989608765, + "memory(GiB)": 46.71, + "nll_loss": 0.6190873384475708, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.10942590236663818, + "rewards/margins": 1.7278107404708862, + "rewards/rejected": -1.837236762046814, + "step": 43, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.09818688981868898, + "grad_norm": 4.994227886199951, + "learning_rate": 0.00019555555555555556, + "logits/chosen": -0.25192195177078247, + "logits/rejected": -0.32522761821746826, + "logps/chosen": -9.805305480957031, + "logps/rejected": -34.161895751953125, + "loss": 0.9997942447662354, + "memory(GiB)": 46.71, + "nll_loss": 0.6020480990409851, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.30676746368408203, + "rewards/margins": 1.7450387477874756, + "rewards/rejected": -2.0518062114715576, + "step": 44, + "train_speed(iter/s)": 0.011214 + }, + { + "epoch": 0.100418410041841, + "grad_norm": 4.717008590698242, + "learning_rate": 0.0002, + "logits/chosen": -0.33617067337036133, + "logits/rejected": -0.35846105217933655, + "logps/chosen": -9.175792694091797, + "logps/rejected": -21.536231994628906, + "loss": 1.014683485031128, + "memory(GiB)": 46.71, + "nll_loss": 0.5040097236633301, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.014129618182778358, + "rewards/margins": 0.7111825942993164, + "rewards/rejected": -0.6970530152320862, + "step": 45, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.10264993026499303, + "grad_norm": 3.089895248413086, + "learning_rate": 0.0001999993185874369, + "logits/chosen": -0.3643554747104645, + "logits/rejected": -0.3828020393848419, + "logps/chosen": -11.385210990905762, + "logps/rejected": -27.30405044555664, + "loss": 0.9333115220069885, + "memory(GiB)": 46.71, + "nll_loss": 0.5031883120536804, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.08416246622800827, + "rewards/margins": 1.1319228410720825, + "rewards/rejected": -1.216085433959961, + "step": 46, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.10488145048814505, + "grad_norm": 4.295483589172363, + "learning_rate": 0.00019999727435903407, + "logits/chosen": -0.3375833332538605, + "logits/rejected": -0.44685542583465576, + "logps/chosen": -4.36940336227417, + "logps/rejected": -21.149555206298828, + "loss": 0.959221601486206, + "memory(GiB)": 46.71, + "nll_loss": 0.42982059717178345, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.15477100014686584, + "rewards/margins": 0.5941969752311707, + "rewards/rejected": -0.4394259452819824, + "step": 47, + "train_speed(iter/s)": 0.011216 + }, + { + "epoch": 0.10711297071129706, + "grad_norm": 2.4132261276245117, + "learning_rate": 0.0001999938673426507, + "logits/chosen": -0.3103625178337097, + "logits/rejected": -0.4320617616176605, + "logps/chosen": -5.777155876159668, + "logps/rejected": -20.897228240966797, + "loss": 0.9094122648239136, + "memory(GiB)": 46.71, + "nll_loss": 0.41829684376716614, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.18217116594314575, + "rewards/margins": 0.714988112449646, + "rewards/rejected": -0.5328169465065002, + "step": 48, + "train_speed(iter/s)": 0.011216 + }, + { + "epoch": 0.10934449093444909, + "grad_norm": 2.716282367706299, + "learning_rate": 0.00019998909758471852, + "logits/chosen": -0.3802068829536438, + "logits/rejected": -0.4424472749233246, + "logps/chosen": -7.844504356384277, + "logps/rejected": -21.755281448364258, + "loss": 1.0127230882644653, + "memory(GiB)": 46.71, + "nll_loss": 0.5032894611358643, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.07639233767986298, + "rewards/margins": 0.79319167137146, + "rewards/rejected": -0.7167993187904358, + "step": 49, + "train_speed(iter/s)": 0.011217 + }, + { + "epoch": 0.11157601115760112, + "grad_norm": 3.5755205154418945, + "learning_rate": 0.000199982965150241, + "logits/chosen": -0.32785969972610474, + "logits/rejected": -0.384295791387558, + "logps/chosen": -12.152947425842285, + "logps/rejected": -21.354625701904297, + "loss": 1.1883673667907715, + "memory(GiB)": 46.71, + "nll_loss": 0.6465156674385071, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.11994989961385727, + "rewards/margins": 0.5093130469322205, + "rewards/rejected": -0.6292628645896912, + "step": 50, + "train_speed(iter/s)": 0.011217 + }, + { + "epoch": 0.11380753138075314, + "grad_norm": 2.3085532188415527, + "learning_rate": 0.00019997547012279244, + "logits/chosen": -0.3017635941505432, + "logits/rejected": -0.4112682342529297, + "logps/chosen": -6.112311363220215, + "logps/rejected": -27.533615112304688, + "loss": 0.8378341794013977, + "memory(GiB)": 46.71, + "nll_loss": 0.4290671646595001, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1803205907344818, + "rewards/margins": 0.9729282855987549, + "rewards/rejected": -0.7926077246665955, + "step": 51, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.11603905160390517, + "grad_norm": 2.699073076248169, + "learning_rate": 0.00019996661260451704, + "logits/chosen": -0.30682653188705444, + "logits/rejected": -0.3987538814544678, + "logps/chosen": -9.152729034423828, + "logps/rejected": -34.165122985839844, + "loss": 0.8945070505142212, + "memory(GiB)": 46.71, + "nll_loss": 0.5094670653343201, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06719347834587097, + "rewards/margins": 1.5018631219863892, + "rewards/rejected": -1.569056510925293, + "step": 52, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.11827057182705718, + "grad_norm": 2.7043395042419434, + "learning_rate": 0.00019995639271612722, + "logits/chosen": -0.3258576989173889, + "logits/rejected": -0.43161430954933167, + "logps/chosen": -9.04993724822998, + "logps/rejected": -29.138545989990234, + "loss": 0.8318122029304504, + "memory(GiB)": 46.71, + "nll_loss": 0.3884216248989105, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.015111252665519714, + "rewards/margins": 1.0627268552780151, + "rewards/rejected": -1.0778381824493408, + "step": 53, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.1205020920502092, + "grad_norm": 5.87897253036499, + "learning_rate": 0.00019994481059690223, + "logits/chosen": -0.36754727363586426, + "logits/rejected": -0.4476523995399475, + "logps/chosen": -10.40526008605957, + "logps/rejected": -32.984188079833984, + "loss": 1.1996067762374878, + "memory(GiB)": 46.71, + "nll_loss": 0.7492297887802124, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.08549878001213074, + "rewards/margins": 1.944332242012024, + "rewards/rejected": -2.0298309326171875, + "step": 54, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.12273361227336123, + "grad_norm": 2.4889438152313232, + "learning_rate": 0.00019993186640468604, + "logits/chosen": -0.28098922967910767, + "logits/rejected": -0.3998360335826874, + "logps/chosen": -11.18938159942627, + "logps/rejected": -33.78412628173828, + "loss": 0.97004234790802, + "memory(GiB)": 46.71, + "nll_loss": 0.532280445098877, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.10216796398162842, + "rewards/margins": 1.2508039474487305, + "rewards/rejected": -1.3529717922210693, + "step": 55, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.12496513249651325, + "grad_norm": 3.1122195720672607, + "learning_rate": 0.0001999175603158854, + "logits/chosen": -0.25420328974723816, + "logits/rejected": -0.40874406695365906, + "logps/chosen": -9.440892219543457, + "logps/rejected": -36.343971252441406, + "loss": 0.8295825123786926, + "memory(GiB)": 46.71, + "nll_loss": 0.4795616567134857, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.014746717177331448, + "rewards/margins": 1.4991542100906372, + "rewards/rejected": -1.4844075441360474, + "step": 56, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.12719665271966527, + "grad_norm": 3.5490951538085938, + "learning_rate": 0.0001999018925254673, + "logits/chosen": -0.3385506272315979, + "logits/rejected": -0.45762842893600464, + "logps/chosen": -9.239278793334961, + "logps/rejected": -32.54361343383789, + "loss": 0.9066818952560425, + "memory(GiB)": 46.71, + "nll_loss": 0.5777646899223328, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.03782232105731964, + "rewards/margins": 1.4800399541854858, + "rewards/rejected": -1.5178624391555786, + "step": 57, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.1294281729428173, + "grad_norm": 3.0512146949768066, + "learning_rate": 0.00019988486324695628, + "logits/chosen": -0.3186691403388977, + "logits/rejected": -0.4108823835849762, + "logps/chosen": -13.795214653015137, + "logps/rejected": -34.02068328857422, + "loss": 0.8977184295654297, + "memory(GiB)": 46.71, + "nll_loss": 0.49673789739608765, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.021573714911937714, + "rewards/margins": 1.2825357913970947, + "rewards/rejected": -1.3041096925735474, + "step": 58, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.13165969316596932, + "grad_norm": 3.224839925765991, + "learning_rate": 0.00019986647271243163, + "logits/chosen": -0.379732221364975, + "logits/rejected": -0.4699477553367615, + "logps/chosen": -6.0904154777526855, + "logps/rejected": -22.338991165161133, + "loss": 0.921177327632904, + "memory(GiB)": 46.71, + "nll_loss": 0.41462287306785583, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.054278142750263214, + "rewards/margins": 0.6603462100028992, + "rewards/rejected": -0.7146244049072266, + "step": 59, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.13389121338912133, + "grad_norm": 3.088550567626953, + "learning_rate": 0.00019984672117252423, + "logits/chosen": -0.3191598057746887, + "logits/rejected": -0.4799092411994934, + "logps/chosen": -8.916747093200684, + "logps/rejected": -38.1026611328125, + "loss": 0.9097569584846497, + "memory(GiB)": 46.71, + "nll_loss": 0.5499361753463745, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.09366529434919357, + "rewards/margins": 1.7305129766464233, + "rewards/rejected": -1.6368474960327148, + "step": 60, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.13612273361227337, + "grad_norm": 3.0570552349090576, + "learning_rate": 0.00019982560889641296, + "logits/chosen": -0.34574997425079346, + "logits/rejected": -0.5139629244804382, + "logps/chosen": -3.300583600997925, + "logps/rejected": -31.543636322021484, + "loss": 0.6459969878196716, + "memory(GiB)": 46.71, + "nll_loss": 0.29164713621139526, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.10295584052801132, + "rewards/margins": 1.4123960733413696, + "rewards/rejected": -1.3094402551651, + "step": 61, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.13835425383542538, + "grad_norm": 4.404062747955322, + "learning_rate": 0.00019980313617182127, + "logits/chosen": -0.3277460038661957, + "logits/rejected": -0.5349987745285034, + "logps/chosen": -6.637196063995361, + "logps/rejected": -34.88399124145508, + "loss": 0.9588042497634888, + "memory(GiB)": 46.71, + "nll_loss": 0.5974500179290771, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.10418260097503662, + "rewards/margins": 1.5509659051895142, + "rewards/rejected": -1.6551483869552612, + "step": 62, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.14058577405857742, + "grad_norm": 7.597557067871094, + "learning_rate": 0.00019977930330501308, + "logits/chosen": -0.40753576159477234, + "logits/rejected": -0.5000228881835938, + "logps/chosen": -9.73190975189209, + "logps/rejected": -31.304035186767578, + "loss": 1.1911669969558716, + "memory(GiB)": 46.71, + "nll_loss": 0.6490563154220581, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.18217454850673676, + "rewards/margins": 1.1403659582138062, + "rewards/rejected": -1.322540521621704, + "step": 63, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.14281729428172943, + "grad_norm": 5.447345733642578, + "learning_rate": 0.0001997541106207887, + "logits/chosen": -0.4495885968208313, + "logits/rejected": -0.521405816078186, + "logps/chosen": -7.186385631561279, + "logps/rejected": -29.914262771606445, + "loss": 0.9763236045837402, + "memory(GiB)": 46.71, + "nll_loss": 0.5955564379692078, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.02932646870613098, + "rewards/margins": 1.4675655364990234, + "rewards/rejected": -1.496891975402832, + "step": 64, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.14504881450488144, + "grad_norm": 4.937906742095947, + "learning_rate": 0.0001997275584624803, + "logits/chosen": -0.3159691393375397, + "logits/rejected": -0.48153895139694214, + "logps/chosen": -7.167924404144287, + "logps/rejected": -33.78512954711914, + "loss": 0.9605119228363037, + "memory(GiB)": 46.71, + "nll_loss": 0.5666901469230652, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.06481783092021942, + "rewards/margins": 1.3504029512405396, + "rewards/rejected": -1.2855849266052246, + "step": 65, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.14728033472803348, + "grad_norm": 4.266327857971191, + "learning_rate": 0.00019969964719194745, + "logits/chosen": -0.4321137070655823, + "logits/rejected": -0.5152468681335449, + "logps/chosen": -8.84105110168457, + "logps/rejected": -15.863529205322266, + "loss": 1.171226978302002, + "memory(GiB)": 46.71, + "nll_loss": 0.5947294235229492, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.039367154240608215, + "rewards/margins": 0.543604850769043, + "rewards/rejected": -0.504237711429596, + "step": 66, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.1495118549511855, + "grad_norm": 3.343055009841919, + "learning_rate": 0.0001996703771895719, + "logits/chosen": -0.5153965353965759, + "logits/rejected": -0.5072118043899536, + "logps/chosen": -13.175134658813477, + "logps/rejected": -18.902141571044922, + "loss": 1.3753248453140259, + "memory(GiB)": 46.71, + "nll_loss": 0.6952039003372192, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.00693463534116745, + "rewards/margins": 0.5273423194885254, + "rewards/rejected": -0.5204076766967773, + "step": 67, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.15174337517433753, + "grad_norm": 2.8605148792266846, + "learning_rate": 0.00019963974885425266, + "logits/chosen": -0.394383043050766, + "logits/rejected": -0.4663364887237549, + "logps/chosen": -7.570481300354004, + "logps/rejected": -18.867816925048828, + "loss": 1.048596739768982, + "memory(GiB)": 46.71, + "nll_loss": 0.447956383228302, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.06730639934539795, + "rewards/margins": 0.42720258235931396, + "rewards/rejected": -0.359896183013916, + "step": 68, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.15397489539748954, + "grad_norm": 2.4179482460021973, + "learning_rate": 0.0001996077626034003, + "logits/chosen": -0.43737077713012695, + "logits/rejected": -0.49085646867752075, + "logps/chosen": -11.430294036865234, + "logps/rejected": -16.858463287353516, + "loss": 1.1468638181686401, + "memory(GiB)": 46.71, + "nll_loss": 0.5987557172775269, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.13983695209026337, + "rewards/margins": 0.43572792410850525, + "rewards/rejected": -0.2958909869194031, + "step": 69, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.15620641562064155, + "grad_norm": 2.619527578353882, + "learning_rate": 0.00019957441887293156, + "logits/chosen": -0.3810432255268097, + "logits/rejected": -0.4588402509689331, + "logps/chosen": -7.18558931350708, + "logps/rejected": -27.03573989868164, + "loss": 0.9460563659667969, + "memory(GiB)": 46.71, + "nll_loss": 0.4206099212169647, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.027686119079589844, + "rewards/margins": 0.6544371843338013, + "rewards/rejected": -0.6267510652542114, + "step": 70, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.1584379358437936, + "grad_norm": 2.098484754562378, + "learning_rate": 0.0001995397181172631, + "logits/chosen": -0.38461002707481384, + "logits/rejected": -0.5058427453041077, + "logps/chosen": -4.015483856201172, + "logps/rejected": -28.747943878173828, + "loss": 0.784480094909668, + "memory(GiB)": 46.71, + "nll_loss": 0.36208027601242065, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.08316721022129059, + "rewards/margins": 1.0465381145477295, + "rewards/rejected": -0.9633709788322449, + "step": 71, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.1606694560669456, + "grad_norm": 2.3822543621063232, + "learning_rate": 0.0001995036608093056, + "logits/chosen": -0.4334784150123596, + "logits/rejected": -0.4664575457572937, + "logps/chosen": -8.978615760803223, + "logps/rejected": -20.211299896240234, + "loss": 0.8698700070381165, + "memory(GiB)": 46.71, + "nll_loss": 0.3961753249168396, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.03191111981868744, + "rewards/margins": 0.8400025963783264, + "rewards/rejected": -0.8080915808677673, + "step": 72, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.16290097629009762, + "grad_norm": 2.3012869358062744, + "learning_rate": 0.00019946624744045704, + "logits/chosen": -0.3998824954032898, + "logits/rejected": -0.4553842544555664, + "logps/chosen": -11.310371398925781, + "logps/rejected": -30.51799964904785, + "loss": 0.9168710708618164, + "memory(GiB)": 46.71, + "nll_loss": 0.40945351123809814, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.09965535998344421, + "rewards/margins": 1.1768431663513184, + "rewards/rejected": -1.2764984369277954, + "step": 73, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.16513249651324965, + "grad_norm": 2.779480218887329, + "learning_rate": 0.00019942747852059632, + "logits/chosen": -0.3825194835662842, + "logits/rejected": -0.4605410397052765, + "logps/chosen": -9.17955493927002, + "logps/rejected": -35.66179656982422, + "loss": 0.8414367437362671, + "memory(GiB)": 46.71, + "nll_loss": 0.39000657200813293, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10101831704378128, + "rewards/margins": 1.6313400268554688, + "rewards/rejected": -1.732358455657959, + "step": 74, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.16736401673640167, + "grad_norm": 2.800877571105957, + "learning_rate": 0.00019938735457807592, + "logits/chosen": -0.358651727437973, + "logits/rejected": -0.43520039319992065, + "logps/chosen": -10.250865936279297, + "logps/rejected": -31.788986206054688, + "loss": 0.8999710083007812, + "memory(GiB)": 46.71, + "nll_loss": 0.4516286253929138, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.00027565285563468933, + "rewards/margins": 1.1217522621154785, + "rewards/rejected": -1.1220279932022095, + "step": 75, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.1695955369595537, + "grad_norm": 3.3984220027923584, + "learning_rate": 0.00019934587615971506, + "logits/chosen": -0.3334260582923889, + "logits/rejected": -0.4145124852657318, + "logps/chosen": -6.258616924285889, + "logps/rejected": -29.351835250854492, + "loss": 0.9294677376747131, + "memory(GiB)": 46.71, + "nll_loss": 0.4573590159416199, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.005807792767882347, + "rewards/margins": 0.9444352388381958, + "rewards/rejected": -0.9386276006698608, + "step": 76, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.17182705718270572, + "grad_norm": 2.565323829650879, + "learning_rate": 0.00019930304383079204, + "logits/chosen": -0.3684268593788147, + "logits/rejected": -0.4622272551059723, + "logps/chosen": -8.288152694702148, + "logps/rejected": -27.87519645690918, + "loss": 0.9020035266876221, + "memory(GiB)": 46.71, + "nll_loss": 0.4591652750968933, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.04557492583990097, + "rewards/margins": 1.3201135396957397, + "rewards/rejected": -1.2745387554168701, + "step": 77, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.17405857740585773, + "grad_norm": 3.3687551021575928, + "learning_rate": 0.00019925885817503663, + "logits/chosen": -0.3821089267730713, + "logits/rejected": -0.4754073917865753, + "logps/chosen": -5.0986528396606445, + "logps/rejected": -25.687240600585938, + "loss": 0.8081955909729004, + "memory(GiB)": 46.71, + "nll_loss": 0.4151981770992279, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06252564489841461, + "rewards/margins": 1.102398157119751, + "rewards/rejected": -1.039872407913208, + "step": 78, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.17629009762900977, + "grad_norm": 3.3963940143585205, + "learning_rate": 0.00019921331979462197, + "logits/chosen": -0.2969811260700226, + "logits/rejected": -0.43268370628356934, + "logps/chosen": -5.748870372772217, + "logps/rejected": -27.06174659729004, + "loss": 0.8259724378585815, + "memory(GiB)": 46.71, + "nll_loss": 0.39037975668907166, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07963411509990692, + "rewards/margins": 0.9517691135406494, + "rewards/rejected": -0.8721350431442261, + "step": 79, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.17852161785216178, + "grad_norm": 4.5683393478393555, + "learning_rate": 0.00019916642931015662, + "logits/chosen": -0.38007351756095886, + "logits/rejected": -0.44829314947128296, + "logps/chosen": -6.661276817321777, + "logps/rejected": -23.286151885986328, + "loss": 1.0330675840377808, + "memory(GiB)": 46.71, + "nll_loss": 0.5717910528182983, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.01540664117783308, + "rewards/margins": 0.9885216951370239, + "rewards/rejected": -0.9731149673461914, + "step": 80, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.18075313807531382, + "grad_norm": 2.7448818683624268, + "learning_rate": 0.00019911818736067586, + "logits/chosen": -0.362295538187027, + "logits/rejected": -0.4660547375679016, + "logps/chosen": -4.499999046325684, + "logps/rejected": -21.89557647705078, + "loss": 0.7531627416610718, + "memory(GiB)": 46.71, + "nll_loss": 0.40978550910949707, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.23986083269119263, + "rewards/margins": 1.2353352308273315, + "rewards/rejected": -0.9954744577407837, + "step": 81, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.18298465829846583, + "grad_norm": 2.189898729324341, + "learning_rate": 0.00019906859460363307, + "logits/chosen": -0.34852471947669983, + "logits/rejected": -0.4147579073905945, + "logps/chosen": -7.054965972900391, + "logps/rejected": -27.020904541015625, + "loss": 0.6932824850082397, + "memory(GiB)": 46.71, + "nll_loss": 0.40176427364349365, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.19743630290031433, + "rewards/margins": 1.7189998626708984, + "rewards/rejected": -1.5215637683868408, + "step": 82, + "train_speed(iter/s)": 0.011183 + }, + { + "epoch": 0.18521617852161784, + "grad_norm": 6.675446510314941, + "learning_rate": 0.00019901765171489086, + "logits/chosen": -0.2617488205432892, + "logits/rejected": -0.42673808336257935, + "logps/chosen": -10.860867500305176, + "logps/rejected": -47.94599533081055, + "loss": 1.1189333200454712, + "memory(GiB)": 46.71, + "nll_loss": 0.7764029502868652, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.33700618147850037, + "rewards/margins": 2.8077898025512695, + "rewards/rejected": -3.144796133041382, + "step": 83, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.18744769874476988, + "grad_norm": 3.3365304470062256, + "learning_rate": 0.0001989653593887117, + "logits/chosen": -0.33305802941322327, + "logits/rejected": -0.41463255882263184, + "logps/chosen": -6.076395034790039, + "logps/rejected": -32.26319885253906, + "loss": 0.7819843292236328, + "memory(GiB)": 46.71, + "nll_loss": 0.4539300501346588, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10270927101373672, + "rewards/margins": 2.0022926330566406, + "rewards/rejected": -1.899583339691162, + "step": 84, + "train_speed(iter/s)": 0.011185 + }, + { + "epoch": 0.1896792189679219, + "grad_norm": 3.774372100830078, + "learning_rate": 0.00019891171833774854, + "logits/chosen": -0.2869430482387543, + "logits/rejected": -0.38458919525146484, + "logps/chosen": -9.53564453125, + "logps/rejected": -44.279476165771484, + "loss": 0.9625971913337708, + "memory(GiB)": 46.71, + "nll_loss": 0.5546936392784119, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.1114846020936966, + "rewards/margins": 2.2966198921203613, + "rewards/rejected": -2.408104419708252, + "step": 85, + "train_speed(iter/s)": 0.011185 + }, + { + "epoch": 0.19191073919107393, + "grad_norm": 7.161458969116211, + "learning_rate": 0.00019885672929303508, + "logits/chosen": -0.2910900115966797, + "logits/rejected": -0.4121524691581726, + "logps/chosen": -10.86208724975586, + "logps/rejected": -43.04856491088867, + "loss": 0.8533170819282532, + "memory(GiB)": 46.71, + "nll_loss": 0.49533799290657043, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0742935836315155, + "rewards/margins": 2.4323437213897705, + "rewards/rejected": -2.5066375732421875, + "step": 86, + "train_speed(iter/s)": 0.011185 + }, + { + "epoch": 0.19414225941422594, + "grad_norm": 3.097945213317871, + "learning_rate": 0.00019880039300397591, + "logits/chosen": -0.38776522874832153, + "logits/rejected": -0.4260600805282593, + "logps/chosen": -7.062775135040283, + "logps/rejected": -30.766754150390625, + "loss": 0.8838493227958679, + "memory(GiB)": 46.71, + "nll_loss": 0.5182939767837524, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.07534455507993698, + "rewards/margins": 1.623365044593811, + "rewards/rejected": -1.5480204820632935, + "step": 87, + "train_speed(iter/s)": 0.011186 + }, + { + "epoch": 0.19637377963737795, + "grad_norm": 2.158613443374634, + "learning_rate": 0.00019874271023833604, + "logits/chosen": -0.3456321060657501, + "logits/rejected": -0.43574386835098267, + "logps/chosen": -7.558653831481934, + "logps/rejected": -30.61342430114746, + "loss": 0.7866738438606262, + "memory(GiB)": 46.71, + "nll_loss": 0.4206286072731018, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15493661165237427, + "rewards/margins": 1.5608376264572144, + "rewards/rejected": -1.4059010744094849, + "step": 88, + "train_speed(iter/s)": 0.011187 + }, + { + "epoch": 0.19860529986053, + "grad_norm": 3.826850414276123, + "learning_rate": 0.00019868368178223075, + "logits/chosen": -0.3609853982925415, + "logits/rejected": -0.4561440646648407, + "logps/chosen": -9.480233192443848, + "logps/rejected": -24.056900024414062, + "loss": 1.0715264081954956, + "memory(GiB)": 46.71, + "nll_loss": 0.5571523904800415, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.019891131669282913, + "rewards/margins": 0.8333019018173218, + "rewards/rejected": -0.8531929850578308, + "step": 89, + "train_speed(iter/s)": 0.011188 + }, + { + "epoch": 0.200836820083682, + "grad_norm": 2.5776290893554688, + "learning_rate": 0.00019862330844011466, + "logits/chosen": -0.39898231625556946, + "logits/rejected": -0.5255773663520813, + "logps/chosen": -6.868825435638428, + "logps/rejected": -27.24391746520996, + "loss": 0.9221024513244629, + "memory(GiB)": 46.71, + "nll_loss": 0.5272523760795593, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.16320021450519562, + "rewards/margins": 1.3196171522140503, + "rewards/rejected": -1.156416893005371, + "step": 90, + "train_speed(iter/s)": 0.011188 + }, + { + "epoch": 0.20306834030683404, + "grad_norm": 2.8630950450897217, + "learning_rate": 0.00019856159103477086, + "logits/chosen": -0.38147053122520447, + "logits/rejected": -0.49192503094673157, + "logps/chosen": -4.319600582122803, + "logps/rejected": -29.042064666748047, + "loss": 0.6743653416633606, + "memory(GiB)": 46.71, + "nll_loss": 0.2815118730068207, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.06610685586929321, + "rewards/margins": 1.2929189205169678, + "rewards/rejected": -1.2268121242523193, + "step": 91, + "train_speed(iter/s)": 0.011189 + }, + { + "epoch": 0.20529986052998606, + "grad_norm": 2.3517754077911377, + "learning_rate": 0.00019849853040729962, + "logits/chosen": -0.3899337947368622, + "logits/rejected": -0.512879490852356, + "logps/chosen": -3.6735916137695312, + "logps/rejected": -38.82426452636719, + "loss": 0.575823962688446, + "memory(GiB)": 46.71, + "nll_loss": 0.27289441227912903, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.014910156838595867, + "rewards/margins": 2.080000638961792, + "rewards/rejected": -2.0650904178619385, + "step": 92, + "train_speed(iter/s)": 0.011189 + }, + { + "epoch": 0.20753138075313807, + "grad_norm": 7.1787543296813965, + "learning_rate": 0.00019843412741710705, + "logits/chosen": -0.4152039885520935, + "logits/rejected": -0.5459296107292175, + "logps/chosen": -9.160662651062012, + "logps/rejected": -33.41329574584961, + "loss": 0.9777874946594238, + "memory(GiB)": 46.71, + "nll_loss": 0.5870996713638306, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06245291978120804, + "rewards/margins": 1.6734349727630615, + "rewards/rejected": -1.7358877658843994, + "step": 93, + "train_speed(iter/s)": 0.01119 + }, + { + "epoch": 0.2097629009762901, + "grad_norm": 6.694753646850586, + "learning_rate": 0.00019836838294189327, + "logits/chosen": -0.2928682565689087, + "logits/rejected": -0.5464470386505127, + "logps/chosen": -4.742891788482666, + "logps/rejected": -48.63407516479492, + "loss": 0.6743770241737366, + "memory(GiB)": 46.71, + "nll_loss": 0.4215732514858246, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.016849342733621597, + "rewards/margins": 2.5316309928894043, + "rewards/rejected": -2.548480272293091, + "step": 94, + "train_speed(iter/s)": 0.011189 + }, + { + "epoch": 0.21199442119944212, + "grad_norm": 3.5523533821105957, + "learning_rate": 0.00019830129787764048, + "logits/chosen": -0.43435996770858765, + "logits/rejected": -0.5884309411048889, + "logps/chosen": -7.528013229370117, + "logps/rejected": -40.399600982666016, + "loss": 0.655762255191803, + "memory(GiB)": 46.71, + "nll_loss": 0.36153632402420044, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.16746671497821808, + "rewards/margins": 2.1469664573669434, + "rewards/rejected": -2.3144330978393555, + "step": 95, + "train_speed(iter/s)": 0.011189 + }, + { + "epoch": 0.21422594142259413, + "grad_norm": 3.652083396911621, + "learning_rate": 0.00019823287313860087, + "logits/chosen": -0.39443954825401306, + "logits/rejected": -0.5311736464500427, + "logps/chosen": -10.99143123626709, + "logps/rejected": -34.9860725402832, + "loss": 1.1298080682754517, + "memory(GiB)": 46.71, + "nll_loss": 0.6828101873397827, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.050042107701301575, + "rewards/margins": 1.6755130290985107, + "rewards/rejected": -1.725555181503296, + "step": 96, + "train_speed(iter/s)": 0.011189 + }, + { + "epoch": 0.21645746164574617, + "grad_norm": 3.373784065246582, + "learning_rate": 0.00019816310965728388, + "logits/chosen": -0.4609702229499817, + "logits/rejected": -0.534326434135437, + "logps/chosen": -9.729392051696777, + "logps/rejected": -26.942209243774414, + "loss": 0.970382571220398, + "memory(GiB)": 46.71, + "nll_loss": 0.4684171676635742, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06363824754953384, + "rewards/margins": 1.244420051574707, + "rewards/rejected": -1.308058261871338, + "step": 97, + "train_speed(iter/s)": 0.01119 + }, + { + "epoch": 0.21868898186889818, + "grad_norm": 2.9718940258026123, + "learning_rate": 0.00019809200838444383, + "logits/chosen": -0.4923046827316284, + "logits/rejected": -0.5684210062026978, + "logps/chosen": -7.5212883949279785, + "logps/rejected": -26.84278678894043, + "loss": 0.7905946373939514, + "memory(GiB)": 46.71, + "nll_loss": 0.4011859893798828, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.1688103973865509, + "rewards/margins": 1.1646181344985962, + "rewards/rejected": -0.9958077669143677, + "step": 98, + "train_speed(iter/s)": 0.011191 + }, + { + "epoch": 0.22092050209205022, + "grad_norm": 4.091674327850342, + "learning_rate": 0.0001980195702890667, + "logits/chosen": -0.41893988847732544, + "logits/rejected": -0.5414128303527832, + "logps/chosen": -7.620563507080078, + "logps/rejected": -34.5891227722168, + "loss": 0.8988033533096313, + "memory(GiB)": 46.71, + "nll_loss": 0.4130387604236603, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0692744329571724, + "rewards/margins": 1.639555811882019, + "rewards/rejected": -1.7088301181793213, + "step": 99, + "train_speed(iter/s)": 0.011191 + }, + { + "epoch": 0.22315202231520223, + "grad_norm": 4.4936113357543945, + "learning_rate": 0.00019794579635835704, + "logits/chosen": -0.4513593912124634, + "logits/rejected": -0.5386379361152649, + "logps/chosen": -7.520744323730469, + "logps/rejected": -21.09442710876465, + "loss": 1.0648761987686157, + "memory(GiB)": 46.71, + "nll_loss": 0.551341712474823, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.06751161813735962, + "rewards/margins": 0.5798344016075134, + "rewards/rejected": -0.5123227834701538, + "step": 100, + "train_speed(iter/s)": 0.011192 + }, + { + "epoch": 0.22538354253835424, + "grad_norm": 3.2135934829711914, + "learning_rate": 0.00019787068759772458, + "logits/chosen": -0.30221307277679443, + "logits/rejected": -0.4815448820590973, + "logps/chosen": -5.885772705078125, + "logps/rejected": -42.119300842285156, + "loss": 0.7016221880912781, + "memory(GiB)": 46.71, + "nll_loss": 0.3754323422908783, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.016318581998348236, + "rewards/margins": 1.9625664949417114, + "rewards/rejected": -1.9462475776672363, + "step": 101, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.22761506276150628, + "grad_norm": 2.6394968032836914, + "learning_rate": 0.0001977942450307703, + "logits/chosen": -0.4444178342819214, + "logits/rejected": -0.550942599773407, + "logps/chosen": -2.515904188156128, + "logps/rejected": -24.57438087463379, + "loss": 0.5894904136657715, + "memory(GiB)": 46.71, + "nll_loss": 0.2374342381954193, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13210514187812805, + "rewards/margins": 1.279597520828247, + "rewards/rejected": -1.1474921703338623, + "step": 102, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.2298465829846583, + "grad_norm": 3.511199712753296, + "learning_rate": 0.00019771646969927276, + "logits/chosen": -0.37454891204833984, + "logits/rejected": -0.5178055763244629, + "logps/chosen": -11.968851089477539, + "logps/rejected": -43.35523986816406, + "loss": 0.9422297477722168, + "memory(GiB)": 46.71, + "nll_loss": 0.643075704574585, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.14428094029426575, + "rewards/margins": 2.009899377822876, + "rewards/rejected": -2.1541800498962402, + "step": 103, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.23207810320781033, + "grad_norm": 3.9855709075927734, + "learning_rate": 0.00019763736266317373, + "logits/chosen": -0.4065098166465759, + "logits/rejected": -0.4758867025375366, + "logps/chosen": -10.250575065612793, + "logps/rejected": -30.894758224487305, + "loss": 1.0436267852783203, + "memory(GiB)": 46.71, + "nll_loss": 0.6723781228065491, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.054463766515254974, + "rewards/margins": 1.513323187828064, + "rewards/rejected": -1.4588593244552612, + "step": 104, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.23430962343096234, + "grad_norm": 2.9467523097991943, + "learning_rate": 0.00019755692500056377, + "logits/chosen": -0.3900934159755707, + "logits/rejected": -0.47925370931625366, + "logps/chosen": -7.484468460083008, + "logps/rejected": -43.62319564819336, + "loss": 0.7645498514175415, + "memory(GiB)": 46.71, + "nll_loss": 0.4648410677909851, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.10762156546115875, + "rewards/margins": 2.406116247177124, + "rewards/rejected": -2.513737916946411, + "step": 105, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.23654114365411436, + "grad_norm": 4.7432637214660645, + "learning_rate": 0.0001974751578076675, + "logits/chosen": -0.4480246603488922, + "logits/rejected": -0.5329591631889343, + "logps/chosen": -8.911030769348145, + "logps/rejected": -36.37174987792969, + "loss": 1.0600045919418335, + "memory(GiB)": 46.71, + "nll_loss": 0.6955873966217041, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.00720573216676712, + "rewards/margins": 2.03155517578125, + "rewards/rejected": -2.0243494510650635, + "step": 106, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.2387726638772664, + "grad_norm": 5.058660984039307, + "learning_rate": 0.0001973920621988288, + "logits/chosen": -0.3936890661716461, + "logits/rejected": -0.4737499952316284, + "logps/chosen": -9.114482879638672, + "logps/rejected": -38.9593505859375, + "loss": 0.9397415518760681, + "memory(GiB)": 46.71, + "nll_loss": 0.619577169418335, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.1711510568857193, + "rewards/margins": 2.256584882736206, + "rewards/rejected": -2.4277358055114746, + "step": 107, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.2410041841004184, + "grad_norm": 5.576498985290527, + "learning_rate": 0.00019730763930649557, + "logits/chosen": -0.3989205062389374, + "logits/rejected": -0.5156173706054688, + "logps/chosen": -7.493094444274902, + "logps/rejected": -38.66194152832031, + "loss": 1.1445399522781372, + "memory(GiB)": 46.71, + "nll_loss": 0.7500584125518799, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.078619584441185, + "rewards/margins": 2.2690536975860596, + "rewards/rejected": -2.347673177719116, + "step": 108, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.24323570432357045, + "grad_norm": 5.548501491546631, + "learning_rate": 0.0001972218902812041, + "logits/chosen": -0.4444655776023865, + "logits/rejected": -0.5106954574584961, + "logps/chosen": -10.32758903503418, + "logps/rejected": -29.883926391601562, + "loss": 1.1218242645263672, + "memory(GiB)": 46.71, + "nll_loss": 0.7221024036407471, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.10507391393184662, + "rewards/margins": 1.7042087316513062, + "rewards/rejected": -1.8092825412750244, + "step": 109, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.24546722454672246, + "grad_norm": 4.309199333190918, + "learning_rate": 0.0001971348162915637, + "logits/chosen": -0.3248450756072998, + "logits/rejected": -0.4501338601112366, + "logps/chosen": -12.836463928222656, + "logps/rejected": -46.953514099121094, + "loss": 1.0559560060501099, + "memory(GiB)": 46.71, + "nll_loss": 0.6316272020339966, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.07269488275051117, + "rewards/margins": 1.5467530488967896, + "rewards/rejected": -1.6194478273391724, + "step": 110, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.24769874476987447, + "grad_norm": 3.218020439147949, + "learning_rate": 0.0001970464185242406, + "logits/chosen": -0.39539581537246704, + "logits/rejected": -0.48628556728363037, + "logps/chosen": -8.726967811584473, + "logps/rejected": -29.990245819091797, + "loss": 0.9488601088523865, + "memory(GiB)": 46.71, + "nll_loss": 0.6031092405319214, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.023079559206962585, + "rewards/margins": 1.3200984001159668, + "rewards/rejected": -1.3431780338287354, + "step": 111, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.2499302649930265, + "grad_norm": 2.6869122982025146, + "learning_rate": 0.00019695669818394176, + "logits/chosen": -0.41482657194137573, + "logits/rejected": -0.4805757999420166, + "logps/chosen": -6.431066036224365, + "logps/rejected": -25.98788833618164, + "loss": 0.8483121991157532, + "memory(GiB)": 46.71, + "nll_loss": 0.37977075576782227, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.08659300208091736, + "rewards/margins": 0.9510751962661743, + "rewards/rejected": -0.8644822835922241, + "step": 112, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.25216178521617855, + "grad_norm": 2.551693916320801, + "learning_rate": 0.00019686565649339852, + "logits/chosen": -0.3926604092121124, + "logits/rejected": -0.4559895694255829, + "logps/chosen": -7.078885078430176, + "logps/rejected": -27.05070686340332, + "loss": 0.8254140019416809, + "memory(GiB)": 46.71, + "nll_loss": 0.4410409927368164, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.20977970957756042, + "rewards/margins": 1.3954874277114868, + "rewards/rejected": -1.1857078075408936, + "step": 113, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.25439330543933053, + "grad_norm": 2.873304843902588, + "learning_rate": 0.0001967732946933499, + "logits/chosen": -0.40831753611564636, + "logits/rejected": -0.43265292048454285, + "logps/chosen": -9.373550415039062, + "logps/rejected": -18.696136474609375, + "loss": 1.1380702257156372, + "memory(GiB)": 46.71, + "nll_loss": 0.5557818412780762, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.030941195785999298, + "rewards/margins": 0.6564021110534668, + "rewards/rejected": -0.6873433589935303, + "step": 114, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.25662482566248257, + "grad_norm": 2.8062174320220947, + "learning_rate": 0.00019667961404252573, + "logits/chosen": -0.39943259954452515, + "logits/rejected": -0.4698421061038971, + "logps/chosen": -6.604292869567871, + "logps/rejected": -27.13275909423828, + "loss": 0.904656708240509, + "memory(GiB)": 46.71, + "nll_loss": 0.40879639983177185, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.06587034463882446, + "rewards/margins": 1.2184947729110718, + "rewards/rejected": -1.1526243686676025, + "step": 115, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.2588563458856346, + "grad_norm": 2.269819974899292, + "learning_rate": 0.00019658461581762948, + "logits/chosen": -0.37201863527297974, + "logits/rejected": -0.49889877438545227, + "logps/chosen": -5.184394836425781, + "logps/rejected": -33.83866500854492, + "loss": 0.6697641015052795, + "memory(GiB)": 46.71, + "nll_loss": 0.3163735568523407, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.12605488300323486, + "rewards/margins": 1.7490179538726807, + "rewards/rejected": -1.6229631900787354, + "step": 116, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.2610878661087866, + "grad_norm": 2.150320291519165, + "learning_rate": 0.0001964883013133208, + "logits/chosen": -0.3641276955604553, + "logits/rejected": -0.46924924850463867, + "logps/chosen": -6.894084930419922, + "logps/rejected": -30.72964096069336, + "loss": 0.7019263505935669, + "memory(GiB)": 46.71, + "nll_loss": 0.3518219292163849, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11152325570583344, + "rewards/margins": 1.5766544342041016, + "rewards/rejected": -1.465131163597107, + "step": 117, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.26331938633193863, + "grad_norm": 1.940995693206787, + "learning_rate": 0.00019639067184219796, + "logits/chosen": -0.4233744740486145, + "logits/rejected": -0.49646085500717163, + "logps/chosen": -2.5980770587921143, + "logps/rejected": -22.459583282470703, + "loss": 0.5635585784912109, + "memory(GiB)": 46.71, + "nll_loss": 0.23038321733474731, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.20914426445960999, + "rewards/margins": 1.3377940654754639, + "rewards/rejected": -1.1286499500274658, + "step": 118, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.26555090655509067, + "grad_norm": 4.13163423538208, + "learning_rate": 0.00019629172873477995, + "logits/chosen": -0.43275851011276245, + "logits/rejected": -0.5510309934616089, + "logps/chosen": -7.118757724761963, + "logps/rejected": -33.80887222290039, + "loss": 0.9356746077537537, + "memory(GiB)": 46.71, + "nll_loss": 0.5518943071365356, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06265924125909805, + "rewards/margins": 1.4980642795562744, + "rewards/rejected": -1.5607234239578247, + "step": 119, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.26778242677824265, + "grad_norm": 4.10536003112793, + "learning_rate": 0.00019619147333948823, + "logits/chosen": -0.3524538576602936, + "logits/rejected": -0.5940331220626831, + "logps/chosen": -3.5720577239990234, + "logps/rejected": -37.1466064453125, + "loss": 0.6390153169631958, + "memory(GiB)": 46.71, + "nll_loss": 0.33055761456489563, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.012503441423177719, + "rewards/margins": 1.8064663410186768, + "rewards/rejected": -1.818969964981079, + "step": 120, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.2700139470013947, + "grad_norm": 3.469205856323242, + "learning_rate": 0.00019608990702262862, + "logits/chosen": -0.43667861819267273, + "logits/rejected": -0.5170078873634338, + "logps/chosen": -10.792959213256836, + "logps/rejected": -33.0848503112793, + "loss": 0.915253758430481, + "memory(GiB)": 46.71, + "nll_loss": 0.5493744015693665, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0627615749835968, + "rewards/margins": 1.9415191411972046, + "rewards/rejected": -2.0042805671691895, + "step": 121, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.27224546722454673, + "grad_norm": 3.509106159210205, + "learning_rate": 0.0001959870311683723, + "logits/chosen": -0.42921414971351624, + "logits/rejected": -0.5258873105049133, + "logps/chosen": -11.711372375488281, + "logps/rejected": -33.04314041137695, + "loss": 0.928779125213623, + "memory(GiB)": 46.71, + "nll_loss": 0.5003984570503235, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.20121777057647705, + "rewards/margins": 1.2945905923843384, + "rewards/rejected": -1.4958083629608154, + "step": 122, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.2744769874476987, + "grad_norm": 3.868866443634033, + "learning_rate": 0.00019588284717873738, + "logits/chosen": -0.47573891282081604, + "logits/rejected": -0.5573767423629761, + "logps/chosen": -11.404234886169434, + "logps/rejected": -44.85041427612305, + "loss": 0.868504524230957, + "memory(GiB)": 46.71, + "nll_loss": 0.5618444085121155, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06724707782268524, + "rewards/margins": 2.6722822189331055, + "rewards/rejected": -2.7395291328430176, + "step": 123, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.27670850767085076, + "grad_norm": 2.392876625061035, + "learning_rate": 0.00019577735647356928, + "logits/chosen": -0.4367309510707855, + "logits/rejected": -0.5319121479988098, + "logps/chosen": -6.822817802429199, + "logps/rejected": -31.786712646484375, + "loss": 0.7306314706802368, + "memory(GiB)": 46.71, + "nll_loss": 0.40483999252319336, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.14910897612571716, + "rewards/margins": 1.7431702613830566, + "rewards/rejected": -1.594061255455017, + "step": 124, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.2789400278940028, + "grad_norm": 3.1055872440338135, + "learning_rate": 0.00019567056049052203, + "logits/chosen": -0.34188365936279297, + "logits/rejected": -0.5163190364837646, + "logps/chosen": -4.810876369476318, + "logps/rejected": -44.724796295166016, + "loss": 0.5886633396148682, + "memory(GiB)": 46.71, + "nll_loss": 0.33786720037460327, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.056971918791532516, + "rewards/margins": 2.44872784614563, + "rewards/rejected": -2.391756296157837, + "step": 125, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.28117154811715483, + "grad_norm": 5.503366947174072, + "learning_rate": 0.00019556246068503795, + "logits/chosen": -0.46696051955223083, + "logits/rejected": -0.5219815969467163, + "logps/chosen": -12.945513725280762, + "logps/rejected": -39.0778923034668, + "loss": 0.9357361197471619, + "memory(GiB)": 46.71, + "nll_loss": 0.6219462752342224, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.06543703377246857, + "rewards/margins": 2.5177855491638184, + "rewards/rejected": -2.5832223892211914, + "step": 126, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.2834030683403068, + "grad_norm": 4.39600133895874, + "learning_rate": 0.00019545305853032848, + "logits/chosen": -0.40930306911468506, + "logits/rejected": -0.5165880918502808, + "logps/chosen": -8.137343406677246, + "logps/rejected": -36.05834197998047, + "loss": 0.9704120755195618, + "memory(GiB)": 46.71, + "nll_loss": 0.6009897589683533, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06638351082801819, + "rewards/margins": 1.870111107826233, + "rewards/rejected": -1.9364944696426392, + "step": 127, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.28563458856345886, + "grad_norm": 2.2071218490600586, + "learning_rate": 0.0001953423555173536, + "logits/chosen": -0.3733062446117401, + "logits/rejected": -0.5163143277168274, + "logps/chosen": -5.431221008300781, + "logps/rejected": -50.760562896728516, + "loss": 0.5296469926834106, + "memory(GiB)": 46.71, + "nll_loss": 0.33633342385292053, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.02796844020485878, + "rewards/margins": 2.690657377243042, + "rewards/rejected": -2.662688732147217, + "step": 128, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.2878661087866109, + "grad_norm": 3.5191235542297363, + "learning_rate": 0.0001952303531548018, + "logits/chosen": -0.36135807633399963, + "logits/rejected": -0.46837830543518066, + "logps/chosen": -5.7369561195373535, + "logps/rejected": -40.44853591918945, + "loss": 0.591960072517395, + "memory(GiB)": 46.71, + "nll_loss": 0.2807152271270752, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.08742895722389221, + "rewards/margins": 2.5908262729644775, + "rewards/rejected": -2.503397226333618, + "step": 129, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.2900976290097629, + "grad_norm": 3.575482130050659, + "learning_rate": 0.00019511705296906945, + "logits/chosen": -0.3611457347869873, + "logits/rejected": -0.503516435623169, + "logps/chosen": -5.223241329193115, + "logps/rejected": -37.931541442871094, + "loss": 0.534879207611084, + "memory(GiB)": 46.71, + "nll_loss": 0.29983922839164734, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0478663444519043, + "rewards/margins": 2.279771327972412, + "rewards/rejected": -2.231905221939087, + "step": 130, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.2923291492329149, + "grad_norm": 3.9460039138793945, + "learning_rate": 0.00019500245650423987, + "logits/chosen": -0.3660318851470947, + "logits/rejected": -0.4796612560749054, + "logps/chosen": -11.665671348571777, + "logps/rejected": -49.753868103027344, + "loss": 0.9670472145080566, + "memory(GiB)": 46.71, + "nll_loss": 0.7097752690315247, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.1927420198917389, + "rewards/margins": 3.0559210777282715, + "rewards/rejected": -3.2486634254455566, + "step": 131, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.29456066945606696, + "grad_norm": 6.069462776184082, + "learning_rate": 0.00019488656532206256, + "logits/chosen": -0.32610565423965454, + "logits/rejected": -0.4974173605442047, + "logps/chosen": -6.3772382736206055, + "logps/rejected": -48.19947814941406, + "loss": 0.575228214263916, + "memory(GiB)": 46.71, + "nll_loss": 0.3116891086101532, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06776230037212372, + "rewards/margins": 2.7837750911712646, + "rewards/rejected": -2.8515372276306152, + "step": 132, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.29679218967921894, + "grad_norm": 8.746431350708008, + "learning_rate": 0.00019476938100193166, + "logits/chosen": -0.3666072189807892, + "logits/rejected": -0.4251011908054352, + "logps/chosen": -11.349700927734375, + "logps/rejected": -45.43285369873047, + "loss": 0.9895598292350769, + "memory(GiB)": 46.71, + "nll_loss": 0.6815392971038818, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.039151158183813095, + "rewards/margins": 2.9056763648986816, + "rewards/rejected": -2.9448275566101074, + "step": 133, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.299023709902371, + "grad_norm": 2.3924574851989746, + "learning_rate": 0.00019465090514086448, + "logits/chosen": -0.3149105906486511, + "logits/rejected": -0.4992123246192932, + "logps/chosen": -6.0071845054626465, + "logps/rejected": -48.10839080810547, + "loss": 0.524428129196167, + "memory(GiB)": 46.71, + "nll_loss": 0.34304654598236084, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.037540286779403687, + "rewards/margins": 2.7853667736053467, + "rewards/rejected": -2.74782657623291, + "step": 134, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.301255230125523, + "grad_norm": 5.539307594299316, + "learning_rate": 0.00019453113935347983, + "logits/chosen": -0.31063827872276306, + "logits/rejected": -0.4415431618690491, + "logps/chosen": -9.355257987976074, + "logps/rejected": -32.0480842590332, + "loss": 0.9339735507965088, + "memory(GiB)": 46.71, + "nll_loss": 0.5960800647735596, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.02493138611316681, + "rewards/margins": 1.7281333208084106, + "rewards/rejected": -1.753064751625061, + "step": 135, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.30348675034867506, + "grad_norm": 2.773390531539917, + "learning_rate": 0.000194410085271976, + "logits/chosen": -0.32995009422302246, + "logits/rejected": -0.47322651743888855, + "logps/chosen": -7.31096076965332, + "logps/rejected": -39.725521087646484, + "loss": 0.7643193006515503, + "memory(GiB)": 46.71, + "nll_loss": 0.4867798089981079, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06533154845237732, + "rewards/margins": 2.221080780029297, + "rewards/rejected": -2.1557490825653076, + "step": 136, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.30571827057182704, + "grad_norm": 3.071211099624634, + "learning_rate": 0.00019428774454610843, + "logits/chosen": -0.3505553901195526, + "logits/rejected": -0.45163339376449585, + "logps/chosen": -7.793257713317871, + "logps/rejected": -34.78621292114258, + "loss": 0.726679265499115, + "memory(GiB)": 46.71, + "nll_loss": 0.4261872172355652, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06730077415704727, + "rewards/margins": 2.1191506385803223, + "rewards/rejected": -2.0518500804901123, + "step": 137, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.3079497907949791, + "grad_norm": 2.8342206478118896, + "learning_rate": 0.00019416411884316722, + "logits/chosen": -0.23897941410541534, + "logits/rejected": -0.4108068645000458, + "logps/chosen": -5.271795272827148, + "logps/rejected": -33.89106750488281, + "loss": 0.6566328406333923, + "memory(GiB)": 46.71, + "nll_loss": 0.3124980330467224, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.06509758532047272, + "rewards/margins": 1.5957400798797607, + "rewards/rejected": -1.5306426286697388, + "step": 138, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.3101813110181311, + "grad_norm": 2.3552298545837402, + "learning_rate": 0.00019403920984795454, + "logits/chosen": -0.22743001580238342, + "logits/rejected": -0.3986920714378357, + "logps/chosen": -4.359319686889648, + "logps/rejected": -34.49266815185547, + "loss": 0.7009214162826538, + "memory(GiB)": 46.71, + "nll_loss": 0.37971875071525574, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.1354263424873352, + "rewards/margins": 1.9385743141174316, + "rewards/rejected": -1.8031479120254517, + "step": 139, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.3124128312412831, + "grad_norm": 3.9349329471588135, + "learning_rate": 0.00019391301926276156, + "logits/chosen": -0.25204330682754517, + "logits/rejected": -0.40545159578323364, + "logps/chosen": -4.7128987312316895, + "logps/rejected": -39.89329147338867, + "loss": 0.7504162192344666, + "memory(GiB)": 46.71, + "nll_loss": 0.41977450251579285, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.04398459568619728, + "rewards/margins": 2.3563807010650635, + "rewards/rejected": -2.400365114212036, + "step": 140, + "train_speed(iter/s)": 0.011182 + }, + { + "epoch": 0.31464435146443515, + "grad_norm": 2.3725013732910156, + "learning_rate": 0.00019378554880734527, + "logits/chosen": -0.25585880875587463, + "logits/rejected": -0.40883561968803406, + "logps/chosen": -5.454152584075928, + "logps/rejected": -47.51259994506836, + "loss": 0.6222885251045227, + "memory(GiB)": 46.71, + "nll_loss": 0.3938242197036743, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09286467730998993, + "rewards/margins": 3.4271411895751953, + "rewards/rejected": -3.3342764377593994, + "step": 141, + "train_speed(iter/s)": 0.011183 + }, + { + "epoch": 0.3168758716875872, + "grad_norm": 3.706570863723755, + "learning_rate": 0.00019365680021890506, + "logits/chosen": -0.26097267866134644, + "logits/rejected": -0.3484109938144684, + "logps/chosen": -8.48320198059082, + "logps/rejected": -31.970016479492188, + "loss": 1.0012670755386353, + "memory(GiB)": 46.71, + "nll_loss": 0.5183718204498291, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.14288514852523804, + "rewards/margins": 1.521733045578003, + "rewards/rejected": -1.6646180152893066, + "step": 142, + "train_speed(iter/s)": 0.011183 + }, + { + "epoch": 0.31910739191073917, + "grad_norm": 4.2724609375, + "learning_rate": 0.0001935267752520591, + "logits/chosen": -0.294653058052063, + "logits/rejected": -0.40346813201904297, + "logps/chosen": -9.981515884399414, + "logps/rejected": -44.07274627685547, + "loss": 0.8689901828765869, + "memory(GiB)": 46.71, + "nll_loss": 0.5607314705848694, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.2522781491279602, + "rewards/margins": 2.5947649478912354, + "rewards/rejected": -2.847043037414551, + "step": 143, + "train_speed(iter/s)": 0.011183 + }, + { + "epoch": 0.3213389121338912, + "grad_norm": 6.414367198944092, + "learning_rate": 0.00019339547567882024, + "logits/chosen": -0.20774677395820618, + "logits/rejected": -0.3707664906978607, + "logps/chosen": -9.347281455993652, + "logps/rejected": -48.28213882446289, + "loss": 0.9361674189567566, + "memory(GiB)": 46.71, + "nll_loss": 0.667937159538269, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.2894432842731476, + "rewards/margins": 3.465684175491333, + "rewards/rejected": -3.1762406826019287, + "step": 144, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.32357043235704325, + "grad_norm": 4.899329662322998, + "learning_rate": 0.0001932629032885721, + "logits/chosen": -0.26781731843948364, + "logits/rejected": -0.36117565631866455, + "logps/chosen": -12.092925071716309, + "logps/rejected": -45.867366790771484, + "loss": 0.9857743382453918, + "memory(GiB)": 46.71, + "nll_loss": 0.6580345630645752, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.3960377275943756, + "rewards/margins": 2.526942729949951, + "rewards/rejected": -2.9229800701141357, + "step": 145, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.32580195258019523, + "grad_norm": 2.351534843444824, + "learning_rate": 0.0001931290598880445, + "logits/chosen": -0.21401235461235046, + "logits/rejected": -0.3749268651008606, + "logps/chosen": -6.150831699371338, + "logps/rejected": -38.348854064941406, + "loss": 0.6613858342170715, + "memory(GiB)": 46.71, + "nll_loss": 0.4117865562438965, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.14331001043319702, + "rewards/margins": 2.4033961296081543, + "rewards/rejected": -2.2600862979888916, + "step": 146, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.32803347280334727, + "grad_norm": 3.732591152191162, + "learning_rate": 0.00019299394730128895, + "logits/chosen": -0.2112840712070465, + "logits/rejected": -0.29343438148498535, + "logps/chosen": -12.490756034851074, + "logps/rejected": -47.45105743408203, + "loss": 0.8992500901222229, + "memory(GiB)": 46.71, + "nll_loss": 0.5636640787124634, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10838784277439117, + "rewards/margins": 2.9080140590667725, + "rewards/rejected": -3.016401767730713, + "step": 147, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.3302649930264993, + "grad_norm": 3.371856927871704, + "learning_rate": 0.00019285756736965367, + "logits/chosen": -0.2626963257789612, + "logits/rejected": -0.3471822142601013, + "logps/chosen": -7.939330101013184, + "logps/rejected": -30.135757446289062, + "loss": 0.9493183493614197, + "memory(GiB)": 46.71, + "nll_loss": 0.5397716760635376, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.016782023012638092, + "rewards/margins": 1.6466865539550781, + "rewards/rejected": -1.6299041509628296, + "step": 148, + "train_speed(iter/s)": 0.011184 + }, + { + "epoch": 0.33249651324965135, + "grad_norm": 2.4161276817321777, + "learning_rate": 0.00019271992195175875, + "logits/chosen": -0.3086288869380951, + "logits/rejected": -0.41329652070999146, + "logps/chosen": -8.446435928344727, + "logps/rejected": -36.33450698852539, + "loss": 0.9108085632324219, + "memory(GiB)": 46.71, + "nll_loss": 0.5454064607620239, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.148118793964386, + "rewards/margins": 2.209937810897827, + "rewards/rejected": -2.061818838119507, + "step": 149, + "train_speed(iter/s)": 0.011185 + }, + { + "epoch": 0.33472803347280333, + "grad_norm": 3.4835684299468994, + "learning_rate": 0.00019258101292347042, + "logits/chosen": -0.22059586644172668, + "logits/rejected": -0.3747985363006592, + "logps/chosen": -8.812771797180176, + "logps/rejected": -44.0800895690918, + "loss": 0.7637215852737427, + "memory(GiB)": 46.71, + "nll_loss": 0.5271515250205994, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.041512295603752136, + "rewards/margins": 2.4112486839294434, + "rewards/rejected": -2.369736433029175, + "step": 150, + "train_speed(iter/s)": 0.011185 + }, + { + "epoch": 0.33695955369595537, + "grad_norm": 2.394838333129883, + "learning_rate": 0.00019244084217787588, + "logits/chosen": -0.24826237559318542, + "logits/rejected": -0.3791409730911255, + "logps/chosen": -8.393878936767578, + "logps/rejected": -45.125953674316406, + "loss": 0.7294331192970276, + "memory(GiB)": 46.71, + "nll_loss": 0.4361075460910797, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18614013493061066, + "rewards/margins": 2.6040918827056885, + "rewards/rejected": -2.417951822280884, + "step": 151, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.3391910739191074, + "grad_norm": 2.946138620376587, + "learning_rate": 0.00019229941162525726, + "logits/chosen": -0.2787875831127167, + "logits/rejected": -0.3120588958263397, + "logps/chosen": -9.058300018310547, + "logps/rejected": -30.770740509033203, + "loss": 0.9096699357032776, + "memory(GiB)": 46.71, + "nll_loss": 0.46970880031585693, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.08315091580152512, + "rewards/margins": 1.5154523849487305, + "rewards/rejected": -1.5986034870147705, + "step": 152, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.3414225941422594, + "grad_norm": 2.37446928024292, + "learning_rate": 0.00019215672319306565, + "logits/chosen": -0.29989758133888245, + "logits/rejected": -0.372408926486969, + "logps/chosen": -6.495873928070068, + "logps/rejected": -32.42622375488281, + "loss": 0.7089872360229492, + "memory(GiB)": 46.71, + "nll_loss": 0.3819705545902252, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0852634608745575, + "rewards/margins": 1.8665060997009277, + "rewards/rejected": -1.7812427282333374, + "step": 153, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.34365411436541143, + "grad_norm": 1.6320723295211792, + "learning_rate": 0.00019201277882589487, + "logits/chosen": -0.2685769498348236, + "logits/rejected": -0.4338504374027252, + "logps/chosen": -4.207334995269775, + "logps/rejected": -54.69655990600586, + "loss": 0.4186314344406128, + "memory(GiB)": 46.71, + "nll_loss": 0.23857831954956055, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.104047991335392, + "rewards/margins": 3.6357991695404053, + "rewards/rejected": -3.5317511558532715, + "step": 154, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.3458856345885635, + "grad_norm": 2.5706071853637695, + "learning_rate": 0.00019186758048545498, + "logits/chosen": -0.31499671936035156, + "logits/rejected": -0.4452478885650635, + "logps/chosen": -8.811934471130371, + "logps/rejected": -41.911869049072266, + "loss": 0.8359880447387695, + "memory(GiB)": 46.71, + "nll_loss": 0.5718236565589905, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.00384414941072464, + "rewards/margins": 2.2188119888305664, + "rewards/rejected": -2.214968204498291, + "step": 155, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.34811715481171546, + "grad_norm": 3.741011619567871, + "learning_rate": 0.00019172113015054532, + "logits/chosen": -0.42364075779914856, + "logits/rejected": -0.48594316840171814, + "logps/chosen": -9.2898588180542, + "logps/rejected": -39.12040710449219, + "loss": 0.7001189589500427, + "memory(GiB)": 46.71, + "nll_loss": 0.43081939220428467, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.08180733025074005, + "rewards/margins": 2.411517381668091, + "rewards/rejected": -2.329710006713867, + "step": 156, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.3503486750348675, + "grad_norm": 2.2846014499664307, + "learning_rate": 0.00019157342981702792, + "logits/chosen": -0.34239232540130615, + "logits/rejected": -0.5085087418556213, + "logps/chosen": -3.7471375465393066, + "logps/rejected": -39.35340118408203, + "loss": 0.537229597568512, + "memory(GiB)": 46.71, + "nll_loss": 0.32565146684646606, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18335413932800293, + "rewards/margins": 2.308189868927002, + "rewards/rejected": -2.124835968017578, + "step": 157, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.35258019525801954, + "grad_norm": 2.8782355785369873, + "learning_rate": 0.0001914244814978001, + "logits/chosen": -0.36600109934806824, + "logits/rejected": -0.4807679057121277, + "logps/chosen": -12.43294906616211, + "logps/rejected": -36.17433547973633, + "loss": 0.7901278734207153, + "memory(GiB)": 46.71, + "nll_loss": 0.5272985100746155, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.03357819840312004, + "rewards/margins": 2.113076686859131, + "rewards/rejected": -2.079498291015625, + "step": 158, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.3548117154811716, + "grad_norm": 4.173655033111572, + "learning_rate": 0.00019127428722276685, + "logits/chosen": -0.29084256291389465, + "logits/rejected": -0.49927467107772827, + "logps/chosen": -8.242868423461914, + "logps/rejected": -54.8849983215332, + "loss": 0.7512383460998535, + "memory(GiB)": 46.71, + "nll_loss": 0.5454739928245544, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08500596880912781, + "rewards/margins": 3.6066102981567383, + "rewards/rejected": -3.521604299545288, + "step": 159, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.35704323570432356, + "grad_norm": 2.355539083480835, + "learning_rate": 0.0001911228490388136, + "logits/chosen": -0.3650870621204376, + "logits/rejected": -0.5237394571304321, + "logps/chosen": -10.184951782226562, + "logps/rejected": -48.1030387878418, + "loss": 0.7325472831726074, + "memory(GiB)": 46.71, + "nll_loss": 0.5253479480743408, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15602552890777588, + "rewards/margins": 3.2045021057128906, + "rewards/rejected": -3.048476457595825, + "step": 160, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.3592747559274756, + "grad_norm": 4.184498310089111, + "learning_rate": 0.00019097016900977788, + "logits/chosen": -0.41671380400657654, + "logits/rejected": -0.5081248879432678, + "logps/chosen": -10.973024368286133, + "logps/rejected": -47.46335983276367, + "loss": 0.5701618194580078, + "memory(GiB)": 46.71, + "nll_loss": 0.3158893883228302, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09667229652404785, + "rewards/margins": 3.3210151195526123, + "rewards/rejected": -3.2243423461914062, + "step": 161, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.36150627615062764, + "grad_norm": 4.150604724884033, + "learning_rate": 0.00019081624921642157, + "logits/chosen": -0.36136606335639954, + "logits/rejected": -0.5436823964118958, + "logps/chosen": -7.6842756271362305, + "logps/rejected": -45.09693908691406, + "loss": 0.7053879499435425, + "memory(GiB)": 46.71, + "nll_loss": 0.4340396523475647, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07871361076831818, + "rewards/margins": 2.7361950874328613, + "rewards/rejected": -2.6574814319610596, + "step": 162, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3637377963737796, + "grad_norm": 3.704453706741333, + "learning_rate": 0.00019066109175640224, + "logits/chosen": -0.40510815382003784, + "logits/rejected": -0.4312695860862732, + "logps/chosen": -12.371721267700195, + "logps/rejected": -50.284141540527344, + "loss": 0.7488542795181274, + "memory(GiB)": 46.71, + "nll_loss": 0.4703201949596405, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1792062222957611, + "rewards/margins": 3.446262836456299, + "rewards/rejected": -3.267056941986084, + "step": 163, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.36596931659693166, + "grad_norm": 2.7603952884674072, + "learning_rate": 0.00019050469874424476, + "logits/chosen": -0.32453101873397827, + "logits/rejected": -0.4822920560836792, + "logps/chosen": -8.248761177062988, + "logps/rejected": -45.44584655761719, + "loss": 0.8071178197860718, + "memory(GiB)": 46.71, + "nll_loss": 0.4910924434661865, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.050887446850538254, + "rewards/margins": 3.054812431335449, + "rewards/rejected": -3.00392484664917, + "step": 164, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3682008368200837, + "grad_norm": 5.73638916015625, + "learning_rate": 0.0001903470723113124, + "logits/chosen": -0.37211766839027405, + "logits/rejected": -0.36960119009017944, + "logps/chosen": -13.764412879943848, + "logps/rejected": -28.488418579101562, + "loss": 1.0636341571807861, + "memory(GiB)": 46.71, + "nll_loss": 0.5649504661560059, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17467395961284637, + "rewards/margins": 1.285156488418579, + "rewards/rejected": -1.459830403327942, + "step": 165, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3704323570432357, + "grad_norm": 3.937652587890625, + "learning_rate": 0.00019018821460577777, + "logits/chosen": -0.2960231602191925, + "logits/rejected": -0.4136447012424469, + "logps/chosen": -12.963543891906738, + "logps/rejected": -43.19397735595703, + "loss": 0.8952102065086365, + "memory(GiB)": 46.71, + "nll_loss": 0.5876767039299011, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.3215300142765045, + "rewards/margins": 2.2675259113311768, + "rewards/rejected": -2.5890560150146484, + "step": 166, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3726638772663877, + "grad_norm": 2.5968947410583496, + "learning_rate": 0.00019002812779259364, + "logits/chosen": -0.31316545605659485, + "logits/rejected": -0.510556161403656, + "logps/chosen": -6.747772216796875, + "logps/rejected": -37.75751876831055, + "loss": 0.7592855095863342, + "memory(GiB)": 46.71, + "nll_loss": 0.47158926725387573, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.17958350479602814, + "rewards/margins": 2.317811965942383, + "rewards/rejected": -2.138228416442871, + "step": 167, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.37489539748953976, + "grad_norm": 9.810136795043945, + "learning_rate": 0.00018986681405346322, + "logits/chosen": -0.28982365131378174, + "logits/rejected": -0.48794350028038025, + "logps/chosen": -9.523417472839355, + "logps/rejected": -53.576263427734375, + "loss": 0.8609976172447205, + "memory(GiB)": 46.71, + "nll_loss": 0.55437833070755, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.018897701054811478, + "rewards/margins": 3.141695499420166, + "rewards/rejected": -3.160593032836914, + "step": 168, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.37712691771269174, + "grad_norm": 3.2620904445648193, + "learning_rate": 0.00018970427558681082, + "logits/chosen": -0.2701128125190735, + "logits/rejected": -0.5124972462654114, + "logps/chosen": -7.385637283325195, + "logps/rejected": -66.33370971679688, + "loss": 0.595870852470398, + "memory(GiB)": 46.71, + "nll_loss": 0.4602699279785156, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2253667265176773, + "rewards/margins": 4.247678756713867, + "rewards/rejected": -4.022312164306641, + "step": 169, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3793584379358438, + "grad_norm": 2.7175841331481934, + "learning_rate": 0.0001895405146077514, + "logits/chosen": -0.3605695366859436, + "logits/rejected": -0.521330714225769, + "logps/chosen": -5.586010456085205, + "logps/rejected": -52.13179397583008, + "loss": 0.652187705039978, + "memory(GiB)": 46.71, + "nll_loss": 0.32753562927246094, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04342030733823776, + "rewards/margins": 3.0307912826538086, + "rewards/rejected": -3.074211597442627, + "step": 170, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.3815899581589958, + "grad_norm": 3.162261486053467, + "learning_rate": 0.00018937553334806077, + "logits/chosen": -0.29991668462753296, + "logits/rejected": -0.4432262182235718, + "logps/chosen": -9.342361450195312, + "logps/rejected": -48.89967346191406, + "loss": 0.7870311737060547, + "memory(GiB)": 46.71, + "nll_loss": 0.543519139289856, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10350081324577332, + "rewards/margins": 3.252532720565796, + "rewards/rejected": -3.149031639099121, + "step": 171, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.38382147838214786, + "grad_norm": 11.82007122039795, + "learning_rate": 0.000189209334056145, + "logits/chosen": -0.3029250502586365, + "logits/rejected": -0.4914131164550781, + "logps/chosen": -5.792529582977295, + "logps/rejected": -52.44511413574219, + "loss": 0.6816909313201904, + "memory(GiB)": 46.71, + "nll_loss": 0.5198902487754822, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.058606795966625214, + "rewards/margins": 3.3576695919036865, + "rewards/rejected": -3.299063205718994, + "step": 172, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.38605299860529985, + "grad_norm": 2.9290287494659424, + "learning_rate": 0.00018904191899700977, + "logits/chosen": -0.40706759691238403, + "logits/rejected": -0.45858079195022583, + "logps/chosen": -6.328141212463379, + "logps/rejected": -34.812583923339844, + "loss": 0.7009620666503906, + "memory(GiB)": 46.71, + "nll_loss": 0.44378477334976196, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12429780513048172, + "rewards/margins": 2.033411979675293, + "rewards/rejected": -1.909114122390747, + "step": 173, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.3882845188284519, + "grad_norm": 3.7593777179718018, + "learning_rate": 0.0001888732904522296, + "logits/chosen": -0.380102276802063, + "logits/rejected": -0.4688776731491089, + "logps/chosen": -5.4205002784729, + "logps/rejected": -42.66455078125, + "loss": 0.705348014831543, + "memory(GiB)": 46.71, + "nll_loss": 0.48004987835884094, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09237709641456604, + "rewards/margins": 2.7925949096679688, + "rewards/rejected": -2.7002179622650146, + "step": 174, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.3905160390516039, + "grad_norm": 4.404231548309326, + "learning_rate": 0.00018870345071991663, + "logits/chosen": -0.42475855350494385, + "logits/rejected": -0.49599534273147583, + "logps/chosen": -4.999125957489014, + "logps/rejected": -31.932191848754883, + "loss": 0.5820786356925964, + "memory(GiB)": 46.71, + "nll_loss": 0.3192511200904846, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.01989058405160904, + "rewards/margins": 1.8925269842147827, + "rewards/rejected": -1.8726365566253662, + "step": 175, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.3927475592747559, + "grad_norm": 4.2940239906311035, + "learning_rate": 0.00018853240211468944, + "logits/chosen": -0.393295019865036, + "logits/rejected": -0.4561651647090912, + "logps/chosen": -12.356813430786133, + "logps/rejected": -33.95934295654297, + "loss": 0.9481363892555237, + "memory(GiB)": 46.71, + "nll_loss": 0.5598008036613464, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.053550563752651215, + "rewards/margins": 1.7341490983963013, + "rewards/rejected": -1.680598497390747, + "step": 176, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.39497907949790795, + "grad_norm": 5.875354766845703, + "learning_rate": 0.0001883601469676414, + "logits/chosen": -0.36429429054260254, + "logits/rejected": -0.5053366422653198, + "logps/chosen": -4.228557586669922, + "logps/rejected": -41.48766326904297, + "loss": 0.5440742373466492, + "memory(GiB)": 46.71, + "nll_loss": 0.34625735878944397, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.046092353761196136, + "rewards/margins": 2.599119186401367, + "rewards/rejected": -2.5530271530151367, + "step": 177, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.39721059972106, + "grad_norm": 2.555330753326416, + "learning_rate": 0.00018818668762630888, + "logits/chosen": -0.40784329175949097, + "logits/rejected": -0.5250139236450195, + "logps/chosen": -6.325516700744629, + "logps/rejected": -37.756141662597656, + "loss": 0.5609665513038635, + "memory(GiB)": 46.71, + "nll_loss": 0.36217477917671204, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06898089498281479, + "rewards/margins": 2.456758499145508, + "rewards/rejected": -2.38777756690979, + "step": 178, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.39944211994421197, + "grad_norm": 2.5570666790008545, + "learning_rate": 0.00018801202645463947, + "logits/chosen": -0.49009108543395996, + "logits/rejected": -0.5421456694602966, + "logps/chosen": -7.800501823425293, + "logps/rejected": -31.90920639038086, + "loss": 0.7761667966842651, + "memory(GiB)": 46.71, + "nll_loss": 0.5564302206039429, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22300021350383759, + "rewards/margins": 2.3164584636688232, + "rewards/rejected": -2.093458414077759, + "step": 179, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.401673640167364, + "grad_norm": 3.062514066696167, + "learning_rate": 0.00018783616583295943, + "logits/chosen": -0.3902238607406616, + "logits/rejected": -0.5166201591491699, + "logps/chosen": -5.271551132202148, + "logps/rejected": -43.41081237792969, + "loss": 0.586236834526062, + "memory(GiB)": 46.71, + "nll_loss": 0.3715082108974457, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10219533741474152, + "rewards/margins": 2.8101999759674072, + "rewards/rejected": -2.7080047130584717, + "step": 180, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.40390516039051605, + "grad_norm": 2.8764405250549316, + "learning_rate": 0.0001876591081579415, + "logits/chosen": -0.4361119270324707, + "logits/rejected": -0.5355557203292847, + "logps/chosen": -5.954665184020996, + "logps/rejected": -40.49996566772461, + "loss": 0.6423499584197998, + "memory(GiB)": 46.71, + "nll_loss": 0.3932062089443207, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.11792459338903427, + "rewards/margins": 2.8236496448516846, + "rewards/rejected": -2.7057251930236816, + "step": 181, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.4061366806136681, + "grad_norm": 3.4861021041870117, + "learning_rate": 0.0001874808558425722, + "logits/chosen": -0.44822514057159424, + "logits/rejected": -0.5208377838134766, + "logps/chosen": -8.507841110229492, + "logps/rejected": -38.145362854003906, + "loss": 0.7365833520889282, + "memory(GiB)": 46.71, + "nll_loss": 0.45295223593711853, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.18383249640464783, + "rewards/margins": 2.3255980014801025, + "rewards/rejected": -2.141765594482422, + "step": 182, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.40836820083682007, + "grad_norm": 3.0177836418151855, + "learning_rate": 0.00018730141131611882, + "logits/chosen": -0.4113612771034241, + "logits/rejected": -0.5111112594604492, + "logps/chosen": -7.983984470367432, + "logps/rejected": -39.916988372802734, + "loss": 0.7822529077529907, + "memory(GiB)": 46.71, + "nll_loss": 0.48471909761428833, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.010233555920422077, + "rewards/margins": 2.3506059646606445, + "rewards/rejected": -2.340372323989868, + "step": 183, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.4105997210599721, + "grad_norm": 2.848646879196167, + "learning_rate": 0.0001871207770240965, + "logits/chosen": -0.41031956672668457, + "logits/rejected": -0.5503991842269897, + "logps/chosen": -6.160996437072754, + "logps/rejected": -49.401180267333984, + "loss": 0.7412950992584229, + "memory(GiB)": 46.71, + "nll_loss": 0.5057318806648254, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.11709622293710709, + "rewards/margins": 2.803727388381958, + "rewards/rejected": -2.686631441116333, + "step": 184, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.41283124128312415, + "grad_norm": 3.7771029472351074, + "learning_rate": 0.00018693895542823476, + "logits/chosen": -0.4378977417945862, + "logits/rejected": -0.5446797609329224, + "logps/chosen": -3.902487277984619, + "logps/rejected": -38.302616119384766, + "loss": 0.6036807298660278, + "memory(GiB)": 46.71, + "nll_loss": 0.3306537866592407, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07923398911952972, + "rewards/margins": 2.431417465209961, + "rewards/rejected": -2.3521835803985596, + "step": 185, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.41506276150627613, + "grad_norm": 2.6410114765167236, + "learning_rate": 0.00018675594900644394, + "logits/chosen": -0.4403599798679352, + "logits/rejected": -0.534117579460144, + "logps/chosen": -7.202261447906494, + "logps/rejected": -40.587371826171875, + "loss": 0.6803833842277527, + "memory(GiB)": 46.71, + "nll_loss": 0.4558661878108978, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11624391376972198, + "rewards/margins": 2.7324059009552, + "rewards/rejected": -2.616162061691284, + "step": 186, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.4172942817294282, + "grad_norm": 27.184694290161133, + "learning_rate": 0.0001865717602527816, + "logits/chosen": -0.3826313018798828, + "logits/rejected": -0.5364388227462769, + "logps/chosen": -5.527843952178955, + "logps/rejected": -49.32733917236328, + "loss": 0.7175433039665222, + "memory(GiB)": 46.71, + "nll_loss": 0.5420017242431641, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13603699207305908, + "rewards/margins": 3.4389665126800537, + "rewards/rejected": -3.302929401397705, + "step": 187, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.4195258019525802, + "grad_norm": 2.5546696186065674, + "learning_rate": 0.00018638639167741834, + "logits/chosen": -0.432650625705719, + "logits/rejected": -0.5771442651748657, + "logps/chosen": -6.985705375671387, + "logps/rejected": -55.28477478027344, + "loss": 0.5891926884651184, + "memory(GiB)": 46.71, + "nll_loss": 0.41381514072418213, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.2026616781949997, + "rewards/margins": 4.167919635772705, + "rewards/rejected": -3.9652576446533203, + "step": 188, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.4217573221757322, + "grad_norm": 2.758708953857422, + "learning_rate": 0.00018619984580660367, + "logits/chosen": -0.41937315464019775, + "logits/rejected": -0.5321905612945557, + "logps/chosen": -6.989339828491211, + "logps/rejected": -58.12518310546875, + "loss": 0.5879656672477722, + "memory(GiB)": 46.71, + "nll_loss": 0.36345142126083374, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.023294318467378616, + "rewards/margins": 4.040982246398926, + "rewards/rejected": -4.064276695251465, + "step": 189, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.42398884239888424, + "grad_norm": 11.678010940551758, + "learning_rate": 0.00018601212518263156, + "logits/chosen": -0.39216288924217224, + "logits/rejected": -0.4594748020172119, + "logps/chosen": -9.29268741607666, + "logps/rejected": -54.65095520019531, + "loss": 0.818715512752533, + "memory(GiB)": 46.71, + "nll_loss": 0.6221151947975159, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13676013052463531, + "rewards/margins": 4.018932819366455, + "rewards/rejected": -3.8821723461151123, + "step": 190, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.4262203626220363, + "grad_norm": 3.9322588443756104, + "learning_rate": 0.00018582323236380588, + "logits/chosen": -0.4796799421310425, + "logits/rejected": -0.5020566582679749, + "logps/chosen": -14.528449058532715, + "logps/rejected": -43.279300689697266, + "loss": 0.9906633496284485, + "memory(GiB)": 46.71, + "nll_loss": 0.6794536709785461, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.2910754680633545, + "rewards/margins": 2.7544803619384766, + "rewards/rejected": -3.045555591583252, + "step": 191, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.42845188284518826, + "grad_norm": 5.284152507781982, + "learning_rate": 0.00018563316992440543, + "logits/chosen": -0.4428099989891052, + "logits/rejected": -0.5434530973434448, + "logps/chosen": -6.242356777191162, + "logps/rejected": -46.20015335083008, + "loss": 0.8040673732757568, + "memory(GiB)": 46.71, + "nll_loss": 0.4949415922164917, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.05515892803668976, + "rewards/margins": 2.9070026874542236, + "rewards/rejected": -2.9621615409851074, + "step": 192, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.4306834030683403, + "grad_norm": 11.493171691894531, + "learning_rate": 0.00018544194045464886, + "logits/chosen": -0.38638293743133545, + "logits/rejected": -0.4842715561389923, + "logps/chosen": -6.438551425933838, + "logps/rejected": -44.880149841308594, + "loss": 0.7885379791259766, + "memory(GiB)": 46.71, + "nll_loss": 0.5359107255935669, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.19068023562431335, + "rewards/margins": 2.8829379081726074, + "rewards/rejected": -2.692257881164551, + "step": 193, + "train_speed(iter/s)": 0.011179 + }, + { + "epoch": 0.43291492329149234, + "grad_norm": 2.3137385845184326, + "learning_rate": 0.00018524954656065946, + "logits/chosen": -0.4063287675380707, + "logits/rejected": -0.5103312730789185, + "logps/chosen": -7.378753662109375, + "logps/rejected": -45.85976791381836, + "loss": 0.45938000082969666, + "memory(GiB)": 46.71, + "nll_loss": 0.3061022460460663, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08831208944320679, + "rewards/margins": 3.123901605606079, + "rewards/rejected": -3.0355896949768066, + "step": 194, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.4351464435146444, + "grad_norm": 4.795388221740723, + "learning_rate": 0.00018505599086442956, + "logits/chosen": -0.38035646080970764, + "logits/rejected": -0.5444158315658569, + "logps/chosen": -2.437512159347534, + "logps/rejected": -43.048858642578125, + "loss": 0.5650367736816406, + "memory(GiB)": 46.71, + "nll_loss": 0.33980026841163635, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13193824887275696, + "rewards/margins": 2.7752439975738525, + "rewards/rejected": -2.643305540084839, + "step": 195, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.43737796373779636, + "grad_norm": 2.4795217514038086, + "learning_rate": 0.0001848612760037848, + "logits/chosen": -0.3798023462295532, + "logits/rejected": -0.5086287260055542, + "logps/chosen": -4.153357982635498, + "logps/rejected": -47.3394775390625, + "loss": 0.4092462360858917, + "memory(GiB)": 46.71, + "nll_loss": 0.22234448790550232, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1915455460548401, + "rewards/margins": 2.8723514080047607, + "rewards/rejected": -2.6808059215545654, + "step": 196, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.4396094839609484, + "grad_norm": 4.820924282073975, + "learning_rate": 0.0001846654046323482, + "logits/chosen": -0.41246578097343445, + "logits/rejected": -0.4796575903892517, + "logps/chosen": -10.3787841796875, + "logps/rejected": -37.14994812011719, + "loss": 0.9857741594314575, + "memory(GiB)": 46.71, + "nll_loss": 0.669836163520813, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.05453118681907654, + "rewards/margins": 2.0403127670288086, + "rewards/rejected": -2.094843864440918, + "step": 197, + "train_speed(iter/s)": 0.01118 + }, + { + "epoch": 0.44184100418410044, + "grad_norm": 3.00054931640625, + "learning_rate": 0.0001844683794195041, + "logits/chosen": -0.3768506646156311, + "logits/rejected": -0.5438990592956543, + "logps/chosen": -6.3500471115112305, + "logps/rejected": -49.48595428466797, + "loss": 0.6618455648422241, + "memory(GiB)": 46.71, + "nll_loss": 0.4003591537475586, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.02325659617781639, + "rewards/margins": 3.2074074745178223, + "rewards/rejected": -3.1841506958007812, + "step": 198, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.4440725244072524, + "grad_norm": 5.5015339851379395, + "learning_rate": 0.00018427020305036157, + "logits/chosen": -0.43915611505508423, + "logits/rejected": -0.5123714804649353, + "logps/chosen": -5.833986759185791, + "logps/rejected": -41.57982635498047, + "loss": 0.7684890031814575, + "memory(GiB)": 46.71, + "nll_loss": 0.4649631679058075, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04928220808506012, + "rewards/margins": 2.619971752166748, + "rewards/rejected": -2.6692538261413574, + "step": 199, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.44630404463040446, + "grad_norm": 4.47312068939209, + "learning_rate": 0.00018407087822571794, + "logits/chosen": -0.41636520624160767, + "logits/rejected": -0.5203005075454712, + "logps/chosen": -9.286731719970703, + "logps/rejected": -41.12049102783203, + "loss": 0.8878820538520813, + "memory(GiB)": 46.71, + "nll_loss": 0.6213392615318298, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.1641845703125, + "rewards/margins": 2.593937397003174, + "rewards/rejected": -2.4297525882720947, + "step": 200, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.4485355648535565, + "grad_norm": 2.523305654525757, + "learning_rate": 0.00018387040766202202, + "logits/chosen": -0.43077439069747925, + "logits/rejected": -0.5606883764266968, + "logps/chosen": -9.173616409301758, + "logps/rejected": -54.819252014160156, + "loss": 0.6095305681228638, + "memory(GiB)": 46.71, + "nll_loss": 0.43501588702201843, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17188499867916107, + "rewards/margins": 3.581376791000366, + "rewards/rejected": -3.409491539001465, + "step": 201, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.4507670850767085, + "grad_norm": 10.484657287597656, + "learning_rate": 0.00018366879409133702, + "logits/chosen": -0.46561458706855774, + "logits/rejected": -0.5806446075439453, + "logps/chosen": -9.230829238891602, + "logps/rejected": -50.296417236328125, + "loss": 0.8357149362564087, + "memory(GiB)": 46.71, + "nll_loss": 0.5318824052810669, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04186338931322098, + "rewards/margins": 3.0313618183135986, + "rewards/rejected": -3.0732247829437256, + "step": 202, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.4529986052998605, + "grad_norm": 3.0593442916870117, + "learning_rate": 0.00018346604026130335, + "logits/chosen": -0.4838520884513855, + "logits/rejected": -0.563227117061615, + "logps/chosen": -6.622610092163086, + "logps/rejected": -33.531341552734375, + "loss": 0.7672325372695923, + "memory(GiB)": 46.71, + "nll_loss": 0.47228333353996277, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16688543558120728, + "rewards/margins": 2.1572673320770264, + "rewards/rejected": -1.9903819561004639, + "step": 203, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.45523012552301256, + "grad_norm": 3.2974071502685547, + "learning_rate": 0.00018326214893510113, + "logits/chosen": -0.48938241600990295, + "logits/rejected": -0.5406135320663452, + "logps/chosen": -6.855633735656738, + "logps/rejected": -32.516056060791016, + "loss": 0.7795042991638184, + "memory(GiB)": 46.71, + "nll_loss": 0.463142991065979, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.058080192655324936, + "rewards/margins": 2.187054395675659, + "rewards/rejected": -2.128974199295044, + "step": 204, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.45746164574616455, + "grad_norm": 2.7023813724517822, + "learning_rate": 0.00018305712289141263, + "logits/chosen": -0.503454864025116, + "logits/rejected": -0.5921227931976318, + "logps/chosen": -7.617156982421875, + "logps/rejected": -42.7328987121582, + "loss": 0.6725366711616516, + "memory(GiB)": 46.71, + "nll_loss": 0.3977656364440918, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02913985215127468, + "rewards/margins": 2.757948398590088, + "rewards/rejected": -2.728808641433716, + "step": 205, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.4596931659693166, + "grad_norm": 3.1167116165161133, + "learning_rate": 0.00018285096492438424, + "logits/chosen": -0.5165062546730042, + "logits/rejected": -0.6217053532600403, + "logps/chosen": -7.3234543800354, + "logps/rejected": -50.999603271484375, + "loss": 0.6318544745445251, + "memory(GiB)": 46.71, + "nll_loss": 0.3780009150505066, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.00554865226149559, + "rewards/margins": 3.287566661834717, + "rewards/rejected": -3.2820186614990234, + "step": 206, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.4619246861924686, + "grad_norm": 9.524746894836426, + "learning_rate": 0.00018264367784358854, + "logits/chosen": -0.4920929968357086, + "logits/rejected": -0.5764138102531433, + "logps/chosen": -7.860991954803467, + "logps/rejected": -42.47827911376953, + "loss": 0.7360129356384277, + "memory(GiB)": 46.71, + "nll_loss": 0.5072849988937378, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2539522051811218, + "rewards/margins": 2.9447669982910156, + "rewards/rejected": -2.690814733505249, + "step": 207, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.46415620641562066, + "grad_norm": 3.1537561416625977, + "learning_rate": 0.00018243526447398595, + "logits/chosen": -0.49637073278427124, + "logits/rejected": -0.5572264194488525, + "logps/chosen": -7.864004135131836, + "logps/rejected": -36.40027618408203, + "loss": 0.701461911201477, + "memory(GiB)": 46.71, + "nll_loss": 0.4231345057487488, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.05218251794576645, + "rewards/margins": 2.490140438079834, + "rewards/rejected": -2.437958002090454, + "step": 208, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.46638772663877265, + "grad_norm": 5.860493183135986, + "learning_rate": 0.00018222572765588625, + "logits/chosen": -0.4768957793712616, + "logits/rejected": -0.5333793759346008, + "logps/chosen": -14.414131164550781, + "logps/rejected": -43.239593505859375, + "loss": 0.8101565837860107, + "memory(GiB)": 46.71, + "nll_loss": 0.49351921677589417, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.05914236605167389, + "rewards/margins": 2.7074344158172607, + "rewards/rejected": -2.648291826248169, + "step": 209, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.4686192468619247, + "grad_norm": 2.79776930809021, + "learning_rate": 0.00018201507024490988, + "logits/chosen": -0.4704314172267914, + "logits/rejected": -0.567833662033081, + "logps/chosen": -14.108932495117188, + "logps/rejected": -44.836395263671875, + "loss": 0.8302231431007385, + "memory(GiB)": 46.71, + "nll_loss": 0.6059545278549194, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17180928587913513, + "rewards/margins": 2.3872344493865967, + "rewards/rejected": -2.2154252529144287, + "step": 210, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.4708507670850767, + "grad_norm": 1.4490225315093994, + "learning_rate": 0.0001818032951119489, + "logits/chosen": -0.45868298411369324, + "logits/rejected": -0.6028948426246643, + "logps/chosen": -3.0846667289733887, + "logps/rejected": -51.68486404418945, + "loss": 0.36262989044189453, + "memory(GiB)": 46.71, + "nll_loss": 0.25144004821777344, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2839798927307129, + "rewards/margins": 3.57785964012146, + "rewards/rejected": -3.293879747390747, + "step": 211, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4730822873082287, + "grad_norm": 2.7278997898101807, + "learning_rate": 0.00018159040514312803, + "logits/chosen": -0.5122844576835632, + "logits/rejected": -0.5693905353546143, + "logps/chosen": -8.715449333190918, + "logps/rejected": -31.896915435791016, + "loss": 0.7132190465927124, + "memory(GiB)": 46.71, + "nll_loss": 0.39171797037124634, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.2175295054912567, + "rewards/margins": 2.153326988220215, + "rewards/rejected": -1.9357972145080566, + "step": 212, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.47531380753138075, + "grad_norm": 2.76104474067688, + "learning_rate": 0.00018137640323976536, + "logits/chosen": -0.5042344331741333, + "logits/rejected": -0.5739261507987976, + "logps/chosen": -12.65793228149414, + "logps/rejected": -42.124107360839844, + "loss": 0.7560016512870789, + "memory(GiB)": 46.71, + "nll_loss": 0.5502416491508484, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.03646032512187958, + "rewards/margins": 2.767151355743408, + "rewards/rejected": -2.7306909561157227, + "step": 213, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4775453277545328, + "grad_norm": 4.021885871887207, + "learning_rate": 0.00018116129231833249, + "logits/chosen": -0.466595321893692, + "logits/rejected": -0.5636504888534546, + "logps/chosen": -8.149341583251953, + "logps/rejected": -38.28250503540039, + "loss": 0.7789570093154907, + "memory(GiB)": 46.71, + "nll_loss": 0.4304181933403015, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0456395149230957, + "rewards/margins": 2.337042808532715, + "rewards/rejected": -2.2914035320281982, + "step": 214, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4797768479776848, + "grad_norm": 2.7031266689300537, + "learning_rate": 0.00018094507531041517, + "logits/chosen": -0.5194039940834045, + "logits/rejected": -0.5763382315635681, + "logps/chosen": -12.234911918640137, + "logps/rejected": -41.63436508178711, + "loss": 0.6696392893791199, + "memory(GiB)": 46.71, + "nll_loss": 0.397487074136734, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.020977359265089035, + "rewards/margins": 2.5402636528015137, + "rewards/rejected": -2.5192863941192627, + "step": 215, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4820083682008368, + "grad_norm": 4.239037990570068, + "learning_rate": 0.00018072775516267306, + "logits/chosen": -0.4820006191730499, + "logits/rejected": -0.5642452239990234, + "logps/chosen": -5.411629676818848, + "logps/rejected": -49.11735534667969, + "loss": 0.6698524951934814, + "memory(GiB)": 46.71, + "nll_loss": 0.4091280698776245, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.007008063141256571, + "rewards/margins": 2.734046459197998, + "rewards/rejected": -2.7270381450653076, + "step": 216, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.48423988842398885, + "grad_norm": 2.3891801834106445, + "learning_rate": 0.00018050933483679976, + "logits/chosen": -0.48481085896492004, + "logits/rejected": -0.6319360136985779, + "logps/chosen": -6.989506721496582, + "logps/rejected": -56.728424072265625, + "loss": 0.601786732673645, + "memory(GiB)": 46.71, + "nll_loss": 0.35297003388404846, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.13874617218971252, + "rewards/margins": 3.526907205581665, + "rewards/rejected": -3.3881607055664062, + "step": 217, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4864714086471409, + "grad_norm": 6.219696044921875, + "learning_rate": 0.0001802898173094824, + "logits/chosen": -0.5356330871582031, + "logits/rejected": -0.6473451852798462, + "logps/chosen": -8.472892761230469, + "logps/rejected": -51.16962432861328, + "loss": 0.783514142036438, + "memory(GiB)": 46.71, + "nll_loss": 0.5371319651603699, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06705744564533234, + "rewards/margins": 3.750828266143799, + "rewards/rejected": -3.6837711334228516, + "step": 218, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.4887029288702929, + "grad_norm": 6.937330722808838, + "learning_rate": 0.0001800692055723609, + "logits/chosen": -0.6238746047019958, + "logits/rejected": -0.6939642429351807, + "logps/chosen": -4.523331642150879, + "logps/rejected": -37.64277267456055, + "loss": 0.6180140376091003, + "memory(GiB)": 46.71, + "nll_loss": 0.3569566607475281, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.11646758764982224, + "rewards/margins": 2.8116490840911865, + "rewards/rejected": -2.695181369781494, + "step": 219, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.4909344490934449, + "grad_norm": 4.587141036987305, + "learning_rate": 0.0001798475026319875, + "logits/chosen": -0.5524365901947021, + "logits/rejected": -0.7282678484916687, + "logps/chosen": -4.9026618003845215, + "logps/rejected": -58.83467483520508, + "loss": 0.5599265098571777, + "memory(GiB)": 46.71, + "nll_loss": 0.3686913251876831, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08274698257446289, + "rewards/margins": 4.399111747741699, + "rewards/rejected": -4.316364765167236, + "step": 220, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.49316596931659695, + "grad_norm": 19.358779907226562, + "learning_rate": 0.00017962471150978563, + "logits/chosen": -0.6010394096374512, + "logits/rejected": -0.7070757150650024, + "logps/chosen": -7.744332313537598, + "logps/rejected": -56.298973083496094, + "loss": 1.0826630592346191, + "memory(GiB)": 46.71, + "nll_loss": 0.7130725979804993, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.036551520228385925, + "rewards/margins": 4.22437858581543, + "rewards/rejected": -4.260930061340332, + "step": 221, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.49539748953974894, + "grad_norm": 1.9726710319519043, + "learning_rate": 0.00017940083524200858, + "logits/chosen": -0.5176704525947571, + "logits/rejected": -0.6265690326690674, + "logps/chosen": -8.897793769836426, + "logps/rejected": -51.35803985595703, + "loss": 0.5861097574234009, + "memory(GiB)": 46.71, + "nll_loss": 0.4163362979888916, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3185967803001404, + "rewards/margins": 3.5779831409454346, + "rewards/rejected": -3.2593865394592285, + "step": 222, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.497629009762901, + "grad_norm": 2.865757465362549, + "learning_rate": 0.00017917587687969847, + "logits/chosen": -0.5248193740844727, + "logits/rejected": -0.6183274984359741, + "logps/chosen": -11.439458847045898, + "logps/rejected": -43.991424560546875, + "loss": 0.8551426529884338, + "memory(GiB)": 46.71, + "nll_loss": 0.6576440334320068, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24198690056800842, + "rewards/margins": 3.191100597381592, + "rewards/rejected": -2.9491138458251953, + "step": 223, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.499860529986053, + "grad_norm": 2.936124324798584, + "learning_rate": 0.00017894983948864431, + "logits/chosen": -0.5203258991241455, + "logits/rejected": -0.6186901926994324, + "logps/chosen": -8.132195472717285, + "logps/rejected": -41.595703125, + "loss": 0.7639778256416321, + "memory(GiB)": 46.71, + "nll_loss": 0.4688246548175812, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.03628700599074364, + "rewards/margins": 2.6348683834075928, + "rewards/rejected": -2.598581314086914, + "step": 224, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.502092050209205, + "grad_norm": 3.2935783863067627, + "learning_rate": 0.0001787227261493405, + "logits/chosen": -0.49744516611099243, + "logits/rejected": -0.6266440749168396, + "logps/chosen": -6.2615861892700195, + "logps/rejected": -52.76634216308594, + "loss": 0.5845623016357422, + "memory(GiB)": 46.71, + "nll_loss": 0.36887243390083313, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10732433944940567, + "rewards/margins": 3.68102765083313, + "rewards/rejected": -3.5737037658691406, + "step": 225, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.5043235704323571, + "grad_norm": 2.1799988746643066, + "learning_rate": 0.0001784945399569447, + "logits/chosen": -0.4298093914985657, + "logits/rejected": -0.6444836854934692, + "logps/chosen": -3.285033941268921, + "logps/rejected": -62.29474639892578, + "loss": 0.3315008282661438, + "memory(GiB)": 46.71, + "nll_loss": 0.21272185444831848, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2050013542175293, + "rewards/margins": 4.134213924407959, + "rewards/rejected": -3.9292120933532715, + "step": 226, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.5065550906555091, + "grad_norm": 1.7241166830062866, + "learning_rate": 0.00017826528402123564, + "logits/chosen": -0.48651599884033203, + "logits/rejected": -0.6437865495681763, + "logps/chosen": -2.7627596855163574, + "logps/rejected": -52.23710632324219, + "loss": 0.3623155653476715, + "memory(GiB)": 46.71, + "nll_loss": 0.25128594040870667, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20487122237682343, + "rewards/margins": 3.8199470043182373, + "rewards/rejected": -3.6150758266448975, + "step": 227, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.5087866108786611, + "grad_norm": 2.1939537525177, + "learning_rate": 0.00017803496146657084, + "logits/chosen": -0.4696894884109497, + "logits/rejected": -0.648972749710083, + "logps/chosen": -6.876445770263672, + "logps/rejected": -58.56797409057617, + "loss": 0.6069039106369019, + "memory(GiB)": 46.71, + "nll_loss": 0.48535749316215515, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30394309759140015, + "rewards/margins": 4.106232643127441, + "rewards/rejected": -3.8022897243499756, + "step": 228, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.5110181311018132, + "grad_norm": 2.2029025554656982, + "learning_rate": 0.00017780357543184397, + "logits/chosen": -0.5041614174842834, + "logits/rejected": -0.6466209292411804, + "logps/chosen": -6.405285835266113, + "logps/rejected": -66.68818664550781, + "loss": 0.530963122844696, + "memory(GiB)": 46.71, + "nll_loss": 0.36968475580215454, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1667797863483429, + "rewards/margins": 5.374972343444824, + "rewards/rejected": -5.208192825317383, + "step": 229, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.5132496513249651, + "grad_norm": 2.9095795154571533, + "learning_rate": 0.000177571129070442, + "logits/chosen": -0.5456891655921936, + "logits/rejected": -0.5758720636367798, + "logps/chosen": -9.691763877868652, + "logps/rejected": -49.30378723144531, + "loss": 0.6552941203117371, + "memory(GiB)": 46.71, + "nll_loss": 0.4235466718673706, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.05060029774904251, + "rewards/margins": 3.4885663986206055, + "rewards/rejected": -3.4379661083221436, + "step": 230, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.5154811715481171, + "grad_norm": 2.6140189170837402, + "learning_rate": 0.00017733762555020234, + "logits/chosen": -0.526019811630249, + "logits/rejected": -0.6267409324645996, + "logps/chosen": -7.562019348144531, + "logps/rejected": -59.53757858276367, + "loss": 0.45963698625564575, + "memory(GiB)": 46.71, + "nll_loss": 0.3218066692352295, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11903621256351471, + "rewards/margins": 4.636847019195557, + "rewards/rejected": -4.517810821533203, + "step": 231, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.5177126917712692, + "grad_norm": 7.271316051483154, + "learning_rate": 0.0001771030680533697, + "logits/chosen": -0.4977180063724518, + "logits/rejected": -0.6347447633743286, + "logps/chosen": -6.1880202293396, + "logps/rejected": -57.32111358642578, + "loss": 0.6974989771842957, + "memory(GiB)": 46.71, + "nll_loss": 0.41052868962287903, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06481020152568817, + "rewards/margins": 4.053691864013672, + "rewards/rejected": -3.988882064819336, + "step": 232, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.5199442119944212, + "grad_norm": 6.688237190246582, + "learning_rate": 0.00017686745977655254, + "logits/chosen": -0.5404477715492249, + "logits/rejected": -0.591978907585144, + "logps/chosen": -7.557736873626709, + "logps/rejected": -53.3175048828125, + "loss": 0.6092088222503662, + "memory(GiB)": 46.71, + "nll_loss": 0.40488630533218384, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.009284242987632751, + "rewards/margins": 4.1996588706970215, + "rewards/rejected": -4.190374851226807, + "step": 233, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5221757322175732, + "grad_norm": 2.909499168395996, + "learning_rate": 0.00017663080393067967, + "logits/chosen": -0.41550740599632263, + "logits/rejected": -0.6418639421463013, + "logps/chosen": -9.175922393798828, + "logps/rejected": -79.41323852539062, + "loss": 0.7257620096206665, + "memory(GiB)": 46.71, + "nll_loss": 0.47780197858810425, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.10455615818500519, + "rewards/margins": 5.70869255065918, + "rewards/rejected": -5.6041364669799805, + "step": 234, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5244072524407253, + "grad_norm": 24.34648895263672, + "learning_rate": 0.00017639310374095642, + "logits/chosen": -0.474945992231369, + "logits/rejected": -0.6495036482810974, + "logps/chosen": -9.139482498168945, + "logps/rejected": -61.76782989501953, + "loss": 0.7843772172927856, + "memory(GiB)": 46.71, + "nll_loss": 0.6071273684501648, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08099109679460526, + "rewards/margins": 4.556737422943115, + "rewards/rejected": -4.475746154785156, + "step": 235, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5266387726638773, + "grad_norm": 3.9490129947662354, + "learning_rate": 0.00017615436244682067, + "logits/chosen": -0.48849430680274963, + "logits/rejected": -0.6626330614089966, + "logps/chosen": -6.473770618438721, + "logps/rejected": -63.68229675292969, + "loss": 0.7651278972625732, + "memory(GiB)": 46.71, + "nll_loss": 0.5939050912857056, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.05846280977129936, + "rewards/margins": 4.720561981201172, + "rewards/rejected": -4.662098407745361, + "step": 236, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5288702928870292, + "grad_norm": 2.3226640224456787, + "learning_rate": 0.0001759145833018988, + "logits/chosen": -0.5129532814025879, + "logits/rejected": -0.6029571294784546, + "logps/chosen": -11.16122055053711, + "logps/rejected": -65.20454406738281, + "loss": 0.6758726239204407, + "memory(GiB)": 46.71, + "nll_loss": 0.44432199001312256, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06612987071275711, + "rewards/margins": 4.690124034881592, + "rewards/rejected": -4.62399435043335, + "step": 237, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5311018131101813, + "grad_norm": 4.086884498596191, + "learning_rate": 0.00017567376957396125, + "logits/chosen": -0.4485701620578766, + "logits/rejected": -0.6184876561164856, + "logps/chosen": -9.818546295166016, + "logps/rejected": -64.69741821289062, + "loss": 0.841502845287323, + "memory(GiB)": 46.71, + "nll_loss": 0.6483633518218994, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15229403972625732, + "rewards/margins": 4.889117240905762, + "rewards/rejected": -4.736823558807373, + "step": 238, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5333333333333333, + "grad_norm": 5.736257076263428, + "learning_rate": 0.00017543192454487795, + "logits/chosen": -0.5332171320915222, + "logits/rejected": -0.6654965877532959, + "logps/chosen": -8.099287986755371, + "logps/rejected": -68.04266357421875, + "loss": 0.6135991215705872, + "memory(GiB)": 46.71, + "nll_loss": 0.47898998856544495, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.012953575700521469, + "rewards/margins": 5.387064456939697, + "rewards/rejected": -5.374111175537109, + "step": 239, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5355648535564853, + "grad_norm": 2.407405138015747, + "learning_rate": 0.0001751890515105738, + "logits/chosen": -0.5446051955223083, + "logits/rejected": -0.6599375009536743, + "logps/chosen": -5.214753150939941, + "logps/rejected": -70.1378173828125, + "loss": 0.6014682054519653, + "memory(GiB)": 46.71, + "nll_loss": 0.4095730781555176, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1154915988445282, + "rewards/margins": 5.451560020446777, + "rewards/rejected": -5.336068153381348, + "step": 240, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5377963737796374, + "grad_norm": 5.806000232696533, + "learning_rate": 0.00017494515378098347, + "logits/chosen": -0.5061202049255371, + "logits/rejected": -0.6401721835136414, + "logps/chosen": -7.594027996063232, + "logps/rejected": -72.66096496582031, + "loss": 0.7293487191200256, + "memory(GiB)": 46.71, + "nll_loss": 0.509558379650116, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.20420843362808228, + "rewards/margins": 5.691208839416504, + "rewards/rejected": -5.895417213439941, + "step": 241, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5400278940027894, + "grad_norm": 20.491920471191406, + "learning_rate": 0.00017470023468000654, + "logits/chosen": -0.5332112312316895, + "logits/rejected": -0.6652044653892517, + "logps/chosen": -4.244685649871826, + "logps/rejected": -67.0967788696289, + "loss": 0.4344191253185272, + "memory(GiB)": 46.71, + "nll_loss": 0.27993929386138916, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.039192792028188705, + "rewards/margins": 5.225668907165527, + "rewards/rejected": -5.18647575378418, + "step": 242, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5422594142259414, + "grad_norm": 1.5459198951721191, + "learning_rate": 0.00017445429754546209, + "logits/chosen": -0.4623527526855469, + "logits/rejected": -0.5880333185195923, + "logps/chosen": -6.493972301483154, + "logps/rejected": -61.213016510009766, + "loss": 0.5002475380897522, + "memory(GiB)": 46.71, + "nll_loss": 0.3536112904548645, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.23787753283977509, + "rewards/margins": 4.298797607421875, + "rewards/rejected": -4.060920715332031, + "step": 243, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5444909344490935, + "grad_norm": 10.530829429626465, + "learning_rate": 0.00017420734572904309, + "logits/chosen": -0.4680861234664917, + "logits/rejected": -0.6114482879638672, + "logps/chosen": -6.2320733070373535, + "logps/rejected": -51.97254180908203, + "loss": 0.6590880155563354, + "memory(GiB)": 46.71, + "nll_loss": 0.4217822849750519, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1015012115240097, + "rewards/margins": 3.5804498195648193, + "rewards/rejected": -3.4789485931396484, + "step": 244, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5467224546722455, + "grad_norm": 4.540682792663574, + "learning_rate": 0.00017395938259627102, + "logits/chosen": -0.4017217457294464, + "logits/rejected": -0.5551414489746094, + "logps/chosen": -4.653961658477783, + "logps/rejected": -60.12483215332031, + "loss": 0.417266309261322, + "memory(GiB)": 46.71, + "nll_loss": 0.26140686869621277, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2787860631942749, + "rewards/margins": 3.9614157676696777, + "rewards/rejected": -3.682629108428955, + "step": 245, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5489539748953974, + "grad_norm": 2.821626663208008, + "learning_rate": 0.00017371041152644976, + "logits/chosen": -0.46805882453918457, + "logits/rejected": -0.5824432373046875, + "logps/chosen": -7.418338775634766, + "logps/rejected": -49.86051940917969, + "loss": 0.7901802659034729, + "memory(GiB)": 46.71, + "nll_loss": 0.5593172311782837, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.2499873787164688, + "rewards/margins": 3.6353776454925537, + "rewards/rejected": -3.385390043258667, + "step": 246, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.5511854951185495, + "grad_norm": 2.110387086868286, + "learning_rate": 0.00017346043591261957, + "logits/chosen": -0.441771000623703, + "logits/rejected": -0.6270588040351868, + "logps/chosen": -7.553977012634277, + "logps/rejected": -58.14564514160156, + "loss": 0.569815993309021, + "memory(GiB)": 46.71, + "nll_loss": 0.40740612149238586, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2955455183982849, + "rewards/margins": 3.5217554569244385, + "rewards/rejected": -3.226210117340088, + "step": 247, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5534170153417015, + "grad_norm": 2.2252297401428223, + "learning_rate": 0.0001732094591615109, + "logits/chosen": -0.3921899199485779, + "logits/rejected": -0.5801397562026978, + "logps/chosen": -8.181288719177246, + "logps/rejected": -63.77415084838867, + "loss": 0.5467262268066406, + "memory(GiB)": 46.71, + "nll_loss": 0.37672924995422363, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2877017557621002, + "rewards/margins": 4.03120231628418, + "rewards/rejected": -3.7435007095336914, + "step": 248, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5556485355648536, + "grad_norm": 2.368830680847168, + "learning_rate": 0.00017295748469349804, + "logits/chosen": -0.4548183083534241, + "logits/rejected": -0.6428787112236023, + "logps/chosen": -5.975998878479004, + "logps/rejected": -64.11628723144531, + "loss": 0.6057455539703369, + "memory(GiB)": 46.71, + "nll_loss": 0.5057395696640015, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24285957217216492, + "rewards/margins": 4.522610664367676, + "rewards/rejected": -4.279751300811768, + "step": 249, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5578800557880056, + "grad_norm": 5.085129261016846, + "learning_rate": 0.00017270451594255233, + "logits/chosen": -0.5124991536140442, + "logits/rejected": -0.6001466512680054, + "logps/chosen": -5.997179985046387, + "logps/rejected": -46.71837615966797, + "loss": 0.5066366791725159, + "memory(GiB)": 46.71, + "nll_loss": 0.31767114996910095, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16393466293811798, + "rewards/margins": 3.424452066421509, + "rewards/rejected": -3.260517120361328, + "step": 250, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.5601115760111576, + "grad_norm": 3.277879476547241, + "learning_rate": 0.00017245055635619542, + "logits/chosen": -0.48801007866859436, + "logits/rejected": -0.5720742344856262, + "logps/chosen": -6.175948143005371, + "logps/rejected": -54.81993865966797, + "loss": 0.5065844058990479, + "memory(GiB)": 46.71, + "nll_loss": 0.3731592297554016, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12712717056274414, + "rewards/margins": 4.286194801330566, + "rewards/rejected": -4.159067153930664, + "step": 251, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5623430962343097, + "grad_norm": 6.859767913818359, + "learning_rate": 0.00017219560939545246, + "logits/chosen": -0.522152304649353, + "logits/rejected": -0.5365570187568665, + "logps/chosen": -14.133954048156738, + "logps/rejected": -37.61845016479492, + "loss": 0.8155819177627563, + "memory(GiB)": 46.71, + "nll_loss": 0.4886224865913391, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.2092638611793518, + "rewards/margins": 2.255005359649658, + "rewards/rejected": -2.4642696380615234, + "step": 252, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5645746164574617, + "grad_norm": 8.044095039367676, + "learning_rate": 0.0001719396785348046, + "logits/chosen": -0.45501673221588135, + "logits/rejected": -0.5942042469978333, + "logps/chosen": -8.521533966064453, + "logps/rejected": -53.618995666503906, + "loss": 0.8110232353210449, + "memory(GiB)": 46.71, + "nll_loss": 0.42624324560165405, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1203906238079071, + "rewards/margins": 3.3475189208984375, + "rewards/rejected": -3.467909574508667, + "step": 253, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5668061366806136, + "grad_norm": 13.483397483825684, + "learning_rate": 0.000171682767262142, + "logits/chosen": -0.44267958402633667, + "logits/rejected": -0.6059361100196838, + "logps/chosen": -5.3091816902160645, + "logps/rejected": -49.56216049194336, + "loss": 0.6120542287826538, + "memory(GiB)": 46.71, + "nll_loss": 0.413876473903656, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14617875218391418, + "rewards/margins": 3.480231523513794, + "rewards/rejected": -3.334052562713623, + "step": 254, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5690376569037657, + "grad_norm": 2.9796292781829834, + "learning_rate": 0.00017142487907871593, + "logits/chosen": -0.441107839345932, + "logits/rejected": -0.5549041628837585, + "logps/chosen": -6.088810920715332, + "logps/rejected": -41.22234344482422, + "loss": 0.556708574295044, + "memory(GiB)": 46.71, + "nll_loss": 0.3824915587902069, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2537227272987366, + "rewards/margins": 2.6484460830688477, + "rewards/rejected": -2.3947231769561768, + "step": 255, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5712691771269177, + "grad_norm": 1.7595632076263428, + "learning_rate": 0.00017116601749909142, + "logits/chosen": -0.4316076636314392, + "logits/rejected": -0.5781916379928589, + "logps/chosen": -8.001547813415527, + "logps/rejected": -45.277442932128906, + "loss": 0.5357844829559326, + "memory(GiB)": 46.71, + "nll_loss": 0.348554790019989, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2612729072570801, + "rewards/margins": 2.8041718006134033, + "rewards/rejected": -2.5428988933563232, + "step": 256, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.5735006973500697, + "grad_norm": 2.4086761474609375, + "learning_rate": 0.0001709061860510991, + "logits/chosen": -0.47363919019699097, + "logits/rejected": -0.575410008430481, + "logps/chosen": -6.791253089904785, + "logps/rejected": -40.982913970947266, + "loss": 0.5101409554481506, + "memory(GiB)": 46.71, + "nll_loss": 0.28215673565864563, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08847678452730179, + "rewards/margins": 2.6485981941223145, + "rewards/rejected": -2.5601210594177246, + "step": 257, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5757322175732218, + "grad_norm": 2.0793402194976807, + "learning_rate": 0.00017064538827578723, + "logits/chosen": -0.4602135419845581, + "logits/rejected": -0.5832696557044983, + "logps/chosen": -6.382266044616699, + "logps/rejected": -46.76750946044922, + "loss": 0.5784123539924622, + "memory(GiB)": 46.71, + "nll_loss": 0.413840115070343, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2313905507326126, + "rewards/margins": 3.05240797996521, + "rewards/rejected": -2.821016788482666, + "step": 258, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5779637377963738, + "grad_norm": 3.499621629714966, + "learning_rate": 0.00017038362772737347, + "logits/chosen": -0.419292688369751, + "logits/rejected": -0.610460102558136, + "logps/chosen": -5.339640140533447, + "logps/rejected": -38.10063171386719, + "loss": 0.5521044731140137, + "memory(GiB)": 46.71, + "nll_loss": 0.3070164620876312, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0932367667555809, + "rewards/margins": 2.2065370082855225, + "rewards/rejected": -2.113300323486328, + "step": 259, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5801952580195258, + "grad_norm": 2.3477797508239746, + "learning_rate": 0.00017012090797319628, + "logits/chosen": -0.46017396450042725, + "logits/rejected": -0.6195208430290222, + "logps/chosen": -8.076725006103516, + "logps/rejected": -50.54127883911133, + "loss": 0.4606272578239441, + "memory(GiB)": 46.71, + "nll_loss": 0.2921941876411438, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.22055713832378387, + "rewards/margins": 3.12892484664917, + "rewards/rejected": -2.908367872238159, + "step": 260, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5824267782426779, + "grad_norm": 2.4851467609405518, + "learning_rate": 0.0001698572325936665, + "logits/chosen": -0.4670488238334656, + "logits/rejected": -0.6289732456207275, + "logps/chosen": -7.322833061218262, + "logps/rejected": -50.35585021972656, + "loss": 0.5902070999145508, + "memory(GiB)": 46.71, + "nll_loss": 0.42221367359161377, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1646100878715515, + "rewards/margins": 3.3369765281677246, + "rewards/rejected": -3.1723666191101074, + "step": 261, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5846582984658298, + "grad_norm": 4.714338779449463, + "learning_rate": 0.00016959260518221845, + "logits/chosen": -0.47086095809936523, + "logits/rejected": -0.5858362913131714, + "logps/chosen": -8.72909927368164, + "logps/rejected": -44.19210433959961, + "loss": 0.6342991590499878, + "memory(GiB)": 46.71, + "nll_loss": 0.4329380691051483, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.02014285698533058, + "rewards/margins": 2.7174415588378906, + "rewards/rejected": -2.737584114074707, + "step": 262, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5868898186889818, + "grad_norm": 1.444083571434021, + "learning_rate": 0.000169327029345261, + "logits/chosen": -0.39938443899154663, + "logits/rejected": -0.6433777809143066, + "logps/chosen": -3.40937876701355, + "logps/rejected": -59.414127349853516, + "loss": 0.3323577046394348, + "memory(GiB)": 46.71, + "nll_loss": 0.22196492552757263, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2753269672393799, + "rewards/margins": 3.9262943267822266, + "rewards/rejected": -3.650967597961426, + "step": 263, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5891213389121339, + "grad_norm": 2.6938703060150146, + "learning_rate": 0.00016906050870212834, + "logits/chosen": -0.5094009637832642, + "logits/rejected": -0.6173782348632812, + "logps/chosen": -7.775401592254639, + "logps/rejected": -54.587913513183594, + "loss": 0.6427381634712219, + "memory(GiB)": 46.71, + "nll_loss": 0.46833711862564087, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10746394097805023, + "rewards/margins": 3.8728220462799072, + "rewards/rejected": -3.7653582096099854, + "step": 264, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5913528591352859, + "grad_norm": 5.664610862731934, + "learning_rate": 0.00016879304688503086, + "logits/chosen": -0.4606212079524994, + "logits/rejected": -0.6145158410072327, + "logps/chosen": -7.418776512145996, + "logps/rejected": -52.65889358520508, + "loss": 0.5809510946273804, + "memory(GiB)": 46.71, + "nll_loss": 0.4256901443004608, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.11412282288074493, + "rewards/margins": 3.720710515975952, + "rewards/rejected": -3.6065876483917236, + "step": 265, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.5935843793584379, + "grad_norm": 2.40004825592041, + "learning_rate": 0.0001685246475390053, + "logits/chosen": -0.4467337131500244, + "logits/rejected": -0.6297879219055176, + "logps/chosen": -8.128316879272461, + "logps/rejected": -51.549537658691406, + "loss": 0.6336352229118347, + "memory(GiB)": 46.71, + "nll_loss": 0.4354446232318878, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07178286463022232, + "rewards/margins": 3.265444755554199, + "rewards/rejected": -3.19366192817688, + "step": 266, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.59581589958159, + "grad_norm": 2.6514225006103516, + "learning_rate": 0.00016825531432186543, + "logits/chosen": -0.41302669048309326, + "logits/rejected": -0.5715283751487732, + "logps/chosen": -6.686316013336182, + "logps/rejected": -57.61973190307617, + "loss": 0.4782642722129822, + "memory(GiB)": 46.71, + "nll_loss": 0.35745877027511597, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13967037200927734, + "rewards/margins": 4.3413496017456055, + "rewards/rejected": -4.201679706573486, + "step": 267, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.598047419804742, + "grad_norm": 1.7012206315994263, + "learning_rate": 0.00016798505090415199, + "logits/chosen": -0.45638614892959595, + "logits/rejected": -0.5790466666221619, + "logps/chosen": -8.776070594787598, + "logps/rejected": -52.691070556640625, + "loss": 0.523406982421875, + "memory(GiB)": 46.71, + "nll_loss": 0.3796443045139313, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3116152584552765, + "rewards/margins": 4.0619001388549805, + "rewards/rejected": -3.750284194946289, + "step": 268, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.600278940027894, + "grad_norm": 2.13728666305542, + "learning_rate": 0.00016771386096908278, + "logits/chosen": -0.45061424374580383, + "logits/rejected": -0.6187809109687805, + "logps/chosen": -6.508578300476074, + "logps/rejected": -55.61383056640625, + "loss": 0.5749614238739014, + "memory(GiB)": 46.71, + "nll_loss": 0.4280470609664917, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.026448212563991547, + "rewards/margins": 4.147185802459717, + "rewards/rejected": -4.1207380294799805, + "step": 269, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.602510460251046, + "grad_norm": 2.3225080966949463, + "learning_rate": 0.00016744174821250237, + "logits/chosen": -0.411843478679657, + "logits/rejected": -0.56050044298172, + "logps/chosen": -11.897720336914062, + "logps/rejected": -57.52003479003906, + "loss": 0.796038031578064, + "memory(GiB)": 46.71, + "nll_loss": 0.5885191559791565, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.023488737642765045, + "rewards/margins": 3.5510995388031006, + "rewards/rejected": -3.5276105403900146, + "step": 270, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.604741980474198, + "grad_norm": 11.049079895019531, + "learning_rate": 0.00016716871634283174, + "logits/chosen": -0.3955327868461609, + "logits/rejected": -0.608815610408783, + "logps/chosen": -4.626939296722412, + "logps/rejected": -62.783973693847656, + "loss": 0.44843360781669617, + "memory(GiB)": 46.71, + "nll_loss": 0.3314272165298462, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13732756674289703, + "rewards/margins": 4.360987663269043, + "rewards/rejected": -4.223659992218018, + "step": 271, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6069735006973501, + "grad_norm": 2.914240598678589, + "learning_rate": 0.00016689476908101782, + "logits/chosen": -0.38342612981796265, + "logits/rejected": -0.6166298985481262, + "logps/chosen": -3.6991920471191406, + "logps/rejected": -56.026390075683594, + "loss": 0.5108941793441772, + "memory(GiB)": 46.71, + "nll_loss": 0.32679831981658936, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10832832008600235, + "rewards/margins": 4.004990577697754, + "rewards/rejected": -3.896662712097168, + "step": 272, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6092050209205021, + "grad_norm": 8.852380752563477, + "learning_rate": 0.00016661991016048275, + "logits/chosen": -0.5051642060279846, + "logits/rejected": -0.6240350604057312, + "logps/chosen": -5.203716278076172, + "logps/rejected": -55.715755462646484, + "loss": 0.5184271335601807, + "memory(GiB)": 46.71, + "nll_loss": 0.43319958448410034, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15009382367134094, + "rewards/margins": 4.303383827209473, + "rewards/rejected": -4.153290271759033, + "step": 273, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6114365411436541, + "grad_norm": 4.726907253265381, + "learning_rate": 0.00016634414332707297, + "logits/chosen": -0.4815787672996521, + "logits/rejected": -0.6398111581802368, + "logps/chosen": -4.289473056793213, + "logps/rejected": -61.950286865234375, + "loss": 0.4551790952682495, + "memory(GiB)": 46.71, + "nll_loss": 0.3091404438018799, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.039064548909664154, + "rewards/margins": 4.923999309539795, + "rewards/rejected": -4.884934425354004, + "step": 274, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6136680613668062, + "grad_norm": 7.542313098907471, + "learning_rate": 0.00016606747233900815, + "logits/chosen": -0.44271501898765564, + "logits/rejected": -0.5747901797294617, + "logps/chosen": -7.7364325523376465, + "logps/rejected": -53.02996826171875, + "loss": 0.744888424873352, + "memory(GiB)": 46.71, + "nll_loss": 0.5044885873794556, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.024614796042442322, + "rewards/margins": 3.7873635292053223, + "rewards/rejected": -3.811978816986084, + "step": 275, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6158995815899582, + "grad_norm": 2.8804731369018555, + "learning_rate": 0.00016578990096683004, + "logits/chosen": -0.42431771755218506, + "logits/rejected": -0.5855906009674072, + "logps/chosen": -8.4268217086792, + "logps/rejected": -61.900108337402344, + "loss": 0.6437846422195435, + "memory(GiB)": 46.71, + "nll_loss": 0.46338486671447754, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.010673696175217628, + "rewards/margins": 4.697712421417236, + "rewards/rejected": -4.687038898468018, + "step": 276, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6181311018131102, + "grad_norm": 16.423973083496094, + "learning_rate": 0.00016551143299335105, + "logits/chosen": -0.4473971724510193, + "logits/rejected": -0.556106448173523, + "logps/chosen": -8.695633888244629, + "logps/rejected": -54.007286071777344, + "loss": 0.7244383096694946, + "memory(GiB)": 46.71, + "nll_loss": 0.45499277114868164, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0047096069902181625, + "rewards/margins": 3.7970499992370605, + "rewards/rejected": -3.7923407554626465, + "step": 277, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6203626220362622, + "grad_norm": 4.081517696380615, + "learning_rate": 0.00016523207221360266, + "logits/chosen": -0.3338325619697571, + "logits/rejected": -0.5568884015083313, + "logps/chosen": -3.983449697494507, + "logps/rejected": -66.22798919677734, + "loss": 0.39685380458831787, + "memory(GiB)": 46.71, + "nll_loss": 0.227124884724617, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1147797703742981, + "rewards/margins": 4.806873321533203, + "rewards/rejected": -4.692093849182129, + "step": 278, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.6225941422594142, + "grad_norm": 1.6026629209518433, + "learning_rate": 0.0001649518224347838, + "logits/chosen": -0.45107001066207886, + "logits/rejected": -0.6225546002388, + "logps/chosen": -2.7264139652252197, + "logps/rejected": -68.20640563964844, + "loss": 0.2848789095878601, + "memory(GiB)": 46.71, + "nll_loss": 0.1853371113538742, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16342315077781677, + "rewards/margins": 5.450206756591797, + "rewards/rejected": -5.2867841720581055, + "step": 279, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6248256624825662, + "grad_norm": 2.1419737339019775, + "learning_rate": 0.0001646706874762089, + "logits/chosen": -0.4340333938598633, + "logits/rejected": -0.6073229312896729, + "logps/chosen": -4.69732141494751, + "logps/rejected": -70.4720687866211, + "loss": 0.42912930250167847, + "memory(GiB)": 46.71, + "nll_loss": 0.33446142077445984, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18399639427661896, + "rewards/margins": 5.338755130767822, + "rewards/rejected": -5.154758453369141, + "step": 280, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6270571827057183, + "grad_norm": 4.532802581787109, + "learning_rate": 0.0001643886711692557, + "logits/chosen": -0.4100894331932068, + "logits/rejected": -0.5919958353042603, + "logps/chosen": -4.708090782165527, + "logps/rejected": -55.577186584472656, + "loss": 0.5581774711608887, + "memory(GiB)": 46.71, + "nll_loss": 0.4139384627342224, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19322583079338074, + "rewards/margins": 3.968625068664551, + "rewards/rejected": -3.7753994464874268, + "step": 281, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6292887029288703, + "grad_norm": 2.134107828140259, + "learning_rate": 0.00016410577735731345, + "logits/chosen": -0.4810371994972229, + "logits/rejected": -0.6207767128944397, + "logps/chosen": -6.703675746917725, + "logps/rejected": -54.464988708496094, + "loss": 0.6276415586471558, + "memory(GiB)": 46.71, + "nll_loss": 0.47040659189224243, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06548688560724258, + "rewards/margins": 3.9635109901428223, + "rewards/rejected": -3.898024320602417, + "step": 282, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6315202231520223, + "grad_norm": 2.135093927383423, + "learning_rate": 0.00016382200989572998, + "logits/chosen": -0.5298405885696411, + "logits/rejected": -0.6139633059501648, + "logps/chosen": -8.422263145446777, + "logps/rejected": -48.18453598022461, + "loss": 0.7457632422447205, + "memory(GiB)": 46.71, + "nll_loss": 0.5003448128700256, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06589431315660477, + "rewards/margins": 3.5135788917541504, + "rewards/rejected": -3.4476842880249023, + "step": 283, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6337517433751744, + "grad_norm": 4.251437664031982, + "learning_rate": 0.00016353737265175963, + "logits/chosen": -0.5017900466918945, + "logits/rejected": -0.6134066581726074, + "logps/chosen": -9.804008483886719, + "logps/rejected": -52.773406982421875, + "loss": 0.8265900611877441, + "memory(GiB)": 46.71, + "nll_loss": 0.6167674660682678, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04522796720266342, + "rewards/margins": 3.658693552017212, + "rewards/rejected": -3.6134657859802246, + "step": 284, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.6359832635983264, + "grad_norm": 10.930916786193848, + "learning_rate": 0.00016325186950451022, + "logits/chosen": -0.5018693804740906, + "logits/rejected": -0.5625038146972656, + "logps/chosen": -6.777836799621582, + "logps/rejected": -42.862220764160156, + "loss": 0.7121441960334778, + "memory(GiB)": 46.71, + "nll_loss": 0.4460356831550598, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.041172679513692856, + "rewards/margins": 3.1301026344299316, + "rewards/rejected": -3.0889296531677246, + "step": 285, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6382147838214783, + "grad_norm": 2.1487908363342285, + "learning_rate": 0.00016296550434489036, + "logits/chosen": -0.47381335496902466, + "logits/rejected": -0.6210272312164307, + "logps/chosen": -7.876993179321289, + "logps/rejected": -67.23505401611328, + "loss": 0.3986350893974304, + "memory(GiB)": 46.71, + "nll_loss": 0.31996262073516846, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07089027762413025, + "rewards/margins": 4.680482864379883, + "rewards/rejected": -4.751373291015625, + "step": 286, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6404463040446304, + "grad_norm": 7.843413829803467, + "learning_rate": 0.0001626782810755565, + "logits/chosen": -0.3885619044303894, + "logits/rejected": -0.5521731972694397, + "logps/chosen": -6.2263922691345215, + "logps/rejected": -72.29328155517578, + "loss": 0.608590304851532, + "memory(GiB)": 46.71, + "nll_loss": 0.4712628126144409, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1404297947883606, + "rewards/margins": 4.971117973327637, + "rewards/rejected": -4.8306884765625, + "step": 287, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6426778242677824, + "grad_norm": 2.2685890197753906, + "learning_rate": 0.00016239020361085945, + "logits/chosen": -0.3876466155052185, + "logits/rejected": -0.5563790202140808, + "logps/chosen": -6.572446823120117, + "logps/rejected": -58.97338104248047, + "loss": 0.5379603505134583, + "memory(GiB)": 46.71, + "nll_loss": 0.3794347047805786, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.163069948554039, + "rewards/margins": 4.21791934967041, + "rewards/rejected": -4.054849624633789, + "step": 288, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6449093444909344, + "grad_norm": 2.607456922531128, + "learning_rate": 0.0001621012758767913, + "logits/chosen": -0.40551966428756714, + "logits/rejected": -0.5489031672477722, + "logps/chosen": -5.411971569061279, + "logps/rejected": -56.15552520751953, + "loss": 0.5132619738578796, + "memory(GiB)": 46.71, + "nll_loss": 0.3366192579269409, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16010434925556183, + "rewards/margins": 3.999542713165283, + "rewards/rejected": -3.8394384384155273, + "step": 289, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6471408647140865, + "grad_norm": 2.2857182025909424, + "learning_rate": 0.0001618115018109318, + "logits/chosen": -0.4304552376270294, + "logits/rejected": -0.602503776550293, + "logps/chosen": -4.60065221786499, + "logps/rejected": -54.26784133911133, + "loss": 0.5486267805099487, + "memory(GiB)": 46.71, + "nll_loss": 0.37609028816223145, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08251349627971649, + "rewards/margins": 3.8639562129974365, + "rewards/rejected": -3.781442165374756, + "step": 290, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6493723849372385, + "grad_norm": 1.324331283569336, + "learning_rate": 0.0001615208853623947, + "logits/chosen": -0.46335089206695557, + "logits/rejected": -0.597943127155304, + "logps/chosen": -3.3643994331359863, + "logps/rejected": -59.1411247253418, + "loss": 0.34732431173324585, + "memory(GiB)": 46.71, + "nll_loss": 0.19164429605007172, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.28088483214378357, + "rewards/margins": 4.684600830078125, + "rewards/rejected": -4.403716087341309, + "step": 291, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6516039051603905, + "grad_norm": 3.25237774848938, + "learning_rate": 0.00016122943049177405, + "logits/chosen": -0.4758291244506836, + "logits/rejected": -0.5325515866279602, + "logps/chosen": -13.164379119873047, + "logps/rejected": -49.068603515625, + "loss": 0.7119560837745667, + "memory(GiB)": 46.71, + "nll_loss": 0.4212368130683899, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.031183507293462753, + "rewards/margins": 3.5282320976257324, + "rewards/rejected": -3.4970486164093018, + "step": 292, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6538354253835426, + "grad_norm": 1.9222995042800903, + "learning_rate": 0.00016093714117109, + "logits/chosen": -0.42248421907424927, + "logits/rejected": -0.6226801872253418, + "logps/chosen": -5.316409111022949, + "logps/rejected": -63.825557708740234, + "loss": 0.5835841298103333, + "memory(GiB)": 46.71, + "nll_loss": 0.46457451581954956, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2873809039592743, + "rewards/margins": 4.637524604797363, + "rewards/rejected": -4.3501434326171875, + "step": 293, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6560669456066945, + "grad_norm": 2.3776681423187256, + "learning_rate": 0.00016064402138373489, + "logits/chosen": -0.399809867143631, + "logits/rejected": -0.5734027028083801, + "logps/chosen": -5.731583118438721, + "logps/rejected": -65.56729125976562, + "loss": 0.5586244463920593, + "memory(GiB)": 46.71, + "nll_loss": 0.41006267070770264, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09848666191101074, + "rewards/margins": 4.583428382873535, + "rewards/rejected": -4.4849419593811035, + "step": 294, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6582984658298466, + "grad_norm": 2.861478567123413, + "learning_rate": 0.0001603500751244188, + "logits/chosen": -0.466978520154953, + "logits/rejected": -0.5577737092971802, + "logps/chosen": -10.857172966003418, + "logps/rejected": -45.819541931152344, + "loss": 0.6700180768966675, + "memory(GiB)": 46.71, + "nll_loss": 0.44924986362457275, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.08089883625507355, + "rewards/margins": 3.153740882873535, + "rewards/rejected": -3.2346394062042236, + "step": 295, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.6605299860529986, + "grad_norm": 2.3550102710723877, + "learning_rate": 0.00016005530639911524, + "logits/chosen": -0.42725443840026855, + "logits/rejected": -0.6156260967254639, + "logps/chosen": -4.741040229797363, + "logps/rejected": -73.73209381103516, + "loss": 0.41657161712646484, + "memory(GiB)": 46.71, + "nll_loss": 0.3108736276626587, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.20638638734817505, + "rewards/margins": 5.785445690155029, + "rewards/rejected": -5.57905912399292, + "step": 296, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.6627615062761506, + "grad_norm": 2.7637619972229004, + "learning_rate": 0.00015975971922500644, + "logits/chosen": -0.4365236759185791, + "logits/rejected": -0.5944050550460815, + "logps/chosen": -5.906551837921143, + "logps/rejected": -64.35894775390625, + "loss": 0.5716227293014526, + "memory(GiB)": 46.71, + "nll_loss": 0.3685966730117798, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14344872534275055, + "rewards/margins": 4.830230236053467, + "rewards/rejected": -4.68678092956543, + "step": 297, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.6649930264993027, + "grad_norm": 3.1376020908355713, + "learning_rate": 0.00015946331763042867, + "logits/chosen": -0.43532899022102356, + "logits/rejected": -0.5989662408828735, + "logps/chosen": -6.79918909072876, + "logps/rejected": -50.513153076171875, + "loss": 0.552319347858429, + "memory(GiB)": 46.71, + "nll_loss": 0.3487764298915863, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16602559387683868, + "rewards/margins": 3.815488338470459, + "rewards/rejected": -3.6494626998901367, + "step": 298, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.6672245467224547, + "grad_norm": 4.621730804443359, + "learning_rate": 0.00015916610565481736, + "logits/chosen": -0.46591538190841675, + "logits/rejected": -0.653298020362854, + "logps/chosen": -2.609394073486328, + "logps/rejected": -72.14924621582031, + "loss": 0.33943986892700195, + "memory(GiB)": 46.71, + "nll_loss": 0.26801469922065735, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17833705246448517, + "rewards/margins": 5.548633098602295, + "rewards/rejected": -5.370296478271484, + "step": 299, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.6694560669456067, + "grad_norm": 3.571917772293091, + "learning_rate": 0.00015886808734865202, + "logits/chosen": -0.4337051510810852, + "logits/rejected": -0.5879015922546387, + "logps/chosen": -6.756601333618164, + "logps/rejected": -48.003501892089844, + "loss": 0.7254310846328735, + "memory(GiB)": 46.71, + "nll_loss": 0.5357213616371155, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1567414551973343, + "rewards/margins": 3.4603137969970703, + "rewards/rejected": -3.303572654724121, + "step": 300, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.6694560669456067, + "eval_logits/chosen": -0.4272937476634979, + "eval_logits/rejected": -0.5822427272796631, + "eval_logps/chosen": -7.934236526489258, + "eval_logps/rejected": -61.74112319946289, + "eval_loss": 0.603725254535675, + "eval_nll_loss": 0.4224081039428711, + "eval_rewards/accuracies": 0.9166666865348816, + "eval_rewards/chosen": 0.0887039378285408, + "eval_rewards/margins": 4.4511003494262695, + "eval_rewards/rejected": -4.362396717071533, + "eval_runtime": 157.1594, + "eval_samples_per_second": 0.916, + "eval_steps_per_second": 0.458, + "step": 300 + }, + { + "epoch": 0.6716875871687588, + "grad_norm": 10.792269706726074, + "learning_rate": 0.00015856926677340093, + "logits/chosen": -0.5080382823944092, + "logits/rejected": -0.574877142906189, + "logps/chosen": -9.988602638244629, + "logps/rejected": -45.06884002685547, + "loss": 0.8157339692115784, + "memory(GiB)": 46.71, + "nll_loss": 0.5001208186149597, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.010560978204011917, + "rewards/margins": 2.8719606399536133, + "rewards/rejected": -2.882521867752075, + "step": 301, + "train_speed(iter/s)": 0.011101 + }, + { + "epoch": 0.6739191073919107, + "grad_norm": 3.010406494140625, + "learning_rate": 0.000158269648001466, + "logits/chosen": -0.39880508184432983, + "logits/rejected": -0.5890823602676392, + "logps/chosen": -3.5330333709716797, + "logps/rejected": -63.52682113647461, + "loss": 0.358871728181839, + "memory(GiB)": 46.71, + "nll_loss": 0.22527243196964264, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3649945855140686, + "rewards/margins": 4.8490753173828125, + "rewards/rejected": -4.484080791473389, + "step": 302, + "train_speed(iter/s)": 0.011101 + }, + { + "epoch": 0.6761506276150627, + "grad_norm": 3.7259819507598877, + "learning_rate": 0.0001579692351161272, + "logits/chosen": -0.32966724038124084, + "logits/rejected": -0.5503748655319214, + "logps/chosen": -7.193322658538818, + "logps/rejected": -73.0583724975586, + "loss": 0.584179162979126, + "memory(GiB)": 46.71, + "nll_loss": 0.3847492039203644, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.020800679922103882, + "rewards/margins": 4.682149887084961, + "rewards/rejected": -4.661348819732666, + "step": 303, + "train_speed(iter/s)": 0.011102 + }, + { + "epoch": 0.6783821478382148, + "grad_norm": 1.916205644607544, + "learning_rate": 0.00015766803221148673, + "logits/chosen": -0.4750916063785553, + "logits/rejected": -0.5531684756278992, + "logps/chosen": -6.722357749938965, + "logps/rejected": -47.454002380371094, + "loss": 0.5085262060165405, + "memory(GiB)": 46.71, + "nll_loss": 0.36307400465011597, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06748831272125244, + "rewards/margins": 3.6778817176818848, + "rewards/rejected": -3.610393524169922, + "step": 304, + "train_speed(iter/s)": 0.011102 + }, + { + "epoch": 0.6806136680613668, + "grad_norm": 2.678760051727295, + "learning_rate": 0.0001573660433924135, + "logits/chosen": -0.4101259410381317, + "logits/rejected": -0.5861769914627075, + "logps/chosen": -6.400012016296387, + "logps/rejected": -61.600013732910156, + "loss": 0.4950736463069916, + "memory(GiB)": 46.71, + "nll_loss": 0.2904612123966217, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.03141636401414871, + "rewards/margins": 4.274345397949219, + "rewards/rejected": -4.305761814117432, + "step": 305, + "train_speed(iter/s)": 0.011102 + }, + { + "epoch": 0.6828451882845188, + "grad_norm": 1.9203068017959595, + "learning_rate": 0.000157063272774487, + "logits/chosen": -0.319125771522522, + "logits/rejected": -0.5144772529602051, + "logps/chosen": -6.5023651123046875, + "logps/rejected": -61.782554626464844, + "loss": 0.5285742878913879, + "memory(GiB)": 46.71, + "nll_loss": 0.41383132338523865, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27395856380462646, + "rewards/margins": 4.111924648284912, + "rewards/rejected": -3.837965965270996, + "step": 306, + "train_speed(iter/s)": 0.011103 + }, + { + "epoch": 0.6850767085076709, + "grad_norm": 1.909354329109192, + "learning_rate": 0.00015675972448394129, + "logits/chosen": -0.43325644731521606, + "logits/rejected": -0.5877099633216858, + "logps/chosen": -5.1956939697265625, + "logps/rejected": -52.69513702392578, + "loss": 0.46260273456573486, + "memory(GiB)": 46.71, + "nll_loss": 0.3058718740940094, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1890178620815277, + "rewards/margins": 3.9786057472229004, + "rewards/rejected": -3.78958797454834, + "step": 307, + "train_speed(iter/s)": 0.011103 + }, + { + "epoch": 0.6873082287308229, + "grad_norm": 1.6346402168273926, + "learning_rate": 0.00015645540265760873, + "logits/chosen": -0.49507996439933777, + "logits/rejected": -0.6308517456054688, + "logps/chosen": -7.058663368225098, + "logps/rejected": -52.11885070800781, + "loss": 0.4992927014827728, + "memory(GiB)": 46.71, + "nll_loss": 0.3881046772003174, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1510738879442215, + "rewards/margins": 4.04608678817749, + "rewards/rejected": -3.895012855529785, + "step": 308, + "train_speed(iter/s)": 0.011103 + }, + { + "epoch": 0.6895397489539749, + "grad_norm": 19.10862159729004, + "learning_rate": 0.00015615031144286366, + "logits/chosen": -0.44366782903671265, + "logits/rejected": -0.6311182379722595, + "logps/chosen": -6.96903657913208, + "logps/rejected": -56.563377380371094, + "loss": 0.798547089099884, + "memory(GiB)": 46.71, + "nll_loss": 0.6148461103439331, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.17197120189666748, + "rewards/margins": 4.0539164543151855, + "rewards/rejected": -3.8819453716278076, + "step": 309, + "train_speed(iter/s)": 0.011104 + }, + { + "epoch": 0.691771269177127, + "grad_norm": 13.525169372558594, + "learning_rate": 0.00015584445499756578, + "logits/chosen": -0.3848733603954315, + "logits/rejected": -0.6370626091957092, + "logps/chosen": -5.978826999664307, + "logps/rejected": -67.30372619628906, + "loss": 0.49153995513916016, + "memory(GiB)": 46.71, + "nll_loss": 0.3625016510486603, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.03675977140665054, + "rewards/margins": 4.50251579284668, + "rewards/rejected": -4.465755939483643, + "step": 310, + "train_speed(iter/s)": 0.011104 + }, + { + "epoch": 0.6940027894002789, + "grad_norm": 2.2359046936035156, + "learning_rate": 0.00015553783749000364, + "logits/chosen": -0.4552823305130005, + "logits/rejected": -0.5677109956741333, + "logps/chosen": -9.264768600463867, + "logps/rejected": -38.399925231933594, + "loss": 0.7099106907844543, + "memory(GiB)": 46.71, + "nll_loss": 0.4691702723503113, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.028204074129462242, + "rewards/margins": 2.4503822326660156, + "rewards/rejected": -2.478586435317993, + "step": 311, + "train_speed(iter/s)": 0.011104 + }, + { + "epoch": 0.6962343096234309, + "grad_norm": 3.2684123516082764, + "learning_rate": 0.0001552304630988376, + "logits/chosen": -0.49416208267211914, + "logits/rejected": -0.6434434652328491, + "logps/chosen": -5.097368240356445, + "logps/rejected": -50.68689727783203, + "loss": 0.5480148196220398, + "memory(GiB)": 46.71, + "nll_loss": 0.42048290371894836, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22367961704730988, + "rewards/margins": 3.8028666973114014, + "rewards/rejected": -3.5791869163513184, + "step": 312, + "train_speed(iter/s)": 0.011105 + }, + { + "epoch": 0.698465829846583, + "grad_norm": 3.828305959701538, + "learning_rate": 0.00015492233601304312, + "logits/chosen": -0.49165451526641846, + "logits/rejected": -0.624678373336792, + "logps/chosen": -8.968114852905273, + "logps/rejected": -50.55556869506836, + "loss": 0.6498446464538574, + "memory(GiB)": 46.71, + "nll_loss": 0.5071594715118408, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.045434772968292236, + "rewards/margins": 3.5770413875579834, + "rewards/rejected": -3.531606674194336, + "step": 313, + "train_speed(iter/s)": 0.011105 + }, + { + "epoch": 0.700697350069735, + "grad_norm": 5.604126930236816, + "learning_rate": 0.00015461346043185356, + "logits/chosen": -0.48774653673171997, + "logits/rejected": -0.6481919884681702, + "logps/chosen": -9.898344993591309, + "logps/rejected": -48.79402160644531, + "loss": 0.7247034311294556, + "memory(GiB)": 46.71, + "nll_loss": 0.5549950003623962, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.010192573070526123, + "rewards/margins": 3.000518321990967, + "rewards/rejected": -2.990325927734375, + "step": 314, + "train_speed(iter/s)": 0.011105 + }, + { + "epoch": 0.702928870292887, + "grad_norm": 1.8141809701919556, + "learning_rate": 0.00015430384056470294, + "logits/chosen": -0.5386990904808044, + "logits/rejected": -0.570178747177124, + "logps/chosen": -7.325660705566406, + "logps/rejected": -47.475189208984375, + "loss": 0.5058916807174683, + "memory(GiB)": 46.71, + "nll_loss": 0.37710830569267273, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24004720151424408, + "rewards/margins": 3.8797364234924316, + "rewards/rejected": -3.6396889686584473, + "step": 315, + "train_speed(iter/s)": 0.011106 + }, + { + "epoch": 0.7051603905160391, + "grad_norm": 2.351076126098633, + "learning_rate": 0.00015399348063116857, + "logits/chosen": -0.42870375514030457, + "logits/rejected": -0.6148338913917542, + "logps/chosen": -9.085245132446289, + "logps/rejected": -62.97663879394531, + "loss": 0.7104619741439819, + "memory(GiB)": 46.71, + "nll_loss": 0.5596457719802856, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.00714380107820034, + "rewards/margins": 4.58463716506958, + "rewards/rejected": -4.577493190765381, + "step": 316, + "train_speed(iter/s)": 0.011106 + }, + { + "epoch": 0.707391910739191, + "grad_norm": 5.977499961853027, + "learning_rate": 0.0001536823848609137, + "logits/chosen": -0.5215577483177185, + "logits/rejected": -0.6443260312080383, + "logps/chosen": -6.4677910804748535, + "logps/rejected": -42.727333068847656, + "loss": 0.8084423542022705, + "memory(GiB)": 46.71, + "nll_loss": 0.5551162362098694, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12047205865383148, + "rewards/margins": 3.0706582069396973, + "rewards/rejected": -2.950186252593994, + "step": 317, + "train_speed(iter/s)": 0.011107 + }, + { + "epoch": 0.7096234309623431, + "grad_norm": 2.851249933242798, + "learning_rate": 0.00015337055749362955, + "logits/chosen": -0.46593403816223145, + "logits/rejected": -0.6074923872947693, + "logps/chosen": -7.754675388336182, + "logps/rejected": -41.88108444213867, + "loss": 0.7191582322120667, + "memory(GiB)": 46.71, + "nll_loss": 0.43810340762138367, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15869829058647156, + "rewards/margins": 2.73551607131958, + "rewards/rejected": -2.576817750930786, + "step": 318, + "train_speed(iter/s)": 0.011107 + }, + { + "epoch": 0.7118549511854951, + "grad_norm": 3.1140987873077393, + "learning_rate": 0.0001530580027789779, + "logits/chosen": -0.4373013973236084, + "logits/rejected": -0.6295573711395264, + "logps/chosen": -6.664909839630127, + "logps/rejected": -62.491798400878906, + "loss": 0.513994574546814, + "memory(GiB)": 46.71, + "nll_loss": 0.3923044204711914, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.03877401351928711, + "rewards/margins": 4.21281623840332, + "rewards/rejected": -4.174041748046875, + "step": 319, + "train_speed(iter/s)": 0.011107 + }, + { + "epoch": 0.7140864714086471, + "grad_norm": 2.3320319652557373, + "learning_rate": 0.0001527447249765329, + "logits/chosen": -0.5398510098457336, + "logits/rejected": -0.612176775932312, + "logps/chosen": -7.770809173583984, + "logps/rejected": -43.53167724609375, + "loss": 0.652389645576477, + "memory(GiB)": 46.71, + "nll_loss": 0.45456767082214355, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09153411537408829, + "rewards/margins": 3.321568250656128, + "rewards/rejected": -3.230034351348877, + "step": 320, + "train_speed(iter/s)": 0.011108 + }, + { + "epoch": 0.7163179916317992, + "grad_norm": 4.979187965393066, + "learning_rate": 0.00015243072835572318, + "logits/chosen": -0.43154215812683105, + "logits/rejected": -0.634253740310669, + "logps/chosen": -7.028312683105469, + "logps/rejected": -55.400917053222656, + "loss": 0.6826280951499939, + "memory(GiB)": 46.71, + "nll_loss": 0.5510784387588501, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.01225266233086586, + "rewards/margins": 4.13831090927124, + "rewards/rejected": -4.126058578491211, + "step": 321, + "train_speed(iter/s)": 0.011108 + }, + { + "epoch": 0.7185495118549512, + "grad_norm": 2.975884199142456, + "learning_rate": 0.00015211601719577358, + "logits/chosen": -0.4353257417678833, + "logits/rejected": -0.5795725584030151, + "logps/chosen": -7.880040168762207, + "logps/rejected": -64.30177307128906, + "loss": 0.5798292756080627, + "memory(GiB)": 46.71, + "nll_loss": 0.47467976808547974, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2598690688610077, + "rewards/margins": 4.767513751983643, + "rewards/rejected": -4.5076446533203125, + "step": 322, + "train_speed(iter/s)": 0.011108 + }, + { + "epoch": 0.7207810320781032, + "grad_norm": 1.4080758094787598, + "learning_rate": 0.00015180059578564681, + "logits/chosen": -0.4497103989124298, + "logits/rejected": -0.5721160173416138, + "logps/chosen": -5.973843574523926, + "logps/rejected": -56.87316131591797, + "loss": 0.45731985569000244, + "memory(GiB)": 46.71, + "nll_loss": 0.34768402576446533, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2195209562778473, + "rewards/margins": 4.106788158416748, + "rewards/rejected": -3.8872671127319336, + "step": 323, + "train_speed(iter/s)": 0.011109 + }, + { + "epoch": 0.7230125523012553, + "grad_norm": 1.3606942892074585, + "learning_rate": 0.0001514844684239852, + "logits/chosen": -0.47787460684776306, + "logits/rejected": -0.5935323238372803, + "logps/chosen": -5.027652740478516, + "logps/rejected": -65.97366333007812, + "loss": 0.41865256428718567, + "memory(GiB)": 46.71, + "nll_loss": 0.3325335681438446, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24971242249011993, + "rewards/margins": 5.141018867492676, + "rewards/rejected": -4.891305923461914, + "step": 324, + "train_speed(iter/s)": 0.011109 + }, + { + "epoch": 0.7252440725244073, + "grad_norm": 1.528626561164856, + "learning_rate": 0.00015116763941905176, + "logits/chosen": -0.4019933044910431, + "logits/rejected": -0.6332893371582031, + "logps/chosen": -4.702518939971924, + "logps/rejected": -69.18038177490234, + "loss": 0.3573070764541626, + "memory(GiB)": 46.71, + "nll_loss": 0.2868815064430237, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12700967490673065, + "rewards/margins": 5.253981113433838, + "rewards/rejected": -5.126971244812012, + "step": 325, + "train_speed(iter/s)": 0.011109 + }, + { + "epoch": 0.7274755927475592, + "grad_norm": 3.2162506580352783, + "learning_rate": 0.00015085011308867182, + "logits/chosen": -0.37698668241500854, + "logits/rejected": -0.5214061141014099, + "logps/chosen": -7.11295223236084, + "logps/rejected": -66.23783111572266, + "loss": 0.4662770628929138, + "memory(GiB)": 46.71, + "nll_loss": 0.3477156460285187, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17707261443138123, + "rewards/margins": 5.171353340148926, + "rewards/rejected": -4.994279861450195, + "step": 326, + "train_speed(iter/s)": 0.01111 + }, + { + "epoch": 0.7297071129707113, + "grad_norm": 9.275425910949707, + "learning_rate": 0.00015053189376017408, + "logits/chosen": -0.37949830293655396, + "logits/rejected": -0.5809910893440247, + "logps/chosen": -3.868182897567749, + "logps/rejected": -58.96842956542969, + "loss": 0.6602414846420288, + "memory(GiB)": 46.71, + "nll_loss": 0.5071837902069092, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18503929674625397, + "rewards/margins": 4.177866458892822, + "rewards/rejected": -3.9928274154663086, + "step": 327, + "train_speed(iter/s)": 0.01111 + }, + { + "epoch": 0.7319386331938633, + "grad_norm": 4.461760997772217, + "learning_rate": 0.00015021298577033135, + "logits/chosen": -0.3945280909538269, + "logits/rejected": -0.5231155157089233, + "logps/chosen": -7.571595191955566, + "logps/rejected": -62.64643096923828, + "loss": 0.5055414438247681, + "memory(GiB)": 46.71, + "nll_loss": 0.39454159140586853, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22002458572387695, + "rewards/margins": 4.510626316070557, + "rewards/rejected": -4.290602207183838, + "step": 328, + "train_speed(iter/s)": 0.01111 + }, + { + "epoch": 0.7341701534170153, + "grad_norm": 7.703919887542725, + "learning_rate": 0.00014989339346530196, + "logits/chosen": -0.37174245715141296, + "logits/rejected": -0.5489162802696228, + "logps/chosen": -8.076615333557129, + "logps/rejected": -67.34407806396484, + "loss": 0.9827853441238403, + "memory(GiB)": 46.71, + "nll_loss": 0.7718144059181213, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.10434423387050629, + "rewards/margins": 4.429540157318115, + "rewards/rejected": -4.533884048461914, + "step": 329, + "train_speed(iter/s)": 0.01111 + }, + { + "epoch": 0.7364016736401674, + "grad_norm": 2.2533440589904785, + "learning_rate": 0.00014957312120057005, + "logits/chosen": -0.4813147187232971, + "logits/rejected": -0.5636628270149231, + "logps/chosen": -7.013173580169678, + "logps/rejected": -51.493812561035156, + "loss": 0.5757937431335449, + "memory(GiB)": 46.71, + "nll_loss": 0.43985041975975037, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11506544798612595, + "rewards/margins": 3.8710973262786865, + "rewards/rejected": -3.7560319900512695, + "step": 330, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7386331938633194, + "grad_norm": 1.4830918312072754, + "learning_rate": 0.00014925217334088662, + "logits/chosen": -0.3828532099723816, + "logits/rejected": -0.5140236616134644, + "logps/chosen": -5.039735317230225, + "logps/rejected": -56.69969177246094, + "loss": 0.40610191226005554, + "memory(GiB)": 46.71, + "nll_loss": 0.28494367003440857, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19726376235485077, + "rewards/margins": 4.019721031188965, + "rewards/rejected": -3.8224573135375977, + "step": 331, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7408647140864714, + "grad_norm": 1.8645910024642944, + "learning_rate": 0.00014893055426020967, + "logits/chosen": -0.438978910446167, + "logits/rejected": -0.5011396408081055, + "logps/chosen": -8.443188667297363, + "logps/rejected": -50.85751724243164, + "loss": 0.6127409934997559, + "memory(GiB)": 46.71, + "nll_loss": 0.4498739540576935, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07569620013237, + "rewards/margins": 3.518016815185547, + "rewards/rejected": -3.4423205852508545, + "step": 332, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7430962343096235, + "grad_norm": 2.0387418270111084, + "learning_rate": 0.00014860826834164488, + "logits/chosen": -0.4517824053764343, + "logits/rejected": -0.5715868473052979, + "logps/chosen": -6.337647914886475, + "logps/rejected": -49.289581298828125, + "loss": 0.5002683997154236, + "memory(GiB)": 46.71, + "nll_loss": 0.3809012770652771, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22188112139701843, + "rewards/margins": 3.3370914459228516, + "rewards/rejected": -3.1152102947235107, + "step": 333, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.7453277545327754, + "grad_norm": 2.6439507007598877, + "learning_rate": 0.00014828531997738574, + "logits/chosen": -0.37341347336769104, + "logits/rejected": -0.5413724184036255, + "logps/chosen": -6.403278827667236, + "logps/rejected": -57.33209991455078, + "loss": 0.6404682397842407, + "memory(GiB)": 46.71, + "nll_loss": 0.46989884972572327, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.22387410700321198, + "rewards/margins": 3.842700481414795, + "rewards/rejected": -3.618826389312744, + "step": 334, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.7475592747559274, + "grad_norm": 5.668474197387695, + "learning_rate": 0.0001479617135686536, + "logits/chosen": -0.38817811012268066, + "logits/rejected": -0.49394482374191284, + "logps/chosen": -5.707259178161621, + "logps/rejected": -43.85125732421875, + "loss": 0.6102214455604553, + "memory(GiB)": 46.71, + "nll_loss": 0.4218248426914215, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10298778116703033, + "rewards/margins": 3.0357441902160645, + "rewards/rejected": -2.9327564239501953, + "step": 335, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.7497907949790795, + "grad_norm": 2.4494247436523438, + "learning_rate": 0.00014763745352563805, + "logits/chosen": -0.46720805764198303, + "logits/rejected": -0.5431802272796631, + "logps/chosen": -6.758922100067139, + "logps/rejected": -46.58302307128906, + "loss": 0.5638653039932251, + "memory(GiB)": 46.71, + "nll_loss": 0.3755757510662079, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18551230430603027, + "rewards/margins": 3.3216769695281982, + "rewards/rejected": -3.136164426803589, + "step": 336, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.7520223152022315, + "grad_norm": 2.7223474979400635, + "learning_rate": 0.00014731254426743632, + "logits/chosen": -0.45732632279396057, + "logits/rejected": -0.5414117574691772, + "logps/chosen": -9.951481819152832, + "logps/rejected": -40.086883544921875, + "loss": 0.7714645266532898, + "memory(GiB)": 46.71, + "nll_loss": 0.5327011942863464, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12452726066112518, + "rewards/margins": 2.818586826324463, + "rewards/rejected": -2.6940598487854004, + "step": 337, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.7542538354253835, + "grad_norm": 3.634488344192505, + "learning_rate": 0.0001469869902219935, + "logits/chosen": -0.4285876750946045, + "logits/rejected": -0.5543171167373657, + "logps/chosen": -7.366232395172119, + "logps/rejected": -61.6998291015625, + "loss": 0.7139219641685486, + "memory(GiB)": 46.71, + "nll_loss": 0.4911733567714691, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.01174296997487545, + "rewards/margins": 4.718740463256836, + "rewards/rejected": -4.706998348236084, + "step": 338, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.7564853556485356, + "grad_norm": 2.3047475814819336, + "learning_rate": 0.00014666079582604185, + "logits/chosen": -0.436638742685318, + "logits/rejected": -0.6006594896316528, + "logps/chosen": -3.1358702182769775, + "logps/rejected": -67.4930648803711, + "loss": 0.36137089133262634, + "memory(GiB)": 46.71, + "nll_loss": 0.29073479771614075, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2920692265033722, + "rewards/margins": 5.5858154296875, + "rewards/rejected": -5.293745994567871, + "step": 339, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.7587168758716876, + "grad_norm": 1.8145880699157715, + "learning_rate": 0.00014633396552504063, + "logits/chosen": -0.3648812472820282, + "logits/rejected": -0.5664454698562622, + "logps/chosen": -6.136991500854492, + "logps/rejected": -75.88450622558594, + "loss": 0.44054728746414185, + "memory(GiB)": 46.71, + "nll_loss": 0.3295823931694031, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3330274820327759, + "rewards/margins": 6.021694183349609, + "rewards/rejected": -5.688666343688965, + "step": 340, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.7609483960948397, + "grad_norm": 3.6075730323791504, + "learning_rate": 0.00014600650377311522, + "logits/chosen": -0.37762531638145447, + "logits/rejected": -0.5749399065971375, + "logps/chosen": -7.936388969421387, + "logps/rejected": -68.10006713867188, + "loss": 0.6828188896179199, + "memory(GiB)": 46.71, + "nll_loss": 0.5689438581466675, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.08158844709396362, + "rewards/margins": 4.6608757972717285, + "rewards/rejected": -4.742464542388916, + "step": 341, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.7631799163179916, + "grad_norm": 4.307149887084961, + "learning_rate": 0.00014567841503299674, + "logits/chosen": -0.4181312322616577, + "logits/rejected": -0.5500895977020264, + "logps/chosen": -9.939905166625977, + "logps/rejected": -49.43525314331055, + "loss": 0.7494221925735474, + "memory(GiB)": 46.71, + "nll_loss": 0.5927161574363708, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.21191245317459106, + "rewards/margins": 4.02361536026001, + "rewards/rejected": -3.8117032051086426, + "step": 342, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.7654114365411436, + "grad_norm": 3.3493688106536865, + "learning_rate": 0.0001453497037759609, + "logits/chosen": -0.3912985920906067, + "logits/rejected": -0.49747738242149353, + "logps/chosen": -7.1425886154174805, + "logps/rejected": -56.73575973510742, + "loss": 0.5522685050964355, + "memory(GiB)": 46.71, + "nll_loss": 0.39775705337524414, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04426157474517822, + "rewards/margins": 4.1686835289001465, + "rewards/rejected": -4.124422073364258, + "step": 343, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.7676429567642957, + "grad_norm": 2.5058019161224365, + "learning_rate": 0.00014502037448176734, + "logits/chosen": -0.3858453631401062, + "logits/rejected": -0.5628204941749573, + "logps/chosen": -3.7287096977233887, + "logps/rejected": -63.88730239868164, + "loss": 0.356404185295105, + "memory(GiB)": 46.71, + "nll_loss": 0.24477878212928772, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13228493928909302, + "rewards/margins": 4.9347429275512695, + "rewards/rejected": -4.802457809448242, + "step": 344, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.7698744769874477, + "grad_norm": 2.141977310180664, + "learning_rate": 0.00014469043163859842, + "logits/chosen": -0.3882080316543579, + "logits/rejected": -0.5043464303016663, + "logps/chosen": -5.224626064300537, + "logps/rejected": -58.283935546875, + "loss": 0.5537149310112, + "memory(GiB)": 46.71, + "nll_loss": 0.4310363829135895, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22599749267101288, + "rewards/margins": 4.411967754364014, + "rewards/rejected": -4.185970306396484, + "step": 345, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.7721059972105997, + "grad_norm": 3.1156811714172363, + "learning_rate": 0.00014435987974299814, + "logits/chosen": -0.3683311939239502, + "logits/rejected": -0.5538841485977173, + "logps/chosen": -4.879660129547119, + "logps/rejected": -59.70796203613281, + "loss": 0.45609307289123535, + "memory(GiB)": 46.71, + "nll_loss": 0.3242284059524536, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16118085384368896, + "rewards/margins": 4.647848129272461, + "rewards/rejected": -4.486668109893799, + "step": 346, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.7743375174337518, + "grad_norm": 1.7269766330718994, + "learning_rate": 0.00014402872329981076, + "logits/chosen": -0.4169507622718811, + "logits/rejected": -0.5397988557815552, + "logps/chosen": -4.958333969116211, + "logps/rejected": -52.016361236572266, + "loss": 0.3880004584789276, + "memory(GiB)": 46.71, + "nll_loss": 0.2464018315076828, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.29084649682044983, + "rewards/margins": 4.126380443572998, + "rewards/rejected": -3.835533857345581, + "step": 347, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.7765690376569038, + "grad_norm": 2.476193904876709, + "learning_rate": 0.00014369696682211948, + "logits/chosen": -0.3376789689064026, + "logits/rejected": -0.5607038140296936, + "logps/chosen": -3.450556516647339, + "logps/rejected": -76.77299499511719, + "loss": 0.4519619345664978, + "memory(GiB)": 46.71, + "nll_loss": 0.3531690835952759, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.032610513269901276, + "rewards/margins": 5.81792688369751, + "rewards/rejected": -5.850537300109863, + "step": 348, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.7788005578800558, + "grad_norm": 2.383699417114258, + "learning_rate": 0.00014336461483118498, + "logits/chosen": -0.43573546409606934, + "logits/rejected": -0.5547890663146973, + "logps/chosen": -4.641900062561035, + "logps/rejected": -60.39362335205078, + "loss": 0.4820173978805542, + "memory(GiB)": 46.71, + "nll_loss": 0.30041489005088806, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12179512530565262, + "rewards/margins": 4.60038948059082, + "rewards/rejected": -4.478594779968262, + "step": 349, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.7810320781032078, + "grad_norm": 4.002810478210449, + "learning_rate": 0.00014303167185638366, + "logits/chosen": -0.35828980803489685, + "logits/rejected": -0.4897570013999939, + "logps/chosen": -7.969301223754883, + "logps/rejected": -63.727901458740234, + "loss": 0.5416594743728638, + "memory(GiB)": 46.71, + "nll_loss": 0.39018237590789795, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11720268428325653, + "rewards/margins": 4.261350154876709, + "rewards/rejected": -4.1441473960876465, + "step": 350, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.7832635983263598, + "grad_norm": 11.636247634887695, + "learning_rate": 0.00014269814243514608, + "logits/chosen": -0.38541609048843384, + "logits/rejected": -0.5292185544967651, + "logps/chosen": -6.552927494049072, + "logps/rejected": -65.6147232055664, + "loss": 0.5698133707046509, + "memory(GiB)": 46.71, + "nll_loss": 0.41967496275901794, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14759182929992676, + "rewards/margins": 4.941274642944336, + "rewards/rejected": -4.79368257522583, + "step": 351, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7854951185495118, + "grad_norm": 1.7822130918502808, + "learning_rate": 0.00014236403111289494, + "logits/chosen": -0.3575177788734436, + "logits/rejected": -0.5745211839675903, + "logps/chosen": -4.144627571105957, + "logps/rejected": -68.5383071899414, + "loss": 0.36631515622138977, + "memory(GiB)": 46.71, + "nll_loss": 0.2836896777153015, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25237223505973816, + "rewards/margins": 5.278615951538086, + "rewards/rejected": -5.026243686676025, + "step": 352, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7877266387726639, + "grad_norm": 10.725011825561523, + "learning_rate": 0.0001420293424429833, + "logits/chosen": -0.37747955322265625, + "logits/rejected": -0.5390688180923462, + "logps/chosen": -7.815685272216797, + "logps/rejected": -62.77814483642578, + "loss": 0.5161012411117554, + "memory(GiB)": 46.71, + "nll_loss": 0.40942439436912537, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2935142517089844, + "rewards/margins": 4.381824493408203, + "rewards/rejected": -4.088310241699219, + "step": 353, + "train_speed(iter/s)": 0.011111 + }, + { + "epoch": 0.7899581589958159, + "grad_norm": 9.515501022338867, + "learning_rate": 0.00014169408098663248, + "logits/chosen": -0.3902387022972107, + "logits/rejected": -0.5715078115463257, + "logps/chosen": -9.20953369140625, + "logps/rejected": -63.03540802001953, + "loss": 0.9012687802314758, + "memory(GiB)": 46.71, + "nll_loss": 0.6767682433128357, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.24745047092437744, + "rewards/margins": 4.600369453430176, + "rewards/rejected": -4.352919101715088, + "step": 354, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.7921896792189679, + "grad_norm": 2.6279428005218506, + "learning_rate": 0.00014135825131286983, + "logits/chosen": -0.37668153643608093, + "logits/rejected": -0.5125768184661865, + "logps/chosen": -8.645218849182129, + "logps/rejected": -70.78175354003906, + "loss": 0.5147952437400818, + "memory(GiB)": 46.71, + "nll_loss": 0.3659819960594177, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.3793887495994568, + "rewards/margins": 5.663691997528076, + "rewards/rejected": -5.284303665161133, + "step": 355, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.79442119944212, + "grad_norm": 3.9836838245391846, + "learning_rate": 0.0001410218579984665, + "logits/chosen": -0.40391817688941956, + "logits/rejected": -0.6119477152824402, + "logps/chosen": -7.379840850830078, + "logps/rejected": -66.17074584960938, + "loss": 0.6494410037994385, + "memory(GiB)": 46.71, + "nll_loss": 0.5213847160339355, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09886465221643448, + "rewards/margins": 5.152929306030273, + "rewards/rejected": -5.054064750671387, + "step": 356, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.796652719665272, + "grad_norm": 2.2495243549346924, + "learning_rate": 0.0001406849056278752, + "logits/chosen": -0.37667348980903625, + "logits/rejected": -0.5073565244674683, + "logps/chosen": -4.928195476531982, + "logps/rejected": -60.2727165222168, + "loss": 0.6239266991615295, + "memory(GiB)": 46.71, + "nll_loss": 0.3930074870586395, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.15948930382728577, + "rewards/margins": 4.717278480529785, + "rewards/rejected": -4.557788848876953, + "step": 357, + "train_speed(iter/s)": 0.011112 + }, + { + "epoch": 0.7988842398884239, + "grad_norm": 1.8445396423339844, + "learning_rate": 0.00014034739879316736, + "logits/chosen": -0.3938915729522705, + "logits/rejected": -0.5046184062957764, + "logps/chosen": -5.023759365081787, + "logps/rejected": -70.5897216796875, + "loss": 0.3982272744178772, + "memory(GiB)": 46.71, + "nll_loss": 0.24949610233306885, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.19955502450466156, + "rewards/margins": 5.79535436630249, + "rewards/rejected": -5.595798969268799, + "step": 358, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.801115760111576, + "grad_norm": 3.1103875637054443, + "learning_rate": 0.00014000934209397104, + "logits/chosen": -0.32392001152038574, + "logits/rejected": -0.5450155138969421, + "logps/chosen": -3.857600450515747, + "logps/rejected": -85.79165649414062, + "loss": 0.33195358514785767, + "memory(GiB)": 46.71, + "nll_loss": 0.2682299017906189, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3685230016708374, + "rewards/margins": 6.954622745513916, + "rewards/rejected": -6.586099624633789, + "step": 359, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.803347280334728, + "grad_norm": 1.984371542930603, + "learning_rate": 0.0001396707401374078, + "logits/chosen": -0.3701359033584595, + "logits/rejected": -0.5381411910057068, + "logps/chosen": -6.86074686050415, + "logps/rejected": -71.24162292480469, + "loss": 0.5251645445823669, + "memory(GiB)": 46.71, + "nll_loss": 0.3750622272491455, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10443415492773056, + "rewards/margins": 5.1036553382873535, + "rewards/rejected": -4.9992218017578125, + "step": 360, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.80557880055788, + "grad_norm": 2.3934128284454346, + "learning_rate": 0.0001393315975380302, + "logits/chosen": -0.3280276954174042, + "logits/rejected": -0.5281027555465698, + "logps/chosen": -8.700102806091309, + "logps/rejected": -62.87435531616211, + "loss": 0.6916484236717224, + "memory(GiB)": 46.71, + "nll_loss": 0.5558599829673767, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24834400415420532, + "rewards/margins": 4.944179058074951, + "rewards/rejected": -4.695835113525391, + "step": 361, + "train_speed(iter/s)": 0.011113 + }, + { + "epoch": 0.8078103207810321, + "grad_norm": 3.060940980911255, + "learning_rate": 0.00013899191891775885, + "logits/chosen": -0.3458540439605713, + "logits/rejected": -0.5299981236457825, + "logps/chosen": -9.838567733764648, + "logps/rejected": -68.1929702758789, + "loss": 0.6143630743026733, + "memory(GiB)": 46.71, + "nll_loss": 0.46228617429733276, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1387019157409668, + "rewards/margins": 5.114952087402344, + "rewards/rejected": -4.976249694824219, + "step": 362, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.8100418410041841, + "grad_norm": 1.7798267602920532, + "learning_rate": 0.00013865170890581924, + "logits/chosen": -0.3505061864852905, + "logits/rejected": -0.516235888004303, + "logps/chosen": -4.807265758514404, + "logps/rejected": -61.87629318237305, + "loss": 0.45331934094429016, + "memory(GiB)": 46.71, + "nll_loss": 0.3245879113674164, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08876000344753265, + "rewards/margins": 4.82363748550415, + "rewards/rejected": -4.734877109527588, + "step": 363, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.8122733612273362, + "grad_norm": 1.6590206623077393, + "learning_rate": 0.000138310972138679, + "logits/chosen": -0.3180076777935028, + "logits/rejected": -0.4814963936805725, + "logps/chosen": -4.849047660827637, + "logps/rejected": -65.47941589355469, + "loss": 0.44194936752319336, + "memory(GiB)": 46.71, + "nll_loss": 0.3421885371208191, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15211261808872223, + "rewards/margins": 5.10720157623291, + "rewards/rejected": -4.955089092254639, + "step": 364, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.8145048814504882, + "grad_norm": 2.0548856258392334, + "learning_rate": 0.00013796971325998433, + "logits/chosen": -0.34006184339523315, + "logits/rejected": -0.508549690246582, + "logps/chosen": -6.979956150054932, + "logps/rejected": -58.9586181640625, + "loss": 0.5468458533287048, + "memory(GiB)": 46.71, + "nll_loss": 0.41807255148887634, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27598223090171814, + "rewards/margins": 4.633675575256348, + "rewards/rejected": -4.357693195343018, + "step": 365, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.8167364016736401, + "grad_norm": 4.551876068115234, + "learning_rate": 0.000137627936920497, + "logits/chosen": -0.31743019819259644, + "logits/rejected": -0.5004254579544067, + "logps/chosen": -4.453533172607422, + "logps/rejected": -80.82231140136719, + "loss": 0.38986608386039734, + "memory(GiB)": 46.71, + "nll_loss": 0.2782318592071533, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11348968744277954, + "rewards/margins": 6.528149127960205, + "rewards/rejected": -6.41465950012207, + "step": 366, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.8189679218967922, + "grad_norm": 2.048898935317993, + "learning_rate": 0.00013728564777803088, + "logits/chosen": -0.3985055685043335, + "logits/rejected": -0.5710167288780212, + "logps/chosen": -6.307327747344971, + "logps/rejected": -73.59968566894531, + "loss": 0.40161988139152527, + "memory(GiB)": 46.71, + "nll_loss": 0.33988818526268005, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1631365567445755, + "rewards/margins": 5.702443599700928, + "rewards/rejected": -5.539307117462158, + "step": 367, + "train_speed(iter/s)": 0.011115 + }, + { + "epoch": 0.8211994421199442, + "grad_norm": 2.9747700691223145, + "learning_rate": 0.00013694285049738834, + "logits/chosen": -0.42140811681747437, + "logits/rejected": -0.5209057927131653, + "logps/chosen": -9.372599601745605, + "logps/rejected": -61.16340255737305, + "loss": 0.6916413307189941, + "memory(GiB)": 46.71, + "nll_loss": 0.5204302072525024, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.040431585162878036, + "rewards/margins": 4.626062393188477, + "rewards/rejected": -4.585630893707275, + "step": 368, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.8234309623430962, + "grad_norm": 1.5708239078521729, + "learning_rate": 0.0001365995497502969, + "logits/chosen": -0.414267361164093, + "logits/rejected": -0.511968731880188, + "logps/chosen": -6.139432430267334, + "logps/rejected": -67.94561767578125, + "loss": 0.3955412805080414, + "memory(GiB)": 46.71, + "nll_loss": 0.2775593101978302, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12219292670488358, + "rewards/margins": 5.711038112640381, + "rewards/rejected": -5.588845729827881, + "step": 369, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.8256624825662483, + "grad_norm": 1.9238094091415405, + "learning_rate": 0.00013625575021534536, + "logits/chosen": -0.41731297969818115, + "logits/rejected": -0.5028038024902344, + "logps/chosen": -8.249049186706543, + "logps/rejected": -66.31785583496094, + "loss": 0.47513440251350403, + "memory(GiB)": 46.71, + "nll_loss": 0.32550662755966187, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15111449360847473, + "rewards/margins": 5.429738521575928, + "rewards/rejected": -5.278623580932617, + "step": 370, + "train_speed(iter/s)": 0.011116 + }, + { + "epoch": 0.8278940027894003, + "grad_norm": 1.6814898252487183, + "learning_rate": 0.00013591145657792017, + "logits/chosen": -0.29436764121055603, + "logits/rejected": -0.5296704173088074, + "logps/chosen": -4.545007705688477, + "logps/rejected": -76.24856567382812, + "loss": 0.4239902198314667, + "memory(GiB)": 46.71, + "nll_loss": 0.3380081057548523, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07384507358074188, + "rewards/margins": 5.721249580383301, + "rewards/rejected": -5.647404670715332, + "step": 371, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.8301255230125523, + "grad_norm": 2.181011199951172, + "learning_rate": 0.0001355666735301416, + "logits/chosen": -0.3410585820674896, + "logits/rejected": -0.5049437880516052, + "logps/chosen": -8.976798057556152, + "logps/rejected": -67.74906921386719, + "loss": 0.7756314277648926, + "memory(GiB)": 46.71, + "nll_loss": 0.5791795253753662, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.05143262445926666, + "rewards/margins": 4.400791168212891, + "rewards/rejected": -4.452223300933838, + "step": 372, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.8323570432357044, + "grad_norm": 13.526124000549316, + "learning_rate": 0.00013522140577079954, + "logits/chosen": -0.3707951009273529, + "logits/rejected": -0.45664089918136597, + "logps/chosen": -15.304136276245117, + "logps/rejected": -53.893157958984375, + "loss": 1.1949626207351685, + "memory(GiB)": 46.71, + "nll_loss": 0.8217987418174744, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.09464707970619202, + "rewards/margins": 3.907057523727417, + "rewards/rejected": -3.812410354614258, + "step": 373, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.8345885634588563, + "grad_norm": 1.608467698097229, + "learning_rate": 0.0001348756580052899, + "logits/chosen": -0.317430317401886, + "logits/rejected": -0.4931824207305908, + "logps/chosen": -9.879939079284668, + "logps/rejected": -59.61829376220703, + "loss": 0.6055309176445007, + "memory(GiB)": 46.71, + "nll_loss": 0.4914649426937103, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.3871683180332184, + "rewards/margins": 4.3960676193237305, + "rewards/rejected": -4.008899211883545, + "step": 374, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.8368200836820083, + "grad_norm": 6.419092178344727, + "learning_rate": 0.00013452943494554997, + "logits/chosen": -0.3663273751735687, + "logits/rejected": -0.5518556237220764, + "logps/chosen": -6.749516010284424, + "logps/rejected": -65.08462524414062, + "loss": 0.6154893636703491, + "memory(GiB)": 46.71, + "nll_loss": 0.4859183728694916, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1840987205505371, + "rewards/margins": 4.825078964233398, + "rewards/rejected": -4.640980243682861, + "step": 375, + "train_speed(iter/s)": 0.011117 + }, + { + "epoch": 0.8390516039051604, + "grad_norm": 1.5220005512237549, + "learning_rate": 0.0001341827413099947, + "logits/chosen": -0.43759799003601074, + "logits/rejected": -0.5578930974006653, + "logps/chosen": -4.9777512550354, + "logps/rejected": -63.857357025146484, + "loss": 0.37524834275245667, + "memory(GiB)": 46.71, + "nll_loss": 0.2966943383216858, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15793432295322418, + "rewards/margins": 4.674270153045654, + "rewards/rejected": -4.516335487365723, + "step": 376, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.8412831241283124, + "grad_norm": 3.5475573539733887, + "learning_rate": 0.00013383558182345203, + "logits/chosen": -0.4074645936489105, + "logits/rejected": -0.5047129988670349, + "logps/chosen": -7.616944313049316, + "logps/rejected": -54.710601806640625, + "loss": 0.5815810561180115, + "memory(GiB)": 46.71, + "nll_loss": 0.4423309564590454, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15139777958393097, + "rewards/margins": 3.9785633087158203, + "rewards/rejected": -3.8271658420562744, + "step": 377, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.8435146443514644, + "grad_norm": 1.7984713315963745, + "learning_rate": 0.00013348796121709862, + "logits/chosen": -0.3888586461544037, + "logits/rejected": -0.5558137893676758, + "logps/chosen": -7.5062079429626465, + "logps/rejected": -63.299095153808594, + "loss": 0.5085947513580322, + "memory(GiB)": 46.71, + "nll_loss": 0.39699020981788635, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09221147745847702, + "rewards/margins": 4.765577793121338, + "rewards/rejected": -4.673365592956543, + "step": 378, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.8457461645746165, + "grad_norm": 1.936753273010254, + "learning_rate": 0.00013313988422839553, + "logits/chosen": -0.43031805753707886, + "logits/rejected": -0.5588259696960449, + "logps/chosen": -5.764710903167725, + "logps/rejected": -54.861000061035156, + "loss": 0.5527252554893494, + "memory(GiB)": 46.71, + "nll_loss": 0.40945884585380554, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1853952407836914, + "rewards/margins": 4.320432662963867, + "rewards/rejected": -4.135037422180176, + "step": 379, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.8479776847977685, + "grad_norm": 1.9498895406723022, + "learning_rate": 0.00013279135560102337, + "logits/chosen": -0.3983275592327118, + "logits/rejected": -0.5249882936477661, + "logps/chosen": -6.359404563903809, + "logps/rejected": -64.9413070678711, + "loss": 0.3789876103401184, + "memory(GiB)": 46.71, + "nll_loss": 0.2830629348754883, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08346953988075256, + "rewards/margins": 4.957228183746338, + "rewards/rejected": -4.873758792877197, + "step": 380, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.8502092050209205, + "grad_norm": 2.092665910720825, + "learning_rate": 0.00013244238008481784, + "logits/chosen": -0.35154277086257935, + "logits/rejected": -0.5239943861961365, + "logps/chosen": -4.158918380737305, + "logps/rejected": -74.18276977539062, + "loss": 0.36264023184776306, + "memory(GiB)": 46.71, + "nll_loss": 0.3030133843421936, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09875375032424927, + "rewards/margins": 5.863274097442627, + "rewards/rejected": -5.764520168304443, + "step": 381, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.8524407252440726, + "grad_norm": 1.9038679599761963, + "learning_rate": 0.000132092962435705, + "logits/chosen": -0.40422144532203674, + "logits/rejected": -0.5475745797157288, + "logps/chosen": -7.517289161682129, + "logps/rejected": -59.79158020019531, + "loss": 0.5256230235099792, + "memory(GiB)": 46.71, + "nll_loss": 0.4094223380088806, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08417153358459473, + "rewards/margins": 4.489100933074951, + "rewards/rejected": -4.404929161071777, + "step": 382, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.8546722454672245, + "grad_norm": 1.3534390926361084, + "learning_rate": 0.00013174310741563628, + "logits/chosen": -0.36429983377456665, + "logits/rejected": -0.5562334060668945, + "logps/chosen": -8.207029342651367, + "logps/rejected": -68.52938842773438, + "loss": 0.4683631658554077, + "memory(GiB)": 46.71, + "nll_loss": 0.41341280937194824, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32433998584747314, + "rewards/margins": 5.395326614379883, + "rewards/rejected": -5.070986747741699, + "step": 383, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.8569037656903765, + "grad_norm": 2.5620298385620117, + "learning_rate": 0.00013139281979252394, + "logits/chosen": -0.3951599895954132, + "logits/rejected": -0.586998462677002, + "logps/chosen": -3.382761001586914, + "logps/rejected": -63.8618278503418, + "loss": 0.36399391293525696, + "memory(GiB)": 46.71, + "nll_loss": 0.29559269547462463, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1514458805322647, + "rewards/margins": 5.087501049041748, + "rewards/rejected": -4.936054706573486, + "step": 384, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.8591352859135286, + "grad_norm": 2.94246244430542, + "learning_rate": 0.00013104210434017557, + "logits/chosen": -0.3674653470516205, + "logits/rejected": -0.511353075504303, + "logps/chosen": -8.046361923217773, + "logps/rejected": -65.10374450683594, + "loss": 0.7132191061973572, + "memory(GiB)": 46.71, + "nll_loss": 0.5216770172119141, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.01664729230105877, + "rewards/margins": 4.854707717895508, + "rewards/rejected": -4.8713555335998535, + "step": 385, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.8613668061366806, + "grad_norm": 6.434685707092285, + "learning_rate": 0.0001306909658382296, + "logits/chosen": -0.400129497051239, + "logits/rejected": -0.5450380444526672, + "logps/chosen": -4.818359851837158, + "logps/rejected": -59.016693115234375, + "loss": 0.37835314869880676, + "memory(GiB)": 46.71, + "nll_loss": 0.2632123529911041, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14880885183811188, + "rewards/margins": 4.848893165588379, + "rewards/rejected": -4.7000837326049805, + "step": 386, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.8635983263598327, + "grad_norm": 2.7335994243621826, + "learning_rate": 0.0001303394090720897, + "logits/chosen": -0.37752431631088257, + "logits/rejected": -0.5266053676605225, + "logps/chosen": -6.391074180603027, + "logps/rejected": -61.5793342590332, + "loss": 0.7463357448577881, + "memory(GiB)": 46.71, + "nll_loss": 0.5113351941108704, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.08021123707294464, + "rewards/margins": 4.718035697937012, + "rewards/rejected": -4.637824535369873, + "step": 387, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.8658298465829847, + "grad_norm": 8.42420482635498, + "learning_rate": 0.0001299874388328598, + "logits/chosen": -0.37399792671203613, + "logits/rejected": -0.518165111541748, + "logps/chosen": -7.431341171264648, + "logps/rejected": -62.52376174926758, + "loss": 0.8066068887710571, + "memory(GiB)": 46.71, + "nll_loss": 0.564358115196228, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10468573868274689, + "rewards/margins": 4.341407775878906, + "rewards/rejected": -4.236721992492676, + "step": 388, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.8680613668061367, + "grad_norm": 4.146071434020996, + "learning_rate": 0.00012963505991727878, + "logits/chosen": -0.33883821964263916, + "logits/rejected": -0.5248036980628967, + "logps/chosen": -5.620823383331299, + "logps/rejected": -52.28062057495117, + "loss": 0.45467111468315125, + "memory(GiB)": 46.71, + "nll_loss": 0.3179868757724762, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15395142138004303, + "rewards/margins": 3.748572826385498, + "rewards/rejected": -3.594621181488037, + "step": 389, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.8702928870292888, + "grad_norm": 1.5783072710037231, + "learning_rate": 0.00012928227712765504, + "logits/chosen": -0.41484951972961426, + "logits/rejected": -0.5287652015686035, + "logps/chosen": -4.470883846282959, + "logps/rejected": -57.34123992919922, + "loss": 0.389884889125824, + "memory(GiB)": 46.71, + "nll_loss": 0.19639500975608826, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.149226576089859, + "rewards/margins": 4.101458549499512, + "rewards/rejected": -3.9522321224212646, + "step": 390, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.8725244072524407, + "grad_norm": 2.507016658782959, + "learning_rate": 0.0001289290952718011, + "logits/chosen": -0.38369786739349365, + "logits/rejected": -0.5141218304634094, + "logps/chosen": -5.142464637756348, + "logps/rejected": -68.59282684326172, + "loss": 0.3908143639564514, + "memory(GiB)": 46.71, + "nll_loss": 0.2560312747955322, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09192930907011032, + "rewards/margins": 5.197025299072266, + "rewards/rejected": -5.105095863342285, + "step": 391, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.8747559274755927, + "grad_norm": 1.4557801485061646, + "learning_rate": 0.00012857551916296798, + "logits/chosen": -0.3842438757419586, + "logits/rejected": -0.5369118452072144, + "logps/chosen": -5.160379409790039, + "logps/rejected": -64.20812225341797, + "loss": 0.3677409589290619, + "memory(GiB)": 46.71, + "nll_loss": 0.29911279678344727, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20458275079727173, + "rewards/margins": 4.682780742645264, + "rewards/rejected": -4.478198528289795, + "step": 392, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.8769874476987448, + "grad_norm": 2.624765396118164, + "learning_rate": 0.00012822155361977977, + "logits/chosen": -0.34929391741752625, + "logits/rejected": -0.5693681240081787, + "logps/chosen": -3.473264694213867, + "logps/rejected": -66.60023498535156, + "loss": 0.4702262878417969, + "memory(GiB)": 46.71, + "nll_loss": 0.3658883571624756, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11007699370384216, + "rewards/margins": 5.03789758682251, + "rewards/rejected": -4.927820682525635, + "step": 393, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.8792189679218968, + "grad_norm": 2.4931700229644775, + "learning_rate": 0.00012786720346616784, + "logits/chosen": -0.42144328355789185, + "logits/rejected": -0.5153274536132812, + "logps/chosen": -9.284778594970703, + "logps/rejected": -53.68639373779297, + "loss": 0.6390541791915894, + "memory(GiB)": 46.71, + "nll_loss": 0.4495624005794525, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14184999465942383, + "rewards/margins": 3.5801758766174316, + "rewards/rejected": -3.438325881958008, + "step": 394, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.8814504881450488, + "grad_norm": 2.3873162269592285, + "learning_rate": 0.00012751247353130508, + "logits/chosen": -0.4466644525527954, + "logits/rejected": -0.5331275463104248, + "logps/chosen": -7.677053451538086, + "logps/rejected": -51.20628356933594, + "loss": 0.6434856057167053, + "memory(GiB)": 46.71, + "nll_loss": 0.4827214479446411, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.30882352590560913, + "rewards/margins": 4.188982009887695, + "rewards/rejected": -3.8801589012145996, + "step": 395, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.8836820083682009, + "grad_norm": 1.492531657218933, + "learning_rate": 0.00012715736864954018, + "logits/chosen": -0.36164212226867676, + "logits/rejected": -0.5238646268844604, + "logps/chosen": -6.853821754455566, + "logps/rejected": -60.264591217041016, + "loss": 0.5100601315498352, + "memory(GiB)": 46.71, + "nll_loss": 0.398866206407547, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4058721661567688, + "rewards/margins": 4.440138339996338, + "rewards/rejected": -4.034265995025635, + "step": 396, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.8859135285913529, + "grad_norm": 1.4941059350967407, + "learning_rate": 0.00012680189366033172, + "logits/chosen": -0.34664303064346313, + "logits/rejected": -0.5415948629379272, + "logps/chosen": -6.301570892333984, + "logps/rejected": -77.05431365966797, + "loss": 0.3252749443054199, + "memory(GiB)": 46.71, + "nll_loss": 0.24337539076805115, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09985675662755966, + "rewards/margins": 5.931595802307129, + "rewards/rejected": -5.8317389488220215, + "step": 397, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.8881450488145048, + "grad_norm": 2.231262683868408, + "learning_rate": 0.00012644605340818218, + "logits/chosen": -0.42692190408706665, + "logits/rejected": -0.5745521783828735, + "logps/chosen": -6.572040557861328, + "logps/rejected": -62.152503967285156, + "loss": 0.5078067183494568, + "memory(GiB)": 46.71, + "nll_loss": 0.3654176592826843, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1571953296661377, + "rewards/margins": 4.710308074951172, + "rewards/rejected": -4.553112506866455, + "step": 398, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.8903765690376569, + "grad_norm": 2.9185144901275635, + "learning_rate": 0.00012608985274257186, + "logits/chosen": -0.4496538043022156, + "logits/rejected": -0.5780081152915955, + "logps/chosen": -7.991746425628662, + "logps/rejected": -55.10255813598633, + "loss": 0.6926721334457397, + "memory(GiB)": 46.71, + "nll_loss": 0.5558676719665527, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2853441536426544, + "rewards/margins": 4.151731014251709, + "rewards/rejected": -3.866386890411377, + "step": 399, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.8926080892608089, + "grad_norm": 1.938307523727417, + "learning_rate": 0.00012573329651789297, + "logits/chosen": -0.43678951263427734, + "logits/rejected": -0.5458590984344482, + "logps/chosen": -4.191629409790039, + "logps/rejected": -52.17749786376953, + "loss": 0.4270729720592499, + "memory(GiB)": 46.71, + "nll_loss": 0.28124403953552246, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24446210265159607, + "rewards/margins": 4.054318904876709, + "rewards/rejected": -3.80985689163208, + "step": 400, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.8948396094839609, + "grad_norm": 2.4259486198425293, + "learning_rate": 0.00012537638959338337, + "logits/chosen": -0.4583006203174591, + "logits/rejected": -0.5670384168624878, + "logps/chosen": -7.806771278381348, + "logps/rejected": -54.58396911621094, + "loss": 0.5598142743110657, + "memory(GiB)": 46.71, + "nll_loss": 0.39694616198539734, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14714935421943665, + "rewards/margins": 3.9732398986816406, + "rewards/rejected": -3.826090097427368, + "step": 401, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.897071129707113, + "grad_norm": 2.4113872051239014, + "learning_rate": 0.00012501913683306027, + "logits/chosen": -0.3870125412940979, + "logits/rejected": -0.6248172521591187, + "logps/chosen": -4.8534321784973145, + "logps/rejected": -79.6197280883789, + "loss": 0.4482497572898865, + "memory(GiB)": 46.71, + "nll_loss": 0.34496423602104187, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.02209564670920372, + "rewards/margins": 5.9457011222839355, + "rewards/rejected": -5.923605918884277, + "step": 402, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.899302649930265, + "grad_norm": 1.6545324325561523, + "learning_rate": 0.000124661543105654, + "logits/chosen": -0.4177844524383545, + "logits/rejected": -0.6554238796234131, + "logps/chosen": -2.01737642288208, + "logps/rejected": -65.77202606201172, + "loss": 0.29415401816368103, + "memory(GiB)": 46.71, + "nll_loss": 0.1746412217617035, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2591646909713745, + "rewards/margins": 5.28652811050415, + "rewards/rejected": -5.027364253997803, + "step": 403, + "train_speed(iter/s)": 0.011118 + }, + { + "epoch": 0.901534170153417, + "grad_norm": 2.9395556449890137, + "learning_rate": 0.00012430361328454178, + "logits/chosen": -0.4731079339981079, + "logits/rejected": -0.5910559892654419, + "logps/chosen": -7.420808792114258, + "logps/rejected": -57.41754913330078, + "loss": 0.5733405351638794, + "memory(GiB)": 46.71, + "nll_loss": 0.44550999999046326, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22910183668136597, + "rewards/margins": 4.69669771194458, + "rewards/rejected": -4.467596530914307, + "step": 404, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.9037656903765691, + "grad_norm": 1.5645016431808472, + "learning_rate": 0.00012394535224768117, + "logits/chosen": -0.37732237577438354, + "logits/rejected": -0.5927914381027222, + "logps/chosen": -4.943865776062012, + "logps/rejected": -61.957462310791016, + "loss": 0.3866198658943176, + "memory(GiB)": 46.71, + "nll_loss": 0.3022453784942627, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16032618284225464, + "rewards/margins": 4.264319896697998, + "rewards/rejected": -4.103993892669678, + "step": 405, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.905997210599721, + "grad_norm": 5.462214469909668, + "learning_rate": 0.00012358676487754354, + "logits/chosen": -0.3900837004184723, + "logits/rejected": -0.5914356708526611, + "logps/chosen": -7.273135185241699, + "logps/rejected": -64.65262603759766, + "loss": 0.6411268711090088, + "memory(GiB)": 46.71, + "nll_loss": 0.507996141910553, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24620768427848816, + "rewards/margins": 5.043286323547363, + "rewards/rejected": -4.7970781326293945, + "step": 406, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.908228730822873, + "grad_norm": 3.6455790996551514, + "learning_rate": 0.0001232278560610477, + "logits/chosen": -0.3219720125198364, + "logits/rejected": -0.6039551496505737, + "logps/chosen": -5.803981304168701, + "logps/rejected": -76.50419616699219, + "loss": 0.5097870826721191, + "memory(GiB)": 46.71, + "nll_loss": 0.4448806047439575, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11342814564704895, + "rewards/margins": 5.36799430847168, + "rewards/rejected": -5.254566192626953, + "step": 407, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.9104602510460251, + "grad_norm": 2.2974917888641357, + "learning_rate": 0.00012286863068949321, + "logits/chosen": -0.4430183470249176, + "logits/rejected": -0.5634139776229858, + "logps/chosen": -8.480108261108398, + "logps/rejected": -49.45329666137695, + "loss": 0.681178867816925, + "memory(GiB)": 46.71, + "nll_loss": 0.49261921644210815, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27699458599090576, + "rewards/margins": 3.6769356727600098, + "rewards/rejected": -3.3999409675598145, + "step": 408, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.9126917712691771, + "grad_norm": 1.8317338228225708, + "learning_rate": 0.00012250909365849367, + "logits/chosen": -0.4659299850463867, + "logits/rejected": -0.590533971786499, + "logps/chosen": -8.896913528442383, + "logps/rejected": -65.60546112060547, + "loss": 0.50341796875, + "memory(GiB)": 46.71, + "nll_loss": 0.47141921520233154, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3340601921081543, + "rewards/margins": 5.3535990715026855, + "rewards/rejected": -5.019538402557373, + "step": 409, + "train_speed(iter/s)": 0.011119 + }, + { + "epoch": 0.9149232914923291, + "grad_norm": 1.3863641023635864, + "learning_rate": 0.00012214924986791003, + "logits/chosen": -0.44458577036857605, + "logits/rejected": -0.6526033878326416, + "logps/chosen": -3.0500755310058594, + "logps/rejected": -65.8653335571289, + "loss": 0.41676628589630127, + "memory(GiB)": 46.71, + "nll_loss": 0.33434438705444336, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22075964510440826, + "rewards/margins": 5.357534885406494, + "rewards/rejected": -5.136775016784668, + "step": 410, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.9171548117154812, + "grad_norm": 3.5864763259887695, + "learning_rate": 0.00012178910422178394, + "logits/chosen": -0.40339091420173645, + "logits/rejected": -0.5973271131515503, + "logps/chosen": -4.365366458892822, + "logps/rejected": -65.15888977050781, + "loss": 0.5601470470428467, + "memory(GiB)": 46.71, + "nll_loss": 0.35508766770362854, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.05216889828443527, + "rewards/margins": 4.944633960723877, + "rewards/rejected": -4.892464637756348, + "step": 411, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.9193863319386332, + "grad_norm": 2.7527568340301514, + "learning_rate": 0.00012142866162827073, + "logits/chosen": -0.4656250774860382, + "logits/rejected": -0.5733537077903748, + "logps/chosen": -8.669557571411133, + "logps/rejected": -48.257286071777344, + "loss": 0.6992867588996887, + "memory(GiB)": 46.71, + "nll_loss": 0.582659900188446, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.34467658400535583, + "rewards/margins": 3.845956563949585, + "rewards/rejected": -3.501279830932617, + "step": 412, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.9216178521617853, + "grad_norm": 3.534930944442749, + "learning_rate": 0.00012106792699957263, + "logits/chosen": -0.417633056640625, + "logits/rejected": -0.4791659712791443, + "logps/chosen": -9.570728302001953, + "logps/rejected": -47.421165466308594, + "loss": 0.5923347473144531, + "memory(GiB)": 46.71, + "nll_loss": 0.43664753437042236, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2927669882774353, + "rewards/margins": 3.77238130569458, + "rewards/rejected": -3.4796142578125, + "step": 413, + "train_speed(iter/s)": 0.01112 + }, + { + "epoch": 0.9238493723849373, + "grad_norm": 3.092557668685913, + "learning_rate": 0.00012070690525187186, + "logits/chosen": -0.3355872929096222, + "logits/rejected": -0.5605220198631287, + "logps/chosen": -6.468746185302734, + "logps/rejected": -71.57562255859375, + "loss": 0.4770223796367645, + "memory(GiB)": 46.71, + "nll_loss": 0.374657541513443, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2624046206474304, + "rewards/margins": 5.457033634185791, + "rewards/rejected": -5.194628715515137, + "step": 414, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.9260808926080892, + "grad_norm": 1.8237637281417847, + "learning_rate": 0.0001203456013052634, + "logits/chosen": -0.34914714097976685, + "logits/rejected": -0.5329669117927551, + "logps/chosen": -9.495757102966309, + "logps/rejected": -73.02941131591797, + "loss": 0.4823957085609436, + "memory(GiB)": 46.71, + "nll_loss": 0.38109683990478516, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24142882227897644, + "rewards/margins": 5.460197448730469, + "rewards/rejected": -5.218769073486328, + "step": 415, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.9283124128312413, + "grad_norm": 1.783011555671692, + "learning_rate": 0.00011998402008368827, + "logits/chosen": -0.36979907751083374, + "logits/rejected": -0.5743371844291687, + "logps/chosen": -5.173707962036133, + "logps/rejected": -68.01774597167969, + "loss": 0.48563554883003235, + "memory(GiB)": 46.71, + "nll_loss": 0.4055817723274231, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09396127611398697, + "rewards/margins": 5.202694892883301, + "rewards/rejected": -5.108734130859375, + "step": 416, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.9305439330543933, + "grad_norm": 2.155675172805786, + "learning_rate": 0.00011962216651486624, + "logits/chosen": -0.35840052366256714, + "logits/rejected": -0.5781437754631042, + "logps/chosen": -5.040507793426514, + "logps/rejected": -63.19990158081055, + "loss": 0.4736884832382202, + "memory(GiB)": 46.71, + "nll_loss": 0.34291768074035645, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12170419096946716, + "rewards/margins": 4.674435138702393, + "rewards/rejected": -4.552731037139893, + "step": 417, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.9327754532775453, + "grad_norm": 3.280628204345703, + "learning_rate": 0.0001192600455302286, + "logits/chosen": -0.3779280185699463, + "logits/rejected": -0.5500413775444031, + "logps/chosen": -3.9350452423095703, + "logps/rejected": -73.46235656738281, + "loss": 0.3550153076648712, + "memory(GiB)": 46.71, + "nll_loss": 0.23411351442337036, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15285731852054596, + "rewards/margins": 5.822902679443359, + "rewards/rejected": -5.670045375823975, + "step": 418, + "train_speed(iter/s)": 0.011121 + }, + { + "epoch": 0.9350069735006974, + "grad_norm": 11.438630104064941, + "learning_rate": 0.0001188976620648512, + "logits/chosen": -0.30616092681884766, + "logits/rejected": -0.570376455783844, + "logps/chosen": -8.577475547790527, + "logps/rejected": -71.0198745727539, + "loss": 0.6161710619926453, + "memory(GiB)": 46.71, + "nll_loss": 0.5294251441955566, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22826698422431946, + "rewards/margins": 5.515110015869141, + "rewards/rejected": -5.286843776702881, + "step": 419, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.9372384937238494, + "grad_norm": 1.640845775604248, + "learning_rate": 0.00011853502105738692, + "logits/chosen": -0.2870388925075531, + "logits/rejected": -0.5356206893920898, + "logps/chosen": -4.021060943603516, + "logps/rejected": -72.19401550292969, + "loss": 0.38995295763015747, + "memory(GiB)": 46.71, + "nll_loss": 0.27541545033454895, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2398616522550583, + "rewards/margins": 5.579682350158691, + "rewards/rejected": -5.339820384979248, + "step": 420, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.9394700139470014, + "grad_norm": 3.161043167114258, + "learning_rate": 0.00011817212744999848, + "logits/chosen": -0.3637564182281494, + "logits/rejected": -0.5138117671012878, + "logps/chosen": -7.268462657928467, + "logps/rejected": -71.36225891113281, + "loss": 0.48019376397132874, + "memory(GiB)": 46.71, + "nll_loss": 0.3564393222332001, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3627624213695526, + "rewards/margins": 5.711356163024902, + "rewards/rejected": -5.3485941886901855, + "step": 421, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.9417015341701535, + "grad_norm": 2.0086593627929688, + "learning_rate": 0.00011780898618829122, + "logits/chosen": -0.42384350299835205, + "logits/rejected": -0.507055401802063, + "logps/chosen": -6.711613655090332, + "logps/rejected": -52.270172119140625, + "loss": 0.5925105214118958, + "memory(GiB)": 46.71, + "nll_loss": 0.38617631793022156, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.05421391874551773, + "rewards/margins": 4.0297932624816895, + "rewards/rejected": -3.9755795001983643, + "step": 422, + "train_speed(iter/s)": 0.011122 + }, + { + "epoch": 0.9439330543933054, + "grad_norm": 2.4280357360839844, + "learning_rate": 0.0001174456022212454, + "logits/chosen": -0.40838170051574707, + "logits/rejected": -0.5214443802833557, + "logps/chosen": -5.486069202423096, + "logps/rejected": -65.27120208740234, + "loss": 0.4892706274986267, + "memory(GiB)": 46.71, + "nll_loss": 0.39567333459854126, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19619357585906982, + "rewards/margins": 5.227800369262695, + "rewards/rejected": -5.031606674194336, + "step": 423, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9461645746164574, + "grad_norm": 11.169919967651367, + "learning_rate": 0.00011708198050114916, + "logits/chosen": -0.35955384373664856, + "logits/rejected": -0.5468286275863647, + "logps/chosen": -6.013669490814209, + "logps/rejected": -68.45548248291016, + "loss": 0.6080888509750366, + "memory(GiB)": 46.71, + "nll_loss": 0.47363466024398804, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.22281761467456818, + "rewards/margins": 5.330093860626221, + "rewards/rejected": -5.107276439666748, + "step": 424, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9483960948396095, + "grad_norm": 1.9644421339035034, + "learning_rate": 0.00011671812598353058, + "logits/chosen": -0.4213840961456299, + "logits/rejected": -0.5007994174957275, + "logps/chosen": -10.670544624328613, + "logps/rejected": -61.778160095214844, + "loss": 0.45855849981307983, + "memory(GiB)": 46.71, + "nll_loss": 0.32925111055374146, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3085613250732422, + "rewards/margins": 4.932414531707764, + "rewards/rejected": -4.62385368347168, + "step": 425, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9506276150627615, + "grad_norm": 2.347611665725708, + "learning_rate": 0.00011635404362709044, + "logits/chosen": -0.39732852578163147, + "logits/rejected": -0.529360830783844, + "logps/chosen": -6.590923309326172, + "logps/rejected": -63.24262619018555, + "loss": 0.4288868308067322, + "memory(GiB)": 46.71, + "nll_loss": 0.33523815870285034, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24915055930614471, + "rewards/margins": 5.274869918823242, + "rewards/rejected": -5.025720119476318, + "step": 426, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9528591352859135, + "grad_norm": 1.9003043174743652, + "learning_rate": 0.00011598973839363461, + "logits/chosen": -0.34907066822052, + "logits/rejected": -0.48841482400894165, + "logps/chosen": -5.798367500305176, + "logps/rejected": -69.36884307861328, + "loss": 0.4100010395050049, + "memory(GiB)": 46.71, + "nll_loss": 0.30011022090911865, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34520187973976135, + "rewards/margins": 5.412018775939941, + "rewards/rejected": -5.066817283630371, + "step": 427, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9550906555090656, + "grad_norm": 3.1005938053131104, + "learning_rate": 0.00011562521524800631, + "logits/chosen": -0.4364015460014343, + "logits/rejected": -0.5649189949035645, + "logps/chosen": -6.55172872543335, + "logps/rejected": -69.01336669921875, + "loss": 0.44750624895095825, + "memory(GiB)": 46.71, + "nll_loss": 0.34643852710723877, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.35352104902267456, + "rewards/margins": 5.5874505043029785, + "rewards/rejected": -5.23392915725708, + "step": 428, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9573221757322176, + "grad_norm": 3.2297468185424805, + "learning_rate": 0.00011526047915801863, + "logits/chosen": -0.276310533285141, + "logits/rejected": -0.5583088397979736, + "logps/chosen": -3.0886363983154297, + "logps/rejected": -91.3880386352539, + "loss": 0.2494816780090332, + "memory(GiB)": 46.71, + "nll_loss": 0.20614728331565857, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.150741308927536, + "rewards/margins": 7.035988807678223, + "rewards/rejected": -6.885247230529785, + "step": 429, + "train_speed(iter/s)": 0.011123 + }, + { + "epoch": 0.9595536959553695, + "grad_norm": 5.799070358276367, + "learning_rate": 0.00011489553509438657, + "logits/chosen": -0.3131294548511505, + "logits/rejected": -0.5742377042770386, + "logps/chosen": -4.343443393707275, + "logps/rejected": -93.7580337524414, + "loss": 0.34438133239746094, + "memory(GiB)": 46.71, + "nll_loss": 0.31319302320480347, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15708959102630615, + "rewards/margins": 7.72693395614624, + "rewards/rejected": -7.569845199584961, + "step": 430, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9617852161785216, + "grad_norm": 6.478673458099365, + "learning_rate": 0.00011453038803065958, + "logits/chosen": -0.2996297776699066, + "logits/rejected": -0.48306334018707275, + "logps/chosen": -11.021463394165039, + "logps/rejected": -77.5682373046875, + "loss": 0.5547783374786377, + "memory(GiB)": 46.71, + "nll_loss": 0.49281901121139526, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.328575074672699, + "rewards/margins": 6.5455145835876465, + "rewards/rejected": -6.2169389724731445, + "step": 431, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9640167364016736, + "grad_norm": 3.0196564197540283, + "learning_rate": 0.00011416504294315358, + "logits/chosen": -0.3967401683330536, + "logits/rejected": -0.5093855857849121, + "logps/chosen": -11.165374755859375, + "logps/rejected": -76.78058624267578, + "loss": 0.6156377196311951, + "memory(GiB)": 46.71, + "nll_loss": 0.506489634513855, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.19617614150047302, + "rewards/margins": 5.939903259277344, + "rewards/rejected": -6.136078834533691, + "step": 432, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9662482566248256, + "grad_norm": 10.77358341217041, + "learning_rate": 0.00011379950481088317, + "logits/chosen": -0.38451313972473145, + "logits/rejected": -0.5707836151123047, + "logps/chosen": -10.529471397399902, + "logps/rejected": -57.81949234008789, + "loss": 1.0786452293395996, + "memory(GiB)": 46.71, + "nll_loss": 0.8908371925354004, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.011326950043439865, + "rewards/margins": 4.2585625648498535, + "rewards/rejected": -4.2472357749938965, + "step": 433, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9684797768479777, + "grad_norm": 2.3110084533691406, + "learning_rate": 0.00011343377861549393, + "logits/chosen": -0.30477893352508545, + "logits/rejected": -0.6313607692718506, + "logps/chosen": -4.472183704376221, + "logps/rejected": -78.5675277709961, + "loss": 0.4345514476299286, + "memory(GiB)": 46.71, + "nll_loss": 0.36963409185409546, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17769955098628998, + "rewards/margins": 6.19083309173584, + "rewards/rejected": -6.0131330490112305, + "step": 434, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9707112970711297, + "grad_norm": 1.3594601154327393, + "learning_rate": 0.00011306786934119433, + "logits/chosen": -0.35890060663223267, + "logits/rejected": -0.5802974700927734, + "logps/chosen": -7.719422817230225, + "logps/rejected": -70.55865478515625, + "loss": 0.46955499053001404, + "memory(GiB)": 46.71, + "nll_loss": 0.38888028264045715, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.42027416825294495, + "rewards/margins": 5.581704139709473, + "rewards/rejected": -5.161430358886719, + "step": 435, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9729428172942818, + "grad_norm": 3.3852317333221436, + "learning_rate": 0.00011270178197468789, + "logits/chosen": -0.36220884323120117, + "logits/rejected": -0.5496484637260437, + "logps/chosen": -5.796716690063477, + "logps/rejected": -68.04342651367188, + "loss": 0.4212677478790283, + "memory(GiB)": 46.71, + "nll_loss": 0.3155061900615692, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12593546509742737, + "rewards/margins": 5.303625583648682, + "rewards/rejected": -5.177689552307129, + "step": 436, + "train_speed(iter/s)": 0.011124 + }, + { + "epoch": 0.9751743375174338, + "grad_norm": 2.4877283573150635, + "learning_rate": 0.00011233552150510523, + "logits/chosen": -0.35733774304389954, + "logits/rejected": -0.5322983860969543, + "logps/chosen": -5.1205010414123535, + "logps/rejected": -65.1980209350586, + "loss": 0.4579578936100006, + "memory(GiB)": 46.71, + "nll_loss": 0.3468402028083801, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.29084041714668274, + "rewards/margins": 5.020003795623779, + "rewards/rejected": -4.729162693023682, + "step": 437, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.9774058577405857, + "grad_norm": 2.9676342010498047, + "learning_rate": 0.000111969092923936, + "logits/chosen": -0.36691799759864807, + "logits/rejected": -0.5556584596633911, + "logps/chosen": -8.225430488586426, + "logps/rejected": -63.60042190551758, + "loss": 0.625101625919342, + "memory(GiB)": 46.71, + "nll_loss": 0.457225501537323, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27358606457710266, + "rewards/margins": 5.005985260009766, + "rewards/rejected": -4.7323994636535645, + "step": 438, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.9796373779637378, + "grad_norm": 2.531219482421875, + "learning_rate": 0.00011160250122496105, + "logits/chosen": -0.4066939353942871, + "logits/rejected": -0.5794554352760315, + "logps/chosen": -4.745359420776367, + "logps/rejected": -61.60368347167969, + "loss": 0.44670411944389343, + "memory(GiB)": 46.71, + "nll_loss": 0.32622507214546204, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.29540878534317017, + "rewards/margins": 4.935802459716797, + "rewards/rejected": -4.6403937339782715, + "step": 439, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.9818688981868898, + "grad_norm": 1.3419512510299683, + "learning_rate": 0.00011123575140418414, + "logits/chosen": -0.3593522906303406, + "logits/rejected": -0.5466734170913696, + "logps/chosen": -6.114975929260254, + "logps/rejected": -71.61012268066406, + "loss": 0.3524559438228607, + "memory(GiB)": 46.71, + "nll_loss": 0.30310600996017456, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08320396393537521, + "rewards/margins": 5.538945198059082, + "rewards/rejected": -5.455740928649902, + "step": 440, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.9841004184100418, + "grad_norm": 1.8977224826812744, + "learning_rate": 0.00011086884845976395, + "logits/chosen": -0.37484806776046753, + "logits/rejected": -0.5711151361465454, + "logps/chosen": -4.339727401733398, + "logps/rejected": -63.90647506713867, + "loss": 0.3327326774597168, + "memory(GiB)": 46.71, + "nll_loss": 0.2596513032913208, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33903539180755615, + "rewards/margins": 5.095975399017334, + "rewards/rejected": -4.756939888000488, + "step": 441, + "train_speed(iter/s)": 0.011125 + }, + { + "epoch": 0.9863319386331939, + "grad_norm": 1.1265037059783936, + "learning_rate": 0.00011050179739194608, + "logits/chosen": -0.37898650765419006, + "logits/rejected": -0.6023902297019958, + "logps/chosen": -4.500143527984619, + "logps/rejected": -78.0557861328125, + "loss": 0.31579047441482544, + "memory(GiB)": 46.71, + "nll_loss": 0.273048460483551, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14017829298973083, + "rewards/margins": 6.283621311187744, + "rewards/rejected": -6.143443584442139, + "step": 442, + "train_speed(iter/s)": 0.011126 + }, + { + "epoch": 0.9885634588563459, + "grad_norm": 1.4137746095657349, + "learning_rate": 0.00011013460320299458, + "logits/chosen": -0.37528711557388306, + "logits/rejected": -0.5533393025398254, + "logps/chosen": -6.6545915603637695, + "logps/rejected": -63.71732711791992, + "loss": 0.4715850353240967, + "memory(GiB)": 46.71, + "nll_loss": 0.3994143605232239, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28346672654151917, + "rewards/margins": 5.119399070739746, + "rewards/rejected": -4.83593225479126, + "step": 443, + "train_speed(iter/s)": 0.011126 + }, + { + "epoch": 0.9907949790794979, + "grad_norm": 7.294280529022217, + "learning_rate": 0.00010976727089712424, + "logits/chosen": -0.43255531787872314, + "logits/rejected": -0.5511634945869446, + "logps/chosen": -5.552098751068115, + "logps/rejected": -59.277099609375, + "loss": 0.3468395173549652, + "memory(GiB)": 46.71, + "nll_loss": 0.29405713081359863, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.062217921018600464, + "rewards/margins": 4.812880039215088, + "rewards/rejected": -4.750662803649902, + "step": 444, + "train_speed(iter/s)": 0.011126 + }, + { + "epoch": 0.99302649930265, + "grad_norm": 5.627692222595215, + "learning_rate": 0.00010939980548043193, + "logits/chosen": -0.3476877808570862, + "logits/rejected": -0.5666697025299072, + "logps/chosen": -7.349039077758789, + "logps/rejected": -76.4934310913086, + "loss": 0.6328080892562866, + "memory(GiB)": 46.71, + "nll_loss": 0.5473927855491638, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0043669589795172215, + "rewards/margins": 5.797153949737549, + "rewards/rejected": -5.792786598205566, + "step": 445, + "train_speed(iter/s)": 0.011126 + }, + { + "epoch": 0.995258019525802, + "grad_norm": 2.142103672027588, + "learning_rate": 0.00010903221196082874, + "logits/chosen": -0.3306184411048889, + "logits/rejected": -0.5126135349273682, + "logps/chosen": -4.500226974487305, + "logps/rejected": -60.817535400390625, + "loss": 0.4388377368450165, + "memory(GiB)": 46.71, + "nll_loss": 0.36833876371383667, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16421036422252655, + "rewards/margins": 4.736064434051514, + "rewards/rejected": -4.571854114532471, + "step": 446, + "train_speed(iter/s)": 0.011126 + }, + { + "epoch": 0.9974895397489539, + "grad_norm": 2.382169246673584, + "learning_rate": 0.00010866449534797149, + "logits/chosen": -0.2875777781009674, + "logits/rejected": -0.5500590801239014, + "logps/chosen": -6.952953815460205, + "logps/rejected": -84.34664154052734, + "loss": 0.3859177827835083, + "memory(GiB)": 46.71, + "nll_loss": 0.3263322412967682, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31038764119148254, + "rewards/margins": 6.625545501708984, + "rewards/rejected": -6.315158367156982, + "step": 447, + "train_speed(iter/s)": 0.011127 + }, + { + "epoch": 0.999721059972106, + "grad_norm": 1.7435284852981567, + "learning_rate": 0.00010829666065319457, + "logits/chosen": -0.40691694617271423, + "logits/rejected": -0.5538883209228516, + "logps/chosen": -7.799534797668457, + "logps/rejected": -62.28192901611328, + "loss": 0.5093610882759094, + "memory(GiB)": 46.71, + "nll_loss": 0.4454079270362854, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5661957263946533, + "rewards/margins": 5.261015892028809, + "rewards/rejected": -4.694820880889893, + "step": 448, + "train_speed(iter/s)": 0.011127 + }, + { + "epoch": 1.0, + "grad_norm": 1.7435284852981567, + "learning_rate": 0.00010792871288944164, + "logits/chosen": -0.44425058364868164, + "logits/rejected": -0.567035436630249, + "logps/chosen": -1.6898080110549927, + "logps/rejected": -61.33506393432617, + "loss": 0.019901663064956665, + "memory(GiB)": 46.71, + "nll_loss": 0.15361957252025604, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4040493369102478, + "rewards/margins": 5.698512077331543, + "rewards/rejected": -5.294462203979492, + "step": 449, + "train_speed(iter/s)": 0.011149 + }, + { + "epoch": 1.002231520223152, + "grad_norm": 1.3548859357833862, + "learning_rate": 0.00010756065707119729, + "logits/chosen": -0.33319422602653503, + "logits/rejected": -0.5021852850914001, + "logps/chosen": -5.385560035705566, + "logps/rejected": -74.84062194824219, + "loss": 0.3303074538707733, + "memory(GiB)": 46.71, + "nll_loss": 0.2911924123764038, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42078158259391785, + "rewards/margins": 6.181977272033691, + "rewards/rejected": -5.761194705963135, + "step": 450, + "train_speed(iter/s)": 0.011149 + } + ], + "logging_steps": 1, + "max_steps": 896, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.469740707410084e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}