diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,5451 @@ +{ + "best_global_step": 300, + "best_metric": 0.54357338, + "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1+3_dpo_absgt14_noidk__pari0.8_0608/v0-20250610-025054/checkpoint-300", + "epoch": 1.8009032743695896, + "eval_steps": 300, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0060218291305984195, + "grad_norm": 7.15871000289917, + "learning_rate": 1.1764705882352942e-05, + "logits/chosen": -0.32459306716918945, + "logits/rejected": -0.5000401735305786, + "logps/chosen": -3.3705074787139893, + "logps/rejected": -19.54801368713379, + "loss": 1.1142030954360962, + "memory(GiB)": 46.13, + "nll_loss": 0.4210559129714966, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1, + "train_speed(iter/s)": 0.005428 + }, + { + "epoch": 0.012043658261196839, + "grad_norm": 6.311911106109619, + "learning_rate": 2.3529411764705884e-05, + "logits/chosen": -0.41802966594696045, + "logits/rejected": -0.5096024870872498, + "logps/chosen": -8.828028678894043, + "logps/rejected": -13.557395935058594, + "loss": 1.4290869235992432, + "memory(GiB)": 46.13, + "nll_loss": 0.7359397411346436, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2, + "train_speed(iter/s)": 0.005545 + }, + { + "epoch": 0.018065487391795258, + "grad_norm": 6.693342685699463, + "learning_rate": 3.529411764705883e-05, + "logits/chosen": -0.3694347143173218, + "logits/rejected": -0.4798021912574768, + "logps/chosen": -6.542965412139893, + "logps/rejected": -17.62507438659668, + "loss": 1.2220425605773926, + "memory(GiB)": 46.13, + "nll_loss": 0.5347560048103333, + "rewards/accuracies": 0.734375, + "rewards/chosen": 0.007337508723139763, + "rewards/margins": 0.011953208595514297, + "rewards/rejected": -0.004615699406713247, + "step": 3, + "train_speed(iter/s)": 0.005572 + }, + { + "epoch": 0.024087316522393678, + "grad_norm": 4.8458251953125, + "learning_rate": 4.705882352941177e-05, + "logits/chosen": -0.36237555742263794, + "logits/rejected": -0.5201014280319214, + "logps/chosen": -5.53680944442749, + "logps/rejected": -16.475929260253906, + "loss": 1.2521781921386719, + "memory(GiB)": 46.13, + "nll_loss": 0.579364538192749, + "rewards/accuracies": 0.734375, + "rewards/chosen": 0.01270484272390604, + "rewards/margins": 0.04416074603796005, + "rewards/rejected": -0.03145590424537659, + "step": 4, + "train_speed(iter/s)": 0.005594 + }, + { + "epoch": 0.030109145652992095, + "grad_norm": 3.684136152267456, + "learning_rate": 5.882352941176471e-05, + "logits/chosen": -0.3457816541194916, + "logits/rejected": -0.48865967988967896, + "logps/chosen": -4.437034606933594, + "logps/rejected": -20.260162353515625, + "loss": 0.9991621375083923, + "memory(GiB)": 46.13, + "nll_loss": 0.38671040534973145, + "rewards/accuracies": 0.734375, + "rewards/chosen": 0.002155877649784088, + "rewards/margins": 0.2208954095840454, + "rewards/rejected": -0.21873953938484192, + "step": 5, + "train_speed(iter/s)": 0.005606 + }, + { + "epoch": 0.036130974783590515, + "grad_norm": 6.8094706535339355, + "learning_rate": 7.058823529411765e-05, + "logits/chosen": -0.37496131658554077, + "logits/rejected": -0.499918133020401, + "logps/chosen": -5.312520980834961, + "logps/rejected": -18.596221923828125, + "loss": 1.1320774555206299, + "memory(GiB)": 46.13, + "nll_loss": 0.5494582056999207, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11241161823272705, + "rewards/margins": 0.37036794424057007, + "rewards/rejected": -0.4827795624732971, + "step": 6, + "train_speed(iter/s)": 0.005617 + }, + { + "epoch": 0.042152803914188935, + "grad_norm": 6.563366889953613, + "learning_rate": 8.23529411764706e-05, + "logits/chosen": -0.3803170323371887, + "logits/rejected": -0.43647128343582153, + "logps/chosen": -12.155686378479004, + "logps/rejected": -19.252357482910156, + "loss": 1.275801420211792, + "memory(GiB)": 46.13, + "nll_loss": 0.6294467449188232, + "rewards/accuracies": 0.609375, + "rewards/chosen": -0.2581723928451538, + "rewards/margins": 0.26009732484817505, + "rewards/rejected": -0.5182697176933289, + "step": 7, + "train_speed(iter/s)": 0.005615 + }, + { + "epoch": 0.048174633044787356, + "grad_norm": 5.275461673736572, + "learning_rate": 9.411764705882353e-05, + "logits/chosen": -0.30787912011146545, + "logits/rejected": -0.420376718044281, + "logps/chosen": -7.271793842315674, + "logps/rejected": -17.376087188720703, + "loss": 1.2506061792373657, + "memory(GiB)": 46.13, + "nll_loss": 0.6039291024208069, + "rewards/accuracies": 0.609375, + "rewards/chosen": -0.07998734712600708, + "rewards/margins": 0.2555825710296631, + "rewards/rejected": -0.3355698883533478, + "step": 8, + "train_speed(iter/s)": 0.005615 + }, + { + "epoch": 0.054196462175385776, + "grad_norm": 3.0540287494659424, + "learning_rate": 0.00010588235294117647, + "logits/chosen": -0.30379173159599304, + "logits/rejected": -0.41216912865638733, + "logps/chosen": -6.820768356323242, + "logps/rejected": -16.3853816986084, + "loss": 1.1634117364883423, + "memory(GiB)": 46.13, + "nll_loss": 0.5367689728736877, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.04989926144480705, + "rewards/margins": 0.1963551640510559, + "rewards/rejected": -0.14645591378211975, + "step": 9, + "train_speed(iter/s)": 0.005615 + }, + { + "epoch": 0.06021829130598419, + "grad_norm": 2.3425395488739014, + "learning_rate": 0.00011764705882352942, + "logits/chosen": -0.3369804918766022, + "logits/rejected": -0.4523884356021881, + "logps/chosen": -7.222663402557373, + "logps/rejected": -17.98470687866211, + "loss": 1.0803574323654175, + "memory(GiB)": 46.13, + "nll_loss": 0.4635474979877472, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0936385914683342, + "rewards/margins": 0.19075465202331543, + "rewards/rejected": -0.09711606800556183, + "step": 10, + "train_speed(iter/s)": 0.005616 + }, + { + "epoch": 0.06624012043658262, + "grad_norm": 2.2927637100219727, + "learning_rate": 0.00012941176470588237, + "logits/chosen": -0.37337860465049744, + "logits/rejected": -0.4716382920742035, + "logps/chosen": -8.593006134033203, + "logps/rejected": -17.979745864868164, + "loss": 1.1164621114730835, + "memory(GiB)": 46.13, + "nll_loss": 0.49796244502067566, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.06070924177765846, + "rewards/margins": 0.19886666536331177, + "rewards/rejected": -0.1381574273109436, + "step": 11, + "train_speed(iter/s)": 0.005619 + }, + { + "epoch": 0.07226194956718103, + "grad_norm": 2.380297899246216, + "learning_rate": 0.0001411764705882353, + "logits/chosen": -0.4056618809700012, + "logits/rejected": -0.47582757472991943, + "logps/chosen": -7.429853916168213, + "logps/rejected": -16.317821502685547, + "loss": 1.167059302330017, + "memory(GiB)": 46.13, + "nll_loss": 0.5555555820465088, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.0010902846697717905, + "rewards/margins": 0.21717122197151184, + "rewards/rejected": -0.2182615101337433, + "step": 12, + "train_speed(iter/s)": 0.005622 + }, + { + "epoch": 0.07828377869777944, + "grad_norm": 1.7392877340316772, + "learning_rate": 0.00015294117647058822, + "logits/chosen": -0.4110736846923828, + "logits/rejected": -0.4770917296409607, + "logps/chosen": -8.422821044921875, + "logps/rejected": -16.356956481933594, + "loss": 1.1002569198608398, + "memory(GiB)": 46.13, + "nll_loss": 0.5357962250709534, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.12992499768733978, + "rewards/margins": 0.3372637927532196, + "rewards/rejected": -0.20733879506587982, + "step": 13, + "train_speed(iter/s)": 0.005625 + }, + { + "epoch": 0.08430560782837787, + "grad_norm": 5.562183856964111, + "learning_rate": 0.0001647058823529412, + "logits/chosen": -0.32676243782043457, + "logits/rejected": -0.4272843599319458, + "logps/chosen": -8.393424034118652, + "logps/rejected": -19.974212646484375, + "loss": 1.156798005104065, + "memory(GiB)": 46.13, + "nll_loss": 0.6415989995002747, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.03989575803279877, + "rewards/margins": 0.5587274432182312, + "rewards/rejected": -0.5986231565475464, + "step": 14, + "train_speed(iter/s)": 0.005627 + }, + { + "epoch": 0.09032743695897628, + "grad_norm": 3.5578579902648926, + "learning_rate": 0.00017647058823529413, + "logits/chosen": -0.36884137988090515, + "logits/rejected": -0.47317999601364136, + "logps/chosen": -6.998829364776611, + "logps/rejected": -18.39483642578125, + "loss": 1.265438199043274, + "memory(GiB)": 46.13, + "nll_loss": 0.7606177926063538, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.002129599452018738, + "rewards/margins": 0.5497386455535889, + "rewards/rejected": -0.5476090908050537, + "step": 15, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.09634926608957471, + "grad_norm": 3.6414742469787598, + "learning_rate": 0.00018823529411764707, + "logits/chosen": -0.3789006471633911, + "logits/rejected": -0.4743453860282898, + "logps/chosen": -6.895637512207031, + "logps/rejected": -20.256675720214844, + "loss": 1.064023494720459, + "memory(GiB)": 46.13, + "nll_loss": 0.5308047533035278, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.055957891047000885, + "rewards/margins": 0.40152865648269653, + "rewards/rejected": -0.34557074308395386, + "step": 16, + "train_speed(iter/s)": 0.005632 + }, + { + "epoch": 0.10237109522017313, + "grad_norm": 3.263599157333374, + "learning_rate": 0.0002, + "logits/chosen": -0.28460752964019775, + "logits/rejected": -0.42314791679382324, + "logps/chosen": -5.295862197875977, + "logps/rejected": -21.666519165039062, + "loss": 0.9750788807868958, + "memory(GiB)": 46.13, + "nll_loss": 0.5031211376190186, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08322687447071075, + "rewards/margins": 0.5906113386154175, + "rewards/rejected": -0.5073844194412231, + "step": 17, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.10839292435077155, + "grad_norm": 2.0240275859832764, + "learning_rate": 0.00019999502669559432, + "logits/chosen": -0.3283552825450897, + "logits/rejected": -0.40596985816955566, + "logps/chosen": -6.424624919891357, + "logps/rejected": -17.71906852722168, + "loss": 0.9781989455223083, + "memory(GiB)": 46.13, + "nll_loss": 0.47217291593551636, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.0207663644105196, + "rewards/margins": 0.5476341247558594, + "rewards/rejected": -0.568400502204895, + "step": 18, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.11441475348136997, + "grad_norm": 1.682889699935913, + "learning_rate": 0.00019998010727705236, + "logits/chosen": -0.2821420729160309, + "logits/rejected": -0.38918256759643555, + "logps/chosen": -5.882609844207764, + "logps/rejected": -20.25113296508789, + "loss": 0.9472936391830444, + "memory(GiB)": 46.13, + "nll_loss": 0.4797293543815613, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.06502628326416016, + "rewards/margins": 0.6772950291633606, + "rewards/rejected": -0.6122686862945557, + "step": 19, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.12043658261196838, + "grad_norm": 2.9386143684387207, + "learning_rate": 0.00019995524322835034, + "logits/chosen": -0.2649421691894531, + "logits/rejected": -0.34010952711105347, + "logps/chosen": -10.95669174194336, + "logps/rejected": -24.044157028198242, + "loss": 1.1134816408157349, + "memory(GiB)": 46.13, + "nll_loss": 0.546178936958313, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.18016880750656128, + "rewards/margins": 0.5494988560676575, + "rewards/rejected": -0.729667603969574, + "step": 20, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.1264584117425668, + "grad_norm": 1.6700323820114136, + "learning_rate": 0.00019992043702261793, + "logits/chosen": -0.2561277151107788, + "logits/rejected": -0.3126121163368225, + "logps/chosen": -6.485073566436768, + "logps/rejected": -19.772449493408203, + "loss": 0.9205772876739502, + "memory(GiB)": 46.13, + "nll_loss": 0.42214635014533997, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.0028960183262825012, + "rewards/margins": 0.6638913750648499, + "rewards/rejected": -0.6667873859405518, + "step": 21, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.13248024087316523, + "grad_norm": 2.0114383697509766, + "learning_rate": 0.00019987569212189224, + "logits/chosen": -0.2601704001426697, + "logits/rejected": -0.3789623975753784, + "logps/chosen": -6.645152568817139, + "logps/rejected": -21.876020431518555, + "loss": 1.1017377376556396, + "memory(GiB)": 46.13, + "nll_loss": 0.6103405356407166, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.04465393349528313, + "rewards/margins": 0.6134660840034485, + "rewards/rejected": -0.5688121318817139, + "step": 22, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.13850207000376363, + "grad_norm": 3.031888961791992, + "learning_rate": 0.0001998210129767735, + "logits/chosen": -0.26405903697013855, + "logits/rejected": -0.3669392466545105, + "logps/chosen": -6.921745300292969, + "logps/rejected": -21.70663070678711, + "loss": 0.9795225858688354, + "memory(GiB)": 46.13, + "nll_loss": 0.4959058463573456, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.10062012821435928, + "rewards/margins": 0.6365569829940796, + "rewards/rejected": -0.5359368324279785, + "step": 23, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.14452389913436206, + "grad_norm": 2.15742564201355, + "learning_rate": 0.00019975640502598244, + "logits/chosen": -0.2354922890663147, + "logits/rejected": -0.36152032017707825, + "logps/chosen": -4.718395709991455, + "logps/rejected": -24.57033348083496, + "loss": 0.7706338763237, + "memory(GiB)": 46.13, + "nll_loss": 0.3560357987880707, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.09812817722558975, + "rewards/margins": 0.9238024950027466, + "rewards/rejected": -0.825674295425415, + "step": 24, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.1505457282649605, + "grad_norm": 3.1512320041656494, + "learning_rate": 0.0001996818746958191, + "logits/chosen": -0.21456243097782135, + "logits/rejected": -0.34130796790122986, + "logps/chosen": -6.996722221374512, + "logps/rejected": -30.622554779052734, + "loss": 0.8399394750595093, + "memory(GiB)": 46.13, + "nll_loss": 0.49700063467025757, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.024456650018692017, + "rewards/margins": 1.4392318725585938, + "rewards/rejected": -1.4636887311935425, + "step": 25, + "train_speed(iter/s)": 0.005628 + }, + { + "epoch": 0.1565675573955589, + "grad_norm": 2.9886889457702637, + "learning_rate": 0.00019959742939952392, + "logits/chosen": -0.2622988224029541, + "logits/rejected": -0.3251183331012726, + "logps/chosen": -12.495084762573242, + "logps/rejected": -25.176090240478516, + "loss": 1.1550263166427612, + "memory(GiB)": 46.13, + "nll_loss": 0.7094165086746216, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.19754531979560852, + "rewards/margins": 1.1228585243225098, + "rewards/rejected": -1.3204039335250854, + "step": 26, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.16258938652615731, + "grad_norm": 2.7348685264587402, + "learning_rate": 0.00019950307753654017, + "logits/chosen": -0.23797279596328735, + "logits/rejected": -0.3161388337612152, + "logps/chosen": -8.345640182495117, + "logps/rejected": -24.480266571044922, + "loss": 1.0113354921340942, + "memory(GiB)": 46.13, + "nll_loss": 0.5643945336341858, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.05142675340175629, + "rewards/margins": 1.0910050868988037, + "rewards/rejected": -1.0395784378051758, + "step": 27, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.16861121565675574, + "grad_norm": 3.4209959506988525, + "learning_rate": 0.00019939882849167852, + "logits/chosen": -0.2834317684173584, + "logits/rejected": -0.4013361930847168, + "logps/chosen": -6.27006196975708, + "logps/rejected": -23.9411678314209, + "loss": 0.9620530605316162, + "memory(GiB)": 46.13, + "nll_loss": 0.5452341437339783, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.08509417623281479, + "rewards/margins": 1.0887510776519775, + "rewards/rejected": -1.0036569833755493, + "step": 28, + "train_speed(iter/s)": 0.005632 + }, + { + "epoch": 0.17463304478735417, + "grad_norm": 3.021099805831909, + "learning_rate": 0.00019928469263418374, + "logits/chosen": -0.29641368985176086, + "logits/rejected": -0.3944772779941559, + "logps/chosen": -6.065124988555908, + "logps/rejected": -19.367380142211914, + "loss": 0.8545750379562378, + "memory(GiB)": 46.13, + "nll_loss": 0.3696524500846863, + "rewards/accuracies": 0.859375, + "rewards/chosen": -0.008128602989017963, + "rewards/margins": 0.7214774489402771, + "rewards/rejected": -0.729606032371521, + "step": 29, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.18065487391795257, + "grad_norm": 2.237170457839966, + "learning_rate": 0.00019916068131670302, + "logits/chosen": -0.2665432393550873, + "logits/rejected": -0.41999131441116333, + "logps/chosen": -3.6829960346221924, + "logps/rejected": -24.998981475830078, + "loss": 0.7719818353652954, + "memory(GiB)": 46.13, + "nll_loss": 0.3786780834197998, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.15382343530654907, + "rewards/margins": 1.0922845602035522, + "rewards/rejected": -0.9384610652923584, + "step": 30, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.186676703048551, + "grad_norm": 2.299628496170044, + "learning_rate": 0.00019902680687415705, + "logits/chosen": -0.33755987882614136, + "logits/rejected": -0.37323832511901855, + "logps/chosen": -11.234481811523438, + "logps/rejected": -24.92943000793457, + "loss": 1.0785183906555176, + "memory(GiB)": 46.13, + "nll_loss": 0.6123806834220886, + "rewards/accuracies": 0.765625, + "rewards/chosen": 0.03546600416302681, + "rewards/margins": 0.9260610342025757, + "rewards/rejected": -0.890595018863678, + "step": 31, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.19269853217914942, + "grad_norm": 2.0755157470703125, + "learning_rate": 0.00019888308262251285, + "logits/chosen": -0.2504952847957611, + "logits/rejected": -0.3852018713951111, + "logps/chosen": -5.893071174621582, + "logps/rejected": -33.010154724121094, + "loss": 0.7333768010139465, + "memory(GiB)": 46.13, + "nll_loss": 0.4039708375930786, + "rewards/accuracies": 0.859375, + "rewards/chosen": -0.07270973920822144, + "rewards/margins": 1.7266591787338257, + "rewards/rejected": -1.7993686199188232, + "step": 32, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.19872036130974782, + "grad_norm": 3.686795711517334, + "learning_rate": 0.00019872952285745959, + "logits/chosen": -0.30149099230766296, + "logits/rejected": -0.4157900810241699, + "logps/chosen": -7.993706703186035, + "logps/rejected": -27.117816925048828, + "loss": 1.0753562450408936, + "memory(GiB)": 46.13, + "nll_loss": 0.6016828417778015, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.10546419769525528, + "rewards/margins": 1.0370161533355713, + "rewards/rejected": -1.1424803733825684, + "step": 33, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.20474219044034625, + "grad_norm": 3.085048198699951, + "learning_rate": 0.0001985661428529863, + "logits/chosen": -0.31711238622665405, + "logits/rejected": -0.41008925437927246, + "logps/chosen": -6.8641557693481445, + "logps/rejected": -27.393281936645508, + "loss": 1.054256558418274, + "memory(GiB)": 46.13, + "nll_loss": 0.6109166741371155, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.06303416192531586, + "rewards/margins": 1.1662019491195679, + "rewards/rejected": -1.229236125946045, + "step": 34, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.21076401957094468, + "grad_norm": 1.7560195922851562, + "learning_rate": 0.00019839295885986296, + "logits/chosen": -0.337604820728302, + "logits/rejected": -0.39799487590789795, + "logps/chosen": -6.864170551300049, + "logps/rejected": -20.806344985961914, + "loss": 0.9619123339653015, + "memory(GiB)": 46.13, + "nll_loss": 0.48513123393058777, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.10183387249708176, + "rewards/margins": 0.7188172340393066, + "rewards/rejected": -0.6169832944869995, + "step": 35, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.2167858487015431, + "grad_norm": 2.6364471912384033, + "learning_rate": 0.0001982099881040239, + "logits/chosen": -0.36966580152511597, + "logits/rejected": -0.43631935119628906, + "logps/chosen": -6.647594451904297, + "logps/rejected": -21.291913986206055, + "loss": 0.9380239844322205, + "memory(GiB)": 46.13, + "nll_loss": 0.4613775312900543, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.06358329951763153, + "rewards/margins": 0.7893975973129272, + "rewards/rejected": -0.7258143424987793, + "step": 36, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.2228076778321415, + "grad_norm": 2.354177474975586, + "learning_rate": 0.00019801724878485438, + "logits/chosen": -0.36403897404670715, + "logits/rejected": -0.45796939730644226, + "logps/chosen": -5.62083625793457, + "logps/rejected": -23.648834228515625, + "loss": 0.9243438243865967, + "memory(GiB)": 46.13, + "nll_loss": 0.46970799565315247, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.06422886252403259, + "rewards/margins": 0.8610265851020813, + "rewards/rejected": -0.7967977523803711, + "step": 37, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.22882950696273993, + "grad_norm": 2.2988836765289307, + "learning_rate": 0.00019781476007338058, + "logits/chosen": -0.3704385459423065, + "logits/rejected": -0.47973009943962097, + "logps/chosen": -4.231719017028809, + "logps/rejected": -20.56240463256836, + "loss": 0.916353702545166, + "memory(GiB)": 46.13, + "nll_loss": 0.4209356904029846, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.06186368316411972, + "rewards/margins": 0.7635293006896973, + "rewards/rejected": -0.7016656398773193, + "step": 38, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.23485133609333836, + "grad_norm": 2.009106397628784, + "learning_rate": 0.00019760254211036244, + "logits/chosen": -0.3372359573841095, + "logits/rejected": -0.453107088804245, + "logps/chosen": -5.785426616668701, + "logps/rejected": -23.19124412536621, + "loss": 0.9124959707260132, + "memory(GiB)": 46.13, + "nll_loss": 0.44838690757751465, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.04832564294338226, + "rewards/margins": 0.937610387802124, + "rewards/rejected": -0.8892846703529358, + "step": 39, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.24087316522393676, + "grad_norm": 2.5412213802337646, + "learning_rate": 0.00019738061600429064, + "logits/chosen": -0.39427676796913147, + "logits/rejected": -0.48203372955322266, + "logps/chosen": -5.785412311553955, + "logps/rejected": -19.031665802001953, + "loss": 0.9683349132537842, + "memory(GiB)": 46.13, + "nll_loss": 0.5110080242156982, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.022132933139801025, + "rewards/margins": 0.8074027299880981, + "rewards/rejected": -0.7852697372436523, + "step": 40, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.24689499435453519, + "grad_norm": 2.0049972534179688, + "learning_rate": 0.00019714900382928675, + "logits/chosen": -0.3044354021549225, + "logits/rejected": -0.41962650418281555, + "logps/chosen": -4.612288475036621, + "logps/rejected": -24.228839874267578, + "loss": 0.8179229497909546, + "memory(GiB)": 46.13, + "nll_loss": 0.4278942048549652, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.07943032681941986, + "rewards/margins": 1.1561895608901978, + "rewards/rejected": -1.0767593383789062, + "step": 41, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.2529168234851336, + "grad_norm": 2.0874667167663574, + "learning_rate": 0.0001969077286229078, + "logits/chosen": -0.2736699879169464, + "logits/rejected": -0.4168338179588318, + "logps/chosen": -7.539179801940918, + "logps/rejected": -30.339994430541992, + "loss": 0.9962757229804993, + "memory(GiB)": 46.13, + "nll_loss": 0.6084927320480347, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.046928200870752335, + "rewards/margins": 1.4784610271453857, + "rewards/rejected": -1.431532859802246, + "step": 42, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.258938652615732, + "grad_norm": 3.2748661041259766, + "learning_rate": 0.00019665681438385473, + "logits/chosen": -0.26249000430107117, + "logits/rejected": -0.3955192565917969, + "logps/chosen": -6.726841449737549, + "logps/rejected": -26.447267532348633, + "loss": 0.9923527836799622, + "memory(GiB)": 46.13, + "nll_loss": 0.5636712312698364, + "rewards/accuracies": 0.859375, + "rewards/chosen": -0.02616805024445057, + "rewards/margins": 1.2797741889953613, + "rewards/rejected": -1.3059422969818115, + "step": 43, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.26496048174633047, + "grad_norm": 4.499722957611084, + "learning_rate": 0.00019639628606958533, + "logits/chosen": -0.3380294442176819, + "logits/rejected": -0.4303723871707916, + "logps/chosen": -7.925315856933594, + "logps/rejected": -23.916906356811523, + "loss": 1.136991024017334, + "memory(GiB)": 46.13, + "nll_loss": 0.6508135795593262, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.1142488494515419, + "rewards/margins": 1.0733367204666138, + "rewards/rejected": -1.1875855922698975, + "step": 44, + "train_speed(iter/s)": 0.005638 + }, + { + "epoch": 0.27098231087692887, + "grad_norm": 3.695734739303589, + "learning_rate": 0.0001961261695938319, + "logits/chosen": -0.2465212196111679, + "logits/rejected": -0.39715391397476196, + "logps/chosen": -7.8324666023254395, + "logps/rejected": -24.97869110107422, + "loss": 0.9623777270317078, + "memory(GiB)": 46.13, + "nll_loss": 0.5307109951972961, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.08475106954574585, + "rewards/margins": 0.9390297532081604, + "rewards/rejected": -0.8542786836624146, + "step": 45, + "train_speed(iter/s)": 0.005638 + }, + { + "epoch": 0.27700414000752727, + "grad_norm": 3.1841602325439453, + "learning_rate": 0.00019584649182402357, + "logits/chosen": -0.22775670886039734, + "logits/rejected": -0.3454330265522003, + "logps/chosen": -6.5436859130859375, + "logps/rejected": -24.402027130126953, + "loss": 1.0284608602523804, + "memory(GiB)": 46.13, + "nll_loss": 0.5513083338737488, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.05229588598012924, + "rewards/margins": 0.9507507085800171, + "rewards/rejected": -0.8984547853469849, + "step": 46, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.2830259691381257, + "grad_norm": 2.0325071811676025, + "learning_rate": 0.0001955572805786141, + "logits/chosen": -0.19194024801254272, + "logits/rejected": -0.32474127411842346, + "logps/chosen": -4.646670341491699, + "logps/rejected": -27.415727615356445, + "loss": 0.7930896878242493, + "memory(GiB)": 46.13, + "nll_loss": 0.38733866810798645, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.11078543961048126, + "rewards/margins": 1.1206859350204468, + "rewards/rejected": -1.009900450706482, + "step": 47, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.2890477982687241, + "grad_norm": 2.07770037651062, + "learning_rate": 0.0001952585646243146, + "logits/chosen": -0.25676336884498596, + "logits/rejected": -0.3494555950164795, + "logps/chosen": -7.388433933258057, + "logps/rejected": -23.070579528808594, + "loss": 0.8534475564956665, + "memory(GiB)": 46.13, + "nll_loss": 0.42261388897895813, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.006194199435412884, + "rewards/margins": 0.9558709859848022, + "rewards/rejected": -0.9496768712997437, + "step": 48, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.2950696273993225, + "grad_norm": 1.6862146854400635, + "learning_rate": 0.00019495037367323262, + "logits/chosen": -0.22853028774261475, + "logits/rejected": -0.3647540211677551, + "logps/chosen": -3.3972930908203125, + "logps/rejected": -25.391393661499023, + "loss": 0.7584111094474792, + "memory(GiB)": 46.13, + "nll_loss": 0.35165295004844666, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.12043334543704987, + "rewards/margins": 1.1430784463882446, + "rewards/rejected": -1.022645115852356, + "step": 49, + "train_speed(iter/s)": 0.005638 + }, + { + "epoch": 0.301091456529921, + "grad_norm": 3.027331829071045, + "learning_rate": 0.00019463273837991643, + "logits/chosen": -0.2110910564661026, + "logits/rejected": -0.38482916355133057, + "logps/chosen": -4.55904483795166, + "logps/rejected": -31.877506256103516, + "loss": 0.675049901008606, + "memory(GiB)": 46.13, + "nll_loss": 0.32799023389816284, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.05019056051969528, + "rewards/margins": 1.6790026426315308, + "rewards/rejected": -1.6288120746612549, + "step": 50, + "train_speed(iter/s)": 0.005638 + }, + { + "epoch": 0.3071132856605194, + "grad_norm": 2.950259208679199, + "learning_rate": 0.00019430569033830605, + "logits/chosen": -0.19638891518115997, + "logits/rejected": -0.3516000807285309, + "logps/chosen": -7.3598952293396, + "logps/rejected": -32.14203643798828, + "loss": 0.9957353472709656, + "memory(GiB)": 46.13, + "nll_loss": 0.6411410570144653, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.031813204288482666, + "rewards/margins": 1.6832952499389648, + "rewards/rejected": -1.7151083946228027, + "step": 51, + "train_speed(iter/s)": 0.005628 + }, + { + "epoch": 0.3131351147911178, + "grad_norm": 3.4817123413085938, + "learning_rate": 0.00019396926207859084, + "logits/chosen": -0.2925015687942505, + "logits/rejected": -0.4453135132789612, + "logps/chosen": -6.357159614562988, + "logps/rejected": -34.39080047607422, + "loss": 0.9899907112121582, + "memory(GiB)": 46.13, + "nll_loss": 0.6184481382369995, + "rewards/accuracies": 0.828125, + "rewards/chosen": -0.06884388625621796, + "rewards/margins": 1.8842136859893799, + "rewards/rejected": -1.9530572891235352, + "step": 52, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.31915694392171623, + "grad_norm": 3.4239518642425537, + "learning_rate": 0.00019362348706397373, + "logits/chosen": -0.22639772295951843, + "logits/rejected": -0.4221537709236145, + "logps/chosen": -5.600902080535889, + "logps/rejected": -36.40532684326172, + "loss": 0.8088786602020264, + "memory(GiB)": 46.13, + "nll_loss": 0.4771510660648346, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.0569700188934803, + "rewards/margins": 1.8676366806030273, + "rewards/rejected": -1.810666561126709, + "step": 53, + "train_speed(iter/s)": 0.005628 + }, + { + "epoch": 0.32517877305231463, + "grad_norm": 3.8685495853424072, + "learning_rate": 0.00019326839968734279, + "logits/chosen": -0.2569487690925598, + "logits/rejected": -0.40343666076660156, + "logps/chosen": -5.900696277618408, + "logps/rejected": -36.28150177001953, + "loss": 0.7994194030761719, + "memory(GiB)": 46.13, + "nll_loss": 0.43986907601356506, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.030100831761956215, + "rewards/margins": 1.9989190101623535, + "rewards/rejected": -1.968818187713623, + "step": 54, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.3312006021829131, + "grad_norm": 3.0355138778686523, + "learning_rate": 0.00019290403526785025, + "logits/chosen": -0.25347286462783813, + "logits/rejected": -0.3757534623146057, + "logps/chosen": -7.442959308624268, + "logps/rejected": -32.45537567138672, + "loss": 0.903397798538208, + "memory(GiB)": 46.13, + "nll_loss": 0.531738817691803, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.0414905920624733, + "rewards/margins": 1.6564455032348633, + "rewards/rejected": -1.614954948425293, + "step": 55, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.3372224313135115, + "grad_norm": 3.739659070968628, + "learning_rate": 0.00019253043004739968, + "logits/chosen": -0.20877386629581451, + "logits/rejected": -0.37541496753692627, + "logps/chosen": -7.142071723937988, + "logps/rejected": -32.15766906738281, + "loss": 0.9619933366775513, + "memory(GiB)": 46.13, + "nll_loss": 0.4930208921432495, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.08227778971195221, + "rewards/margins": 1.5348973274230957, + "rewards/rejected": -1.6171751022338867, + "step": 56, + "train_speed(iter/s)": 0.005628 + }, + { + "epoch": 0.3432442604441099, + "grad_norm": 2.4167847633361816, + "learning_rate": 0.00019214762118704076, + "logits/chosen": -0.32553619146347046, + "logits/rejected": -0.44049111008644104, + "logps/chosen": -6.96627140045166, + "logps/rejected": -24.777843475341797, + "loss": 0.8523041605949402, + "memory(GiB)": 46.13, + "nll_loss": 0.44423431158065796, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.08760333806276321, + "rewards/margins": 1.0987879037857056, + "rewards/rejected": -1.011184573173523, + "step": 57, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.34926608957470834, + "grad_norm": 1.849902629852295, + "learning_rate": 0.00019175564676327339, + "logits/chosen": -0.29927390813827515, + "logits/rejected": -0.4095182418823242, + "logps/chosen": -6.824729919433594, + "logps/rejected": -31.636598587036133, + "loss": 0.827016294002533, + "memory(GiB)": 46.13, + "nll_loss": 0.432888925075531, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.10196143388748169, + "rewards/margins": 1.6724345684051514, + "rewards/rejected": -1.5704729557037354, + "step": 58, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.35528791870530674, + "grad_norm": 2.3601560592651367, + "learning_rate": 0.0001913545457642601, + "logits/chosen": -0.31936806440353394, + "logits/rejected": -0.42600923776626587, + "logps/chosen": -6.396060943603516, + "logps/rejected": -26.946306228637695, + "loss": 0.9836419224739075, + "memory(GiB)": 46.13, + "nll_loss": 0.5654138326644897, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.013314278796315193, + "rewards/margins": 1.2707021236419678, + "rewards/rejected": -1.2573878765106201, + "step": 59, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.36130974783590514, + "grad_norm": 2.7022287845611572, + "learning_rate": 0.00019094435808594823, + "logits/chosen": -0.3565162420272827, + "logits/rejected": -0.48129239678382874, + "logps/chosen": -4.523666858673096, + "logps/rejected": -30.854751586914062, + "loss": 0.7463247776031494, + "memory(GiB)": 46.13, + "nll_loss": 0.41281038522720337, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.06385684013366699, + "rewards/margins": 1.8442981243133545, + "rewards/rejected": -1.7804412841796875, + "step": 60, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.3673315769665036, + "grad_norm": 2.549546957015991, + "learning_rate": 0.0001905251245281015, + "logits/chosen": -0.2870449125766754, + "logits/rejected": -0.4699952006340027, + "logps/chosen": -3.242504596710205, + "logps/rejected": -35.58238220214844, + "loss": 0.6698994636535645, + "memory(GiB)": 46.13, + "nll_loss": 0.3660232126712799, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.03870222344994545, + "rewards/margins": 1.8160982131958008, + "rewards/rejected": -1.7773959636688232, + "step": 61, + "train_speed(iter/s)": 0.005631 + }, + { + "epoch": 0.373353406097102, + "grad_norm": 3.344719409942627, + "learning_rate": 0.0001900968867902419, + "logits/chosen": -0.26915398240089417, + "logits/rejected": -0.38059523701667786, + "logps/chosen": -6.996611595153809, + "logps/rejected": -36.200714111328125, + "loss": 0.8631665110588074, + "memory(GiB)": 46.13, + "nll_loss": 0.5069284439086914, + "rewards/accuracies": 0.828125, + "rewards/chosen": -0.0650690495967865, + "rewards/margins": 1.7960646152496338, + "rewards/rejected": -1.8611338138580322, + "step": 62, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.3793752352277004, + "grad_norm": 1.7888697385787964, + "learning_rate": 0.0001896596874675021, + "logits/chosen": -0.25557130575180054, + "logits/rejected": -0.4095707833766937, + "logps/chosen": -5.574047088623047, + "logps/rejected": -31.464292526245117, + "loss": 0.6977128386497498, + "memory(GiB)": 46.13, + "nll_loss": 0.3594370484352112, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.08366499841213226, + "rewards/margins": 1.6534584760665894, + "rewards/rejected": -1.569793462753296, + "step": 63, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.38539706435829885, + "grad_norm": 2.8035733699798584, + "learning_rate": 0.00018921357004638835, + "logits/chosen": -0.22736577689647675, + "logits/rejected": -0.3719921410083771, + "logps/chosen": -6.317353248596191, + "logps/rejected": -30.95716094970703, + "loss": 0.8575116991996765, + "memory(GiB)": 46.13, + "nll_loss": 0.5229077339172363, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.13013550639152527, + "rewards/margins": 1.687712550163269, + "rewards/rejected": -1.5575770139694214, + "step": 64, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.39141889348889725, + "grad_norm": 2.560202121734619, + "learning_rate": 0.00018875857890045543, + "logits/chosen": -0.24293088912963867, + "logits/rejected": -0.35710442066192627, + "logps/chosen": -7.503670692443848, + "logps/rejected": -32.99870300292969, + "loss": 0.8683996796607971, + "memory(GiB)": 46.13, + "nll_loss": 0.49205920100212097, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.03412435203790665, + "rewards/margins": 1.4795939922332764, + "rewards/rejected": -1.4454697370529175, + "step": 65, + "train_speed(iter/s)": 0.005629 + }, + { + "epoch": 0.39744072261949565, + "grad_norm": 1.993003487586975, + "learning_rate": 0.00018829475928589271, + "logits/chosen": -0.23541702330112457, + "logits/rejected": -0.3272259533405304, + "logps/chosen": -5.38151216506958, + "logps/rejected": -29.121570587158203, + "loss": 0.7320932149887085, + "memory(GiB)": 46.13, + "nll_loss": 0.36119604110717773, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.0803036242723465, + "rewards/margins": 1.6109750270843506, + "rewards/rejected": -1.53067147731781, + "step": 66, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.4034625517500941, + "grad_norm": 2.9964606761932373, + "learning_rate": 0.00018782215733702286, + "logits/chosen": -0.20383372902870178, + "logits/rejected": -0.34373044967651367, + "logps/chosen": -6.649240493774414, + "logps/rejected": -31.494646072387695, + "loss": 0.8884384036064148, + "memory(GiB)": 46.13, + "nll_loss": 0.5308991074562073, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.10617589205503464, + "rewards/margins": 1.5828511714935303, + "rewards/rejected": -1.4766751527786255, + "step": 67, + "train_speed(iter/s)": 0.00563 + }, + { + "epoch": 0.4094843808806925, + "grad_norm": 2.774994134902954, + "learning_rate": 0.00018734082006171299, + "logits/chosen": -0.25100839138031006, + "logits/rejected": -0.4020751118659973, + "logps/chosen": -7.144831657409668, + "logps/rejected": -29.9566707611084, + "loss": 0.8242477178573608, + "memory(GiB)": 46.13, + "nll_loss": 0.5139617919921875, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13794061541557312, + "rewards/margins": 1.6364552974700928, + "rewards/rejected": -1.4985146522521973, + "step": 68, + "train_speed(iter/s)": 0.005631 + }, + { + "epoch": 0.41550621001129096, + "grad_norm": 3.4358551502227783, + "learning_rate": 0.0001868507953366989, + "logits/chosen": -0.24417072534561157, + "logits/rejected": -0.36953434348106384, + "logps/chosen": -6.3338799476623535, + "logps/rejected": -32.21156311035156, + "loss": 0.7758980989456177, + "memory(GiB)": 46.13, + "nll_loss": 0.48221129179000854, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14049482345581055, + "rewards/margins": 1.806667447090149, + "rewards/rejected": -1.666172742843628, + "step": 69, + "train_speed(iter/s)": 0.005631 + }, + { + "epoch": 0.42152803914188935, + "grad_norm": 2.821110248565674, + "learning_rate": 0.0001863521319028231, + "logits/chosen": -0.2586533725261688, + "logits/rejected": -0.36297786235809326, + "logps/chosen": -7.11786413192749, + "logps/rejected": -31.115772247314453, + "loss": 0.8807559013366699, + "memory(GiB)": 46.13, + "nll_loss": 0.5770330429077148, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.07292432337999344, + "rewards/margins": 2.026425838470459, + "rewards/rejected": -1.9535014629364014, + "step": 70, + "train_speed(iter/s)": 0.005632 + }, + { + "epoch": 0.42754986827248775, + "grad_norm": 2.6153159141540527, + "learning_rate": 0.00018584487936018661, + "logits/chosen": -0.305365651845932, + "logits/rejected": -0.4076971709728241, + "logps/chosen": -4.401376247406006, + "logps/rejected": -25.76744842529297, + "loss": 0.7574676275253296, + "memory(GiB)": 46.13, + "nll_loss": 0.3759956955909729, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.015199379995465279, + "rewards/margins": 1.5401445627212524, + "rewards/rejected": -1.5553438663482666, + "step": 71, + "train_speed(iter/s)": 0.005632 + }, + { + "epoch": 0.4335716974030862, + "grad_norm": 4.708693027496338, + "learning_rate": 0.00018532908816321558, + "logits/chosen": -0.22257624566555023, + "logits/rejected": -0.36776047945022583, + "logps/chosen": -5.041079998016357, + "logps/rejected": -31.822460174560547, + "loss": 0.7604058980941772, + "memory(GiB)": 46.13, + "nll_loss": 0.3996947705745697, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0631202757358551, + "rewards/margins": 1.8200846910476685, + "rewards/rejected": -1.7569644451141357, + "step": 72, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.4395935265336846, + "grad_norm": 4.16855001449585, + "learning_rate": 0.0001848048096156426, + "logits/chosen": -0.1833086460828781, + "logits/rejected": -0.40944039821624756, + "logps/chosen": -7.540615081787109, + "logps/rejected": -41.107208251953125, + "loss": 1.1337863206863403, + "memory(GiB)": 46.13, + "nll_loss": 0.7324954271316528, + "rewards/accuracies": 0.828125, + "rewards/chosen": -0.13253508508205414, + "rewards/margins": 2.3366596698760986, + "rewards/rejected": -2.4691946506500244, + "step": 73, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.445615355664283, + "grad_norm": 2.806788444519043, + "learning_rate": 0.0001842720958654039, + "logits/chosen": -0.3181215524673462, + "logits/rejected": -0.39911144971847534, + "logps/chosen": -5.607314586639404, + "logps/rejected": -28.428564071655273, + "loss": 0.6960507035255432, + "memory(GiB)": 46.13, + "nll_loss": 0.40211787819862366, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15752148628234863, + "rewards/margins": 1.9513351917266846, + "rewards/rejected": -1.7938138246536255, + "step": 74, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.45163718479488146, + "grad_norm": 3.420902967453003, + "learning_rate": 0.00018373099989945236, + "logits/chosen": -0.22940213978290558, + "logits/rejected": -0.4191284477710724, + "logps/chosen": -6.713252067565918, + "logps/rejected": -37.6962890625, + "loss": 0.920997679233551, + "memory(GiB)": 46.13, + "nll_loss": 0.5668741464614868, + "rewards/accuracies": 0.890625, + "rewards/chosen": -0.0326458215713501, + "rewards/margins": 2.1453044414520264, + "rewards/rejected": -2.177950382232666, + "step": 75, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.45765901392547986, + "grad_norm": 3.6164190769195557, + "learning_rate": 0.0001831815755384869, + "logits/chosen": -0.20445629954338074, + "logits/rejected": -0.3627314567565918, + "logps/chosen": -5.958117485046387, + "logps/rejected": -38.63805389404297, + "loss": 0.884059727191925, + "memory(GiB)": 46.13, + "nll_loss": 0.5545799732208252, + "rewards/accuracies": 0.828125, + "rewards/chosen": -0.049575597047805786, + "rewards/margins": 2.2317745685577393, + "rewards/rejected": -2.2813501358032227, + "step": 76, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.46368084305607826, + "grad_norm": 2.9762485027313232, + "learning_rate": 0.0001826238774315995, + "logits/chosen": -0.23990976810455322, + "logits/rejected": -0.38009804487228394, + "logps/chosen": -5.436191082000732, + "logps/rejected": -34.6729850769043, + "loss": 0.7682054042816162, + "memory(GiB)": 46.13, + "nll_loss": 0.4302901029586792, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.1761879026889801, + "rewards/margins": 2.175489902496338, + "rewards/rejected": -1.9993020296096802, + "step": 77, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.4697026721866767, + "grad_norm": 2.6883339881896973, + "learning_rate": 0.00018205796105083915, + "logits/chosen": -0.27416834235191345, + "logits/rejected": -0.387048602104187, + "logps/chosen": -6.189967155456543, + "logps/rejected": -30.830598831176758, + "loss": 0.8499253988265991, + "memory(GiB)": 46.13, + "nll_loss": 0.46602901816368103, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.06820189207792282, + "rewards/margins": 1.8341140747070312, + "rewards/rejected": -1.7659121751785278, + "step": 78, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.4757245013172751, + "grad_norm": 2.6955106258392334, + "learning_rate": 0.00018148388268569453, + "logits/chosen": -0.22744283080101013, + "logits/rejected": -0.35057857632637024, + "logps/chosen": -4.485665798187256, + "logps/rejected": -29.74733543395996, + "loss": 0.7549667954444885, + "memory(GiB)": 46.13, + "nll_loss": 0.4278162717819214, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.20732006430625916, + "rewards/margins": 2.0227808952331543, + "rewards/rejected": -1.8154606819152832, + "step": 79, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.4817463304478735, + "grad_norm": 2.907402515411377, + "learning_rate": 0.00018090169943749476, + "logits/chosen": -0.13104476034641266, + "logits/rejected": -0.36177706718444824, + "logps/chosen": -3.6902096271514893, + "logps/rejected": -34.731773376464844, + "loss": 0.6847038269042969, + "memory(GiB)": 46.13, + "nll_loss": 0.34349215030670166, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.11521060019731522, + "rewards/margins": 1.8666913509368896, + "rewards/rejected": -1.7514808177947998, + "step": 80, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.48776815957847197, + "grad_norm": 2.823246955871582, + "learning_rate": 0.00018031146921373018, + "logits/chosen": -0.22965751588344574, + "logits/rejected": -0.3291424512863159, + "logps/chosen": -5.761353969573975, + "logps/rejected": -36.051082611083984, + "loss": 0.7157370448112488, + "memory(GiB)": 46.13, + "nll_loss": 0.4026964008808136, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.07981520146131516, + "rewards/margins": 2.387995958328247, + "rewards/rejected": -2.308180809020996, + "step": 81, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.49378998870907037, + "grad_norm": 2.73394513130188, + "learning_rate": 0.00017971325072229226, + "logits/chosen": -0.19317063689231873, + "logits/rejected": -0.40784093737602234, + "logps/chosen": -5.166616916656494, + "logps/rejected": -40.66303634643555, + "loss": 0.6980624198913574, + "memory(GiB)": 46.13, + "nll_loss": 0.4219977855682373, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.030578728765249252, + "rewards/margins": 2.3669514656066895, + "rewards/rejected": -2.3363726139068604, + "step": 82, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.4998118178396688, + "grad_norm": 5.509173393249512, + "learning_rate": 0.00017910710346563416, + "logits/chosen": -0.1950196623802185, + "logits/rejected": -0.3517180383205414, + "logps/chosen": -8.197553634643555, + "logps/rejected": -42.4526252746582, + "loss": 0.9209753274917603, + "memory(GiB)": 46.13, + "nll_loss": 0.608447790145874, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.012901969254016876, + "rewards/margins": 2.6168861389160156, + "rewards/rejected": -2.6039838790893555, + "step": 83, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5058336469702672, + "grad_norm": 3.090348482131958, + "learning_rate": 0.00017849308773485226, + "logits/chosen": -0.2090476155281067, + "logits/rejected": -0.3706829845905304, + "logps/chosen": -10.569226264953613, + "logps/rejected": -43.29594421386719, + "loss": 1.0815891027450562, + "memory(GiB)": 46.13, + "nll_loss": 0.7449980974197388, + "rewards/accuracies": 0.796875, + "rewards/chosen": -0.03622851148247719, + "rewards/margins": 2.7713966369628906, + "rewards/rejected": -2.8076250553131104, + "step": 84, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5118554761008657, + "grad_norm": 3.75370717048645, + "learning_rate": 0.0001778712646036894, + "logits/chosen": -0.2378547489643097, + "logits/rejected": -0.3873806297779083, + "logps/chosen": -8.168487548828125, + "logps/rejected": -37.17033386230469, + "loss": 1.0251762866973877, + "memory(GiB)": 46.13, + "nll_loss": 0.6340765953063965, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.11355286091566086, + "rewards/margins": 2.2919938564300537, + "rewards/rejected": -2.4055469036102295, + "step": 85, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.517877305231464, + "grad_norm": 3.9649507999420166, + "learning_rate": 0.00017724169592245995, + "logits/chosen": -0.20498239994049072, + "logits/rejected": -0.369476854801178, + "logps/chosen": -6.779036521911621, + "logps/rejected": -37.98490905761719, + "loss": 0.797798216342926, + "memory(GiB)": 46.13, + "nll_loss": 0.5137072801589966, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.07833351194858551, + "rewards/margins": 2.144557476043701, + "rewards/rejected": -2.0662238597869873, + "step": 86, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5238991343620625, + "grad_norm": 3.4646809101104736, + "learning_rate": 0.0001766044443118978, + "logits/chosen": -0.22029685974121094, + "logits/rejected": -0.33659541606903076, + "logps/chosen": -5.358316421508789, + "logps/rejected": -24.96926498413086, + "loss": 0.809937596321106, + "memory(GiB)": 46.13, + "nll_loss": 0.3953189551830292, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.08655387908220291, + "rewards/margins": 1.2862589359283447, + "rewards/rejected": -1.1997051239013672, + "step": 87, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5299209634926609, + "grad_norm": 2.3174376487731934, + "learning_rate": 0.00017595957315692782, + "logits/chosen": -0.23393434286117554, + "logits/rejected": -0.37657639384269714, + "logps/chosen": -5.694438934326172, + "logps/rejected": -26.54452896118164, + "loss": 0.8167567253112793, + "memory(GiB)": 46.13, + "nll_loss": 0.46906933188438416, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.12023281306028366, + "rewards/margins": 1.344594120979309, + "rewards/rejected": -1.2243614196777344, + "step": 88, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5359427926232593, + "grad_norm": 2.29378342628479, + "learning_rate": 0.00017530714660036112, + "logits/chosen": -0.23060575127601624, + "logits/rejected": -0.34717997908592224, + "logps/chosen": -8.571242332458496, + "logps/rejected": -27.7784423828125, + "loss": 0.9238011837005615, + "memory(GiB)": 46.13, + "nll_loss": 0.5501088500022888, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.21566246449947357, + "rewards/margins": 1.4910237789154053, + "rewards/rejected": -1.2753612995147705, + "step": 89, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5419646217538577, + "grad_norm": 4.33613395690918, + "learning_rate": 0.00017464722953651504, + "logits/chosen": -0.17152062058448792, + "logits/rejected": -0.3561325669288635, + "logps/chosen": -6.058459281921387, + "logps/rejected": -32.55611801147461, + "loss": 0.8752692341804504, + "memory(GiB)": 46.13, + "nll_loss": 0.5144660472869873, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0842672809958458, + "rewards/margins": 1.5552325248718262, + "rewards/rejected": -1.4709652662277222, + "step": 90, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5479864508844562, + "grad_norm": 2.2437572479248047, + "learning_rate": 0.0001739798876047584, + "logits/chosen": -0.23511910438537598, + "logits/rejected": -0.3851989507675171, + "logps/chosen": -4.1521100997924805, + "logps/rejected": -28.886817932128906, + "loss": 0.7264631390571594, + "memory(GiB)": 46.13, + "nll_loss": 0.36743801832199097, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.10910709202289581, + "rewards/margins": 1.5735148191452026, + "rewards/rejected": -1.4644078016281128, + "step": 91, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5540082800150545, + "grad_norm": 2.482116460800171, + "learning_rate": 0.00017330518718298264, + "logits/chosen": -0.19554060697555542, + "logits/rejected": -0.38464802503585815, + "logps/chosen": -5.105670928955078, + "logps/rejected": -42.76155471801758, + "loss": 0.6233892440795898, + "memory(GiB)": 46.13, + "nll_loss": 0.36078184843063354, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06755763292312622, + "rewards/margins": 2.4900288581848145, + "rewards/rejected": -2.422471523284912, + "step": 92, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.560030109145653, + "grad_norm": 3.629793167114258, + "learning_rate": 0.0001726231953809993, + "logits/chosen": -0.23075315356254578, + "logits/rejected": -0.4017852246761322, + "logps/chosen": -7.799711227416992, + "logps/rejected": -38.58491516113281, + "loss": 0.7807697057723999, + "memory(GiB)": 46.13, + "nll_loss": 0.5262271165847778, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15829423069953918, + "rewards/margins": 2.4986987113952637, + "rewards/rejected": -2.340404510498047, + "step": 93, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.5660519382762514, + "grad_norm": 10.366227149963379, + "learning_rate": 0.0001719339800338651, + "logits/chosen": -0.29187849164009094, + "logits/rejected": -0.42478230595588684, + "logps/chosen": -7.801548004150391, + "logps/rejected": -34.29828643798828, + "loss": 1.1007416248321533, + "memory(GiB)": 46.13, + "nll_loss": 0.7008671164512634, + "rewards/accuracies": 0.828125, + "rewards/chosen": -0.0017978232353925705, + "rewards/margins": 1.9699221849441528, + "rewards/rejected": -1.9717202186584473, + "step": 94, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.5720737674068498, + "grad_norm": 4.7859039306640625, + "learning_rate": 0.0001712376096951345, + "logits/chosen": -0.2881244421005249, + "logits/rejected": -0.39795249700546265, + "logps/chosen": -6.9338459968566895, + "logps/rejected": -34.682186126708984, + "loss": 0.8693129420280457, + "memory(GiB)": 46.13, + "nll_loss": 0.5744621157646179, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12829959392547607, + "rewards/margins": 2.2821671962738037, + "rewards/rejected": -2.153867244720459, + "step": 95, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5780955965374482, + "grad_norm": 2.521221160888672, + "learning_rate": 0.0001705341536300409, + "logits/chosen": -0.2661621868610382, + "logits/rejected": -0.3923559784889221, + "logps/chosen": -4.780788898468018, + "logps/rejected": -28.810710906982422, + "loss": 0.7346460819244385, + "memory(GiB)": 46.13, + "nll_loss": 0.40230822563171387, + "rewards/accuracies": 0.796875, + "rewards/chosen": 0.10683214664459229, + "rewards/margins": 1.7658103704452515, + "rewards/rejected": -1.6589782238006592, + "step": 96, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.5841174256680467, + "grad_norm": 5.023887634277344, + "learning_rate": 0.00016982368180860728, + "logits/chosen": -0.25105586647987366, + "logits/rejected": -0.3866545855998993, + "logps/chosen": -7.320672512054443, + "logps/rejected": -34.95018005371094, + "loss": 0.9987191557884216, + "memory(GiB)": 46.13, + "nll_loss": 0.6328321695327759, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07585513591766357, + "rewards/margins": 2.0516769886016846, + "rewards/rejected": -2.127532482147217, + "step": 97, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.590139254798645, + "grad_norm": 1.8328591585159302, + "learning_rate": 0.00016910626489868649, + "logits/chosen": -0.2209641933441162, + "logits/rejected": -0.398262083530426, + "logps/chosen": -4.149725437164307, + "logps/rejected": -35.193931579589844, + "loss": 0.6401265263557434, + "memory(GiB)": 46.13, + "nll_loss": 0.31758564710617065, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.14480024576187134, + "rewards/margins": 2.074404716491699, + "rewards/rejected": -1.9296045303344727, + "step": 98, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.5961610839292435, + "grad_norm": 1.696344017982483, + "learning_rate": 0.00016838197425893202, + "logits/chosen": -0.2796581983566284, + "logits/rejected": -0.4003712832927704, + "logps/chosen": -5.847466468811035, + "logps/rejected": -30.844961166381836, + "loss": 0.7925845384597778, + "memory(GiB)": 46.13, + "nll_loss": 0.4691714644432068, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15563875436782837, + "rewards/margins": 1.9312589168548584, + "rewards/rejected": -1.7756202220916748, + "step": 99, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.602182913059842, + "grad_norm": 2.3960113525390625, + "learning_rate": 0.00016765088193170053, + "logits/chosen": -0.2749744653701782, + "logits/rejected": -0.39394593238830566, + "logps/chosen": -6.603066444396973, + "logps/rejected": -25.912214279174805, + "loss": 0.9519745707511902, + "memory(GiB)": 46.13, + "nll_loss": 0.5554052591323853, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.11803217232227325, + "rewards/margins": 1.5442473888397217, + "rewards/rejected": -1.4262150526046753, + "step": 100, + "train_speed(iter/s)": 0.005637 + }, + { + "epoch": 0.6082047421904403, + "grad_norm": 2.1803297996520996, + "learning_rate": 0.00016691306063588583, + "logits/chosen": -0.14278045296669006, + "logits/rejected": -0.32641783356666565, + "logps/chosen": -6.367352485656738, + "logps/rejected": -36.09748077392578, + "loss": 0.7241327166557312, + "memory(GiB)": 46.13, + "nll_loss": 0.4310902953147888, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.20738306641578674, + "rewards/margins": 1.9153938293457031, + "rewards/rejected": -1.7080106735229492, + "step": 101, + "train_speed(iter/s)": 0.005632 + }, + { + "epoch": 0.6142265713210388, + "grad_norm": 2.27510142326355, + "learning_rate": 0.00016616858375968595, + "logits/chosen": -0.20140963792800903, + "logits/rejected": -0.34892868995666504, + "logps/chosen": -4.8850579261779785, + "logps/rejected": -34.458396911621094, + "loss": 0.6944241523742676, + "memory(GiB)": 46.13, + "nll_loss": 0.39014381170272827, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.05396907776594162, + "rewards/margins": 2.1584324836730957, + "rewards/rejected": -2.1044633388519287, + "step": 102, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6202484004516372, + "grad_norm": 3.8538124561309814, + "learning_rate": 0.00016541752535330345, + "logits/chosen": -0.15393608808517456, + "logits/rejected": -0.32232990860939026, + "logps/chosen": -7.490941047668457, + "logps/rejected": -38.019405364990234, + "loss": 0.899553656578064, + "memory(GiB)": 46.13, + "nll_loss": 0.5592579245567322, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04624926298856735, + "rewards/margins": 2.2129275798797607, + "rewards/rejected": -2.259176731109619, + "step": 103, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6262702295822355, + "grad_norm": 3.2243363857269287, + "learning_rate": 0.00016465996012157995, + "logits/chosen": -0.1763302981853485, + "logits/rejected": -0.30002710223197937, + "logps/chosen": -7.91542387008667, + "logps/rejected": -35.163421630859375, + "loss": 0.8102483153343201, + "memory(GiB)": 46.13, + "nll_loss": 0.5111361145973206, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.1345573365688324, + "rewards/margins": 2.1800684928894043, + "rewards/rejected": -2.045511245727539, + "step": 104, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.632292058712834, + "grad_norm": 2.4296176433563232, + "learning_rate": 0.0001638959634165656, + "logits/chosen": -0.2042100727558136, + "logits/rejected": -0.3498837947845459, + "logps/chosen": -5.70271635055542, + "logps/rejected": -33.96530532836914, + "loss": 0.7614078521728516, + "memory(GiB)": 46.13, + "nll_loss": 0.46085914969444275, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.10853511095046997, + "rewards/margins": 2.407238483428955, + "rewards/rejected": -2.298703670501709, + "step": 105, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6383138878434325, + "grad_norm": 5.010378837585449, + "learning_rate": 0.0001631256112300239, + "logits/chosen": -0.17106319963932037, + "logits/rejected": -0.3643282651901245, + "logps/chosen": -5.012183666229248, + "logps/rejected": -39.59567642211914, + "loss": 0.7013410329818726, + "memory(GiB)": 46.13, + "nll_loss": 0.4339035153388977, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.07484348863363266, + "rewards/margins": 2.594076156616211, + "rewards/rejected": -2.5192322731018066, + "step": 106, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6443357169740309, + "grad_norm": 3.9870245456695557, + "learning_rate": 0.00016234898018587337, + "logits/chosen": -0.13750208914279938, + "logits/rejected": -0.3746242821216583, + "logps/chosen": -5.721056938171387, + "logps/rejected": -44.44032287597656, + "loss": 0.7298431396484375, + "memory(GiB)": 46.13, + "nll_loss": 0.4898163080215454, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.036943770945072174, + "rewards/margins": 2.8432068824768066, + "rewards/rejected": -2.806262969970703, + "step": 107, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6503575461046293, + "grad_norm": 2.7155423164367676, + "learning_rate": 0.0001615661475325658, + "logits/chosen": -0.1980631947517395, + "logits/rejected": -0.36777210235595703, + "logps/chosen": -5.860571384429932, + "logps/rejected": -45.095069885253906, + "loss": 0.6415181756019592, + "memory(GiB)": 46.13, + "nll_loss": 0.43772590160369873, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.029481690376996994, + "rewards/margins": 2.904158115386963, + "rewards/rejected": -2.874676465988159, + "step": 108, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6563793752352277, + "grad_norm": 3.9366610050201416, + "learning_rate": 0.00016077719113540302, + "logits/chosen": -0.201637402176857, + "logits/rejected": -0.36231857538223267, + "logps/chosen": -8.20331859588623, + "logps/rejected": -47.63356018066406, + "loss": 0.8109980821609497, + "memory(GiB)": 46.13, + "nll_loss": 0.588591456413269, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.11406072974205017, + "rewards/margins": 3.15336275100708, + "rewards/rejected": -3.267423629760742, + "step": 109, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.6624012043658262, + "grad_norm": 1.8952491283416748, + "learning_rate": 0.00015998218946879138, + "logits/chosen": -0.20580148696899414, + "logits/rejected": -0.37684956192970276, + "logps/chosen": -4.04914665222168, + "logps/rejected": -40.22568130493164, + "loss": 0.6388105154037476, + "memory(GiB)": 46.13, + "nll_loss": 0.39933112263679504, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.07669900357723236, + "rewards/margins": 2.701504945755005, + "rewards/rejected": -2.6248061656951904, + "step": 110, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.6684230334964245, + "grad_norm": 2.1146159172058105, + "learning_rate": 0.00015918122160843678, + "logits/chosen": -0.21242693066596985, + "logits/rejected": -0.3542603850364685, + "logps/chosen": -7.400607109069824, + "logps/rejected": -41.83246612548828, + "loss": 0.9009187817573547, + "memory(GiB)": 46.13, + "nll_loss": 0.6562238931655884, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07808566093444824, + "rewards/margins": 2.6408753395080566, + "rewards/rejected": -2.5627896785736084, + "step": 111, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.674444862627023, + "grad_norm": 3.891860008239746, + "learning_rate": 0.000158374367223479, + "logits/chosen": -0.2456948161125183, + "logits/rejected": -0.3476937413215637, + "logps/chosen": -6.91848087310791, + "logps/rejected": -35.34187698364258, + "loss": 0.9391785860061646, + "memory(GiB)": 46.13, + "nll_loss": 0.618550181388855, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.011886654421687126, + "rewards/margins": 2.1392247676849365, + "rewards/rejected": -2.151111602783203, + "step": 112, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.6804666917576214, + "grad_norm": 4.234856128692627, + "learning_rate": 0.00015756170656856737, + "logits/chosen": -0.11636840552091599, + "logits/rejected": -0.23480768501758575, + "logps/chosen": -7.3483710289001465, + "logps/rejected": -33.948822021484375, + "loss": 0.8160063624382019, + "memory(GiB)": 46.13, + "nll_loss": 0.5110282301902771, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.08705638349056244, + "rewards/margins": 2.111095905303955, + "rewards/rejected": -2.0240395069122314, + "step": 113, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.6864885208882198, + "grad_norm": 1.5901049375534058, + "learning_rate": 0.0001567433204758782, + "logits/chosen": -0.15935438871383667, + "logits/rejected": -0.3056758642196655, + "logps/chosen": -5.887468338012695, + "logps/rejected": -39.561500549316406, + "loss": 0.7292969226837158, + "memory(GiB)": 46.13, + "nll_loss": 0.4773525893688202, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.141588032245636, + "rewards/margins": 2.442899703979492, + "rewards/rejected": -2.301311731338501, + "step": 114, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.6925103500188182, + "grad_norm": 2.1147372722625732, + "learning_rate": 0.0001559192903470747, + "logits/chosen": -0.1537177711725235, + "logits/rejected": -0.2976364493370056, + "logps/chosen": -7.735870361328125, + "logps/rejected": -36.14551544189453, + "loss": 0.822877049446106, + "memory(GiB)": 46.13, + "nll_loss": 0.5838936567306519, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16609230637550354, + "rewards/margins": 2.184666633605957, + "rewards/rejected": -2.0185742378234863, + "step": 115, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.6985321791494167, + "grad_norm": 3.8271219730377197, + "learning_rate": 0.00015508969814521025, + "logits/chosen": -0.16476985812187195, + "logits/rejected": -0.2944994568824768, + "logps/chosen": -4.954969882965088, + "logps/rejected": -31.42076301574707, + "loss": 0.8119629621505737, + "memory(GiB)": 46.13, + "nll_loss": 0.4658912420272827, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.09785570204257965, + "rewards/margins": 1.8094596862792969, + "rewards/rejected": -1.711604118347168, + "step": 116, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.704554008280015, + "grad_norm": 1.9744106531143188, + "learning_rate": 0.00015425462638657595, + "logits/chosen": -0.2048090547323227, + "logits/rejected": -0.2695329189300537, + "logps/chosen": -5.02174711227417, + "logps/rejected": -31.327930450439453, + "loss": 0.6611427068710327, + "memory(GiB)": 46.13, + "nll_loss": 0.3884551525115967, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.18278352916240692, + "rewards/margins": 2.1040430068969727, + "rewards/rejected": -1.9212596416473389, + "step": 117, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.7105758374106135, + "grad_norm": 2.7671103477478027, + "learning_rate": 0.00015341415813249288, + "logits/chosen": -0.18153494596481323, + "logits/rejected": -0.31533682346343994, + "logps/chosen": -3.5208582878112793, + "logps/rejected": -28.974889755249023, + "loss": 0.6663014888763428, + "memory(GiB)": 46.13, + "nll_loss": 0.31734898686408997, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.08336202800273895, + "rewards/margins": 1.5573171377182007, + "rewards/rejected": -1.4739550352096558, + "step": 118, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7165976665412119, + "grad_norm": 2.2519025802612305, + "learning_rate": 0.00015256837698105047, + "logits/chosen": -0.18909907341003418, + "logits/rejected": -0.32073426246643066, + "logps/chosen": -5.691015243530273, + "logps/rejected": -38.229949951171875, + "loss": 0.6945815086364746, + "memory(GiB)": 46.13, + "nll_loss": 0.4534215033054352, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.016410622745752335, + "rewards/margins": 2.294341564178467, + "rewards/rejected": -2.3107523918151855, + "step": 119, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7226194956718103, + "grad_norm": 2.6865475177764893, + "learning_rate": 0.00015171736705879126, + "logits/chosen": -0.11694711446762085, + "logits/rejected": -0.30947864055633545, + "logps/chosen": -4.004083156585693, + "logps/rejected": -42.21112060546875, + "loss": 0.5099095702171326, + "memory(GiB)": 46.13, + "nll_loss": 0.2805330157279968, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08305128663778305, + "rewards/margins": 2.5462100505828857, + "rewards/rejected": -2.46315860748291, + "step": 120, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7286413248024087, + "grad_norm": 5.433821201324463, + "learning_rate": 0.00015086121301234316, + "logits/chosen": -0.15977299213409424, + "logits/rejected": -0.30839866399765015, + "logps/chosen": -7.34042501449585, + "logps/rejected": -38.30808639526367, + "loss": 0.9920263886451721, + "memory(GiB)": 46.13, + "nll_loss": 0.6664140224456787, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.023471834138035774, + "rewards/margins": 2.068791151046753, + "rewards/rejected": -2.0922627449035645, + "step": 121, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7346631539330072, + "grad_norm": 1.8786602020263672, + "learning_rate": 0.00015000000000000001, + "logits/chosen": -0.12786100804805756, + "logits/rejected": -0.2967092990875244, + "logps/chosen": -4.349850654602051, + "logps/rejected": -32.289390563964844, + "loss": 0.6910618543624878, + "memory(GiB)": 46.13, + "nll_loss": 0.37633225321769714, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.09151757508516312, + "rewards/margins": 1.6974818706512451, + "rewards/rejected": -1.6059644222259521, + "step": 122, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.7406849830636055, + "grad_norm": 2.327937364578247, + "learning_rate": 0.00014913381368325115, + "logits/chosen": -0.12772715091705322, + "logits/rejected": -0.3204698860645294, + "logps/chosen": -3.4514901638031006, + "logps/rejected": -40.17311096191406, + "loss": 0.5442243814468384, + "memory(GiB)": 46.13, + "nll_loss": 0.2678568661212921, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.06198100745677948, + "rewards/margins": 2.348788022994995, + "rewards/rejected": -2.286807060241699, + "step": 123, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.746706812194204, + "grad_norm": 3.345496892929077, + "learning_rate": 0.0001482627402182611, + "logits/chosen": -0.23605135083198547, + "logits/rejected": -0.3248905539512634, + "logps/chosen": -6.426857948303223, + "logps/rejected": -27.78033447265625, + "loss": 0.8108398914337158, + "memory(GiB)": 46.13, + "nll_loss": 0.4387803077697754, + "rewards/accuracies": 0.828125, + "rewards/chosen": 0.12587979435920715, + "rewards/margins": 1.6669539213180542, + "rewards/rejected": -1.5410741567611694, + "step": 124, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.7527286413248024, + "grad_norm": 2.567275285720825, + "learning_rate": 0.00014738686624729986, + "logits/chosen": -0.19267147779464722, + "logits/rejected": -0.3316629230976105, + "logps/chosen": -5.075592994689941, + "logps/rejected": -33.312400817871094, + "loss": 0.7099701166152954, + "memory(GiB)": 46.13, + "nll_loss": 0.3936714828014374, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.13089658319950104, + "rewards/margins": 1.870408535003662, + "rewards/rejected": -1.7395117282867432, + "step": 125, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7587504704554008, + "grad_norm": 3.0261592864990234, + "learning_rate": 0.00014650627889012507, + "logits/chosen": -0.2155275195837021, + "logits/rejected": -0.37678274512290955, + "logps/chosen": -4.682234764099121, + "logps/rejected": -28.974559783935547, + "loss": 0.7498282790184021, + "memory(GiB)": 46.13, + "nll_loss": 0.4256855249404907, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.15469788014888763, + "rewards/margins": 1.657031774520874, + "rewards/rejected": -1.502333641052246, + "step": 126, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7647722995859992, + "grad_norm": 2.6022913455963135, + "learning_rate": 0.0001456210657353163, + "logits/chosen": -0.19964055716991425, + "logits/rejected": -0.35465607047080994, + "logps/chosen": -3.941540479660034, + "logps/rejected": -32.9716682434082, + "loss": 0.6142888069152832, + "memory(GiB)": 46.13, + "nll_loss": 0.3339819610118866, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.15736861526966095, + "rewards/margins": 2.048983335494995, + "rewards/rejected": -1.8916147947311401, + "step": 127, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7707941287165977, + "grad_norm": 2.2070999145507812, + "learning_rate": 0.00014473131483156327, + "logits/chosen": -0.24945035576820374, + "logits/rejected": -0.32592612504959106, + "logps/chosen": -5.566514015197754, + "logps/rejected": -27.138111114501953, + "loss": 0.7188521027565002, + "memory(GiB)": 46.13, + "nll_loss": 0.4044472575187683, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.16954132914543152, + "rewards/margins": 1.7891260385513306, + "rewards/rejected": -1.6195846796035767, + "step": 128, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.776815957847196, + "grad_norm": 1.9873756170272827, + "learning_rate": 0.00014383711467890774, + "logits/chosen": -0.18453390896320343, + "logits/rejected": -0.3942897319793701, + "logps/chosen": -4.547736167907715, + "logps/rejected": -43.979976654052734, + "loss": 0.5406314730644226, + "memory(GiB)": 46.13, + "nll_loss": 0.3523382544517517, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.08807562291622162, + "rewards/margins": 2.9467825889587402, + "rewards/rejected": -2.8587071895599365, + "step": 129, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.7828377869777945, + "grad_norm": 3.6719274520874023, + "learning_rate": 0.00014293855421994094, + "logits/chosen": -0.20188507437705994, + "logits/rejected": -0.34430867433547974, + "logps/chosen": -5.333648204803467, + "logps/rejected": -41.819515228271484, + "loss": 0.7074599862098694, + "memory(GiB)": 46.13, + "nll_loss": 0.44100311398506165, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.027486218139529228, + "rewards/margins": 2.8748302459716797, + "rewards/rejected": -2.847343921661377, + "step": 130, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.788859616108393, + "grad_norm": 3.5544822216033936, + "learning_rate": 0.00014203572283095657, + "logits/chosen": -0.1257411241531372, + "logits/rejected": -0.35228949785232544, + "logps/chosen": -5.203995704650879, + "logps/rejected": -53.26233673095703, + "loss": 0.763848602771759, + "memory(GiB)": 46.13, + "nll_loss": 0.5348649024963379, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0426684133708477, + "rewards/margins": 3.6239006519317627, + "rewards/rejected": -3.5812320709228516, + "step": 131, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.7948814452389913, + "grad_norm": 2.9171648025512695, + "learning_rate": 0.00014112871031306119, + "logits/chosen": -0.19912837445735931, + "logits/rejected": -0.3755126893520355, + "logps/chosen": -7.589679718017578, + "logps/rejected": -46.79793930053711, + "loss": 0.7475419640541077, + "memory(GiB)": 46.13, + "nll_loss": 0.5176687240600586, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.19966475665569305, + "rewards/margins": 3.243659496307373, + "rewards/rejected": -3.043994426727295, + "step": 132, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8009032743695897, + "grad_norm": 3.5576326847076416, + "learning_rate": 0.00014021760688324176, + "logits/chosen": -0.17760571837425232, + "logits/rejected": -0.34351277351379395, + "logps/chosen": -6.067855358123779, + "logps/rejected": -45.17057800292969, + "loss": 0.7542024850845337, + "memory(GiB)": 46.13, + "nll_loss": 0.46581241488456726, + "rewards/accuracies": 0.859375, + "rewards/chosen": -0.06834451109170914, + "rewards/margins": 2.9498496055603027, + "rewards/rejected": -3.0181939601898193, + "step": 133, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8069251035001882, + "grad_norm": 3.7524359226226807, + "learning_rate": 0.00013930250316539238, + "logits/chosen": -0.24070125818252563, + "logits/rejected": -0.36900123953819275, + "logps/chosen": -6.3118109703063965, + "logps/rejected": -41.5582389831543, + "loss": 0.8333712220191956, + "memory(GiB)": 46.13, + "nll_loss": 0.5454068183898926, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0363120511174202, + "rewards/margins": 2.832289695739746, + "rewards/rejected": -2.7959775924682617, + "step": 134, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8129469326307867, + "grad_norm": 4.076293468475342, + "learning_rate": 0.00013838349018130007, + "logits/chosen": -0.22921916842460632, + "logits/rejected": -0.3980764150619507, + "logps/chosen": -6.020735263824463, + "logps/rejected": -36.49159622192383, + "loss": 0.7660678029060364, + "memory(GiB)": 46.13, + "nll_loss": 0.5027486085891724, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.11397107690572739, + "rewards/margins": 2.2893691062927246, + "rewards/rejected": -2.175398349761963, + "step": 135, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.818968761761385, + "grad_norm": 3.8375930786132812, + "learning_rate": 0.00013746065934159123, + "logits/chosen": -0.21597175300121307, + "logits/rejected": -0.3748128414154053, + "logps/chosen": -5.917651176452637, + "logps/rejected": -34.7623176574707, + "loss": 0.7133091688156128, + "memory(GiB)": 46.13, + "nll_loss": 0.43554598093032837, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14565804600715637, + "rewards/margins": 2.124354124069214, + "rewards/rejected": -1.9786962270736694, + "step": 136, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8249905908919835, + "grad_norm": 3.4924089908599854, + "learning_rate": 0.00013653410243663952, + "logits/chosen": -0.18882140517234802, + "logits/rejected": -0.39621540904045105, + "logps/chosen": -3.5247278213500977, + "logps/rejected": -39.52235794067383, + "loss": 0.5699490308761597, + "memory(GiB)": 46.13, + "nll_loss": 0.3287566304206848, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16140440106391907, + "rewards/margins": 2.0828282833099365, + "rewards/rejected": -1.9214237928390503, + "step": 137, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8310124200225819, + "grad_norm": 2.952988386154175, + "learning_rate": 0.00013560391162743569, + "logits/chosen": -0.1859571784734726, + "logits/rejected": -0.381632536649704, + "logps/chosen": -4.896010398864746, + "logps/rejected": -35.494384765625, + "loss": 0.7929538488388062, + "memory(GiB)": 46.13, + "nll_loss": 0.4508986473083496, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.021028656512498856, + "rewards/margins": 1.990783452987671, + "rewards/rejected": -1.969754695892334, + "step": 138, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8370342491531803, + "grad_norm": 2.0521535873413086, + "learning_rate": 0.00013467017943642073, + "logits/chosen": -0.24490389227867126, + "logits/rejected": -0.42894116044044495, + "logps/chosen": -4.2235565185546875, + "logps/rejected": -38.84959411621094, + "loss": 0.6944494247436523, + "memory(GiB)": 46.13, + "nll_loss": 0.43725934624671936, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12278101593255997, + "rewards/margins": 2.2289106845855713, + "rewards/rejected": -2.1061296463012695, + "step": 139, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8430560782837787, + "grad_norm": 3.664316415786743, + "learning_rate": 0.00013373299873828303, + "logits/chosen": -0.29347366094589233, + "logits/rejected": -0.41366317868232727, + "logps/chosen": -4.828797340393066, + "logps/rejected": -31.14942169189453, + "loss": 0.6968463659286499, + "memory(GiB)": 46.13, + "nll_loss": 0.41966789960861206, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.13118553161621094, + "rewards/margins": 1.9206351041793823, + "rewards/rejected": -1.7894494533538818, + "step": 140, + "train_speed(iter/s)": 0.005635 + }, + { + "epoch": 0.8490779074143772, + "grad_norm": 3.614262580871582, + "learning_rate": 0.00013279246275072046, + "logits/chosen": -0.31295719742774963, + "logits/rejected": -0.4447307288646698, + "logps/chosen": -8.66226577758789, + "logps/rejected": -31.42186737060547, + "loss": 1.0331419706344604, + "memory(GiB)": 46.13, + "nll_loss": 0.6634517312049866, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.08492518961429596, + "rewards/margins": 1.7986928224563599, + "rewards/rejected": -1.713767647743225, + "step": 141, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8550997365449755, + "grad_norm": 1.7650597095489502, + "learning_rate": 0.00013184866502516845, + "logits/chosen": -0.26669561862945557, + "logits/rejected": -0.4871166944503784, + "logps/chosen": -3.3100223541259766, + "logps/rejected": -39.79899215698242, + "loss": 0.5506975650787354, + "memory(GiB)": 46.13, + "nll_loss": 0.29588890075683594, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.10198746621608734, + "rewards/margins": 2.259812831878662, + "rewards/rejected": -2.157825231552124, + "step": 142, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.861121565675574, + "grad_norm": 3.4034411907196045, + "learning_rate": 0.00013090169943749476, + "logits/chosen": -0.3030458390712738, + "logits/rejected": -0.45216381549835205, + "logps/chosen": -5.469812393188477, + "logps/rejected": -35.740806579589844, + "loss": 0.6628592610359192, + "memory(GiB)": 46.13, + "nll_loss": 0.34965524077415466, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07742348313331604, + "rewards/margins": 2.012543201446533, + "rewards/rejected": -1.935119867324829, + "step": 143, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8671433948061724, + "grad_norm": 2.271817922592163, + "learning_rate": 0.00012995166017866193, + "logits/chosen": -0.35303038358688354, + "logits/rejected": -0.4534931778907776, + "logps/chosen": -7.304016590118408, + "logps/rejected": -30.363569259643555, + "loss": 0.8833200931549072, + "memory(GiB)": 46.13, + "nll_loss": 0.5442834496498108, + "rewards/accuracies": 0.765625, + "rewards/chosen": 0.18977457284927368, + "rewards/margins": 2.04044508934021, + "rewards/rejected": -1.850670576095581, + "step": 144, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8731652239367708, + "grad_norm": 1.6546629667282104, + "learning_rate": 0.00012899864174535864, + "logits/chosen": -0.30710387229919434, + "logits/rejected": -0.47071051597595215, + "logps/chosen": -4.921065330505371, + "logps/rejected": -37.83810043334961, + "loss": 0.611689567565918, + "memory(GiB)": 46.13, + "nll_loss": 0.36449986696243286, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14039897918701172, + "rewards/margins": 2.364004373550415, + "rewards/rejected": -2.2236056327819824, + "step": 145, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8791870530673692, + "grad_norm": 1.4133797883987427, + "learning_rate": 0.00012804273893060028, + "logits/chosen": -0.35278186202049255, + "logits/rejected": -0.4680350422859192, + "logps/chosen": -4.455732345581055, + "logps/rejected": -32.905887603759766, + "loss": 0.5695258975028992, + "memory(GiB)": 46.13, + "nll_loss": 0.3188043534755707, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18160954117774963, + "rewards/margins": 2.25382137298584, + "rewards/rejected": -2.072211742401123, + "step": 146, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8852088821979677, + "grad_norm": 1.8506555557250977, + "learning_rate": 0.00012708404681430053, + "logits/chosen": -0.3219718635082245, + "logits/rejected": -0.4345773756504059, + "logps/chosen": -5.016592979431152, + "logps/rejected": -34.69657516479492, + "loss": 0.6405315399169922, + "memory(GiB)": 46.13, + "nll_loss": 0.35303816199302673, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.13450342416763306, + "rewards/margins": 2.0750272274017334, + "rewards/rejected": -1.9405235052108765, + "step": 147, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.891230711328566, + "grad_norm": 2.6582088470458984, + "learning_rate": 0.00012612266075381386, + "logits/chosen": -0.2835950255393982, + "logits/rejected": -0.4708768129348755, + "logps/chosen": -6.048505783081055, + "logps/rejected": -42.608131408691406, + "loss": 0.676105260848999, + "memory(GiB)": 46.13, + "nll_loss": 0.4015389084815979, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.033125825226306915, + "rewards/margins": 2.4881484508514404, + "rewards/rejected": -2.4550228118896484, + "step": 148, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.8972525404591645, + "grad_norm": 3.3612794876098633, + "learning_rate": 0.00012515867637445086, + "logits/chosen": -0.3777806758880615, + "logits/rejected": -0.477069228887558, + "logps/chosen": -6.0037922859191895, + "logps/rejected": -32.48824691772461, + "loss": 0.825937032699585, + "memory(GiB)": 46.13, + "nll_loss": 0.5271134972572327, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07130320370197296, + "rewards/margins": 2.3660011291503906, + "rewards/rejected": -2.2946979999542236, + "step": 149, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.9032743695897629, + "grad_norm": 1.5847569704055786, + "learning_rate": 0.00012419218955996676, + "logits/chosen": -0.31665414571762085, + "logits/rejected": -0.4995279312133789, + "logps/chosen": -6.320664882659912, + "logps/rejected": -52.651451110839844, + "loss": 0.5624623894691467, + "memory(GiB)": 46.13, + "nll_loss": 0.368425577878952, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14041581749916077, + "rewards/margins": 3.4388084411621094, + "rewards/rejected": -3.2983925342559814, + "step": 150, + "train_speed(iter/s)": 0.005636 + }, + { + "epoch": 0.9092961987203613, + "grad_norm": 1.733017921447754, + "learning_rate": 0.00012322329644302426, + "logits/chosen": -0.3288433849811554, + "logits/rejected": -0.5056887865066528, + "logps/chosen": -4.5702128410339355, + "logps/rejected": -42.928321838378906, + "loss": 0.534808337688446, + "memory(GiB)": 46.13, + "nll_loss": 0.3458808958530426, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14961260557174683, + "rewards/margins": 3.0105981826782227, + "rewards/rejected": -2.860985279083252, + "step": 151, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9153180278509597, + "grad_norm": 1.9891310930252075, + "learning_rate": 0.00012225209339563145, + "logits/chosen": -0.34356942772865295, + "logits/rejected": -0.46553438901901245, + "logps/chosen": -8.254244804382324, + "logps/rejected": -43.76982116699219, + "loss": 0.7122747302055359, + "memory(GiB)": 46.13, + "nll_loss": 0.5048399567604065, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.2129802703857422, + "rewards/margins": 3.227698802947998, + "rewards/rejected": -3.0147182941436768, + "step": 152, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9213398569815582, + "grad_norm": 4.222365856170654, + "learning_rate": 0.00012127867701955622, + "logits/chosen": -0.3883526921272278, + "logits/rejected": -0.5774507522583008, + "logps/chosen": -6.422748565673828, + "logps/rejected": -53.63523483276367, + "loss": 0.6593223214149475, + "memory(GiB)": 46.13, + "nll_loss": 0.48106443881988525, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.016244899481534958, + "rewards/margins": 4.012790203094482, + "rewards/rejected": -4.029035568237305, + "step": 153, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9273616861121565, + "grad_norm": 14.615867614746094, + "learning_rate": 0.00012030314413671762, + "logits/chosen": -0.39986497163772583, + "logits/rejected": -0.6344658136367798, + "logps/chosen": -4.43770170211792, + "logps/rejected": -55.2861213684082, + "loss": 0.5306422710418701, + "memory(GiB)": 46.13, + "nll_loss": 0.3766982853412628, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10156996548175812, + "rewards/margins": 3.934446334838867, + "rewards/rejected": -3.832875967025757, + "step": 154, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.933383515242755, + "grad_norm": 4.434772491455078, + "learning_rate": 0.00011932559177955533, + "logits/chosen": -0.4241827726364136, + "logits/rejected": -0.6216625571250916, + "logps/chosen": -5.785084247589111, + "logps/rejected": -56.10906982421875, + "loss": 0.7327705025672913, + "memory(GiB)": 46.13, + "nll_loss": 0.5102080702781677, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.01768992468714714, + "rewards/margins": 4.036444187164307, + "rewards/rejected": -4.018754005432129, + "step": 155, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9394053443733534, + "grad_norm": 3.1120123863220215, + "learning_rate": 0.00011834611718137824, + "logits/chosen": -0.364039808511734, + "logits/rejected": -0.6007083654403687, + "logps/chosen": -5.2767415046691895, + "logps/rejected": -66.11264038085938, + "loss": 0.5869796276092529, + "memory(GiB)": 46.13, + "nll_loss": 0.47315409779548645, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10729408264160156, + "rewards/margins": 4.741492748260498, + "rewards/rejected": -4.6341986656188965, + "step": 156, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9454271735039518, + "grad_norm": 9.935791015625, + "learning_rate": 0.00011736481776669306, + "logits/chosen": -0.43170925974845886, + "logits/rejected": -0.5715627670288086, + "logps/chosen": -9.369845390319824, + "logps/rejected": -57.28983688354492, + "loss": 1.0344016551971436, + "memory(GiB)": 46.13, + "nll_loss": 0.7142276763916016, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.12030352652072906, + "rewards/margins": 3.983743190765381, + "rewards/rejected": -4.1040472984313965, + "step": 157, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9514490026345502, + "grad_norm": 3.4565649032592773, + "learning_rate": 0.00011638179114151377, + "logits/chosen": -0.4627074599266052, + "logits/rejected": -0.6269153952598572, + "logps/chosen": -6.801128387451172, + "logps/rejected": -50.93914031982422, + "loss": 0.7641481757164001, + "memory(GiB)": 46.13, + "nll_loss": 0.5224257111549377, + "rewards/accuracies": 0.859375, + "rewards/chosen": 0.020412985235452652, + "rewards/margins": 3.8560848236083984, + "rewards/rejected": -3.835671901702881, + "step": 158, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9574708317651487, + "grad_norm": 7.00652551651001, + "learning_rate": 0.00011539713508365335, + "logits/chosen": -0.4332171678543091, + "logits/rejected": -0.589859127998352, + "logps/chosen": -6.74440860748291, + "logps/rejected": -51.30781173706055, + "loss": 0.8142269253730774, + "memory(GiB)": 46.13, + "nll_loss": 0.6082335710525513, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.04817220941185951, + "rewards/margins": 3.839707136154175, + "rewards/rejected": -3.791534662246704, + "step": 159, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.963492660895747, + "grad_norm": 4.027029514312744, + "learning_rate": 0.00011441094753299801, + "logits/chosen": -0.4457279443740845, + "logits/rejected": -0.5885544419288635, + "logps/chosen": -7.359399318695068, + "logps/rejected": -45.14234924316406, + "loss": 0.572091281414032, + "memory(GiB)": 46.13, + "nll_loss": 0.35912448167800903, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.04880950227379799, + "rewards/margins": 3.246192216873169, + "rewards/rejected": -3.1973824501037598, + "step": 160, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.9695144900263455, + "grad_norm": 3.832231044769287, + "learning_rate": 0.00011342332658176555, + "logits/chosen": -0.3761252760887146, + "logits/rejected": -0.5621005892753601, + "logps/chosen": -5.201396942138672, + "logps/rejected": -43.30262756347656, + "loss": 0.601041853427887, + "memory(GiB)": 46.13, + "nll_loss": 0.37146201729774475, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.1526133269071579, + "rewards/margins": 3.075632095336914, + "rewards/rejected": -2.92301869392395, + "step": 161, + "train_speed(iter/s)": 0.005634 + }, + { + "epoch": 0.9755363191569439, + "grad_norm": 3.592341184616089, + "learning_rate": 0.00011243437046474853, + "logits/chosen": -0.2978363037109375, + "logits/rejected": -0.5343031883239746, + "logps/chosen": -5.316001892089844, + "logps/rejected": -56.440513610839844, + "loss": 0.5654405951499939, + "memory(GiB)": 46.13, + "nll_loss": 0.40133601427078247, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0053778961300849915, + "rewards/margins": 3.901097536087036, + "rewards/rejected": -3.895719289779663, + "step": 162, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9815581482875423, + "grad_norm": 3.0024077892303467, + "learning_rate": 0.0001114441775495432, + "logits/chosen": -0.3479576110839844, + "logits/rejected": -0.5726731419563293, + "logps/chosen": -5.025125026702881, + "logps/rejected": -43.54122543334961, + "loss": 0.7037834525108337, + "memory(GiB)": 46.13, + "nll_loss": 0.48029187321662903, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18540027737617493, + "rewards/margins": 2.662545919418335, + "rewards/rejected": -2.4771456718444824, + "step": 163, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9875799774181407, + "grad_norm": 5.379988670349121, + "learning_rate": 0.00011045284632676536, + "logits/chosen": -0.3792319893836975, + "logits/rejected": -0.5281450152397156, + "logps/chosen": -8.132346153259277, + "logps/rejected": -42.75868606567383, + "loss": 0.7357421517372131, + "memory(GiB)": 46.13, + "nll_loss": 0.4848954677581787, + "rewards/accuracies": 0.890625, + "rewards/chosen": 0.09983620047569275, + "rewards/margins": 2.911416530609131, + "rewards/rejected": -2.8115804195404053, + "step": 164, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9936018065487392, + "grad_norm": 7.149594306945801, + "learning_rate": 0.00010946047540025372, + "logits/chosen": -0.362922728061676, + "logits/rejected": -0.5330293774604797, + "logps/chosen": -5.657723903656006, + "logps/rejected": -44.190521240234375, + "loss": 0.777948796749115, + "memory(GiB)": 46.13, + "nll_loss": 0.4741102457046509, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0894114077091217, + "rewards/margins": 2.840927839279175, + "rewards/rejected": -2.751516580581665, + "step": 165, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 0.9996236356793377, + "grad_norm": 2.048894166946411, + "learning_rate": 0.00010846716347726233, + "logits/chosen": -0.4384031891822815, + "logits/rejected": -0.5566267371177673, + "logps/chosen": -3.853196382522583, + "logps/rejected": -43.331207275390625, + "loss": 0.5384762287139893, + "memory(GiB)": 46.13, + "nll_loss": 0.346954882144928, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1474352478981018, + "rewards/margins": 3.1648757457733154, + "rewards/rejected": -3.0174405574798584, + "step": 166, + "train_speed(iter/s)": 0.005633 + }, + { + "epoch": 1.0, + "grad_norm": 2.048894166946411, + "learning_rate": 0.00010747300935864243, + "logits/chosen": -0.37010034918785095, + "logits/rejected": -0.6208374500274658, + "logps/chosen": -8.17520809173584, + "logps/rejected": -61.29972839355469, + "loss": 0.12779855728149414, + "memory(GiB)": 46.13, + "nll_loss": 1.3625344038009644, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4605603814125061, + "rewards/margins": 4.346360683441162, + "rewards/rejected": -4.806921005249023, + "step": 167, + "train_speed(iter/s)": 0.005666 + }, + { + "epoch": 1.0060218291305985, + "grad_norm": 13.219114303588867, + "learning_rate": 0.00010647811192901518, + "logits/chosen": -0.34268608689308167, + "logits/rejected": -0.5246624946594238, + "logps/chosen": -4.731266975402832, + "logps/rejected": -46.866214752197266, + "loss": 0.5699129104614258, + "memory(GiB)": 46.13, + "nll_loss": 0.35907092690467834, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.17517316341400146, + "rewards/margins": 3.2273640632629395, + "rewards/rejected": -3.0521905422210693, + "step": 168, + "train_speed(iter/s)": 0.005665 + }, + { + "epoch": 1.012043658261197, + "grad_norm": 1.8148096799850464, + "learning_rate": 0.00010548257014693601, + "logits/chosen": -0.3512643575668335, + "logits/rejected": -0.5287365913391113, + "logps/chosen": -3.7313232421875, + "logps/rejected": -46.643409729003906, + "loss": 0.39852774143218994, + "memory(GiB)": 46.13, + "nll_loss": 0.22772511839866638, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.18213030695915222, + "rewards/margins": 3.319718837738037, + "rewards/rejected": -3.1375885009765625, + "step": 169, + "train_speed(iter/s)": 0.005665 + }, + { + "epoch": 1.0180654873917954, + "grad_norm": 3.1325247287750244, + "learning_rate": 0.00010448648303505151, + "logits/chosen": -0.3514709770679474, + "logits/rejected": -0.5060309171676636, + "logps/chosen": -2.770275592803955, + "logps/rejected": -42.078311920166016, + "loss": 0.4157463312149048, + "memory(GiB)": 46.13, + "nll_loss": 0.2555791139602661, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.31045016646385193, + "rewards/margins": 3.08093523979187, + "rewards/rejected": -2.770484685897827, + "step": 170, + "train_speed(iter/s)": 0.005665 + }, + { + "epoch": 1.0240873165223936, + "grad_norm": 2.593531370162964, + "learning_rate": 0.00010348994967025012, + "logits/chosen": -0.39463332295417786, + "logits/rejected": -0.5413990616798401, + "logps/chosen": -5.247183322906494, + "logps/rejected": -46.65768051147461, + "loss": 0.5715495347976685, + "memory(GiB)": 46.13, + "nll_loss": 0.3746676445007324, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.16656973958015442, + "rewards/margins": 3.0420989990234375, + "rewards/rejected": -2.8755292892456055, + "step": 171, + "train_speed(iter/s)": 0.005665 + }, + { + "epoch": 1.030109145652992, + "grad_norm": 1.7776541709899902, + "learning_rate": 0.0001024930691738073, + "logits/chosen": -0.3912968635559082, + "logits/rejected": -0.514131486415863, + "logps/chosen": -6.068740367889404, + "logps/rejected": -39.78652572631836, + "loss": 0.590811550617218, + "memory(GiB)": 46.13, + "nll_loss": 0.396287202835083, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2542215585708618, + "rewards/margins": 2.7374861240386963, + "rewards/rejected": -2.483264923095703, + "step": 172, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0361309747835905, + "grad_norm": 1.4022067785263062, + "learning_rate": 0.00010149594070152638, + "logits/chosen": -0.3674977123737335, + "logits/rejected": -0.5223217606544495, + "logps/chosen": -2.335606098175049, + "logps/rejected": -41.703453063964844, + "loss": 0.42932039499282837, + "memory(GiB)": 46.13, + "nll_loss": 0.24382930994033813, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.26927071809768677, + "rewards/margins": 2.965996503829956, + "rewards/rejected": -2.696725845336914, + "step": 173, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.042152803914189, + "grad_norm": 2.1577670574188232, + "learning_rate": 0.00010049866343387581, + "logits/chosen": -0.33516278862953186, + "logits/rejected": -0.4972761273384094, + "logps/chosen": -3.2422990798950195, + "logps/rejected": -43.91548538208008, + "loss": 0.4766288697719574, + "memory(GiB)": 46.13, + "nll_loss": 0.2899796962738037, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.25678345561027527, + "rewards/margins": 3.0347728729248047, + "rewards/rejected": -2.777989149093628, + "step": 174, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0481746330447874, + "grad_norm": 1.4100942611694336, + "learning_rate": 9.950133656612421e-05, + "logits/chosen": -0.3753768503665924, + "logits/rejected": -0.49912041425704956, + "logps/chosen": -3.9077322483062744, + "logps/rejected": -36.67354965209961, + "loss": 0.44091612100601196, + "memory(GiB)": 46.13, + "nll_loss": 0.2640770673751831, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.37277600169181824, + "rewards/margins": 2.572587013244629, + "rewards/rejected": -2.1998109817504883, + "step": 175, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0541964621753859, + "grad_norm": 1.346986174583435, + "learning_rate": 9.850405929847366e-05, + "logits/chosen": -0.38442033529281616, + "logits/rejected": -0.5594974160194397, + "logps/chosen": -3.173485040664673, + "logps/rejected": -43.258094787597656, + "loss": 0.43492069840431213, + "memory(GiB)": 46.13, + "nll_loss": 0.28007155656814575, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.28305596113204956, + "rewards/margins": 3.2277960777282715, + "rewards/rejected": -2.9447402954101562, + "step": 176, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.060218291305984, + "grad_norm": 1.9254361391067505, + "learning_rate": 9.750693082619273e-05, + "logits/chosen": -0.3788023889064789, + "logits/rejected": -0.5781509876251221, + "logps/chosen": -2.432941198348999, + "logps/rejected": -43.248146057128906, + "loss": 0.40009191632270813, + "memory(GiB)": 46.13, + "nll_loss": 0.26620903611183167, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.17993758618831635, + "rewards/margins": 3.0553667545318604, + "rewards/rejected": -2.8754286766052246, + "step": 177, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0662401204365826, + "grad_norm": 3.2801098823547363, + "learning_rate": 9.651005032974994e-05, + "logits/chosen": -0.33594202995300293, + "logits/rejected": -0.564365029335022, + "logps/chosen": -3.6123833656311035, + "logps/rejected": -49.69728088378906, + "loss": 0.4477875828742981, + "memory(GiB)": 46.13, + "nll_loss": 0.2894076108932495, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.19575706124305725, + "rewards/margins": 3.35469388961792, + "rewards/rejected": -3.1589369773864746, + "step": 178, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.072261949567181, + "grad_norm": 1.4043627977371216, + "learning_rate": 9.551351696494854e-05, + "logits/chosen": -0.37707191705703735, + "logits/rejected": -0.5827841758728027, + "logps/chosen": -2.930352210998535, + "logps/rejected": -50.44269943237305, + "loss": 0.3742366135120392, + "memory(GiB)": 46.13, + "nll_loss": 0.2148890197277069, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.21785704791545868, + "rewards/margins": 3.5218472480773926, + "rewards/rejected": -3.303990364074707, + "step": 179, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0782837786977795, + "grad_norm": 1.5572144985198975, + "learning_rate": 9.451742985306398e-05, + "logits/chosen": -0.45742160081863403, + "logits/rejected": -0.6289136409759521, + "logps/chosen": -4.650609016418457, + "logps/rejected": -44.8750114440918, + "loss": 0.4705524444580078, + "memory(GiB)": 46.13, + "nll_loss": 0.3358931541442871, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2931283414363861, + "rewards/margins": 3.50144624710083, + "rewards/rejected": -3.2083182334899902, + "step": 180, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.084305607828378, + "grad_norm": 2.9546408653259277, + "learning_rate": 9.352188807098481e-05, + "logits/chosen": -0.4296163320541382, + "logits/rejected": -0.5809882879257202, + "logps/chosen": -4.599586009979248, + "logps/rejected": -42.98283386230469, + "loss": 0.5064187049865723, + "memory(GiB)": 46.13, + "nll_loss": 0.32983240485191345, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.33693230152130127, + "rewards/margins": 3.1911778450012207, + "rewards/rejected": -2.854245662689209, + "step": 181, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.0903274369589764, + "grad_norm": 1.369982123374939, + "learning_rate": 9.252699064135758e-05, + "logits/chosen": -0.5194458961486816, + "logits/rejected": -0.7049198746681213, + "logps/chosen": -2.871542453765869, + "logps/rejected": -51.704444885253906, + "loss": 0.36053964495658875, + "memory(GiB)": 46.13, + "nll_loss": 0.25251564383506775, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.25039854645729065, + "rewards/margins": 4.2018303871154785, + "rewards/rejected": -3.9514317512512207, + "step": 182, + "train_speed(iter/s)": 0.005664 + }, + { + "epoch": 1.0963492660895746, + "grad_norm": 3.087183713912964, + "learning_rate": 9.153283652273768e-05, + "logits/chosen": -0.42322614789009094, + "logits/rejected": -0.687001645565033, + "logps/chosen": -5.198493957519531, + "logps/rejected": -59.92394256591797, + "loss": 0.49455440044403076, + "memory(GiB)": 46.13, + "nll_loss": 0.391512006521225, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.19199909269809723, + "rewards/margins": 4.593501091003418, + "rewards/rejected": -4.401501178741455, + "step": 183, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.102371095220173, + "grad_norm": 2.106433391571045, + "learning_rate": 9.05395245997463e-05, + "logits/chosen": -0.44497790932655334, + "logits/rejected": -0.6745092272758484, + "logps/chosen": -4.499228477478027, + "logps/rejected": -65.7539291381836, + "loss": 0.4104112982749939, + "memory(GiB)": 46.13, + "nll_loss": 0.3255768418312073, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31253090500831604, + "rewards/margins": 5.569586753845215, + "rewards/rejected": -5.257055759429932, + "step": 184, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.1083929243507715, + "grad_norm": 3.478254556655884, + "learning_rate": 8.954715367323468e-05, + "logits/chosen": -0.4755058288574219, + "logits/rejected": -0.7400538325309753, + "logps/chosen": -4.192931175231934, + "logps/rejected": -58.25489044189453, + "loss": 0.49360448122024536, + "memory(GiB)": 46.13, + "nll_loss": 0.3856234550476074, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.23331968486309052, + "rewards/margins": 4.7285542488098145, + "rewards/rejected": -4.495234489440918, + "step": 185, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.11441475348137, + "grad_norm": 3.7779037952423096, + "learning_rate": 8.855582245045683e-05, + "logits/chosen": -0.47070175409317017, + "logits/rejected": -0.6918365955352783, + "logps/chosen": -5.071094036102295, + "logps/rejected": -53.90576934814453, + "loss": 0.6311454772949219, + "memory(GiB)": 46.13, + "nll_loss": 0.44870883226394653, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.04279123246669769, + "rewards/margins": 4.136241912841797, + "rewards/rejected": -4.093451023101807, + "step": 186, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.1204365826119684, + "grad_norm": 7.022064685821533, + "learning_rate": 8.756562953525152e-05, + "logits/chosen": -0.432133287191391, + "logits/rejected": -0.776434063911438, + "logps/chosen": -3.279095411300659, + "logps/rejected": -69.65541076660156, + "loss": 0.3344402611255646, + "memory(GiB)": 46.13, + "nll_loss": 0.2780175805091858, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.25577378273010254, + "rewards/margins": 5.72814416885376, + "rewards/rejected": -5.472370624542236, + "step": 187, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.126458411742567, + "grad_norm": 2.485914945602417, + "learning_rate": 8.657667341823448e-05, + "logits/chosen": -0.49354246258735657, + "logits/rejected": -0.7343668341636658, + "logps/chosen": -4.402050971984863, + "logps/rejected": -56.273170471191406, + "loss": 0.454555481672287, + "memory(GiB)": 46.13, + "nll_loss": 0.3319138288497925, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.23367615044116974, + "rewards/margins": 4.513181209564209, + "rewards/rejected": -4.279505252838135, + "step": 188, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.1324802408731651, + "grad_norm": 1.653633713722229, + "learning_rate": 8.558905246700201e-05, + "logits/chosen": -0.47012466192245483, + "logits/rejected": -0.7377257943153381, + "logps/chosen": -2.28665828704834, + "logps/rejected": -65.824462890625, + "loss": 0.31783220171928406, + "memory(GiB)": 46.13, + "nll_loss": 0.21150296926498413, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2025526762008667, + "rewards/margins": 5.142179012298584, + "rewards/rejected": -4.939626216888428, + "step": 189, + "train_speed(iter/s)": 0.005663 + }, + { + "epoch": 1.1385020700037636, + "grad_norm": 1.506331443786621, + "learning_rate": 8.460286491634663e-05, + "logits/chosen": -0.45535990595817566, + "logits/rejected": -0.6702480316162109, + "logps/chosen": -6.301774978637695, + "logps/rejected": -60.1036262512207, + "loss": 0.41861438751220703, + "memory(GiB)": 46.13, + "nll_loss": 0.35035622119903564, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4653804302215576, + "rewards/margins": 4.745593547821045, + "rewards/rejected": -4.280213356018066, + "step": 190, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.144523899134362, + "grad_norm": 7.771515369415283, + "learning_rate": 8.361820885848624e-05, + "logits/chosen": -0.559324324131012, + "logits/rejected": -0.7053715586662292, + "logps/chosen": -6.118768692016602, + "logps/rejected": -52.78406524658203, + "loss": 0.6141310930252075, + "memory(GiB)": 46.13, + "nll_loss": 0.4540223479270935, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3042090833187103, + "rewards/margins": 4.374350547790527, + "rewards/rejected": -4.070141792297363, + "step": 191, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.1505457282649605, + "grad_norm": 2.293672800064087, + "learning_rate": 8.263518223330697e-05, + "logits/chosen": -0.49931055307388306, + "logits/rejected": -0.7118666768074036, + "logps/chosen": -4.773472785949707, + "logps/rejected": -52.72026824951172, + "loss": 0.43889304995536804, + "memory(GiB)": 46.13, + "nll_loss": 0.34974291920661926, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32993441820144653, + "rewards/margins": 4.337234973907471, + "rewards/rejected": -4.00730037689209, + "step": 192, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.156567557395559, + "grad_norm": 2.9582977294921875, + "learning_rate": 8.165388281862178e-05, + "logits/chosen": -0.5245535373687744, + "logits/rejected": -0.7488053441047668, + "logps/chosen": -4.319539546966553, + "logps/rejected": -49.7554817199707, + "loss": 0.4565528333187103, + "memory(GiB)": 46.13, + "nll_loss": 0.34413373470306396, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2616381347179413, + "rewards/margins": 3.892477512359619, + "rewards/rejected": -3.6308393478393555, + "step": 193, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.1625893865261574, + "grad_norm": 2.9784886837005615, + "learning_rate": 8.067440822044469e-05, + "logits/chosen": -0.548271894454956, + "logits/rejected": -0.7532668113708496, + "logps/chosen": -6.1191020011901855, + "logps/rejected": -50.176265716552734, + "loss": 0.6044948101043701, + "memory(GiB)": 46.13, + "nll_loss": 0.49495670199394226, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20567229390144348, + "rewards/margins": 4.019640922546387, + "rewards/rejected": -3.8139681816101074, + "step": 194, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.1686112156567559, + "grad_norm": 1.896803617477417, + "learning_rate": 7.96968558632824e-05, + "logits/chosen": -0.4820174276828766, + "logits/rejected": -0.7427547574043274, + "logps/chosen": -6.175469875335693, + "logps/rejected": -60.446834564208984, + "loss": 0.5533992648124695, + "memory(GiB)": 46.13, + "nll_loss": 0.4257928133010864, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.25794556736946106, + "rewards/margins": 4.815202236175537, + "rewards/rejected": -4.557256698608398, + "step": 195, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.174633044787354, + "grad_norm": 2.890577554702759, + "learning_rate": 7.872132298044382e-05, + "logits/chosen": -0.47485801577568054, + "logits/rejected": -0.7618663311004639, + "logps/chosen": -2.814539909362793, + "logps/rejected": -54.596343994140625, + "loss": 0.3804952800273895, + "memory(GiB)": 46.13, + "nll_loss": 0.24290579557418823, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.3001152276992798, + "rewards/margins": 4.137256622314453, + "rewards/rejected": -3.8371410369873047, + "step": 196, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.1806548739179525, + "grad_norm": 3.058959722518921, + "learning_rate": 7.774790660436858e-05, + "logits/chosen": -0.5394716262817383, + "logits/rejected": -0.7631605267524719, + "logps/chosen": -5.447549819946289, + "logps/rejected": -47.610145568847656, + "loss": 0.5637054443359375, + "memory(GiB)": 46.13, + "nll_loss": 0.4707457423210144, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3298027515411377, + "rewards/margins": 3.690687656402588, + "rewards/rejected": -3.36088490486145, + "step": 197, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.186676703048551, + "grad_norm": 1.5582202672958374, + "learning_rate": 7.677670355697577e-05, + "logits/chosen": -0.3999323546886444, + "logits/rejected": -0.7442487478256226, + "logps/chosen": -2.5974080562591553, + "logps/rejected": -62.564483642578125, + "loss": 0.33909475803375244, + "memory(GiB)": 46.13, + "nll_loss": 0.25633975863456726, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.32342803478240967, + "rewards/margins": 4.781001091003418, + "rewards/rejected": -4.457572937011719, + "step": 198, + "train_speed(iter/s)": 0.005662 + }, + { + "epoch": 1.1926985321791495, + "grad_norm": 1.2910958528518677, + "learning_rate": 7.580781044003324e-05, + "logits/chosen": -0.4777570068836212, + "logits/rejected": -0.7280046343803406, + "logps/chosen": -3.076066493988037, + "logps/rejected": -55.281494140625, + "loss": 0.32068008184432983, + "memory(GiB)": 46.13, + "nll_loss": 0.2149108499288559, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.284832626581192, + "rewards/margins": 4.372798442840576, + "rewards/rejected": -4.087965488433838, + "step": 199, + "train_speed(iter/s)": 0.005661 + }, + { + "epoch": 1.198720361309748, + "grad_norm": 2.6249725818634033, + "learning_rate": 7.484132362554915e-05, + "logits/chosen": -0.5193163752555847, + "logits/rejected": -0.7307865619659424, + "logps/chosen": -6.235736846923828, + "logps/rejected": -44.48322677612305, + "loss": 0.6363410353660583, + "memory(GiB)": 46.13, + "nll_loss": 0.44810640811920166, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.32792535424232483, + "rewards/margins": 3.486876964569092, + "rewards/rejected": -3.158951759338379, + "step": 200, + "train_speed(iter/s)": 0.005661 + }, + { + "epoch": 1.2047421904403461, + "grad_norm": 2.804173707962036, + "learning_rate": 7.387733924618617e-05, + "logits/chosen": -0.41156628727912903, + "logits/rejected": -0.6807573437690735, + "logps/chosen": -4.78164005279541, + "logps/rejected": -58.340606689453125, + "loss": 0.43140679597854614, + "memory(GiB)": 46.13, + "nll_loss": 0.33691924810409546, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.29949134588241577, + "rewards/margins": 4.446183681488037, + "rewards/rejected": -4.146692276000977, + "step": 201, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.2107640195709446, + "grad_norm": 3.942891836166382, + "learning_rate": 7.291595318569951e-05, + "logits/chosen": -0.5371648669242859, + "logits/rejected": -0.7549322843551636, + "logps/chosen": -7.573888301849365, + "logps/rejected": -50.77197265625, + "loss": 0.8032181262969971, + "memory(GiB)": 46.13, + "nll_loss": 0.6219516396522522, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.18600612878799438, + "rewards/margins": 3.8017468452453613, + "rewards/rejected": -3.6157405376434326, + "step": 202, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.216785848701543, + "grad_norm": 1.7948954105377197, + "learning_rate": 7.195726106939974e-05, + "logits/chosen": -0.49177029728889465, + "logits/rejected": -0.726800799369812, + "logps/chosen": -4.9699907302856445, + "logps/rejected": -60.005218505859375, + "loss": 0.4122886061668396, + "memory(GiB)": 46.13, + "nll_loss": 0.30634433031082153, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.36401644349098206, + "rewards/margins": 4.706804275512695, + "rewards/rejected": -4.342788219451904, + "step": 203, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.2228076778321415, + "grad_norm": 4.069206237792969, + "learning_rate": 7.100135825464139e-05, + "logits/chosen": -0.45375001430511475, + "logits/rejected": -0.7056143283843994, + "logps/chosen": -6.165468692779541, + "logps/rejected": -48.77824783325195, + "loss": 0.5478510856628418, + "memory(GiB)": 46.13, + "nll_loss": 0.4077027440071106, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.40980643033981323, + "rewards/margins": 3.759075403213501, + "rewards/rejected": -3.349268913269043, + "step": 204, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.22882950696274, + "grad_norm": 3.2379000186920166, + "learning_rate": 7.004833982133808e-05, + "logits/chosen": -0.4625844955444336, + "logits/rejected": -0.7497645616531372, + "logps/chosen": -3.2215240001678467, + "logps/rejected": -54.89192199707031, + "loss": 0.4756178557872772, + "memory(GiB)": 46.13, + "nll_loss": 0.3104082942008972, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10858356952667236, + "rewards/margins": 4.205223083496094, + "rewards/rejected": -4.096639156341553, + "step": 205, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.2348513360933384, + "grad_norm": 1.2772859334945679, + "learning_rate": 6.909830056250527e-05, + "logits/chosen": -0.46088141202926636, + "logits/rejected": -0.696212649345398, + "logps/chosen": -3.752298355102539, + "logps/rejected": -51.43436813354492, + "loss": 0.3476787805557251, + "memory(GiB)": 46.13, + "nll_loss": 0.24112334847450256, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3736254572868347, + "rewards/margins": 4.043981552124023, + "rewards/rejected": -3.670356512069702, + "step": 206, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.2408731652239369, + "grad_norm": 3.96500301361084, + "learning_rate": 6.815133497483157e-05, + "logits/chosen": -0.4538869261741638, + "logits/rejected": -0.6587194204330444, + "logps/chosen": -3.7387635707855225, + "logps/rejected": -44.64501953125, + "loss": 0.4004911780357361, + "memory(GiB)": 46.13, + "nll_loss": 0.2752317786216736, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32666444778442383, + "rewards/margins": 3.603152275085449, + "rewards/rejected": -3.2764875888824463, + "step": 207, + "train_speed(iter/s)": 0.005658 + }, + { + "epoch": 1.246894994354535, + "grad_norm": 1.6064969301223755, + "learning_rate": 6.720753724927958e-05, + "logits/chosen": -0.49224531650543213, + "logits/rejected": -0.7212605476379395, + "logps/chosen": -8.330548286437988, + "logps/rejected": -52.39625930786133, + "loss": 0.5388084650039673, + "memory(GiB)": 46.13, + "nll_loss": 0.440222829580307, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.4413294792175293, + "rewards/margins": 4.095629692077637, + "rewards/rejected": -3.6542999744415283, + "step": 208, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.2529168234851336, + "grad_norm": 1.888574481010437, + "learning_rate": 6.626700126171702e-05, + "logits/chosen": -0.46829870343208313, + "logits/rejected": -0.7399269342422485, + "logps/chosen": -3.1671807765960693, + "logps/rejected": -52.16673278808594, + "loss": 0.35412514209747314, + "memory(GiB)": 46.13, + "nll_loss": 0.24468238651752472, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.35541072487831116, + "rewards/margins": 4.117459774017334, + "rewards/rejected": -3.7620487213134766, + "step": 209, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.258938652615732, + "grad_norm": 8.906493186950684, + "learning_rate": 6.532982056357928e-05, + "logits/chosen": -0.5025848150253296, + "logits/rejected": -0.647516131401062, + "logps/chosen": -4.769540309906006, + "logps/rejected": -46.10678482055664, + "loss": 0.487936407327652, + "memory(GiB)": 46.13, + "nll_loss": 0.35828882455825806, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3698485195636749, + "rewards/margins": 3.711308717727661, + "rewards/rejected": -3.3414597511291504, + "step": 210, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.2649604817463305, + "grad_norm": 2.5342342853546143, + "learning_rate": 6.439608837256432e-05, + "logits/chosen": -0.43483102321624756, + "logits/rejected": -0.7881991863250732, + "logps/chosen": -2.7267870903015137, + "logps/rejected": -65.4534683227539, + "loss": 0.3906051218509674, + "memory(GiB)": 46.13, + "nll_loss": 0.29971927404403687, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2297130823135376, + "rewards/margins": 5.093780040740967, + "rewards/rejected": -4.864067077636719, + "step": 211, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.270982310876929, + "grad_norm": 1.950996994972229, + "learning_rate": 6.34658975633605e-05, + "logits/chosen": -0.35362473130226135, + "logits/rejected": -0.6550623774528503, + "logps/chosen": -4.400423526763916, + "logps/rejected": -51.62921905517578, + "loss": 0.49274933338165283, + "memory(GiB)": 46.13, + "nll_loss": 0.3038281798362732, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.2719678282737732, + "rewards/margins": 3.763463258743286, + "rewards/rejected": -3.4914956092834473, + "step": 212, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.2770041400075272, + "grad_norm": 1.5276005268096924, + "learning_rate": 6.25393406584088e-05, + "logits/chosen": -0.40574443340301514, + "logits/rejected": -0.7509509921073914, + "logps/chosen": -3.0956404209136963, + "logps/rejected": -59.7081298828125, + "loss": 0.34677064418792725, + "memory(GiB)": 46.13, + "nll_loss": 0.2606160342693329, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23602759838104248, + "rewards/margins": 4.654949188232422, + "rewards/rejected": -4.418920993804932, + "step": 213, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.2830259691381256, + "grad_norm": 1.9581797122955322, + "learning_rate": 6.161650981869998e-05, + "logits/chosen": -0.4477730691432953, + "logits/rejected": -0.708689272403717, + "logps/chosen": -4.111853122711182, + "logps/rejected": -65.05699157714844, + "loss": 0.45744559168815613, + "memory(GiB)": 46.13, + "nll_loss": 0.333187997341156, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.20490425825119019, + "rewards/margins": 4.838937759399414, + "rewards/rejected": -4.634034156799316, + "step": 214, + "train_speed(iter/s)": 0.005657 + }, + { + "epoch": 1.289047798268724, + "grad_norm": 0.9938692450523376, + "learning_rate": 6.069749683460765e-05, + "logits/chosen": -0.3912752568721771, + "logits/rejected": -0.7120423316955566, + "logps/chosen": -2.6046087741851807, + "logps/rejected": -72.46825408935547, + "loss": 0.28505194187164307, + "memory(GiB)": 46.13, + "nll_loss": 0.23823222517967224, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31369316577911377, + "rewards/margins": 5.755457401275635, + "rewards/rejected": -5.441763877868652, + "step": 215, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.2950696273993225, + "grad_norm": 1.6541746854782104, + "learning_rate": 5.978239311675826e-05, + "logits/chosen": -0.4647775888442993, + "logits/rejected": -0.7389938831329346, + "logps/chosen": -4.554535865783691, + "logps/rejected": -61.292266845703125, + "loss": 0.44827941060066223, + "memory(GiB)": 46.13, + "nll_loss": 0.3300400674343109, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.24124576151371002, + "rewards/margins": 4.951149940490723, + "rewards/rejected": -4.709904193878174, + "step": 216, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.301091456529921, + "grad_norm": 1.3506146669387817, + "learning_rate": 5.887128968693887e-05, + "logits/chosen": -0.5377171635627747, + "logits/rejected": -0.7490763664245605, + "logps/chosen": -4.447143077850342, + "logps/rejected": -55.9873161315918, + "loss": 0.44089001417160034, + "memory(GiB)": 46.13, + "nll_loss": 0.3394245207309723, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.4647373557090759, + "rewards/margins": 4.664052963256836, + "rewards/rejected": -4.199316024780273, + "step": 217, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3071132856605194, + "grad_norm": 3.494126796722412, + "learning_rate": 5.796427716904347e-05, + "logits/chosen": -0.5005620718002319, + "logits/rejected": -0.8207815885543823, + "logps/chosen": -2.703669309616089, + "logps/rejected": -65.87652587890625, + "loss": 0.33874765038490295, + "memory(GiB)": 46.13, + "nll_loss": 0.2589980661869049, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.22972920536994934, + "rewards/margins": 5.275555610656738, + "rewards/rejected": -5.0458269119262695, + "step": 218, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3131351147911179, + "grad_norm": 1.4259244203567505, + "learning_rate": 5.7061445780059074e-05, + "logits/chosen": -0.4546062648296356, + "logits/rejected": -0.7871499061584473, + "logps/chosen": -3.830498218536377, + "logps/rejected": -65.30023956298828, + "loss": 0.31216228008270264, + "memory(GiB)": 46.13, + "nll_loss": 0.2483048439025879, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3124459385871887, + "rewards/margins": 5.200329303741455, + "rewards/rejected": -4.88788366317749, + "step": 219, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3191569439217163, + "grad_norm": 2.587780475616455, + "learning_rate": 5.616288532109225e-05, + "logits/chosen": -0.5172604322433472, + "logits/rejected": -0.7480805516242981, + "logps/chosen": -5.291163921356201, + "logps/rejected": -57.106868743896484, + "loss": 0.5239050388336182, + "memory(GiB)": 46.13, + "nll_loss": 0.406022310256958, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.26827341318130493, + "rewards/margins": 4.4456329345703125, + "rewards/rejected": -4.1773600578308105, + "step": 220, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3251787730523146, + "grad_norm": 1.639807105064392, + "learning_rate": 5.526868516843673e-05, + "logits/chosen": -0.547332227230072, + "logits/rejected": -0.7309650182723999, + "logps/chosen": -5.008947372436523, + "logps/rejected": -56.65232849121094, + "loss": 0.4063935875892639, + "memory(GiB)": 46.13, + "nll_loss": 0.31995317339897156, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2977209985256195, + "rewards/margins": 4.773816108703613, + "rewards/rejected": -4.476095676422119, + "step": 221, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.331200602182913, + "grad_norm": 4.514076232910156, + "learning_rate": 5.43789342646837e-05, + "logits/chosen": -0.4913971424102783, + "logits/rejected": -0.7436814308166504, + "logps/chosen": -4.23374605178833, + "logps/rejected": -58.28513717651367, + "loss": 0.4951242208480835, + "memory(GiB)": 46.13, + "nll_loss": 0.3493507504463196, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2692214250564575, + "rewards/margins": 4.63377046585083, + "rewards/rejected": -4.364549160003662, + "step": 222, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3372224313135115, + "grad_norm": 1.7904651165008545, + "learning_rate": 5.349372110987496e-05, + "logits/chosen": -0.525532603263855, + "logits/rejected": -0.7780842781066895, + "logps/chosen": -3.3643298149108887, + "logps/rejected": -65.86946868896484, + "loss": 0.3287546932697296, + "memory(GiB)": 46.13, + "nll_loss": 0.26226621866226196, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34989917278289795, + "rewards/margins": 5.363834381103516, + "rewards/rejected": -5.013935089111328, + "step": 223, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.34324426044411, + "grad_norm": 1.8052747249603271, + "learning_rate": 5.261313375270014e-05, + "logits/chosen": -0.5206581950187683, + "logits/rejected": -0.7903106212615967, + "logps/chosen": -4.260349273681641, + "logps/rejected": -65.74819946289062, + "loss": 0.4206431806087494, + "memory(GiB)": 46.13, + "nll_loss": 0.3237663805484772, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.32122018933296204, + "rewards/margins": 5.483022212982178, + "rewards/rejected": -5.161801815032959, + "step": 224, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3492660895747084, + "grad_norm": 2.2413487434387207, + "learning_rate": 5.1737259781738936e-05, + "logits/chosen": -0.5118913650512695, + "logits/rejected": -0.7823872566223145, + "logps/chosen": -4.02402400970459, + "logps/rejected": -56.05274200439453, + "loss": 0.4401736855506897, + "memory(GiB)": 46.13, + "nll_loss": 0.34718388319015503, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31864693760871887, + "rewards/margins": 4.531757354736328, + "rewards/rejected": -4.213110446929932, + "step": 225, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.3552879187053066, + "grad_norm": 2.4015703201293945, + "learning_rate": 5.086618631674888e-05, + "logits/chosen": -0.4304109215736389, + "logits/rejected": -0.8146275281906128, + "logps/chosen": -5.407003402709961, + "logps/rejected": -75.61699676513672, + "loss": 0.4439658224582672, + "memory(GiB)": 46.13, + "nll_loss": 0.3463272750377655, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.24775578081607819, + "rewards/margins": 5.974477767944336, + "rewards/rejected": -5.726721286773682, + "step": 226, + "train_speed(iter/s)": 0.005656 + }, + { + "epoch": 1.361309747835905, + "grad_norm": 1.798647165298462, + "learning_rate": 5.000000000000002e-05, + "logits/chosen": -0.41718822717666626, + "logits/rejected": -0.8002769947052002, + "logps/chosen": -3.8272299766540527, + "logps/rejected": -80.6712646484375, + "loss": 0.41253572702407837, + "memory(GiB)": 46.13, + "nll_loss": 0.35548415780067444, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15689942240715027, + "rewards/margins": 6.417231559753418, + "rewards/rejected": -6.260332107543945, + "step": 227, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.3673315769665035, + "grad_norm": 1.539751648902893, + "learning_rate": 4.913878698765686e-05, + "logits/chosen": -0.5234534740447998, + "logits/rejected": -0.7054301500320435, + "logps/chosen": -7.054202079772949, + "logps/rejected": -59.50374221801758, + "loss": 0.46764418482780457, + "memory(GiB)": 46.13, + "nll_loss": 0.379008412361145, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3111666738986969, + "rewards/margins": 4.931512832641602, + "rewards/rejected": -4.620346546173096, + "step": 228, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.373353406097102, + "grad_norm": 4.103943347930908, + "learning_rate": 4.8282632941208725e-05, + "logits/chosen": -0.40814512968063354, + "logits/rejected": -0.7456347942352295, + "logps/chosen": -4.591305255889893, + "logps/rejected": -72.21307373046875, + "loss": 0.372346967458725, + "memory(GiB)": 46.13, + "nll_loss": 0.33533719182014465, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37759003043174744, + "rewards/margins": 5.916057586669922, + "rewards/rejected": -5.5384674072265625, + "step": 229, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.3793752352277004, + "grad_norm": 2.3027875423431396, + "learning_rate": 4.743162301894952e-05, + "logits/chosen": -0.530301034450531, + "logits/rejected": -0.7990034222602844, + "logps/chosen": -4.5230021476745605, + "logps/rejected": -65.62507629394531, + "loss": 0.38261985778808594, + "memory(GiB)": 46.13, + "nll_loss": 0.3187011778354645, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3396291434764862, + "rewards/margins": 5.285628318786621, + "rewards/rejected": -4.945999622344971, + "step": 230, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.385397064358299, + "grad_norm": 3.5107035636901855, + "learning_rate": 4.658584186750713e-05, + "logits/chosen": -0.4405829906463623, + "logits/rejected": -0.7592389583587646, + "logps/chosen": -3.0335636138916016, + "logps/rejected": -64.45790100097656, + "loss": 0.3538115918636322, + "memory(GiB)": 46.13, + "nll_loss": 0.26427680253982544, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.22065187990665436, + "rewards/margins": 5.173749923706055, + "rewards/rejected": -4.953097343444824, + "step": 231, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.3914188934888974, + "grad_norm": 3.794492244720459, + "learning_rate": 4.574537361342407e-05, + "logits/chosen": -0.5307365655899048, + "logits/rejected": -0.8200229406356812, + "logps/chosen": -2.5140573978424072, + "logps/rejected": -63.374088287353516, + "loss": 0.30093103647232056, + "memory(GiB)": 46.13, + "nll_loss": 0.23118244111537933, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2798585295677185, + "rewards/margins": 5.135382175445557, + "rewards/rejected": -4.855523109436035, + "step": 232, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.3974407226194956, + "grad_norm": 3.8283984661102295, + "learning_rate": 4.491030185478976e-05, + "logits/chosen": -0.5696873664855957, + "logits/rejected": -0.7703967690467834, + "logps/chosen": -4.258246421813965, + "logps/rejected": -60.22206497192383, + "loss": 0.4060544967651367, + "memory(GiB)": 46.13, + "nll_loss": 0.3264697194099426, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27311259508132935, + "rewards/margins": 5.165642261505127, + "rewards/rejected": -4.892529487609863, + "step": 233, + "train_speed(iter/s)": 0.005655 + }, + { + "epoch": 1.403462551750094, + "grad_norm": 3.2133429050445557, + "learning_rate": 4.4080709652925336e-05, + "logits/chosen": -0.45184794068336487, + "logits/rejected": -0.7718617916107178, + "logps/chosen": -5.11681604385376, + "logps/rejected": -76.45780181884766, + "loss": 0.43574121594429016, + "memory(GiB)": 46.13, + "nll_loss": 0.3303045332431793, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2558760643005371, + "rewards/margins": 6.22172737121582, + "rewards/rejected": -5.965850830078125, + "step": 234, + "train_speed(iter/s)": 0.005654 + }, + { + "epoch": 1.4094843808806925, + "grad_norm": 9.291942596435547, + "learning_rate": 4.3256679524121834e-05, + "logits/chosen": -0.4269176423549652, + "logits/rejected": -0.7866188883781433, + "logps/chosen": -1.7651253938674927, + "logps/rejected": -71.36811065673828, + "loss": 0.26370862126350403, + "memory(GiB)": 46.13, + "nll_loss": 0.178541362285614, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.21358121931552887, + "rewards/margins": 5.760619163513184, + "rewards/rejected": -5.547037601470947, + "step": 235, + "train_speed(iter/s)": 0.005654 + }, + { + "epoch": 1.415506210011291, + "grad_norm": 1.960109829902649, + "learning_rate": 4.2438293431432665e-05, + "logits/chosen": -0.4432763159275055, + "logits/rejected": -0.7087511420249939, + "logps/chosen": -3.8448731899261475, + "logps/rejected": -70.58623504638672, + "loss": 0.3397082984447479, + "memory(GiB)": 46.13, + "nll_loss": 0.2688691318035126, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3202817142009735, + "rewards/margins": 5.778705596923828, + "rewards/rejected": -5.458423614501953, + "step": 236, + "train_speed(iter/s)": 0.005654 + }, + { + "epoch": 1.4215280391418894, + "grad_norm": 1.571708083152771, + "learning_rate": 4.1625632776521037e-05, + "logits/chosen": -0.3686775863170624, + "logits/rejected": -0.7584090828895569, + "logps/chosen": -3.5422518253326416, + "logps/rejected": -74.61741638183594, + "loss": 0.36048567295074463, + "memory(GiB)": 46.13, + "nll_loss": 0.3007555603981018, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.32932642102241516, + "rewards/margins": 5.9051923751831055, + "rewards/rejected": -5.575865745544434, + "step": 237, + "train_speed(iter/s)": 0.005654 + }, + { + "epoch": 1.4275498682724876, + "grad_norm": 1.7286449670791626, + "learning_rate": 4.081877839156325e-05, + "logits/chosen": -0.3519198000431061, + "logits/rejected": -0.7000710964202881, + "logps/chosen": -4.214243412017822, + "logps/rejected": -73.10647583007812, + "loss": 0.3545505106449127, + "memory(GiB)": 46.13, + "nll_loss": 0.2885700464248657, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.24009957909584045, + "rewards/margins": 5.838498592376709, + "rewards/rejected": -5.5983991622924805, + "step": 238, + "train_speed(iter/s)": 0.005654 + }, + { + "epoch": 1.433571697403086, + "grad_norm": 4.094414234161377, + "learning_rate": 4.001781053120863e-05, + "logits/chosen": -0.5312204360961914, + "logits/rejected": -0.7494320869445801, + "logps/chosen": -3.4775454998016357, + "logps/rejected": -54.855403900146484, + "loss": 0.40386420488357544, + "memory(GiB)": 46.13, + "nll_loss": 0.24309346079826355, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.268097460269928, + "rewards/margins": 4.557480812072754, + "rewards/rejected": -4.289383411407471, + "step": 239, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.4395935265336846, + "grad_norm": 1.7741433382034302, + "learning_rate": 3.9222808864597004e-05, + "logits/chosen": -0.5123165845870972, + "logits/rejected": -0.8100269436836243, + "logps/chosen": -3.479257106781006, + "logps/rejected": -70.73236083984375, + "loss": 0.29427477717399597, + "memory(GiB)": 46.13, + "nll_loss": 0.2662776708602905, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23656059801578522, + "rewards/margins": 5.880814075469971, + "rewards/rejected": -5.644253730773926, + "step": 240, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.445615355664283, + "grad_norm": 1.358980417251587, + "learning_rate": 3.843385246743417e-05, + "logits/chosen": -0.43873101472854614, + "logits/rejected": -0.7396629452705383, + "logps/chosen": -2.895552396774292, + "logps/rejected": -58.965843200683594, + "loss": 0.27671289443969727, + "memory(GiB)": 46.13, + "nll_loss": 0.18713214993476868, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.253683865070343, + "rewards/margins": 4.665976047515869, + "rewards/rejected": -4.412292003631592, + "step": 241, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.4516371847948815, + "grad_norm": 2.561663866043091, + "learning_rate": 3.7651019814126654e-05, + "logits/chosen": -0.5158201456069946, + "logits/rejected": -0.7556857466697693, + "logps/chosen": -6.659661769866943, + "logps/rejected": -61.460845947265625, + "loss": 0.540454626083374, + "memory(GiB)": 46.13, + "nll_loss": 0.43979448080062866, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.351629376411438, + "rewards/margins": 5.123873710632324, + "rewards/rejected": -4.772244453430176, + "step": 242, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.45765901392548, + "grad_norm": 4.672154903411865, + "learning_rate": 3.687438876997612e-05, + "logits/chosen": -0.5328869819641113, + "logits/rejected": -0.78375244140625, + "logps/chosen": -4.969141960144043, + "logps/rejected": -57.542362213134766, + "loss": 0.5365437269210815, + "memory(GiB)": 46.13, + "nll_loss": 0.3501831889152527, + "rewards/accuracies": 0.921875, + "rewards/chosen": 0.1367063820362091, + "rewards/margins": 4.617161273956299, + "rewards/rejected": -4.480454444885254, + "step": 243, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.4636808430560784, + "grad_norm": 7.847040176391602, + "learning_rate": 3.610403658343443e-05, + "logits/chosen": -0.35487285256385803, + "logits/rejected": -0.7575075030326843, + "logps/chosen": -4.001574993133545, + "logps/rejected": -74.3595962524414, + "loss": 0.4401260018348694, + "memory(GiB)": 46.13, + "nll_loss": 0.35397177934646606, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21161355078220367, + "rewards/margins": 5.741542816162109, + "rewards/rejected": -5.529928684234619, + "step": 244, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.4697026721866768, + "grad_norm": 1.5019311904907227, + "learning_rate": 3.534003987842005e-05, + "logits/chosen": -0.5985352993011475, + "logits/rejected": -0.8159947395324707, + "logps/chosen": -3.0694146156311035, + "logps/rejected": -67.13231658935547, + "loss": 0.3305334746837616, + "memory(GiB)": 46.13, + "nll_loss": 0.26731085777282715, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21900442242622375, + "rewards/margins": 5.598498821258545, + "rewards/rejected": -5.3794941902160645, + "step": 245, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.475724501317275, + "grad_norm": 2.49977707862854, + "learning_rate": 3.458247464669657e-05, + "logits/chosen": -0.5261157155036926, + "logits/rejected": -0.7526097893714905, + "logps/chosen": -4.502616882324219, + "logps/rejected": -55.263763427734375, + "loss": 0.38907888531684875, + "memory(GiB)": 46.13, + "nll_loss": 0.29498404264450073, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2758098840713501, + "rewards/margins": 4.53220796585083, + "rewards/rejected": -4.2563982009887695, + "step": 246, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.4817463304478735, + "grad_norm": 1.7574135065078735, + "learning_rate": 3.383141624031408e-05, + "logits/chosen": -0.5520765781402588, + "logits/rejected": -0.7475904822349548, + "logps/chosen": -6.128981590270996, + "logps/rejected": -56.70994186401367, + "loss": 0.43652814626693726, + "memory(GiB)": 46.13, + "nll_loss": 0.3729217052459717, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.38195130228996277, + "rewards/margins": 4.705140113830566, + "rewards/rejected": -4.323188781738281, + "step": 247, + "train_speed(iter/s)": 0.005653 + }, + { + "epoch": 1.487768159578472, + "grad_norm": 1.5766643285751343, + "learning_rate": 3.308693936411421e-05, + "logits/chosen": -0.4617379307746887, + "logits/rejected": -0.6871187686920166, + "logps/chosen": -6.003100872039795, + "logps/rejected": -63.69612503051758, + "loss": 0.4494744539260864, + "memory(GiB)": 46.13, + "nll_loss": 0.3877749443054199, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3871355652809143, + "rewards/margins": 5.155317783355713, + "rewards/rejected": -4.768182277679443, + "step": 248, + "train_speed(iter/s)": 0.005652 + }, + { + "epoch": 1.4937899887090704, + "grad_norm": 2.092433214187622, + "learning_rate": 3.234911806829948e-05, + "logits/chosen": -0.4331311285495758, + "logits/rejected": -0.7093103528022766, + "logps/chosen": -4.4328694343566895, + "logps/rejected": -61.90970230102539, + "loss": 0.3958251476287842, + "memory(GiB)": 46.13, + "nll_loss": 0.3243080973625183, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3177702724933624, + "rewards/margins": 5.146597385406494, + "rewards/rejected": -4.828826904296875, + "step": 249, + "train_speed(iter/s)": 0.005652 + }, + { + "epoch": 1.4998118178396689, + "grad_norm": 2.878459930419922, + "learning_rate": 3.161802574106799e-05, + "logits/chosen": -0.5477363467216492, + "logits/rejected": -0.73226398229599, + "logps/chosen": -4.833172798156738, + "logps/rejected": -54.378639221191406, + "loss": 0.42393556237220764, + "memory(GiB)": 46.13, + "nll_loss": 0.3001037836074829, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.32397884130477905, + "rewards/margins": 4.320741653442383, + "rewards/rejected": -3.99676251411438, + "step": 250, + "train_speed(iter/s)": 0.005652 + }, + { + "epoch": 1.5058336469702671, + "grad_norm": 2.852609872817993, + "learning_rate": 3.089373510131354e-05, + "logits/chosen": -0.4120064973831177, + "logits/rejected": -0.7510567307472229, + "logps/chosen": -3.8237786293029785, + "logps/rejected": -68.50797271728516, + "loss": 0.38543784618377686, + "memory(GiB)": 46.13, + "nll_loss": 0.28033876419067383, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2964167594909668, + "rewards/margins": 5.442708969116211, + "rewards/rejected": -5.146292686462402, + "step": 251, + "train_speed(iter/s)": 0.00565 + }, + { + "epoch": 1.5118554761008656, + "grad_norm": 3.0803050994873047, + "learning_rate": 3.0176318191392726e-05, + "logits/chosen": -0.4008835256099701, + "logits/rejected": -0.7395371198654175, + "logps/chosen": -2.694051742553711, + "logps/rejected": -64.98765563964844, + "loss": 0.4233083128929138, + "memory(GiB)": 46.13, + "nll_loss": 0.28342971205711365, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.18508972227573395, + "rewards/margins": 5.180187702178955, + "rewards/rejected": -4.995098114013672, + "step": 252, + "train_speed(iter/s)": 0.00565 + }, + { + "epoch": 1.517877305231464, + "grad_norm": 3.8462982177734375, + "learning_rate": 2.9465846369959127e-05, + "logits/chosen": -0.44880083203315735, + "logits/rejected": -0.7400986552238464, + "logps/chosen": -4.19825553894043, + "logps/rejected": -68.70524597167969, + "loss": 0.3208198547363281, + "memory(GiB)": 46.13, + "nll_loss": 0.23766814172267914, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2140774428844452, + "rewards/margins": 5.5977983474731445, + "rewards/rejected": -5.383721351623535, + "step": 253, + "train_speed(iter/s)": 0.00565 + }, + { + "epoch": 1.5238991343620625, + "grad_norm": 2.542881488800049, + "learning_rate": 2.876239030486554e-05, + "logits/chosen": -0.5086544752120972, + "logits/rejected": -0.7899590730667114, + "logps/chosen": -3.355459451675415, + "logps/rejected": -63.492759704589844, + "loss": 0.31336328387260437, + "memory(GiB)": 46.13, + "nll_loss": 0.246277317404747, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.22459468245506287, + "rewards/margins": 5.192716121673584, + "rewards/rejected": -4.9681220054626465, + "step": 254, + "train_speed(iter/s)": 0.00565 + }, + { + "epoch": 1.529920963492661, + "grad_norm": 2.262983560562134, + "learning_rate": 2.8066019966134904e-05, + "logits/chosen": -0.4363003671169281, + "logits/rejected": -0.7306255102157593, + "logps/chosen": -2.433912515640259, + "logps/rejected": -67.02496337890625, + "loss": 0.31392139196395874, + "memory(GiB)": 46.13, + "nll_loss": 0.22945323586463928, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2390332818031311, + "rewards/margins": 5.437534332275391, + "rewards/rejected": -5.1985015869140625, + "step": 255, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5359427926232594, + "grad_norm": 2.1863996982574463, + "learning_rate": 2.7376804619000707e-05, + "logits/chosen": -0.5122266411781311, + "logits/rejected": -0.7367725372314453, + "logps/chosen": -5.143320560455322, + "logps/rejected": -58.59758758544922, + "loss": 0.5367969274520874, + "memory(GiB)": 46.13, + "nll_loss": 0.4321354627609253, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3802338242530823, + "rewards/margins": 4.9757304191589355, + "rewards/rejected": -4.59549617767334, + "step": 256, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5419646217538578, + "grad_norm": 2.382030725479126, + "learning_rate": 2.669481281701739e-05, + "logits/chosen": -0.4875583052635193, + "logits/rejected": -0.7425970435142517, + "logps/chosen": -3.6908037662506104, + "logps/rejected": -66.99980163574219, + "loss": 0.3160356283187866, + "memory(GiB)": 46.13, + "nll_loss": 0.24375540018081665, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3102016746997833, + "rewards/margins": 5.104434013366699, + "rewards/rejected": -4.7942328453063965, + "step": 257, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5479864508844563, + "grad_norm": 1.3870384693145752, + "learning_rate": 2.6020112395241624e-05, + "logits/chosen": -0.5185623168945312, + "logits/rejected": -0.7977117300033569, + "logps/chosen": -3.2771565914154053, + "logps/rejected": -66.14156341552734, + "loss": 0.32793527841567993, + "memory(GiB)": 46.13, + "nll_loss": 0.2557143270969391, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3175499141216278, + "rewards/margins": 5.339153289794922, + "rewards/rejected": -5.021603584289551, + "step": 258, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5540082800150545, + "grad_norm": 1.6842197179794312, + "learning_rate": 2.5352770463484987e-05, + "logits/chosen": -0.5024842023849487, + "logits/rejected": -0.7918374538421631, + "logps/chosen": -1.9520148038864136, + "logps/rejected": -62.17351150512695, + "loss": 0.2610028386116028, + "memory(GiB)": 46.13, + "nll_loss": 0.18495017290115356, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.21339333057403564, + "rewards/margins": 5.084794998168945, + "rewards/rejected": -4.871401786804199, + "step": 259, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.560030109145653, + "grad_norm": 2.327899694442749, + "learning_rate": 2.4692853399638917e-05, + "logits/chosen": -0.45503148436546326, + "logits/rejected": -0.7623938918113708, + "logps/chosen": -3.5812909603118896, + "logps/rejected": -72.19535064697266, + "loss": 0.41872018575668335, + "memory(GiB)": 46.13, + "nll_loss": 0.2947246730327606, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15223976969718933, + "rewards/margins": 5.696834564208984, + "rewards/rejected": -5.544594764709473, + "step": 260, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5660519382762514, + "grad_norm": 1.320288896560669, + "learning_rate": 2.4040426843072206e-05, + "logits/chosen": -0.47948870062828064, + "logits/rejected": -0.7575923204421997, + "logps/chosen": -2.7919044494628906, + "logps/rejected": -62.61322784423828, + "loss": 0.34768229722976685, + "memory(GiB)": 46.13, + "nll_loss": 0.23828013241291046, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2685001492500305, + "rewards/margins": 5.049767971038818, + "rewards/rejected": -4.781268119812012, + "step": 261, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5720737674068497, + "grad_norm": 2.9725844860076904, + "learning_rate": 2.339555568810221e-05, + "logits/chosen": -0.46958619356155396, + "logits/rejected": -0.7571808695793152, + "logps/chosen": -4.220214366912842, + "logps/rejected": -70.71320343017578, + "loss": 0.37105321884155273, + "memory(GiB)": 46.13, + "nll_loss": 0.2905551791191101, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.11353044211864471, + "rewards/margins": 5.697925090789795, + "rewards/rejected": -5.5843939781188965, + "step": 262, + "train_speed(iter/s)": 0.005649 + }, + { + "epoch": 1.5780955965374481, + "grad_norm": 2.7388265132904053, + "learning_rate": 2.275830407754006e-05, + "logits/chosen": -0.43985384702682495, + "logits/rejected": -0.7053717374801636, + "logps/chosen": -3.578861951828003, + "logps/rejected": -65.03799438476562, + "loss": 0.34339943528175354, + "memory(GiB)": 46.13, + "nll_loss": 0.2526337504386902, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.23234206438064575, + "rewards/margins": 5.231771469116211, + "rewards/rejected": -4.999429702758789, + "step": 263, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.5841174256680466, + "grad_norm": 2.7772486209869385, + "learning_rate": 2.212873539631061e-05, + "logits/chosen": -0.49956512451171875, + "logits/rejected": -0.7878684997558594, + "logps/chosen": -2.7876102924346924, + "logps/rejected": -65.78087615966797, + "loss": 0.3053550720214844, + "memory(GiB)": 46.13, + "nll_loss": 0.2294946312904358, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21735113859176636, + "rewards/margins": 5.3624958992004395, + "rewards/rejected": -5.145144462585449, + "step": 264, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.590139254798645, + "grad_norm": 1.685667634010315, + "learning_rate": 2.1506912265147772e-05, + "logits/chosen": -0.5206426382064819, + "logits/rejected": -0.7813724279403687, + "logps/chosen": -2.941920757293701, + "logps/rejected": -66.5394287109375, + "loss": 0.2972472906112671, + "memory(GiB)": 46.13, + "nll_loss": 0.2362329214811325, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.33175477385520935, + "rewards/margins": 5.447272777557373, + "rewards/rejected": -5.115517616271973, + "step": 265, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.5961610839292435, + "grad_norm": 3.3678839206695557, + "learning_rate": 2.0892896534365904e-05, + "logits/chosen": -0.3419654369354248, + "logits/rejected": -0.7248679995536804, + "logps/chosen": -3.409761905670166, + "logps/rejected": -71.6166763305664, + "loss": 0.32658249139785767, + "memory(GiB)": 46.13, + "nll_loss": 0.26064980030059814, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.20473551750183105, + "rewards/margins": 5.657358646392822, + "rewards/rejected": -5.452622890472412, + "step": 266, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.602182913059842, + "grad_norm": 3.060711145401001, + "learning_rate": 2.0286749277707782e-05, + "logits/chosen": -0.4216618835926056, + "logits/rejected": -0.6746920347213745, + "logps/chosen": -7.840616703033447, + "logps/rejected": -59.754539489746094, + "loss": 0.5373175740242004, + "memory(GiB)": 46.13, + "nll_loss": 0.4301339387893677, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.4534755051136017, + "rewards/margins": 4.956415176391602, + "rewards/rejected": -4.502939701080322, + "step": 267, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.6082047421904404, + "grad_norm": 1.2512023448944092, + "learning_rate": 1.9688530786269855e-05, + "logits/chosen": -0.48612943291664124, + "logits/rejected": -0.78404700756073, + "logps/chosen": -3.4436018466949463, + "logps/rejected": -64.1688461303711, + "loss": 0.3891962468624115, + "memory(GiB)": 46.13, + "nll_loss": 0.3172661066055298, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27034294605255127, + "rewards/margins": 5.256752967834473, + "rewards/rejected": -4.986410140991211, + "step": 268, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.6142265713210389, + "grad_norm": 1.5459010601043701, + "learning_rate": 1.9098300562505266e-05, + "logits/chosen": -0.4943903386592865, + "logits/rejected": -0.8177123069763184, + "logps/chosen": -2.876307964324951, + "logps/rejected": -72.3165512084961, + "loss": 0.3240829110145569, + "memory(GiB)": 46.13, + "nll_loss": 0.2640679180622101, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.26817601919174194, + "rewards/margins": 5.955280780792236, + "rewards/rejected": -5.687105178833008, + "step": 269, + "train_speed(iter/s)": 0.005648 + }, + { + "epoch": 1.6202484004516373, + "grad_norm": 2.2002980709075928, + "learning_rate": 1.8516117314305524e-05, + "logits/chosen": -0.4193212389945984, + "logits/rejected": -0.7498923540115356, + "logps/chosen": -2.122291088104248, + "logps/rejected": -78.85472106933594, + "loss": 0.29622161388397217, + "memory(GiB)": 46.13, + "nll_loss": 0.21589502692222595, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19214794039726257, + "rewards/margins": 6.349010467529297, + "rewards/rejected": -6.156862735748291, + "step": 270, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.6262702295822355, + "grad_norm": 2.6795237064361572, + "learning_rate": 1.7942038949160854e-05, + "logits/chosen": -0.44892024993896484, + "logits/rejected": -0.6841689348220825, + "logps/chosen": -6.089541435241699, + "logps/rejected": -60.286041259765625, + "loss": 0.5212526321411133, + "memory(GiB)": 46.13, + "nll_loss": 0.32065996527671814, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24887697398662567, + "rewards/margins": 4.777753829956055, + "rewards/rejected": -4.528877258300781, + "step": 271, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.632292058712834, + "grad_norm": 1.3338947296142578, + "learning_rate": 1.7376122568400532e-05, + "logits/chosen": -0.5868579149246216, + "logits/rejected": -0.9036458730697632, + "logps/chosen": -2.3813061714172363, + "logps/rejected": -66.72323608398438, + "loss": 0.29132747650146484, + "memory(GiB)": 46.13, + "nll_loss": 0.23172350227832794, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2914634346961975, + "rewards/margins": 5.641142845153809, + "rewards/rejected": -5.349678993225098, + "step": 272, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.6383138878434325, + "grad_norm": 1.2542729377746582, + "learning_rate": 1.681842446151313e-05, + "logits/chosen": -0.4844375252723694, + "logits/rejected": -0.7457984685897827, + "logps/chosen": -5.260637283325195, + "logps/rejected": -71.04159545898438, + "loss": 0.3770938515663147, + "memory(GiB)": 46.13, + "nll_loss": 0.33192208409309387, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40421298146247864, + "rewards/margins": 5.93914270401001, + "rewards/rejected": -5.534929275512695, + "step": 273, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.644335716974031, + "grad_norm": 2.0362296104431152, + "learning_rate": 1.6269000100547683e-05, + "logits/chosen": -0.4546166658401489, + "logits/rejected": -0.7672044634819031, + "logps/chosen": -3.0699520111083984, + "logps/rejected": -71.65580749511719, + "loss": 0.311799019575119, + "memory(GiB)": 46.13, + "nll_loss": 0.24750828742980957, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31899750232696533, + "rewards/margins": 6.106335163116455, + "rewards/rejected": -5.787337303161621, + "step": 274, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.6503575461046291, + "grad_norm": 3.0854225158691406, + "learning_rate": 1.5727904134596083e-05, + "logits/chosen": -0.5049324035644531, + "logits/rejected": -0.8067833185195923, + "logps/chosen": -3.4519662857055664, + "logps/rejected": -64.27713012695312, + "loss": 0.40550893545150757, + "memory(GiB)": 46.13, + "nll_loss": 0.3140624165534973, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.23077721893787384, + "rewards/margins": 5.18168830871582, + "rewards/rejected": -4.950911045074463, + "step": 275, + "train_speed(iter/s)": 0.005647 + }, + { + "epoch": 1.6563793752352276, + "grad_norm": 2.3667151927948, + "learning_rate": 1.5195190384357404e-05, + "logits/chosen": -0.4429381787776947, + "logits/rejected": -0.7047520279884338, + "logps/chosen": -3.2484500408172607, + "logps/rejected": -63.5272216796875, + "loss": 0.36934536695480347, + "memory(GiB)": 46.13, + "nll_loss": 0.24999630451202393, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.2938525974750519, + "rewards/margins": 5.066824913024902, + "rewards/rejected": -4.772972583770752, + "step": 276, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.662401204365826, + "grad_norm": 2.0824172496795654, + "learning_rate": 1.467091183678444e-05, + "logits/chosen": -0.4420183002948761, + "logits/rejected": -0.7609032988548279, + "logps/chosen": -4.130496025085449, + "logps/rejected": -72.41409301757812, + "loss": 0.3867420256137848, + "memory(GiB)": 46.13, + "nll_loss": 0.31039467453956604, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.35360175371170044, + "rewards/margins": 6.035587310791016, + "rewards/rejected": -5.681984901428223, + "step": 277, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.6684230334964245, + "grad_norm": 1.2125921249389648, + "learning_rate": 1.415512063981339e-05, + "logits/chosen": -0.47004735469818115, + "logits/rejected": -0.7464879751205444, + "logps/chosen": -4.137353420257568, + "logps/rejected": -72.29967498779297, + "loss": 0.38727715611457825, + "memory(GiB)": 46.13, + "nll_loss": 0.32129383087158203, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.5302993059158325, + "rewards/margins": 6.306224822998047, + "rewards/rejected": -5.775925159454346, + "step": 278, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.674444862627023, + "grad_norm": 1.5865390300750732, + "learning_rate": 1.364786809717692e-05, + "logits/chosen": -0.4276161789894104, + "logits/rejected": -0.7880982160568237, + "logps/chosen": -3.557345390319824, + "logps/rejected": -73.96890258789062, + "loss": 0.33818569779396057, + "memory(GiB)": 46.13, + "nll_loss": 0.28797072172164917, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25843125581741333, + "rewards/margins": 6.005448818206787, + "rewards/rejected": -5.747016906738281, + "step": 279, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.6804666917576214, + "grad_norm": 5.5078630447387695, + "learning_rate": 1.3149204663301118e-05, + "logits/chosen": -0.3974856734275818, + "logits/rejected": -0.7452144622802734, + "logps/chosen": -3.9654970169067383, + "logps/rejected": -71.51333618164062, + "loss": 0.35094186663627625, + "memory(GiB)": 46.13, + "nll_loss": 0.2737097442150116, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2904951870441437, + "rewards/margins": 5.682476043701172, + "rewards/rejected": -5.39198112487793, + "step": 280, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.6864885208882199, + "grad_norm": 1.8228479623794556, + "learning_rate": 1.2659179938287035e-05, + "logits/chosen": -0.40838098526000977, + "logits/rejected": -0.7652671933174133, + "logps/chosen": -2.5664548873901367, + "logps/rejected": -64.62358856201172, + "loss": 0.24017123878002167, + "memory(GiB)": 46.13, + "nll_loss": 0.17215371131896973, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.20578134059906006, + "rewards/margins": 5.029583930969238, + "rewards/rejected": -4.823802471160889, + "step": 281, + "train_speed(iter/s)": 0.005646 + }, + { + "epoch": 1.6925103500188183, + "grad_norm": 1.196491003036499, + "learning_rate": 1.2177842662977135e-05, + "logits/chosen": -0.4218314588069916, + "logits/rejected": -0.7560523748397827, + "logps/chosen": -2.7297964096069336, + "logps/rejected": -65.94020080566406, + "loss": 0.2704247236251831, + "memory(GiB)": 46.13, + "nll_loss": 0.1988007128238678, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31327253580093384, + "rewards/margins": 5.445861339569092, + "rewards/rejected": -5.132589340209961, + "step": 282, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.6985321791494168, + "grad_norm": 2.948547840118408, + "learning_rate": 1.1705240714107302e-05, + "logits/chosen": -0.3723593056201935, + "logits/rejected": -0.7366158366203308, + "logps/chosen": -3.405308485031128, + "logps/rejected": -75.39693450927734, + "loss": 0.40320032835006714, + "memory(GiB)": 46.13, + "nll_loss": 0.2922399640083313, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.29622313380241394, + "rewards/margins": 6.236778736114502, + "rewards/rejected": -5.940556049346924, + "step": 283, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.704554008280015, + "grad_norm": 5.310917854309082, + "learning_rate": 1.124142109954459e-05, + "logits/chosen": -0.5253931283950806, + "logits/rejected": -0.8019742965698242, + "logps/chosen": -2.7380077838897705, + "logps/rejected": -66.83124542236328, + "loss": 0.42098596692085266, + "memory(GiB)": 46.13, + "nll_loss": 0.3041377365589142, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.1617843061685562, + "rewards/margins": 5.515201568603516, + "rewards/rejected": -5.353416919708252, + "step": 284, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7105758374106135, + "grad_norm": 1.8457385301589966, + "learning_rate": 1.0786429953611666e-05, + "logits/chosen": -0.4387832283973694, + "logits/rejected": -0.8179764747619629, + "logps/chosen": -2.1042332649230957, + "logps/rejected": -68.6592788696289, + "loss": 0.28756269812583923, + "memory(GiB)": 46.13, + "nll_loss": 0.21063612401485443, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.24719983339309692, + "rewards/margins": 5.616422176361084, + "rewards/rejected": -5.3692216873168945, + "step": 285, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.716597666541212, + "grad_norm": 1.7732220888137817, + "learning_rate": 1.034031253249792e-05, + "logits/chosen": -0.5289000272750854, + "logits/rejected": -0.8081372380256653, + "logps/chosen": -2.455111503601074, + "logps/rejected": -60.26368713378906, + "loss": 0.2956588566303253, + "memory(GiB)": 46.13, + "nll_loss": 0.2198590785264969, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.19829076528549194, + "rewards/margins": 4.948812961578369, + "rewards/rejected": -4.750522136688232, + "step": 286, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7226194956718102, + "grad_norm": 3.0396289825439453, + "learning_rate": 9.903113209758096e-06, + "logits/chosen": -0.5536311268806458, + "logits/rejected": -0.7566096186637878, + "logps/chosen": -4.647332191467285, + "logps/rejected": -53.316307067871094, + "loss": 0.4976504147052765, + "memory(GiB)": 46.13, + "nll_loss": 0.36106935143470764, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3211871385574341, + "rewards/margins": 4.494439601898193, + "rewards/rejected": -4.173252582550049, + "step": 287, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7286413248024086, + "grad_norm": 1.4801286458969116, + "learning_rate": 9.474875471898526e-06, + "logits/chosen": -0.4908003807067871, + "logits/rejected": -0.8340423703193665, + "logps/chosen": -2.6292285919189453, + "logps/rejected": -66.33879089355469, + "loss": 0.2964615523815155, + "memory(GiB)": 46.13, + "nll_loss": 0.23429438471794128, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.22690246999263763, + "rewards/margins": 5.54394006729126, + "rewards/rejected": -5.317037582397461, + "step": 288, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.734663153933007, + "grad_norm": 3.686876058578491, + "learning_rate": 9.055641914051782e-06, + "logits/chosen": -0.5189223885536194, + "logits/rejected": -0.8108649253845215, + "logps/chosen": -3.9785354137420654, + "logps/rejected": -68.66986083984375, + "loss": 0.4266186058521271, + "memory(GiB)": 46.13, + "nll_loss": 0.3564129173755646, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27698981761932373, + "rewards/margins": 5.682110786437988, + "rewards/rejected": -5.405120849609375, + "step": 289, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7406849830636055, + "grad_norm": 4.0403594970703125, + "learning_rate": 8.645454235739903e-06, + "logits/chosen": -0.48102113604545593, + "logits/rejected": -0.8606957197189331, + "logps/chosen": -3.213351249694824, + "logps/rejected": -77.46415710449219, + "loss": 0.3690274953842163, + "memory(GiB)": 46.13, + "nll_loss": 0.31783145666122437, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25699326395988464, + "rewards/margins": 6.534820556640625, + "rewards/rejected": -6.277827262878418, + "step": 290, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.746706812194204, + "grad_norm": 3.175973892211914, + "learning_rate": 8.24435323672661e-06, + "logits/chosen": -0.5072706937789917, + "logits/rejected": -0.8571786284446716, + "logps/chosen": -3.415510654449463, + "logps/rejected": -68.72850036621094, + "loss": 0.346034973859787, + "memory(GiB)": 46.13, + "nll_loss": 0.28102245926856995, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22457678616046906, + "rewards/margins": 5.616724491119385, + "rewards/rejected": -5.392147541046143, + "step": 291, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7527286413248024, + "grad_norm": 3.0831406116485596, + "learning_rate": 7.852378812959227e-06, + "logits/chosen": -0.40548837184906006, + "logits/rejected": -0.818709135055542, + "logps/chosen": -2.9739181995391846, + "logps/rejected": -73.55524444580078, + "loss": 0.31632980704307556, + "memory(GiB)": 46.13, + "nll_loss": 0.22905109822750092, + "rewards/accuracies": 0.953125, + "rewards/chosen": 0.16724324226379395, + "rewards/margins": 5.85488224029541, + "rewards/rejected": -5.687638759613037, + "step": 292, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.758750470455401, + "grad_norm": 1.8791142702102661, + "learning_rate": 7.46956995260033e-06, + "logits/chosen": -0.4915006458759308, + "logits/rejected": -0.8246431946754456, + "logps/chosen": -6.461263179779053, + "logps/rejected": -72.80626678466797, + "loss": 0.43981942534446716, + "memory(GiB)": 46.13, + "nll_loss": 0.3707093298435211, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.22866468131542206, + "rewards/margins": 5.882740020751953, + "rewards/rejected": -5.654075622558594, + "step": 293, + "train_speed(iter/s)": 0.005645 + }, + { + "epoch": 1.7647722995859993, + "grad_norm": 2.1687467098236084, + "learning_rate": 7.09596473214974e-06, + "logits/chosen": -0.4673860967159271, + "logits/rejected": -0.7422292232513428, + "logps/chosen": -5.505043983459473, + "logps/rejected": -82.50333404541016, + "loss": 0.41253846883773804, + "memory(GiB)": 46.13, + "nll_loss": 0.3449009358882904, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.3176400065422058, + "rewards/margins": 6.987927436828613, + "rewards/rejected": -6.670287132263184, + "step": 294, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.7707941287165978, + "grad_norm": 4.685088634490967, + "learning_rate": 6.731600312657238e-06, + "logits/chosen": -0.563675045967102, + "logits/rejected": -0.8160005807876587, + "logps/chosen": -3.448794364929199, + "logps/rejected": -61.5041389465332, + "loss": 0.45613980293273926, + "memory(GiB)": 46.13, + "nll_loss": 0.3632497191429138, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.27636757493019104, + "rewards/margins": 5.065147399902344, + "rewards/rejected": -4.788780212402344, + "step": 295, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.776815957847196, + "grad_norm": 4.02125883102417, + "learning_rate": 6.37651293602628e-06, + "logits/chosen": -0.5208415389060974, + "logits/rejected": -0.7839468121528625, + "logps/chosen": -5.326009750366211, + "logps/rejected": -63.45610809326172, + "loss": 0.501464307308197, + "memory(GiB)": 46.13, + "nll_loss": 0.3729536831378937, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2640419602394104, + "rewards/margins": 5.044376850128174, + "rewards/rejected": -4.780335426330566, + "step": 296, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.7828377869777945, + "grad_norm": 2.1763715744018555, + "learning_rate": 6.030737921409169e-06, + "logits/chosen": -0.48864227533340454, + "logits/rejected": -0.7334147095680237, + "logps/chosen": -5.976287841796875, + "logps/rejected": -67.99011993408203, + "loss": 0.46500977873802185, + "memory(GiB)": 46.13, + "nll_loss": 0.39140668511390686, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.31234222650527954, + "rewards/margins": 5.641515731811523, + "rewards/rejected": -5.329173564910889, + "step": 297, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.788859616108393, + "grad_norm": 11.35409164428711, + "learning_rate": 5.694309661693942e-06, + "logits/chosen": -0.31258460879325867, + "logits/rejected": -0.6516164541244507, + "logps/chosen": -4.385894775390625, + "logps/rejected": -64.1596450805664, + "loss": 0.5977675914764404, + "memory(GiB)": 46.13, + "nll_loss": 0.399192750453949, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13077813386917114, + "rewards/margins": 4.987821102142334, + "rewards/rejected": -4.85704231262207, + "step": 298, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.7948814452389912, + "grad_norm": 1.2614425420761108, + "learning_rate": 5.367261620083575e-06, + "logits/chosen": -0.43785005807876587, + "logits/rejected": -0.7445405125617981, + "logps/chosen": -3.5625617504119873, + "logps/rejected": -69.88119506835938, + "loss": 0.37494975328445435, + "memory(GiB)": 46.13, + "nll_loss": 0.2672398090362549, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.20106351375579834, + "rewards/margins": 5.906167984008789, + "rewards/rejected": -5.705103397369385, + "step": 299, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.8009032743695896, + "grad_norm": 2.9862029552459717, + "learning_rate": 5.049626326767365e-06, + "logits/chosen": -0.4020475745201111, + "logits/rejected": -0.722282350063324, + "logps/chosen": -2.223629951477051, + "logps/rejected": -74.56559753417969, + "loss": 0.2519979476928711, + "memory(GiB)": 46.13, + "nll_loss": 0.18500222265720367, + "rewards/accuracies": 0.984375, + "rewards/chosen": 0.25891339778900146, + "rewards/margins": 6.117902755737305, + "rewards/rejected": -5.858989238739014, + "step": 300, + "train_speed(iter/s)": 0.005644 + }, + { + "epoch": 1.8009032743695896, + "eval_logits/chosen": -0.5320332646369934, + "eval_logits/rejected": -0.7993212342262268, + "eval_logps/chosen": -5.09476375579834, + "eval_logps/rejected": -66.83241271972656, + "eval_loss": 0.5435733795166016, + "eval_nll_loss": 0.39369699358940125, + "eval_rewards/accuracies": 0.9444444179534912, + "eval_rewards/chosen": 0.06985757499933243, + "eval_rewards/margins": 5.177610397338867, + "eval_rewards/rejected": -5.107752323150635, + "eval_runtime": 117.9868, + "eval_samples_per_second": 0.907, + "eval_steps_per_second": 0.229, + "step": 300 + } + ], + "logging_steps": 1, + "max_steps": 332, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.444748519116636e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}