diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,5434 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0488656195462478, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0034904013961605585, + "grad_norm": 8.557130813598633, + "learning_rate": 4.651162790697674e-06, + "logits/chosen": -0.3853919506072998, + "logits/rejected": -0.6177393794059753, + "logps/chosen": -6.8596577644348145, + "logps/rejected": -24.441532135009766, + "loss": 1.0831061601638794, + "memory(GiB)": 34.14, + "nll_loss": 0.38995903730392456, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1, + "train_speed(iter/s)": 0.01012 + }, + { + "epoch": 0.006980802792321117, + "grad_norm": 4.541833400726318, + "learning_rate": 9.302325581395349e-06, + "logits/chosen": -0.4651119112968445, + "logits/rejected": -0.5818633437156677, + "logps/chosen": -7.042446136474609, + "logps/rejected": -15.569698333740234, + "loss": 1.1563867330551147, + "memory(GiB)": 34.14, + "nll_loss": 0.46323955059051514, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2, + "train_speed(iter/s)": 0.010413 + }, + { + "epoch": 0.010471204188481676, + "grad_norm": 5.130979061126709, + "learning_rate": 1.3953488372093024e-05, + "logits/chosen": -0.46299099922180176, + "logits/rejected": -0.584057629108429, + "logps/chosen": -10.678628921508789, + "logps/rejected": -13.385712623596191, + "loss": 1.3326630592346191, + "memory(GiB)": 34.14, + "nll_loss": 0.6397495865821838, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.005845161620527506, + "rewards/margins": 0.0005908504826948047, + "rewards/rejected": 0.005254311487078667, + "step": 3, + "train_speed(iter/s)": 0.010516 + }, + { + "epoch": 0.013961605584642234, + "grad_norm": 9.7429780960083, + "learning_rate": 1.8604651162790697e-05, + "logits/chosen": -0.5326871871948242, + "logits/rejected": -0.6262682676315308, + "logps/chosen": -6.428069114685059, + "logps/rejected": -15.568026542663574, + "loss": 1.1956652402877808, + "memory(GiB)": 34.14, + "nll_loss": 0.5060467720031738, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.006483433302491903, + "rewards/margins": 0.0072298417799174786, + "rewards/rejected": -0.0007464090595021844, + "step": 4, + "train_speed(iter/s)": 0.010579 + }, + { + "epoch": 0.017452006980802792, + "grad_norm": 5.269836902618408, + "learning_rate": 2.3255813953488374e-05, + "logits/chosen": -0.5215513706207275, + "logits/rejected": -0.6214993000030518, + "logps/chosen": -6.4991326332092285, + "logps/rejected": -16.232433319091797, + "loss": 1.144551396369934, + "memory(GiB)": 34.14, + "nll_loss": 0.46053266525268555, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.004076660610735416, + "rewards/margins": 0.018971633166074753, + "rewards/rejected": -0.014894972555339336, + "step": 5, + "train_speed(iter/s)": 0.010622 + }, + { + "epoch": 0.020942408376963352, + "grad_norm": 6.636388301849365, + "learning_rate": 2.7906976744186048e-05, + "logits/chosen": -0.4555979073047638, + "logits/rejected": -0.5607460737228394, + "logps/chosen": -9.820672035217285, + "logps/rejected": -15.622188568115234, + "loss": 1.3625051975250244, + "memory(GiB)": 34.14, + "nll_loss": 0.6840521693229675, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.018183868378400803, + "rewards/margins": 0.03185040131211281, + "rewards/rejected": -0.01366653386503458, + "step": 6, + "train_speed(iter/s)": 0.01064 + }, + { + "epoch": 0.02443280977312391, + "grad_norm": 5.103886127471924, + "learning_rate": 3.2558139534883724e-05, + "logits/chosen": -0.4968104660511017, + "logits/rejected": -0.569257378578186, + "logps/chosen": -10.183512687683105, + "logps/rejected": -15.70079231262207, + "loss": 1.4255497455596924, + "memory(GiB)": 34.14, + "nll_loss": 0.7536435127258301, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.010219119489192963, + "rewards/margins": 0.054266996681690216, + "rewards/rejected": -0.06448611617088318, + "step": 7, + "train_speed(iter/s)": 0.010656 + }, + { + "epoch": 0.027923211169284468, + "grad_norm": 3.999744176864624, + "learning_rate": 3.7209302325581394e-05, + "logits/chosen": -0.4695020914077759, + "logits/rejected": -0.5785450339317322, + "logps/chosen": -8.685606002807617, + "logps/rejected": -19.699125289916992, + "loss": 1.167594075202942, + "memory(GiB)": 34.14, + "nll_loss": 0.5315794944763184, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.033686134964227676, + "rewards/margins": 0.14157818257808685, + "rewards/rejected": -0.17526429891586304, + "step": 8, + "train_speed(iter/s)": 0.010669 + }, + { + "epoch": 0.031413612565445025, + "grad_norm": 7.809542179107666, + "learning_rate": 4.186046511627907e-05, + "logits/chosen": -0.5049821138381958, + "logits/rejected": -0.5677512288093567, + "logps/chosen": -10.067422866821289, + "logps/rejected": -13.192615509033203, + "loss": 1.3344353437423706, + "memory(GiB)": 34.14, + "nll_loss": 0.6376084089279175, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.1379907727241516, + "rewards/margins": 0.05565176531672478, + "rewards/rejected": -0.1936425417661667, + "step": 9, + "train_speed(iter/s)": 0.010683 + }, + { + "epoch": 0.034904013961605584, + "grad_norm": 10.225089073181152, + "learning_rate": 4.651162790697675e-05, + "logits/chosen": -0.5532017350196838, + "logits/rejected": -0.5782958269119263, + "logps/chosen": -13.762276649475098, + "logps/rejected": -13.101286888122559, + "loss": 1.5571683645248413, + "memory(GiB)": 34.14, + "nll_loss": 0.8523505330085754, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.15877792239189148, + "rewards/margins": 0.03356418013572693, + "rewards/rejected": -0.19234208762645721, + "step": 10, + "train_speed(iter/s)": 0.010697 + }, + { + "epoch": 0.038394415357766144, + "grad_norm": 4.407655239105225, + "learning_rate": 5.1162790697674425e-05, + "logits/chosen": -0.4217715263366699, + "logits/rejected": -0.5799385905265808, + "logps/chosen": -6.071174621582031, + "logps/rejected": -21.04494857788086, + "loss": 0.9959170818328857, + "memory(GiB)": 34.14, + "nll_loss": 0.4200224280357361, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.03702637180685997, + "rewards/margins": 0.3200541138648987, + "rewards/rejected": -0.35708048939704895, + "step": 11, + "train_speed(iter/s)": 0.010696 + }, + { + "epoch": 0.041884816753926704, + "grad_norm": 6.385944366455078, + "learning_rate": 5.5813953488372095e-05, + "logits/chosen": -0.5289366841316223, + "logits/rejected": -0.5345426201820374, + "logps/chosen": -11.089645385742188, + "logps/rejected": -13.696191787719727, + "loss": 1.2887790203094482, + "memory(GiB)": 34.14, + "nll_loss": 0.5673516392707825, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.10325971990823746, + "rewards/margins": -0.009972111321985722, + "rewards/rejected": -0.09328760206699371, + "step": 12, + "train_speed(iter/s)": 0.010702 + }, + { + "epoch": 0.04537521815008726, + "grad_norm": 4.892186641693115, + "learning_rate": 6.0465116279069765e-05, + "logits/chosen": -0.48081111907958984, + "logits/rejected": -0.5861090421676636, + "logps/chosen": -7.97854471206665, + "logps/rejected": -18.214845657348633, + "loss": 1.23686683177948, + "memory(GiB)": 34.14, + "nll_loss": 0.5658432841300964, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.07524491846561432, + "rewards/margins": 0.07474859058856964, + "rewards/rejected": -0.14999350905418396, + "step": 13, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 0.04886561954624782, + "grad_norm": 3.9216458797454834, + "learning_rate": 6.511627906976745e-05, + "logits/chosen": -0.4399729073047638, + "logits/rejected": -0.5742396116256714, + "logps/chosen": -3.621753692626953, + "logps/rejected": -17.848834991455078, + "loss": 0.9254244565963745, + "memory(GiB)": 34.14, + "nll_loss": 0.26018136739730835, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.069590725004673, + "rewards/margins": 0.08134818077087402, + "rewards/rejected": -0.01175746414810419, + "step": 14, + "train_speed(iter/s)": 0.010711 + }, + { + "epoch": 0.05235602094240838, + "grad_norm": 4.189650058746338, + "learning_rate": 6.976744186046513e-05, + "logits/chosen": -0.4310145378112793, + "logits/rejected": -0.5189388990402222, + "logps/chosen": -7.985783100128174, + "logps/rejected": -17.343812942504883, + "loss": 1.1174938678741455, + "memory(GiB)": 34.14, + "nll_loss": 0.4816132187843323, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.09267675131559372, + "rewards/margins": 0.1413428634405136, + "rewards/rejected": -0.04866611212491989, + "step": 15, + "train_speed(iter/s)": 0.010712 + }, + { + "epoch": 0.055846422338568937, + "grad_norm": 3.32806658744812, + "learning_rate": 7.441860465116279e-05, + "logits/chosen": -0.43775254487991333, + "logits/rejected": -0.5088982582092285, + "logps/chosen": -8.117635726928711, + "logps/rejected": -19.758106231689453, + "loss": 1.1459051370620728, + "memory(GiB)": 34.14, + "nll_loss": 0.5276587605476379, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.10978943854570389, + "rewards/margins": 0.18414708971977234, + "rewards/rejected": -0.07435765117406845, + "step": 16, + "train_speed(iter/s)": 0.010712 + }, + { + "epoch": 0.059336823734729496, + "grad_norm": 2.725511312484741, + "learning_rate": 7.906976744186047e-05, + "logits/chosen": -0.47936227917671204, + "logits/rejected": -0.5878744721412659, + "logps/chosen": -9.313335418701172, + "logps/rejected": -21.849124908447266, + "loss": 1.058612585067749, + "memory(GiB)": 34.14, + "nll_loss": 0.4116324186325073, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.01833728700876236, + "rewards/margins": 0.10961442440748215, + "rewards/rejected": -0.09127713739871979, + "step": 17, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.06282722513089005, + "grad_norm": 3.7582316398620605, + "learning_rate": 8.372093023255814e-05, + "logits/chosen": -0.476583331823349, + "logits/rejected": -0.5290005803108215, + "logps/chosen": -8.529088020324707, + "logps/rejected": -16.1294002532959, + "loss": 1.1786390542984009, + "memory(GiB)": 34.14, + "nll_loss": 0.578920841217041, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.02365059219300747, + "rewards/margins": 0.23341532051563263, + "rewards/rejected": -0.25706592202186584, + "step": 18, + "train_speed(iter/s)": 0.010716 + }, + { + "epoch": 0.06631762652705062, + "grad_norm": 3.4042398929595947, + "learning_rate": 8.837209302325582e-05, + "logits/chosen": -0.4703082740306854, + "logits/rejected": -0.5102555155754089, + "logps/chosen": -9.428271293640137, + "logps/rejected": -16.359838485717773, + "loss": 1.235347867012024, + "memory(GiB)": 34.14, + "nll_loss": 0.6085215210914612, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.01763956993818283, + "rewards/margins": 0.1921030879020691, + "rewards/rejected": -0.17446354031562805, + "step": 19, + "train_speed(iter/s)": 0.010716 + }, + { + "epoch": 0.06980802792321117, + "grad_norm": 3.1032357215881348, + "learning_rate": 9.30232558139535e-05, + "logits/chosen": -0.401233434677124, + "logits/rejected": -0.494634747505188, + "logps/chosen": -9.093883514404297, + "logps/rejected": -18.11814308166504, + "loss": 1.1991461515426636, + "memory(GiB)": 34.14, + "nll_loss": 0.5884966850280762, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.02572321705520153, + "rewards/margins": 0.21222788095474243, + "rewards/rejected": -0.18650466203689575, + "step": 20, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 0.07329842931937172, + "grad_norm": 3.3876612186431885, + "learning_rate": 9.767441860465116e-05, + "logits/chosen": -0.46866974234580994, + "logits/rejected": -0.530718207359314, + "logps/chosen": -7.192732334136963, + "logps/rejected": -14.595564842224121, + "loss": 1.1225976943969727, + "memory(GiB)": 34.14, + "nll_loss": 0.5212462544441223, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.1131197139620781, + "rewards/margins": 0.2251490354537964, + "rewards/rejected": -0.1120293140411377, + "step": 21, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.07678883071553229, + "grad_norm": 2.6822471618652344, + "learning_rate": 0.00010232558139534885, + "logits/chosen": -0.5048164129257202, + "logits/rejected": -0.5435423851013184, + "logps/chosen": -7.50157356262207, + "logps/rejected": -14.21214485168457, + "loss": 1.088458776473999, + "memory(GiB)": 34.14, + "nll_loss": 0.5341289043426514, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16505159437656403, + "rewards/margins": 0.3283952474594116, + "rewards/rejected": -0.1633436679840088, + "step": 22, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.08027923211169284, + "grad_norm": 2.733980417251587, + "learning_rate": 0.00010697674418604651, + "logits/chosen": -0.40645402669906616, + "logits/rejected": -0.5064697861671448, + "logps/chosen": -8.52833080291748, + "logps/rejected": -21.627185821533203, + "loss": 1.119882345199585, + "memory(GiB)": 34.14, + "nll_loss": 0.6272884607315063, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.21130108833312988, + "rewards/margins": 0.5197649598121643, + "rewards/rejected": -0.3084638714790344, + "step": 23, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.08376963350785341, + "grad_norm": 3.6145081520080566, + "learning_rate": 0.00011162790697674419, + "logits/chosen": -0.40803515911102295, + "logits/rejected": -0.4656044840812683, + "logps/chosen": -7.790849208831787, + "logps/rejected": -19.12059783935547, + "loss": 1.073976755142212, + "memory(GiB)": 34.14, + "nll_loss": 0.5007548332214355, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.021760545670986176, + "rewards/margins": 0.325797438621521, + "rewards/rejected": -0.3040368854999542, + "step": 24, + "train_speed(iter/s)": 0.010727 + }, + { + "epoch": 0.08726003490401396, + "grad_norm": 3.5218026638031006, + "learning_rate": 0.00011627906976744187, + "logits/chosen": -0.38492777943611145, + "logits/rejected": -0.4633912742137909, + "logps/chosen": -9.945892333984375, + "logps/rejected": -22.549875259399414, + "loss": 1.099092960357666, + "memory(GiB)": 34.14, + "nll_loss": 0.6007935404777527, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.18332718312740326, + "rewards/margins": 0.6362053155899048, + "rewards/rejected": -0.8195324540138245, + "step": 25, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 0.09075043630017451, + "grad_norm": 4.300631523132324, + "learning_rate": 0.00012093023255813953, + "logits/chosen": -0.3624393343925476, + "logits/rejected": -0.40366220474243164, + "logps/chosen": -13.089804649353027, + "logps/rejected": -20.56357192993164, + "loss": 1.241166114807129, + "memory(GiB)": 34.14, + "nll_loss": 0.6592291593551636, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10197470337152481, + "rewards/margins": 0.3783476948738098, + "rewards/rejected": -0.4803224205970764, + "step": 26, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 0.09424083769633508, + "grad_norm": 6.326510906219482, + "learning_rate": 0.0001255813953488372, + "logits/chosen": -0.44795599579811096, + "logits/rejected": -0.5086410045623779, + "logps/chosen": -6.848785400390625, + "logps/rejected": -16.293134689331055, + "loss": 1.1223118305206299, + "memory(GiB)": 34.14, + "nll_loss": 0.5288960933685303, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.07493848353624344, + "rewards/margins": 0.3489125967025757, + "rewards/rejected": -0.4238511323928833, + "step": 27, + "train_speed(iter/s)": 0.010732 + }, + { + "epoch": 0.09773123909249563, + "grad_norm": 3.89947509765625, + "learning_rate": 0.0001302325581395349, + "logits/chosen": -0.3942677080631256, + "logits/rejected": -0.44409164786338806, + "logps/chosen": -12.765388488769531, + "logps/rejected": -21.65694236755371, + "loss": 1.2387657165527344, + "memory(GiB)": 34.14, + "nll_loss": 0.7279061675071716, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.10546489804983139, + "rewards/margins": 0.5104267001152039, + "rewards/rejected": -0.6158915758132935, + "step": 28, + "train_speed(iter/s)": 0.010734 + }, + { + "epoch": 0.1012216404886562, + "grad_norm": 2.5956175327301025, + "learning_rate": 0.00013488372093023256, + "logits/chosen": -0.4249032139778137, + "logits/rejected": -0.47334301471710205, + "logps/chosen": -8.39433765411377, + "logps/rejected": -19.74716567993164, + "loss": 1.050766110420227, + "memory(GiB)": 34.14, + "nll_loss": 0.5741204619407654, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.02486160770058632, + "rewards/margins": 0.6779846549034119, + "rewards/rejected": -0.6531230211257935, + "step": 29, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 0.10471204188481675, + "grad_norm": 3.123206377029419, + "learning_rate": 0.00013953488372093025, + "logits/chosen": -0.3965456485748291, + "logits/rejected": -0.5027358531951904, + "logps/chosen": -4.980073928833008, + "logps/rejected": -25.411418914794922, + "loss": 0.8653208613395691, + "memory(GiB)": 34.14, + "nll_loss": 0.4148715138435364, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.021732095628976822, + "rewards/margins": 0.7521398663520813, + "rewards/rejected": -0.73040771484375, + "step": 30, + "train_speed(iter/s)": 0.010738 + }, + { + "epoch": 0.1082024432809773, + "grad_norm": 3.027268409729004, + "learning_rate": 0.00014418604651162791, + "logits/chosen": -0.40849679708480835, + "logits/rejected": -0.47468990087509155, + "logps/chosen": -7.037868499755859, + "logps/rejected": -21.788843154907227, + "loss": 1.0142453908920288, + "memory(GiB)": 34.14, + "nll_loss": 0.48545634746551514, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.026916703209280968, + "rewards/margins": 0.5216760039329529, + "rewards/rejected": -0.5485926866531372, + "step": 31, + "train_speed(iter/s)": 0.010739 + }, + { + "epoch": 0.11169284467713787, + "grad_norm": 4.0368146896362305, + "learning_rate": 0.00014883720930232558, + "logits/chosen": -0.3984982967376709, + "logits/rejected": -0.47634080052375793, + "logps/chosen": -14.304816246032715, + "logps/rejected": -21.326597213745117, + "loss": 1.2883504629135132, + "memory(GiB)": 34.14, + "nll_loss": 0.6627980470657349, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.268436074256897, + "rewards/margins": 0.32418739795684814, + "rewards/rejected": -0.5926234722137451, + "step": 32, + "train_speed(iter/s)": 0.010739 + }, + { + "epoch": 0.11518324607329843, + "grad_norm": 2.3341856002807617, + "learning_rate": 0.00015348837209302327, + "logits/chosen": -0.34923142194747925, + "logits/rejected": -0.45246458053588867, + "logps/chosen": -7.6517744064331055, + "logps/rejected": -26.005094528198242, + "loss": 0.8868998885154724, + "memory(GiB)": 34.14, + "nll_loss": 0.45535364747047424, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.009979171678423882, + "rewards/margins": 0.8548348546028137, + "rewards/rejected": -0.8448556661605835, + "step": 33, + "train_speed(iter/s)": 0.010739 + }, + { + "epoch": 0.11867364746945899, + "grad_norm": 3.8161845207214355, + "learning_rate": 0.00015813953488372093, + "logits/chosen": -0.4433535933494568, + "logits/rejected": -0.4599112570285797, + "logps/chosen": -7.665817737579346, + "logps/rejected": -16.353017807006836, + "loss": 0.9908475279808044, + "memory(GiB)": 34.14, + "nll_loss": 0.5115343332290649, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.05993407219648361, + "rewards/margins": 0.6281672716140747, + "rewards/rejected": -0.5682331919670105, + "step": 34, + "train_speed(iter/s)": 0.010742 + }, + { + "epoch": 0.12216404886561955, + "grad_norm": 2.7297890186309814, + "learning_rate": 0.00016279069767441862, + "logits/chosen": -0.32466650009155273, + "logits/rejected": -0.4645735025405884, + "logps/chosen": -6.396218299865723, + "logps/rejected": -25.87699317932129, + "loss": 0.8697510957717896, + "memory(GiB)": 34.14, + "nll_loss": 0.38066551089286804, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.11624527722597122, + "rewards/margins": 0.6377441883087158, + "rewards/rejected": -0.5214988589286804, + "step": 35, + "train_speed(iter/s)": 0.010743 + }, + { + "epoch": 0.1256544502617801, + "grad_norm": 3.7659413814544678, + "learning_rate": 0.00016744186046511629, + "logits/chosen": -0.3423185348510742, + "logits/rejected": -0.5114607214927673, + "logps/chosen": -6.822861671447754, + "logps/rejected": -25.182849884033203, + "loss": 1.0008474588394165, + "memory(GiB)": 34.14, + "nll_loss": 0.5603237152099609, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.10806835442781448, + "rewards/margins": 0.8264971375465393, + "rewards/rejected": -0.7184286713600159, + "step": 36, + "train_speed(iter/s)": 0.010743 + }, + { + "epoch": 0.12914485165794065, + "grad_norm": 5.439046382904053, + "learning_rate": 0.00017209302325581395, + "logits/chosen": -0.4786157011985779, + "logits/rejected": -0.4922429919242859, + "logps/chosen": -6.435626983642578, + "logps/rejected": -17.2198429107666, + "loss": 0.9384127259254456, + "memory(GiB)": 34.14, + "nll_loss": 0.43419790267944336, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.02293190360069275, + "rewards/margins": 0.5868319869041443, + "rewards/rejected": -0.5639001131057739, + "step": 37, + "train_speed(iter/s)": 0.010745 + }, + { + "epoch": 0.13263525305410123, + "grad_norm": 3.3508763313293457, + "learning_rate": 0.00017674418604651164, + "logits/chosen": -0.42465993762016296, + "logits/rejected": -0.4925902783870697, + "logps/chosen": -8.958169937133789, + "logps/rejected": -20.73528289794922, + "loss": 1.111018180847168, + "memory(GiB)": 34.14, + "nll_loss": 0.7000492811203003, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.15357299149036407, + "rewards/margins": 1.0291643142700195, + "rewards/rejected": -0.8755912780761719, + "step": 38, + "train_speed(iter/s)": 0.010747 + }, + { + "epoch": 0.13612565445026178, + "grad_norm": 3.021416664123535, + "learning_rate": 0.0001813953488372093, + "logits/chosen": -0.4066935181617737, + "logits/rejected": -0.4705404043197632, + "logps/chosen": -9.284646987915039, + "logps/rejected": -20.46940803527832, + "loss": 0.9931483864784241, + "memory(GiB)": 34.14, + "nll_loss": 0.5600231885910034, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.11639773845672607, + "rewards/margins": 0.8292201161384583, + "rewards/rejected": -0.7128223776817322, + "step": 39, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.13961605584642234, + "grad_norm": 3.415731191635132, + "learning_rate": 0.000186046511627907, + "logits/chosen": -0.348653107881546, + "logits/rejected": -0.5043780207633972, + "logps/chosen": -2.844910144805908, + "logps/rejected": -24.771116256713867, + "loss": 0.7038365006446838, + "memory(GiB)": 34.14, + "nll_loss": 0.29491642117500305, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.14402121305465698, + "rewards/margins": 1.1453313827514648, + "rewards/rejected": -1.0013102293014526, + "step": 40, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.1431064572425829, + "grad_norm": 3.0232815742492676, + "learning_rate": 0.00019069767441860466, + "logits/chosen": -0.35887610912323, + "logits/rejected": -0.38277357816696167, + "logps/chosen": -10.264820098876953, + "logps/rejected": -24.83388328552246, + "loss": 0.8683040738105774, + "memory(GiB)": 34.14, + "nll_loss": 0.4005107581615448, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.013488545082509518, + "rewards/margins": 0.8365979194641113, + "rewards/rejected": -0.8231093883514404, + "step": 41, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.14659685863874344, + "grad_norm": 16.639280319213867, + "learning_rate": 0.00019534883720930232, + "logits/chosen": -0.33245426416397095, + "logits/rejected": -0.4030519127845764, + "logps/chosen": -10.0272216796875, + "logps/rejected": -25.483577728271484, + "loss": 1.21886146068573, + "memory(GiB)": 34.14, + "nll_loss": 0.7048445343971252, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.14778879284858704, + "rewards/margins": 1.201229453086853, + "rewards/rejected": -1.3490180969238281, + "step": 42, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.15008726003490402, + "grad_norm": 6.577022075653076, + "learning_rate": 0.0002, + "logits/chosen": -0.29526323080062866, + "logits/rejected": -0.45897889137268066, + "logps/chosen": -5.878642559051514, + "logps/rejected": -39.53807830810547, + "loss": 0.7673805356025696, + "memory(GiB)": 35.89, + "nll_loss": 0.4374830424785614, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.023150455206632614, + "rewards/margins": 1.816420555114746, + "rewards/rejected": -1.8395708799362183, + "step": 43, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.15357766143106458, + "grad_norm": 2.505460262298584, + "learning_rate": 0.00019999925705956715, + "logits/chosen": -0.37890657782554626, + "logits/rejected": -0.44291937351226807, + "logps/chosen": -8.378349304199219, + "logps/rejected": -32.722042083740234, + "loss": 0.8200633525848389, + "memory(GiB)": 35.89, + "nll_loss": 0.5234110355377197, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08434319496154785, + "rewards/margins": 1.911144495010376, + "rewards/rejected": -1.8268014192581177, + "step": 44, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.15706806282722513, + "grad_norm": 6.880366802215576, + "learning_rate": 0.0001999970282493078, + "logits/chosen": -0.328549325466156, + "logits/rejected": -0.4838561713695526, + "logps/chosen": -6.23681116104126, + "logps/rejected": -29.523792266845703, + "loss": 1.0678659677505493, + "memory(GiB)": 35.89, + "nll_loss": 0.5795442461967468, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0050928061828017235, + "rewards/margins": 1.2474907636642456, + "rewards/rejected": -1.2423979043960571, + "step": 45, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.16055846422338568, + "grad_norm": 6.962847709655762, + "learning_rate": 0.00019999331360233944, + "logits/chosen": -0.33589401841163635, + "logits/rejected": -0.45322129130363464, + "logps/chosen": -6.451783657073975, + "logps/rejected": -32.003658294677734, + "loss": 0.9696826338768005, + "memory(GiB)": 35.89, + "nll_loss": 0.5726639032363892, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.014406954869627953, + "rewards/margins": 1.381211757659912, + "rewards/rejected": -1.3956186771392822, + "step": 46, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.16404886561954624, + "grad_norm": 7.938783168792725, + "learning_rate": 0.00019998811317385729, + "logits/chosen": -0.3734630346298218, + "logits/rejected": -0.4592462480068207, + "logps/chosen": -12.47143268585205, + "logps/rejected": -26.8325138092041, + "loss": 1.141100287437439, + "memory(GiB)": 35.89, + "nll_loss": 0.6339684724807739, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.027786267921328545, + "rewards/margins": 0.6427249312400818, + "rewards/rejected": -0.6149386167526245, + "step": 47, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.16753926701570682, + "grad_norm": 2.760986804962158, + "learning_rate": 0.0001999814270411335, + "logits/chosen": -0.43143945932388306, + "logits/rejected": -0.4347304701805115, + "logps/chosen": -9.348878860473633, + "logps/rejected": -24.229524612426758, + "loss": 0.9695644974708557, + "memory(GiB)": 35.89, + "nll_loss": 0.5839965343475342, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.007340218871831894, + "rewards/margins": 1.3898271322250366, + "rewards/rejected": -1.3824869394302368, + "step": 48, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.17102966841186737, + "grad_norm": 11.194586753845215, + "learning_rate": 0.00019997325530351606, + "logits/chosen": -0.3548398017883301, + "logits/rejected": -0.4637337028980255, + "logps/chosen": -16.913610458374023, + "logps/rejected": -33.84115982055664, + "loss": 1.521134614944458, + "memory(GiB)": 35.89, + "nll_loss": 1.0307905673980713, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.28235313296318054, + "rewards/margins": 1.413867712020874, + "rewards/rejected": -1.6962207555770874, + "step": 49, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.17452006980802792, + "grad_norm": 3.4211995601654053, + "learning_rate": 0.00019996359808242722, + "logits/chosen": -0.32980048656463623, + "logits/rejected": -0.5140572190284729, + "logps/chosen": -4.228909492492676, + "logps/rejected": -34.62148666381836, + "loss": 0.6742458343505859, + "memory(GiB)": 35.89, + "nll_loss": 0.37631386518478394, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08238644152879715, + "rewards/margins": 1.9769954681396484, + "rewards/rejected": -1.894608974456787, + "step": 50, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.17801047120418848, + "grad_norm": 4.199666500091553, + "learning_rate": 0.00019995245552136183, + "logits/chosen": -0.38337764143943787, + "logits/rejected": -0.5411950349807739, + "logps/chosen": -7.726480484008789, + "logps/rejected": -33.165184020996094, + "loss": 0.8604421615600586, + "memory(GiB)": 35.89, + "nll_loss": 0.4990411400794983, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.0083555206656456, + "rewards/margins": 1.5133056640625, + "rewards/rejected": -1.5049501657485962, + "step": 51, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.18150087260034903, + "grad_norm": 7.909679412841797, + "learning_rate": 0.00019993982778588506, + "logits/chosen": -0.42613935470581055, + "logits/rejected": -0.5141798257827759, + "logps/chosen": -12.04079818725586, + "logps/rejected": -32.32722091674805, + "loss": 1.079092264175415, + "memory(GiB)": 35.89, + "nll_loss": 0.5544219613075256, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.21235862374305725, + "rewards/margins": 1.353042721748352, + "rewards/rejected": -1.565401315689087, + "step": 52, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.1849912739965096, + "grad_norm": 3.7452335357666016, + "learning_rate": 0.00019992571506363, + "logits/chosen": -0.3618060052394867, + "logits/rejected": -0.5190442800521851, + "logps/chosen": -6.607875347137451, + "logps/rejected": -25.438873291015625, + "loss": 1.0128110647201538, + "memory(GiB)": 35.89, + "nll_loss": 0.5347008109092712, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0013710036873817444, + "rewards/margins": 0.7542194724082947, + "rewards/rejected": -0.7555904388427734, + "step": 53, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.18848167539267016, + "grad_norm": 2.7128043174743652, + "learning_rate": 0.00019991011756429486, + "logits/chosen": -0.43821805715560913, + "logits/rejected": -0.48400819301605225, + "logps/chosen": -8.567977905273438, + "logps/rejected": -19.301103591918945, + "loss": 1.0873574018478394, + "memory(GiB)": 35.89, + "nll_loss": 0.5480538010597229, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.06769907474517822, + "rewards/margins": 0.6852700114250183, + "rewards/rejected": -0.6175709366798401, + "step": 54, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.19197207678883071, + "grad_norm": 2.5323894023895264, + "learning_rate": 0.00019989303551963997, + "logits/chosen": -0.3141109347343445, + "logits/rejected": -0.48292648792266846, + "logps/chosen": -4.789846420288086, + "logps/rejected": -26.765024185180664, + "loss": 0.880359411239624, + "memory(GiB)": 35.89, + "nll_loss": 0.3599545359611511, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09160856902599335, + "rewards/margins": 0.7724493145942688, + "rewards/rejected": -0.6808407306671143, + "step": 55, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.19546247818499127, + "grad_norm": 2.958927869796753, + "learning_rate": 0.00019987446918348411, + "logits/chosen": -0.3392045199871063, + "logits/rejected": -0.4613609313964844, + "logps/chosen": -6.773719310760498, + "logps/rejected": -20.55025291442871, + "loss": 1.0082640647888184, + "memory(GiB)": 35.89, + "nll_loss": 0.44486063718795776, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.06419242918491364, + "rewards/margins": 0.5171071887016296, + "rewards/rejected": -0.4529147446155548, + "step": 56, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.19895287958115182, + "grad_norm": 3.204970359802246, + "learning_rate": 0.00019985441883170094, + "logits/chosen": -0.45556554198265076, + "logits/rejected": -0.47999584674835205, + "logps/chosen": -8.982965469360352, + "logps/rejected": -18.054445266723633, + "loss": 0.996261477470398, + "memory(GiB)": 35.89, + "nll_loss": 0.4766736924648285, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.07613325119018555, + "rewards/margins": 0.774691104888916, + "rewards/rejected": -0.6985578536987305, + "step": 57, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.2024432809773124, + "grad_norm": 3.0290675163269043, + "learning_rate": 0.0001998328847622148, + "logits/chosen": -0.43794357776641846, + "logits/rejected": -0.4973166882991791, + "logps/chosen": -6.941474437713623, + "logps/rejected": -21.18113136291504, + "loss": 0.9271993637084961, + "memory(GiB)": 35.89, + "nll_loss": 0.40610748529434204, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.031344618648290634, + "rewards/margins": 0.7323437929153442, + "rewards/rejected": -0.7009991407394409, + "step": 58, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.20593368237347295, + "grad_norm": 3.3924288749694824, + "learning_rate": 0.0001998098672949963, + "logits/chosen": -0.43890056014060974, + "logits/rejected": -0.47092944383621216, + "logps/chosen": -6.540399074554443, + "logps/rejected": -23.68883514404297, + "loss": 0.8886168003082275, + "memory(GiB)": 35.89, + "nll_loss": 0.42720285058021545, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.062318772077560425, + "rewards/margins": 1.219529151916504, + "rewards/rejected": -1.1572105884552002, + "step": 59, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.2094240837696335, + "grad_norm": 6.436001300811768, + "learning_rate": 0.00019978536677205753, + "logits/chosen": -0.3602336645126343, + "logits/rejected": -0.49971872568130493, + "logps/chosen": -7.300111770629883, + "logps/rejected": -35.16114044189453, + "loss": 0.893642783164978, + "memory(GiB)": 35.89, + "nll_loss": 0.5613251328468323, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.04828762263059616, + "rewards/margins": 2.0058753490448, + "rewards/rejected": -2.0541629791259766, + "step": 60, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.21291448516579406, + "grad_norm": 9.151269912719727, + "learning_rate": 0.0001997593835574472, + "logits/chosen": -0.2997259497642517, + "logits/rejected": -0.4807735085487366, + "logps/chosen": -7.510552406311035, + "logps/rejected": -51.04875946044922, + "loss": 0.7812849879264832, + "memory(GiB)": 35.89, + "nll_loss": 0.5295180082321167, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.27329760789871216, + "rewards/margins": 3.0174202919006348, + "rewards/rejected": -3.2907180786132812, + "step": 61, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2164048865619546, + "grad_norm": 3.0901706218719482, + "learning_rate": 0.00019973191803724483, + "logits/chosen": -0.3464053273200989, + "logits/rejected": -0.43850764632225037, + "logps/chosen": -5.784665107727051, + "logps/rejected": -29.168745040893555, + "loss": 0.6676730513572693, + "memory(GiB)": 35.89, + "nll_loss": 0.2769017219543457, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.09001894295215607, + "rewards/margins": 1.1760774850845337, + "rewards/rejected": -1.0860586166381836, + "step": 62, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2198952879581152, + "grad_norm": 3.0961596965789795, + "learning_rate": 0.00019970297061955533, + "logits/chosen": -0.35332319140434265, + "logits/rejected": -0.5135117173194885, + "logps/chosen": -5.485988616943359, + "logps/rejected": -36.72576141357422, + "loss": 0.7853246331214905, + "memory(GiB)": 35.89, + "nll_loss": 0.5297773480415344, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.02064245566725731, + "rewards/margins": 2.3967862129211426, + "rewards/rejected": -2.37614369392395, + "step": 63, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.22338568935427575, + "grad_norm": 4.25160026550293, + "learning_rate": 0.00019967254173450287, + "logits/chosen": -0.2867816090583801, + "logits/rejected": -0.44146350026130676, + "logps/chosen": -5.827723026275635, + "logps/rejected": -31.073637008666992, + "loss": 0.7783889770507812, + "memory(GiB)": 35.89, + "nll_loss": 0.43023356795310974, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.09779734909534454, + "rewards/margins": 1.5616310834884644, + "rewards/rejected": -1.4638336896896362, + "step": 64, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.2268760907504363, + "grad_norm": 2.711167097091675, + "learning_rate": 0.0001996406318342244, + "logits/chosen": -0.351021945476532, + "logits/rejected": -0.4185492694377899, + "logps/chosen": -6.972557544708252, + "logps/rejected": -29.75743865966797, + "loss": 0.7590102553367615, + "memory(GiB)": 35.89, + "nll_loss": 0.42238616943359375, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07267698645591736, + "rewards/margins": 1.7358276844024658, + "rewards/rejected": -1.663150429725647, + "step": 65, + "train_speed(iter/s)": 0.010761 + }, + { + "epoch": 0.23036649214659685, + "grad_norm": 5.093292236328125, + "learning_rate": 0.00019960724139286308, + "logits/chosen": -0.29837629199028015, + "logits/rejected": -0.38664788007736206, + "logps/chosen": -11.696621894836426, + "logps/rejected": -34.02775192260742, + "loss": 1.0549557209014893, + "memory(GiB)": 35.89, + "nll_loss": 0.7019686102867126, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.16252195835113525, + "rewards/margins": 1.5406640768051147, + "rewards/rejected": -1.70318603515625, + "step": 66, + "train_speed(iter/s)": 0.010761 + }, + { + "epoch": 0.2338568935427574, + "grad_norm": 3.672001838684082, + "learning_rate": 0.00019957237090656102, + "logits/chosen": -0.2561183571815491, + "logits/rejected": -0.4472733438014984, + "logps/chosen": -6.693790912628174, + "logps/rejected": -52.02687072753906, + "loss": 0.6644150614738464, + "memory(GiB)": 35.89, + "nll_loss": 0.44371137022972107, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.018377352505922318, + "rewards/margins": 2.62888765335083, + "rewards/rejected": -2.6472654342651367, + "step": 67, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.23734729493891799, + "grad_norm": 8.564802169799805, + "learning_rate": 0.00019953602089345217, + "logits/chosen": -0.3133094310760498, + "logits/rejected": -0.45969486236572266, + "logps/chosen": -7.815582275390625, + "logps/rejected": -38.529781341552734, + "loss": 1.0037181377410889, + "memory(GiB)": 35.89, + "nll_loss": 0.6282736659049988, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.1540919542312622, + "rewards/margins": 1.7298449277877808, + "rewards/rejected": -1.883936882019043, + "step": 68, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.24083769633507854, + "grad_norm": 12.39240837097168, + "learning_rate": 0.00019949819189365433, + "logits/chosen": -0.3124208152294159, + "logits/rejected": -0.43993785977363586, + "logps/chosen": -7.740966796875, + "logps/rejected": -34.374534606933594, + "loss": 0.9371472001075745, + "memory(GiB)": 35.89, + "nll_loss": 0.46057644486427307, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.0446137860417366, + "rewards/margins": 1.5109288692474365, + "rewards/rejected": -1.5555427074432373, + "step": 69, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.2443280977312391, + "grad_norm": 6.788973331451416, + "learning_rate": 0.00019945888446926143, + "logits/chosen": -0.3286086618900299, + "logits/rejected": -0.40763434767723083, + "logps/chosen": -10.763023376464844, + "logps/rejected": -23.91502571105957, + "loss": 1.182040810585022, + "memory(GiB)": 35.89, + "nll_loss": 0.6381036639213562, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1615467518568039, + "rewards/margins": 0.6939660906791687, + "rewards/rejected": -0.8555128574371338, + "step": 70, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.24781849912739964, + "grad_norm": 4.333127975463867, + "learning_rate": 0.000199418099204335, + "logits/chosen": -0.3585875630378723, + "logits/rejected": -0.44930267333984375, + "logps/chosen": -8.16705322265625, + "logps/rejected": -22.100000381469727, + "loss": 0.9709787368774414, + "memory(GiB)": 35.89, + "nll_loss": 0.5348560214042664, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.04899175092577934, + "rewards/margins": 0.8827412128448486, + "rewards/rejected": -0.9317330121994019, + "step": 71, + "train_speed(iter/s)": 0.010761 + }, + { + "epoch": 0.2513089005235602, + "grad_norm": 3.37380051612854, + "learning_rate": 0.00019937583670489545, + "logits/chosen": -0.33419570326805115, + "logits/rejected": -0.43994009494781494, + "logps/chosen": -9.082391738891602, + "logps/rejected": -28.777481079101562, + "loss": 0.9270099401473999, + "memory(GiB)": 35.89, + "nll_loss": 0.48707476258277893, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.024601396173238754, + "rewards/margins": 1.1600151062011719, + "rewards/rejected": -1.1354137659072876, + "step": 72, + "train_speed(iter/s)": 0.010761 + }, + { + "epoch": 0.2547993019197208, + "grad_norm": 3.593212366104126, + "learning_rate": 0.00019933209759891317, + "logits/chosen": -0.4130091071128845, + "logits/rejected": -0.4818781018257141, + "logps/chosen": -10.08952808380127, + "logps/rejected": -34.58759307861328, + "loss": 0.9288522601127625, + "memory(GiB)": 35.89, + "nll_loss": 0.6054138541221619, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11658284813165665, + "rewards/margins": 2.089298963546753, + "rewards/rejected": -2.2058815956115723, + "step": 73, + "train_speed(iter/s)": 0.010762 + }, + { + "epoch": 0.2582897033158813, + "grad_norm": 3.8672237396240234, + "learning_rate": 0.00019928688253629916, + "logits/chosen": -0.19415511190891266, + "logits/rejected": -0.36130261421203613, + "logps/chosen": -11.009477615356445, + "logps/rejected": -43.973114013671875, + "loss": 1.1013773679733276, + "memory(GiB)": 35.89, + "nll_loss": 0.6544516086578369, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.030637577176094055, + "rewards/margins": 2.5454745292663574, + "rewards/rejected": -2.5761120319366455, + "step": 74, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.2617801047120419, + "grad_norm": 4.066303730010986, + "learning_rate": 0.00019924019218889536, + "logits/chosen": -0.33296871185302734, + "logits/rejected": -0.4902675747871399, + "logps/chosen": -9.64806079864502, + "logps/rejected": -37.88252639770508, + "loss": 0.8953545093536377, + "memory(GiB)": 35.89, + "nll_loss": 0.5640730857849121, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.009408235549926758, + "rewards/margins": 2.1965668201446533, + "rewards/rejected": -2.1871585845947266, + "step": 75, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.26527050610820246, + "grad_norm": 2.3979082107543945, + "learning_rate": 0.00019919202725046477, + "logits/chosen": -0.20567730069160461, + "logits/rejected": -0.34439828991889954, + "logps/chosen": -8.100298881530762, + "logps/rejected": -39.28445816040039, + "loss": 0.7528076767921448, + "memory(GiB)": 39.42, + "nll_loss": 0.39297810196876526, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.025398975238204002, + "rewards/margins": 1.6433254480361938, + "rewards/rejected": -1.6179265975952148, + "step": 76, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.268760907504363, + "grad_norm": 2.7927446365356445, + "learning_rate": 0.00019914238843668096, + "logits/chosen": -0.3428923785686493, + "logits/rejected": -0.4334644675254822, + "logps/chosen": -6.549041271209717, + "logps/rejected": -27.63149070739746, + "loss": 0.796095609664917, + "memory(GiB)": 39.42, + "nll_loss": 0.3915262818336487, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.06563803553581238, + "rewards/margins": 1.2631423473358154, + "rewards/rejected": -1.1975042819976807, + "step": 77, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.27225130890052357, + "grad_norm": 4.248587608337402, + "learning_rate": 0.00019909127648511755, + "logits/chosen": -0.3395402431488037, + "logits/rejected": -0.48165830969810486, + "logps/chosen": -5.904273986816406, + "logps/rejected": -27.16798973083496, + "loss": 0.743380606174469, + "memory(GiB)": 39.42, + "nll_loss": 0.3838931918144226, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10840368270874023, + "rewards/margins": 1.24807870388031, + "rewards/rejected": -1.1396750211715698, + "step": 78, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2757417102966841, + "grad_norm": 2.7059414386749268, + "learning_rate": 0.00019903869215523733, + "logits/chosen": -0.22887416183948517, + "logits/rejected": -0.48973891139030457, + "logps/chosen": -3.493281364440918, + "logps/rejected": -34.020050048828125, + "loss": 0.6441786885261536, + "memory(GiB)": 39.42, + "nll_loss": 0.3432372510433197, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12880486249923706, + "rewards/margins": 1.6592087745666504, + "rewards/rejected": -1.5304038524627686, + "step": 79, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2792321116928447, + "grad_norm": 4.634321212768555, + "learning_rate": 0.0001989846362283807, + "logits/chosen": -0.29562249779701233, + "logits/rejected": -0.34773433208465576, + "logps/chosen": -11.475696563720703, + "logps/rejected": -27.678905487060547, + "loss": 1.1467502117156982, + "memory(GiB)": 39.42, + "nll_loss": 0.6945275664329529, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.10402540862560272, + "rewards/margins": 1.2328475713729858, + "rewards/rejected": -1.336873173713684, + "step": 80, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.28272251308900526, + "grad_norm": 3.8445465564727783, + "learning_rate": 0.00019892910950775435, + "logits/chosen": -0.23248018324375153, + "logits/rejected": -0.35242336988449097, + "logps/chosen": -6.628226280212402, + "logps/rejected": -33.20770263671875, + "loss": 0.8408610224723816, + "memory(GiB)": 39.42, + "nll_loss": 0.5566035509109497, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13027045130729675, + "rewards/margins": 2.089359998703003, + "rewards/rejected": -1.9590896368026733, + "step": 81, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2862129144851658, + "grad_norm": 2.7230985164642334, + "learning_rate": 0.00019887211281841924, + "logits/chosen": -0.2518586218357086, + "logits/rejected": -0.31852811574935913, + "logps/chosen": -8.052734375, + "logps/rejected": -28.10201644897461, + "loss": 0.7522032856941223, + "memory(GiB)": 39.42, + "nll_loss": 0.3988189697265625, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.162813201546669, + "rewards/margins": 1.698049783706665, + "rewards/rejected": -1.5352365970611572, + "step": 82, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.28970331588132636, + "grad_norm": 5.669232368469238, + "learning_rate": 0.00019881364700727823, + "logits/chosen": -0.13075971603393555, + "logits/rejected": -0.3868337869644165, + "logps/chosen": -9.65750503540039, + "logps/rejected": -36.58469772338867, + "loss": 1.048365831375122, + "memory(GiB)": 39.42, + "nll_loss": 0.7252820730209351, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.21855562925338745, + "rewards/margins": 1.9177855253219604, + "rewards/rejected": -1.6992299556732178, + "step": 83, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.2931937172774869, + "grad_norm": 2.559206962585449, + "learning_rate": 0.00019875371294306366, + "logits/chosen": -0.07886800169944763, + "logits/rejected": -0.302437424659729, + "logps/chosen": -6.699373722076416, + "logps/rejected": -43.041194915771484, + "loss": 0.7339796423912048, + "memory(GiB)": 39.42, + "nll_loss": 0.4990921914577484, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12242552638053894, + "rewards/margins": 2.4995968341827393, + "rewards/rejected": -2.377171277999878, + "step": 84, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.29668411867364747, + "grad_norm": 3.492406129837036, + "learning_rate": 0.0001986923115163243, + "logits/chosen": -0.13503603637218475, + "logits/rejected": -0.3026173710823059, + "logps/chosen": -6.315262317657471, + "logps/rejected": -38.55803680419922, + "loss": 0.8880235552787781, + "memory(GiB)": 39.42, + "nll_loss": 0.5917512774467468, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04879331588745117, + "rewards/margins": 1.9771116971969604, + "rewards/rejected": -1.9283183813095093, + "step": 85, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.30017452006980805, + "grad_norm": 3.961932897567749, + "learning_rate": 0.00019862944363941218, + "logits/chosen": -0.10436159372329712, + "logits/rejected": -0.3089469075202942, + "logps/chosen": -6.1503825187683105, + "logps/rejected": -38.23727035522461, + "loss": 0.90598064661026, + "memory(GiB)": 39.42, + "nll_loss": 0.633453905582428, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14362435042858124, + "rewards/margins": 2.058913469314575, + "rewards/rejected": -1.915289044380188, + "step": 86, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.3036649214659686, + "grad_norm": 3.4905996322631836, + "learning_rate": 0.0001985651102464691, + "logits/chosen": -0.11853908002376556, + "logits/rejected": -0.24401453137397766, + "logps/chosen": -7.3659820556640625, + "logps/rejected": -38.6995849609375, + "loss": 0.749198853969574, + "memory(GiB)": 39.42, + "nll_loss": 0.43377092480659485, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.14165769517421722, + "rewards/margins": 2.025956630706787, + "rewards/rejected": -2.167614459991455, + "step": 87, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.30715532286212915, + "grad_norm": 3.6831657886505127, + "learning_rate": 0.00019849931229341258, + "logits/chosen": -0.1332027167081833, + "logits/rejected": -0.24646437168121338, + "logps/chosen": -8.003013610839844, + "logps/rejected": -37.04756164550781, + "loss": 0.8140410780906677, + "memory(GiB)": 39.42, + "nll_loss": 0.44854581356048584, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.034915290772914886, + "rewards/margins": 1.382434606552124, + "rewards/rejected": -1.3475191593170166, + "step": 88, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.3106457242582897, + "grad_norm": 2.4958512783050537, + "learning_rate": 0.00019843205075792186, + "logits/chosen": -0.11535613983869553, + "logits/rejected": -0.246973916888237, + "logps/chosen": -7.1996660232543945, + "logps/rejected": -32.96755599975586, + "loss": 0.7998999357223511, + "memory(GiB)": 39.42, + "nll_loss": 0.49011173844337463, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08970604836940765, + "rewards/margins": 1.6430060863494873, + "rewards/rejected": -1.553299903869629, + "step": 89, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.31413612565445026, + "grad_norm": 2.470099687576294, + "learning_rate": 0.0001983633266394232, + "logits/chosen": -0.13420692086219788, + "logits/rejected": -0.3025938868522644, + "logps/chosen": -10.592232704162598, + "logps/rejected": -43.20408630371094, + "loss": 0.8399574160575867, + "memory(GiB)": 39.42, + "nll_loss": 0.5707088112831116, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.024432912468910217, + "rewards/margins": 1.8845049142837524, + "rewards/rejected": -1.8600718975067139, + "step": 90, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.31762652705061084, + "grad_norm": 2.7454211711883545, + "learning_rate": 0.00019829314095907516, + "logits/chosen": -0.15840838849544525, + "logits/rejected": -0.29640981554985046, + "logps/chosen": -7.358951568603516, + "logps/rejected": -35.788387298583984, + "loss": 0.7901937961578369, + "memory(GiB)": 39.42, + "nll_loss": 0.48892611265182495, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.018669307231903076, + "rewards/margins": 1.815453290939331, + "rewards/rejected": -1.7967840433120728, + "step": 91, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.32111692844677137, + "grad_norm": 2.443572998046875, + "learning_rate": 0.0001982214947597533, + "logits/chosen": -0.13980664312839508, + "logits/rejected": -0.2785203754901886, + "logps/chosen": -7.595067024230957, + "logps/rejected": -29.64536476135254, + "loss": 0.7760062217712402, + "memory(GiB)": 39.42, + "nll_loss": 0.417621374130249, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.18094170093536377, + "rewards/margins": 1.5685675144195557, + "rewards/rejected": -1.3876259326934814, + "step": 92, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.32460732984293195, + "grad_norm": 2.695469856262207, + "learning_rate": 0.00019814838910603481, + "logits/chosen": -0.21802689135074615, + "logits/rejected": -0.30977922677993774, + "logps/chosen": -8.865757942199707, + "logps/rejected": -31.998289108276367, + "loss": 0.8171509504318237, + "memory(GiB)": 39.42, + "nll_loss": 0.46762198209762573, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.08467155694961548, + "rewards/margins": 1.8997126817703247, + "rewards/rejected": -1.815041184425354, + "step": 93, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.32809773123909247, + "grad_norm": 2.3831567764282227, + "learning_rate": 0.0001980738250841826, + "logits/chosen": -0.1939636468887329, + "logits/rejected": -0.33366823196411133, + "logps/chosen": -8.547904968261719, + "logps/rejected": -36.44358444213867, + "loss": 0.8093112707138062, + "memory(GiB)": 39.42, + "nll_loss": 0.46711432933807373, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.023340808227658272, + "rewards/margins": 1.871730923652649, + "rewards/rejected": -1.8483901023864746, + "step": 94, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.33158813263525305, + "grad_norm": 4.449499130249023, + "learning_rate": 0.0001979978038021292, + "logits/chosen": -0.21011316776275635, + "logits/rejected": -0.3800049126148224, + "logps/chosen": -6.6607513427734375, + "logps/rejected": -37.015220642089844, + "loss": 0.7690098285675049, + "memory(GiB)": 39.42, + "nll_loss": 0.3787845969200134, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.022871140390634537, + "rewards/margins": 1.7818647623062134, + "rewards/rejected": -1.7589936256408691, + "step": 95, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.33507853403141363, + "grad_norm": 8.076894760131836, + "learning_rate": 0.00019792032638946027, + "logits/chosen": -0.22930172085762024, + "logits/rejected": -0.363534152507782, + "logps/chosen": -9.168333053588867, + "logps/rejected": -39.97100067138672, + "loss": 0.999840497970581, + "memory(GiB)": 39.42, + "nll_loss": 0.5044428706169128, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19479180872440338, + "rewards/margins": 2.030325174331665, + "rewards/rejected": -2.225116729736328, + "step": 96, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.33856893542757416, + "grad_norm": 2.602322578430176, + "learning_rate": 0.00019784139399739794, + "logits/chosen": -0.23653970658779144, + "logits/rejected": -0.37720808386802673, + "logps/chosen": -7.70982551574707, + "logps/rejected": -46.72187423706055, + "loss": 0.7515106797218323, + "memory(GiB)": 39.42, + "nll_loss": 0.49746939539909363, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.16890637576580048, + "rewards/margins": 2.9459898471832275, + "rewards/rejected": -3.114896297454834, + "step": 97, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.34205933682373474, + "grad_norm": 2.917598009109497, + "learning_rate": 0.00019776100779878345, + "logits/chosen": -0.24879562854766846, + "logits/rejected": -0.4712115526199341, + "logps/chosen": -4.071171760559082, + "logps/rejected": -39.8735237121582, + "loss": 0.6295332908630371, + "memory(GiB)": 39.42, + "nll_loss": 0.3750327527523041, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12702509760856628, + "rewards/margins": 2.281287431716919, + "rewards/rejected": -2.1542623043060303, + "step": 98, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.34554973821989526, + "grad_norm": 3.531543016433716, + "learning_rate": 0.00019767916898805993, + "logits/chosen": -0.3587397038936615, + "logits/rejected": -0.4245099723339081, + "logps/chosen": -9.423014640808105, + "logps/rejected": -32.22254943847656, + "loss": 0.9320547580718994, + "memory(GiB)": 39.42, + "nll_loss": 0.5687791109085083, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0431206040084362, + "rewards/margins": 1.7739802598953247, + "rewards/rejected": -1.8171007633209229, + "step": 99, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.34904013961605584, + "grad_norm": 2.429708242416382, + "learning_rate": 0.00019759587878125467, + "logits/chosen": -0.38257211446762085, + "logits/rejected": -0.46099749207496643, + "logps/chosen": -6.002010345458984, + "logps/rejected": -35.495018005371094, + "loss": 0.7784558534622192, + "memory(GiB)": 39.42, + "nll_loss": 0.4933393895626068, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10934662073850632, + "rewards/margins": 2.241340160369873, + "rewards/rejected": -2.1319937705993652, + "step": 100, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.3525305410122164, + "grad_norm": 2.213499069213867, + "learning_rate": 0.00019751113841596086, + "logits/chosen": -0.37704822421073914, + "logits/rejected": -0.4514527916908264, + "logps/chosen": -6.885778427124023, + "logps/rejected": -27.876874923706055, + "loss": 0.704035758972168, + "memory(GiB)": 39.42, + "nll_loss": 0.4308836758136749, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12775495648384094, + "rewards/margins": 1.5644640922546387, + "rewards/rejected": -1.4367092847824097, + "step": 101, + "train_speed(iter/s)": 0.010742 + }, + { + "epoch": 0.35602094240837695, + "grad_norm": 2.3929998874664307, + "learning_rate": 0.00019742494915131942, + "logits/chosen": -0.36924856901168823, + "logits/rejected": -0.41376832127571106, + "logps/chosen": -5.98624324798584, + "logps/rejected": -36.84775161743164, + "loss": 0.5458755493164062, + "memory(GiB)": 39.42, + "nll_loss": 0.35526004433631897, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1133466586470604, + "rewards/margins": 2.4806315898895264, + "rewards/rejected": -2.3672847747802734, + "step": 102, + "train_speed(iter/s)": 0.010743 + }, + { + "epoch": 0.35951134380453753, + "grad_norm": 5.081653118133545, + "learning_rate": 0.00019733731226800015, + "logits/chosen": -0.2884291112422943, + "logits/rejected": -0.41856223344802856, + "logps/chosen": -7.244772911071777, + "logps/rejected": -43.501136779785156, + "loss": 1.0223764181137085, + "memory(GiB)": 39.42, + "nll_loss": 0.7268551588058472, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.17409339547157288, + "rewards/margins": 2.4381885528564453, + "rewards/rejected": -2.6122817993164062, + "step": 103, + "train_speed(iter/s)": 0.010743 + }, + { + "epoch": 0.36300174520069806, + "grad_norm": 4.38121223449707, + "learning_rate": 0.00019724822906818265, + "logits/chosen": -0.3204030692577362, + "logits/rejected": -0.37165117263793945, + "logps/chosen": -7.093808650970459, + "logps/rejected": -36.024147033691406, + "loss": 0.746134877204895, + "memory(GiB)": 39.42, + "nll_loss": 0.4921882450580597, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.053046248853206635, + "rewards/margins": 2.321000337600708, + "rewards/rejected": -2.267954111099243, + "step": 104, + "train_speed(iter/s)": 0.010744 + }, + { + "epoch": 0.36649214659685864, + "grad_norm": 7.504276752471924, + "learning_rate": 0.0001971577008755372, + "logits/chosen": -0.2759462296962738, + "logits/rejected": -0.3651443123817444, + "logps/chosen": -11.585759162902832, + "logps/rejected": -35.60843276977539, + "loss": 1.0082931518554688, + "memory(GiB)": 39.42, + "nll_loss": 0.7006896734237671, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.08436793088912964, + "rewards/margins": 2.2114996910095215, + "rewards/rejected": -2.127131700515747, + "step": 105, + "train_speed(iter/s)": 0.010745 + }, + { + "epoch": 0.3699825479930192, + "grad_norm": 2.724567174911499, + "learning_rate": 0.00019706572903520494, + "logits/chosen": -0.3390807509422302, + "logits/rejected": -0.3694685101509094, + "logps/chosen": -9.778353691101074, + "logps/rejected": -30.431264877319336, + "loss": 0.8204815983772278, + "memory(GiB)": 39.42, + "nll_loss": 0.487753689289093, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.11946438252925873, + "rewards/margins": 1.8712043762207031, + "rewards/rejected": -1.7517400979995728, + "step": 106, + "train_speed(iter/s)": 0.010746 + }, + { + "epoch": 0.37347294938917974, + "grad_norm": 3.963045358657837, + "learning_rate": 0.00019697231491377774, + "logits/chosen": -0.3571932911872864, + "logits/rejected": -0.41452813148498535, + "logps/chosen": -6.354501724243164, + "logps/rejected": -29.302684783935547, + "loss": 0.8431603908538818, + "memory(GiB)": 39.42, + "nll_loss": 0.46558207273483276, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0002909880131483078, + "rewards/margins": 1.4369157552719116, + "rewards/rejected": -1.437206745147705, + "step": 107, + "train_speed(iter/s)": 0.010746 + }, + { + "epoch": 0.3769633507853403, + "grad_norm": 4.0586652755737305, + "learning_rate": 0.00019687745989927823, + "logits/chosen": -0.3135523796081543, + "logits/rejected": -0.3737596273422241, + "logps/chosen": -9.778253555297852, + "logps/rejected": -28.1949462890625, + "loss": 0.966033935546875, + "memory(GiB)": 39.42, + "nll_loss": 0.6015961170196533, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08021017909049988, + "rewards/margins": 1.4340676069259644, + "rewards/rejected": -1.353857398033142, + "step": 108, + "train_speed(iter/s)": 0.010747 + }, + { + "epoch": 0.38045375218150085, + "grad_norm": 5.568571090698242, + "learning_rate": 0.0001967811654011389, + "logits/chosen": -0.309251606464386, + "logits/rejected": -0.3893003463745117, + "logps/chosen": -7.1915602684021, + "logps/rejected": -27.5125732421875, + "loss": 0.9027649164199829, + "memory(GiB)": 39.42, + "nll_loss": 0.49153536558151245, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.14030015468597412, + "rewards/margins": 1.3549611568450928, + "rewards/rejected": -1.4952614307403564, + "step": 109, + "train_speed(iter/s)": 0.010748 + }, + { + "epoch": 0.38394415357766143, + "grad_norm": 3.212317943572998, + "learning_rate": 0.00019668343285018125, + "logits/chosen": -0.30074530839920044, + "logits/rejected": -0.3879661560058594, + "logps/chosen": -9.096397399902344, + "logps/rejected": -42.42276382446289, + "loss": 0.8074911236763, + "memory(GiB)": 39.42, + "nll_loss": 0.5355080962181091, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.040462881326675415, + "rewards/margins": 2.193300247192383, + "rewards/rejected": -2.2337629795074463, + "step": 110, + "train_speed(iter/s)": 0.010748 + }, + { + "epoch": 0.387434554973822, + "grad_norm": 4.342100620269775, + "learning_rate": 0.0001965842636985946, + "logits/chosen": -0.3064587414264679, + "logits/rejected": -0.41097527742385864, + "logps/chosen": -5.184054374694824, + "logps/rejected": -36.894256591796875, + "loss": 0.7728255987167358, + "memory(GiB)": 39.42, + "nll_loss": 0.5083194971084595, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11109408736228943, + "rewards/margins": 2.293015241622925, + "rewards/rejected": -2.1819212436676025, + "step": 111, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.39092495636998253, + "grad_norm": 9.064605712890625, + "learning_rate": 0.00019648365941991438, + "logits/chosen": -0.35194405913352966, + "logits/rejected": -0.42771002650260925, + "logps/chosen": -8.304715156555176, + "logps/rejected": -32.21118927001953, + "loss": 0.998765766620636, + "memory(GiB)": 39.42, + "nll_loss": 0.6066679954528809, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.15771695971488953, + "rewards/margins": 1.7298005819320679, + "rewards/rejected": -1.572083592414856, + "step": 112, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.3944153577661431, + "grad_norm": 1.8839598894119263, + "learning_rate": 0.00019638162150900027, + "logits/chosen": -0.2599456310272217, + "logits/rejected": -0.3942990303039551, + "logps/chosen": -4.890069484710693, + "logps/rejected": -48.83489227294922, + "loss": 0.5059012770652771, + "memory(GiB)": 39.42, + "nll_loss": 0.31480100750923157, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19593901932239532, + "rewards/margins": 2.834808588027954, + "rewards/rejected": -2.638869524002075, + "step": 113, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.39790575916230364, + "grad_norm": 2.1122896671295166, + "learning_rate": 0.00019627815148201417, + "logits/chosen": -0.35711023211479187, + "logits/rejected": -0.3968570828437805, + "logps/chosen": -7.015074729919434, + "logps/rejected": -31.74431037902832, + "loss": 0.6316798329353333, + "memory(GiB)": 39.42, + "nll_loss": 0.3775865137577057, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09947216510772705, + "rewards/margins": 2.002511739730835, + "rewards/rejected": -1.9030394554138184, + "step": 114, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.4013961605584642, + "grad_norm": 2.927731990814209, + "learning_rate": 0.00019617325087639727, + "logits/chosen": -0.38088345527648926, + "logits/rejected": -0.435588538646698, + "logps/chosen": -6.47522497177124, + "logps/rejected": -24.941457748413086, + "loss": 0.7609400749206543, + "memory(GiB)": 39.42, + "nll_loss": 0.4365846514701843, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.15598955750465393, + "rewards/margins": 1.493629813194275, + "rewards/rejected": -1.3376405239105225, + "step": 115, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.4048865619546248, + "grad_norm": 2.6690480709075928, + "learning_rate": 0.00019606692125084775, + "logits/chosen": -0.1995878666639328, + "logits/rejected": -0.4380982518196106, + "logps/chosen": -4.782075881958008, + "logps/rejected": -42.2791862487793, + "loss": 0.7123544216156006, + "memory(GiB)": 39.42, + "nll_loss": 0.4524919390678406, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12693020701408386, + "rewards/margins": 2.1254653930664062, + "rewards/rejected": -1.9985350370407104, + "step": 116, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.4083769633507853, + "grad_norm": 3.349586248397827, + "learning_rate": 0.00019595916418529706, + "logits/chosen": -0.3445770740509033, + "logits/rejected": -0.4120224118232727, + "logps/chosen": -9.293133735656738, + "logps/rejected": -35.069541931152344, + "loss": 0.7675687074661255, + "memory(GiB)": 39.42, + "nll_loss": 0.47724154591560364, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06645308434963226, + "rewards/margins": 1.9597532749176025, + "rewards/rejected": -1.8933004140853882, + "step": 117, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.4118673647469459, + "grad_norm": 3.534466028213501, + "learning_rate": 0.00019584998128088684, + "logits/chosen": -0.30230188369750977, + "logits/rejected": -0.43015873432159424, + "logps/chosen": -6.687424659729004, + "logps/rejected": -34.876686096191406, + "loss": 0.830184817314148, + "memory(GiB)": 39.42, + "nll_loss": 0.5149248838424683, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07988367974758148, + "rewards/margins": 1.8753106594085693, + "rewards/rejected": -1.795426845550537, + "step": 118, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.41535776614310643, + "grad_norm": 2.4526314735412598, + "learning_rate": 0.00019573937415994506, + "logits/chosen": -0.3823411464691162, + "logits/rejected": -0.3723237216472626, + "logps/chosen": -8.30302906036377, + "logps/rejected": -28.920549392700195, + "loss": 0.6885042190551758, + "memory(GiB)": 39.42, + "nll_loss": 0.37792226672172546, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.050916727632284164, + "rewards/margins": 1.7539997100830078, + "rewards/rejected": -1.7030829191207886, + "step": 119, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.418848167539267, + "grad_norm": 5.966183185577393, + "learning_rate": 0.00019562734446596162, + "logits/chosen": -0.36912596225738525, + "logits/rejected": -0.4693865478038788, + "logps/chosen": -7.145996570587158, + "logps/rejected": -37.43076705932617, + "loss": 0.9455270767211914, + "memory(GiB)": 39.42, + "nll_loss": 0.48743128776550293, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.13650617003440857, + "rewards/margins": 2.0651345252990723, + "rewards/rejected": -2.2016406059265137, + "step": 120, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.4223385689354276, + "grad_norm": 6.053987979888916, + "learning_rate": 0.00019551389386356443, + "logits/chosen": -0.35416483879089355, + "logits/rejected": -0.4060932993888855, + "logps/chosen": -11.305073738098145, + "logps/rejected": -39.34588623046875, + "loss": 1.0363645553588867, + "memory(GiB)": 39.42, + "nll_loss": 0.7409172058105469, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.3775523602962494, + "rewards/margins": 2.277557611465454, + "rewards/rejected": -2.6551101207733154, + "step": 121, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.4258289703315881, + "grad_norm": 3.908426523208618, + "learning_rate": 0.0001953990240384942, + "logits/chosen": -0.2593086361885071, + "logits/rejected": -0.45180562138557434, + "logps/chosen": -7.346467018127441, + "logps/rejected": -48.82087326049805, + "loss": 0.7506681680679321, + "memory(GiB)": 39.42, + "nll_loss": 0.5086178779602051, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.042671557515859604, + "rewards/margins": 2.6901655197143555, + "rewards/rejected": -2.647494077682495, + "step": 122, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.4293193717277487, + "grad_norm": 10.322138786315918, + "learning_rate": 0.00019528273669757972, + "logits/chosen": -0.35214099287986755, + "logits/rejected": -0.4367164373397827, + "logps/chosen": -8.127342224121094, + "logps/rejected": -38.454200744628906, + "loss": 0.8126936554908752, + "memory(GiB)": 39.42, + "nll_loss": 0.5385327935218811, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.10964661836624146, + "rewards/margins": 2.599274158477783, + "rewards/rejected": -2.4896275997161865, + "step": 123, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.4328097731239092, + "grad_norm": 5.38076114654541, + "learning_rate": 0.00019516503356871234, + "logits/chosen": -0.2883262634277344, + "logits/rejected": -0.32226479053497314, + "logps/chosen": -11.945974349975586, + "logps/rejected": -43.99768829345703, + "loss": 0.8885233998298645, + "memory(GiB)": 39.42, + "nll_loss": 0.5691980123519897, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.22344030439853668, + "rewards/margins": 2.4954993724823, + "rewards/rejected": -2.718939781188965, + "step": 124, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.4363001745200698, + "grad_norm": 3.945094347000122, + "learning_rate": 0.00019504591640082035, + "logits/chosen": -0.3206433653831482, + "logits/rejected": -0.3835434913635254, + "logps/chosen": -9.910886764526367, + "logps/rejected": -37.540409088134766, + "loss": 0.8022382855415344, + "memory(GiB)": 39.42, + "nll_loss": 0.5557839870452881, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.04477905482053757, + "rewards/margins": 2.5910792350769043, + "rewards/rejected": -2.6358585357666016, + "step": 125, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.4397905759162304, + "grad_norm": 4.2087531089782715, + "learning_rate": 0.0001949253869638429, + "logits/chosen": -0.28855112195014954, + "logits/rejected": -0.3965805768966675, + "logps/chosen": -2.8932034969329834, + "logps/rejected": -33.3571662902832, + "loss": 0.48199671506881714, + "memory(GiB)": 39.42, + "nll_loss": 0.2255885899066925, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10981371998786926, + "rewards/margins": 2.0570077896118164, + "rewards/rejected": -1.9471940994262695, + "step": 126, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.4432809773123909, + "grad_norm": 3.723099946975708, + "learning_rate": 0.00019480344704870387, + "logits/chosen": -0.4086211025714874, + "logits/rejected": -0.4154241383075714, + "logps/chosen": -11.32919692993164, + "logps/rejected": -25.829769134521484, + "loss": 1.3574812412261963, + "memory(GiB)": 39.42, + "nll_loss": 0.8371793627738953, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.18372564017772675, + "rewards/margins": 1.1997389793395996, + "rewards/rejected": -1.3834646940231323, + "step": 127, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.4467713787085515, + "grad_norm": 3.090599775314331, + "learning_rate": 0.00019468009846728513, + "logits/chosen": -0.3228675127029419, + "logits/rejected": -0.4174826741218567, + "logps/chosen": -5.812500476837158, + "logps/rejected": -34.774322509765625, + "loss": 0.6064198017120361, + "memory(GiB)": 39.42, + "nll_loss": 0.3521398603916168, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.049313537776470184, + "rewards/margins": 2.06463623046875, + "rewards/rejected": -2.0153229236602783, + "step": 128, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.450261780104712, + "grad_norm": 2.82310152053833, + "learning_rate": 0.00019455534305239965, + "logits/chosen": -0.3688339293003082, + "logits/rejected": -0.3691863417625427, + "logps/chosen": -10.617326736450195, + "logps/rejected": -23.13429832458496, + "loss": 0.9776577949523926, + "memory(GiB)": 39.42, + "nll_loss": 0.5499588251113892, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.03131576254963875, + "rewards/margins": 1.1372824907302856, + "rewards/rejected": -1.1059668064117432, + "step": 129, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.4537521815008726, + "grad_norm": 2.351433038711548, + "learning_rate": 0.00019442918265776422, + "logits/chosen": -0.3329261839389801, + "logits/rejected": -0.36310911178588867, + "logps/chosen": -13.333334922790527, + "logps/rejected": -23.961681365966797, + "loss": 0.8306537866592407, + "memory(GiB)": 39.42, + "nll_loss": 0.5025785565376282, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.04980917274951935, + "rewards/margins": 1.3957810401916504, + "rewards/rejected": -1.3459720611572266, + "step": 130, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.4572425828970332, + "grad_norm": 2.437668561935425, + "learning_rate": 0.00019430161915797207, + "logits/chosen": -0.3711564540863037, + "logits/rejected": -0.40104660391807556, + "logps/chosen": -4.970005512237549, + "logps/rejected": -25.718761444091797, + "loss": 0.6583466529846191, + "memory(GiB)": 39.42, + "nll_loss": 0.32833656668663025, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.15192265808582306, + "rewards/margins": 1.5051937103271484, + "rewards/rejected": -1.3532708883285522, + "step": 131, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.4607329842931937, + "grad_norm": 2.258094310760498, + "learning_rate": 0.00019417265444846476, + "logits/chosen": -0.2738487422466278, + "logits/rejected": -0.4087858200073242, + "logps/chosen": -7.829878330230713, + "logps/rejected": -34.82905578613281, + "loss": 0.7665334343910217, + "memory(GiB)": 39.42, + "nll_loss": 0.5376078486442566, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.25566646456718445, + "rewards/margins": 2.1234703063964844, + "rewards/rejected": -1.8678035736083984, + "step": 132, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.4642233856893543, + "grad_norm": 2.1597509384155273, + "learning_rate": 0.00019404229044550433, + "logits/chosen": -0.33906158804893494, + "logits/rejected": -0.4303910434246063, + "logps/chosen": -4.8524885177612305, + "logps/rejected": -30.576656341552734, + "loss": 0.7118875980377197, + "memory(GiB)": 39.42, + "nll_loss": 0.4106830060482025, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.17056593298912048, + "rewards/margins": 1.768500804901123, + "rewards/rejected": -1.5979350805282593, + "step": 133, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4677137870855148, + "grad_norm": 4.846266269683838, + "learning_rate": 0.0001939105290861445, + "logits/chosen": -0.1988743394613266, + "logits/rejected": -0.41542911529541016, + "logps/chosen": -7.35241174697876, + "logps/rejected": -47.594486236572266, + "loss": 0.8350856304168701, + "memory(GiB)": 39.42, + "nll_loss": 0.551296055316925, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09475097805261612, + "rewards/margins": 2.521266222000122, + "rewards/rejected": -2.4265151023864746, + "step": 134, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4712041884816754, + "grad_norm": 2.220877170562744, + "learning_rate": 0.00019377737232820208, + "logits/chosen": -0.2817349433898926, + "logits/rejected": -0.3702714443206787, + "logps/chosen": -7.847001552581787, + "logps/rejected": -44.94153594970703, + "loss": 0.6441789269447327, + "memory(GiB)": 39.42, + "nll_loss": 0.44079264998435974, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16393792629241943, + "rewards/margins": 2.845656394958496, + "rewards/rejected": -2.681718587875366, + "step": 135, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.47469458987783597, + "grad_norm": 2.1187074184417725, + "learning_rate": 0.00019364282215022786, + "logits/chosen": -0.28489771485328674, + "logits/rejected": -0.37565189599990845, + "logps/chosen": -5.1259026527404785, + "logps/rejected": -39.07181930541992, + "loss": 0.5232971906661987, + "memory(GiB)": 39.42, + "nll_loss": 0.31729230284690857, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14290614426136017, + "rewards/margins": 2.855766773223877, + "rewards/rejected": -2.712860345840454, + "step": 136, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4781849912739965, + "grad_norm": 4.652167320251465, + "learning_rate": 0.00019350688055147725, + "logits/chosen": -0.2049810290336609, + "logits/rejected": -0.35532209277153015, + "logps/chosen": -8.197148323059082, + "logps/rejected": -50.812965393066406, + "loss": 0.5982136130332947, + "memory(GiB)": 39.42, + "nll_loss": 0.3783411979675293, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.036504216492176056, + "rewards/margins": 3.0081937313079834, + "rewards/rejected": -2.9716897010803223, + "step": 137, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4816753926701571, + "grad_norm": 1.6813859939575195, + "learning_rate": 0.0001933695495518804, + "logits/chosen": -0.15937744081020355, + "logits/rejected": -0.36899077892303467, + "logps/chosen": -5.227466106414795, + "logps/rejected": -57.80689239501953, + "loss": 0.4328049421310425, + "memory(GiB)": 39.42, + "nll_loss": 0.3204546272754669, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08378630876541138, + "rewards/margins": 3.9659409523010254, + "rewards/rejected": -3.882154703140259, + "step": 138, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4851657940663176, + "grad_norm": 1.7590577602386475, + "learning_rate": 0.00019323083119201235, + "logits/chosen": -0.21911956369876862, + "logits/rejected": -0.3360063135623932, + "logps/chosen": -10.714500427246094, + "logps/rejected": -44.478004455566406, + "loss": 0.6973531246185303, + "memory(GiB)": 39.42, + "nll_loss": 0.5391013622283936, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15974897146224976, + "rewards/margins": 3.1409525871276855, + "rewards/rejected": -2.981203556060791, + "step": 139, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.4886561954624782, + "grad_norm": 4.423391342163086, + "learning_rate": 0.0001930907275330627, + "logits/chosen": -0.21830472350120544, + "logits/rejected": -0.41240787506103516, + "logps/chosen": -6.044207572937012, + "logps/rejected": -47.71708679199219, + "loss": 0.7773061990737915, + "memory(GiB)": 39.42, + "nll_loss": 0.4624183773994446, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.08216652274131775, + "rewards/margins": 2.775662422180176, + "rewards/rejected": -2.8578288555145264, + "step": 140, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.49214659685863876, + "grad_norm": 5.617547988891602, + "learning_rate": 0.00019294924065680488, + "logits/chosen": -0.3345920145511627, + "logits/rejected": -0.4564991593360901, + "logps/chosen": -4.883607387542725, + "logps/rejected": -50.870941162109375, + "loss": 0.49507585167884827, + "memory(GiB)": 39.42, + "nll_loss": 0.3347318768501282, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0749196857213974, + "rewards/margins": 3.8949146270751953, + "rewards/rejected": -3.819995164871216, + "step": 141, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.4956369982547993, + "grad_norm": 6.824800491333008, + "learning_rate": 0.00019280637266556533, + "logits/chosen": -0.255269318819046, + "logits/rejected": -0.38520547747612, + "logps/chosen": -6.951573371887207, + "logps/rejected": -43.31431579589844, + "loss": 0.8777123689651489, + "memory(GiB)": 39.42, + "nll_loss": 0.6212184429168701, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.1106327623128891, + "rewards/margins": 3.119386672973633, + "rewards/rejected": -3.008753538131714, + "step": 142, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.49912739965095987, + "grad_norm": 3.7692058086395264, + "learning_rate": 0.0001926621256821922, + "logits/chosen": -0.15499627590179443, + "logits/rejected": -0.30336061120033264, + "logps/chosen": -11.738297462463379, + "logps/rejected": -56.02423095703125, + "loss": 0.7617011070251465, + "memory(GiB)": 39.42, + "nll_loss": 0.5688201189041138, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.04947775602340698, + "rewards/margins": 3.5284841060638428, + "rewards/rejected": -3.5779623985290527, + "step": 143, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5026178010471204, + "grad_norm": 2.1459271907806396, + "learning_rate": 0.0001925165018500238, + "logits/chosen": -0.21587318181991577, + "logits/rejected": -0.3930494487285614, + "logps/chosen": -9.389811515808105, + "logps/rejected": -52.38633728027344, + "loss": 0.6086324453353882, + "memory(GiB)": 39.42, + "nll_loss": 0.4416413903236389, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.32234904170036316, + "rewards/margins": 3.8869223594665527, + "rewards/rejected": -3.5645735263824463, + "step": 144, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.506108202443281, + "grad_norm": 5.556336402893066, + "learning_rate": 0.00019236950333285683, + "logits/chosen": -0.2211378514766693, + "logits/rejected": -0.3483867645263672, + "logps/chosen": -8.280564308166504, + "logps/rejected": -51.631282806396484, + "loss": 0.7348828315734863, + "memory(GiB)": 39.42, + "nll_loss": 0.48733851313591003, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.00019693002104759216, + "rewards/margins": 3.514716148376465, + "rewards/rejected": -3.514913320541382, + "step": 145, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5095986038394416, + "grad_norm": 2.229085683822632, + "learning_rate": 0.00019222113231491409, + "logits/chosen": -0.2527725100517273, + "logits/rejected": -0.41788095235824585, + "logps/chosen": -10.011974334716797, + "logps/rejected": -52.808170318603516, + "loss": 0.7164273858070374, + "memory(GiB)": 39.42, + "nll_loss": 0.5022612810134888, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3206445574760437, + "rewards/margins": 3.888902425765991, + "rewards/rejected": -3.568258285522461, + "step": 146, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5130890052356021, + "grad_norm": 7.876246452331543, + "learning_rate": 0.00019207139100081212, + "logits/chosen": -0.19432660937309265, + "logits/rejected": -0.33527010679244995, + "logps/chosen": -7.427047252655029, + "logps/rejected": -53.75447082519531, + "loss": 0.760964035987854, + "memory(GiB)": 39.42, + "nll_loss": 0.5109995007514954, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08073252439498901, + "rewards/margins": 3.443918228149414, + "rewards/rejected": -3.524650812149048, + "step": 147, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5165794066317626, + "grad_norm": 4.659561634063721, + "learning_rate": 0.00019192028161552847, + "logits/chosen": -0.18484185636043549, + "logits/rejected": -0.39253929257392883, + "logps/chosen": -8.587300300598145, + "logps/rejected": -52.95454788208008, + "loss": 0.7159382104873657, + "memory(GiB)": 39.42, + "nll_loss": 0.5496724247932434, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18269562721252441, + "rewards/margins": 3.7814650535583496, + "rewards/rejected": -3.598769187927246, + "step": 148, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5200698080279232, + "grad_norm": 3.1019811630249023, + "learning_rate": 0.0001917678064043686, + "logits/chosen": -0.27576160430908203, + "logits/rejected": -0.4377831816673279, + "logps/chosen": -5.784604549407959, + "logps/rejected": -52.2569580078125, + "loss": 0.6684166193008423, + "memory(GiB)": 39.42, + "nll_loss": 0.4899623990058899, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07499371469020844, + "rewards/margins": 3.7692465782165527, + "rewards/rejected": -3.6942529678344727, + "step": 149, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5235602094240838, + "grad_norm": 5.6313676834106445, + "learning_rate": 0.00019161396763293253, + "logits/chosen": -0.24116772413253784, + "logits/rejected": -0.4378838539123535, + "logps/chosen": -8.672993659973145, + "logps/rejected": -52.83271026611328, + "loss": 0.9051390290260315, + "memory(GiB)": 39.42, + "nll_loss": 0.5615367889404297, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.022281643003225327, + "rewards/margins": 3.423705816268921, + "rewards/rejected": -3.4459874629974365, + "step": 150, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5270506108202443, + "grad_norm": 30.42255973815918, + "learning_rate": 0.00019145876758708106, + "logits/chosen": -0.23821839690208435, + "logits/rejected": -0.48250508308410645, + "logps/chosen": -5.898958683013916, + "logps/rejected": -51.13921356201172, + "loss": 1.0185292959213257, + "memory(GiB)": 39.42, + "nll_loss": 0.6116878986358643, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.1395014375448227, + "rewards/margins": 3.174698829650879, + "rewards/rejected": -3.3142004013061523, + "step": 151, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5305410122164049, + "grad_norm": 3.989668369293213, + "learning_rate": 0.00019130220857290202, + "logits/chosen": -0.2887514531612396, + "logits/rejected": -0.4765075743198395, + "logps/chosen": -9.17143726348877, + "logps/rejected": -41.9118766784668, + "loss": 1.0394952297210693, + "memory(GiB)": 39.42, + "nll_loss": 0.7010207772254944, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1842147558927536, + "rewards/margins": 2.678420066833496, + "rewards/rejected": -2.4942049980163574, + "step": 152, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.5340314136125655, + "grad_norm": 3.6354000568389893, + "learning_rate": 0.00019114429291667583, + "logits/chosen": -0.3704184889793396, + "logits/rejected": -0.44566863775253296, + "logps/chosen": -13.917726516723633, + "logps/rejected": -36.536293029785156, + "loss": 0.9129284620285034, + "memory(GiB)": 39.42, + "nll_loss": 0.6521165370941162, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.30949151515960693, + "rewards/margins": 3.0189402103424072, + "rewards/rejected": -2.70944881439209, + "step": 153, + "train_speed(iter/s)": 0.010757 + }, + { + "epoch": 0.537521815008726, + "grad_norm": 3.1834535598754883, + "learning_rate": 0.000190985022964841, + "logits/chosen": -0.36771464347839355, + "logits/rejected": -0.48058295249938965, + "logps/chosen": -4.574065685272217, + "logps/rejected": -36.14601135253906, + "loss": 0.646094024181366, + "memory(GiB)": 39.42, + "nll_loss": 0.40117186307907104, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.26323747634887695, + "rewards/margins": 2.6909196376800537, + "rewards/rejected": -2.4276821613311768, + "step": 154, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5410122164048866, + "grad_norm": 4.318117141723633, + "learning_rate": 0.0001908244010839593, + "logits/chosen": -0.3432493805885315, + "logits/rejected": -0.47267085313796997, + "logps/chosen": -6.45510196685791, + "logps/rejected": -37.24266815185547, + "loss": 0.6357433795928955, + "memory(GiB)": 39.42, + "nll_loss": 0.35821908712387085, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1637301743030548, + "rewards/margins": 2.375438690185547, + "rewards/rejected": -2.2117085456848145, + "step": 155, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5445026178010471, + "grad_norm": 3.158153772354126, + "learning_rate": 0.00019066242966068054, + "logits/chosen": -0.43242546916007996, + "logits/rejected": -0.47301432490348816, + "logps/chosen": -9.04937744140625, + "logps/rejected": -33.77018356323242, + "loss": 0.8393766283988953, + "memory(GiB)": 39.42, + "nll_loss": 0.5358401536941528, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1126929298043251, + "rewards/margins": 2.264133930206299, + "rewards/rejected": -2.1514410972595215, + "step": 156, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5479930191972077, + "grad_norm": 4.308590888977051, + "learning_rate": 0.00019049911110170705, + "logits/chosen": -0.3886866867542267, + "logits/rejected": -0.4790132939815521, + "logps/chosen": -9.111655235290527, + "logps/rejected": -36.21464538574219, + "loss": 0.8003115057945251, + "memory(GiB)": 39.42, + "nll_loss": 0.4989633858203888, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.07737411558628082, + "rewards/margins": 2.306735038757324, + "rewards/rejected": -2.3841090202331543, + "step": 157, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5514834205933682, + "grad_norm": 7.6933746337890625, + "learning_rate": 0.00019033444783375804, + "logits/chosen": -0.34298592805862427, + "logits/rejected": -0.4637518525123596, + "logps/chosen": -8.123557090759277, + "logps/rejected": -50.31752014160156, + "loss": 0.6172691583633423, + "memory(GiB)": 39.42, + "nll_loss": 0.40251481533050537, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.028386808931827545, + "rewards/margins": 3.503352403640747, + "rewards/rejected": -3.4749653339385986, + "step": 158, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5549738219895288, + "grad_norm": 7.182351112365723, + "learning_rate": 0.00019016844230353356, + "logits/chosen": -0.3422991633415222, + "logits/rejected": -0.42033761739730835, + "logps/chosen": -8.310185432434082, + "logps/rejected": -43.29906463623047, + "loss": 0.7140598893165588, + "memory(GiB)": 39.42, + "nll_loss": 0.3994865417480469, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.060791049152612686, + "rewards/margins": 2.747554302215576, + "rewards/rejected": -2.808345079421997, + "step": 159, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5584642233856894, + "grad_norm": 2.537100076675415, + "learning_rate": 0.00019000109697767798, + "logits/chosen": -0.3541107475757599, + "logits/rejected": -0.44592800736427307, + "logps/chosen": -7.220765590667725, + "logps/rejected": -36.86073684692383, + "loss": 0.6231136918067932, + "memory(GiB)": 39.42, + "nll_loss": 0.3817756175994873, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14087501168251038, + "rewards/margins": 2.464402198791504, + "rewards/rejected": -2.3235268592834473, + "step": 160, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5619546247818499, + "grad_norm": 5.032235145568848, + "learning_rate": 0.00018983241434274351, + "logits/chosen": -0.3592570722103119, + "logits/rejected": -0.4694417417049408, + "logps/chosen": -8.956731796264648, + "logps/rejected": -36.53116989135742, + "loss": 0.8788279294967651, + "memory(GiB)": 39.42, + "nll_loss": 0.5014119744300842, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.02851598709821701, + "rewards/margins": 2.0984349250793457, + "rewards/rejected": -2.0699188709259033, + "step": 161, + "train_speed(iter/s)": 0.010758 + }, + { + "epoch": 0.5654450261780105, + "grad_norm": 2.1505980491638184, + "learning_rate": 0.00018966239690515309, + "logits/chosen": -0.40191835165023804, + "logits/rejected": -0.511231541633606, + "logps/chosen": -8.61328125, + "logps/rejected": -42.80579376220703, + "loss": 0.650855302810669, + "memory(GiB)": 39.42, + "nll_loss": 0.3947713077068329, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.1195174977183342, + "rewards/margins": 2.9840264320373535, + "rewards/rejected": -2.864508628845215, + "step": 162, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.568935427574171, + "grad_norm": 2.0593886375427246, + "learning_rate": 0.00018949104719116332, + "logits/chosen": -0.36368370056152344, + "logits/rejected": -0.4502822458744049, + "logps/chosen": -6.056973934173584, + "logps/rejected": -42.781681060791016, + "loss": 0.5314807891845703, + "memory(GiB)": 39.42, + "nll_loss": 0.31094634532928467, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14926032721996307, + "rewards/margins": 2.994180202484131, + "rewards/rejected": -2.8449201583862305, + "step": 163, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5724258289703316, + "grad_norm": 3.314478635787964, + "learning_rate": 0.00018931836774682687, + "logits/chosen": -0.4336795210838318, + "logits/rejected": -0.4525458812713623, + "logps/chosen": -7.387899875640869, + "logps/rejected": -32.55196762084961, + "loss": 0.7532250285148621, + "memory(GiB)": 39.42, + "nll_loss": 0.37785154581069946, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06012080982327461, + "rewards/margins": 2.1696441173553467, + "rewards/rejected": -2.109523296356201, + "step": 164, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5759162303664922, + "grad_norm": 3.4192256927490234, + "learning_rate": 0.00018914436113795448, + "logits/chosen": -0.2911241948604584, + "logits/rejected": -0.4634595513343811, + "logps/chosen": -7.420339584350586, + "logps/rejected": -39.048561096191406, + "loss": 0.8260236978530884, + "memory(GiB)": 39.42, + "nll_loss": 0.5035777688026428, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.03999996930360794, + "rewards/margins": 2.3207309246063232, + "rewards/rejected": -2.280730724334717, + "step": 165, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5794066317626527, + "grad_norm": 4.191897392272949, + "learning_rate": 0.0001889690299500771, + "logits/chosen": -0.32450249791145325, + "logits/rejected": -0.4613388776779175, + "logps/chosen": -9.48214340209961, + "logps/rejected": -39.381065368652344, + "loss": 0.8530201315879822, + "memory(GiB)": 39.42, + "nll_loss": 0.59242844581604, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08060712367296219, + "rewards/margins": 2.242609977722168, + "rewards/rejected": -2.1620030403137207, + "step": 166, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5828970331588132, + "grad_norm": 2.7431366443634033, + "learning_rate": 0.0001887923767884073, + "logits/chosen": -0.3079373836517334, + "logits/rejected": -0.44786494970321655, + "logps/chosen": -5.737804412841797, + "logps/rejected": -36.809600830078125, + "loss": 0.6614618301391602, + "memory(GiB)": 39.42, + "nll_loss": 0.4013901352882385, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3159565329551697, + "rewards/margins": 2.179208755493164, + "rewards/rejected": -1.8632525205612183, + "step": 167, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5863874345549738, + "grad_norm": 3.6514716148376465, + "learning_rate": 0.0001886144042778006, + "logits/chosen": -0.39408621191978455, + "logits/rejected": -0.43902865052223206, + "logps/chosen": -9.832802772521973, + "logps/rejected": -41.6961784362793, + "loss": 0.7166361212730408, + "memory(GiB)": 39.42, + "nll_loss": 0.490989625453949, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17612463235855103, + "rewards/margins": 2.8474600315093994, + "rewards/rejected": -2.671335458755493, + "step": 168, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5898778359511344, + "grad_norm": 3.6428065299987793, + "learning_rate": 0.00018843511506271644, + "logits/chosen": -0.4216082990169525, + "logits/rejected": -0.44014686346054077, + "logps/chosen": -8.206501960754395, + "logps/rejected": -32.93410873413086, + "loss": 0.7580087780952454, + "memory(GiB)": 39.42, + "nll_loss": 0.4095152020454407, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0056315031833946705, + "rewards/margins": 2.0248444080352783, + "rewards/rejected": -2.0192127227783203, + "step": 169, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5933682373472949, + "grad_norm": 3.8498518466949463, + "learning_rate": 0.00018825451180717905, + "logits/chosen": -0.3026273846626282, + "logits/rejected": -0.5032517910003662, + "logps/chosen": -10.3818941116333, + "logps/rejected": -51.91915512084961, + "loss": 1.0770937204360962, + "memory(GiB)": 39.42, + "nll_loss": 0.713114857673645, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18697917461395264, + "rewards/margins": 2.874116897583008, + "rewards/rejected": -3.061095952987671, + "step": 170, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.5968586387434555, + "grad_norm": 1.5238920450210571, + "learning_rate": 0.00018807259719473757, + "logits/chosen": -0.3206470012664795, + "logits/rejected": -0.46274593472480774, + "logps/chosen": -4.761229515075684, + "logps/rejected": -47.91825866699219, + "loss": 0.5495983362197876, + "memory(GiB)": 39.42, + "nll_loss": 0.3658239543437958, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14044716954231262, + "rewards/margins": 3.341961145401001, + "rewards/rejected": -3.2015140056610107, + "step": 171, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6003490401396161, + "grad_norm": 2.570446252822876, + "learning_rate": 0.00018788937392842647, + "logits/chosen": -0.2903798520565033, + "logits/rejected": -0.4074512720108032, + "logps/chosen": -5.276430130004883, + "logps/rejected": -48.5122184753418, + "loss": 0.5867538452148438, + "memory(GiB)": 39.42, + "nll_loss": 0.34603506326675415, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06058134138584137, + "rewards/margins": 3.133744478225708, + "rewards/rejected": -3.0731630325317383, + "step": 172, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6038394415357766, + "grad_norm": 3.918461799621582, + "learning_rate": 0.0001877048447307252, + "logits/chosen": -0.19242779910564423, + "logits/rejected": -0.38455089926719666, + "logps/chosen": -5.807734489440918, + "logps/rejected": -52.73225784301758, + "loss": 0.6217489838600159, + "memory(GiB)": 39.42, + "nll_loss": 0.38360434770584106, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12222804874181747, + "rewards/margins": 3.3160438537597656, + "rewards/rejected": -3.1938157081604004, + "step": 173, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6073298429319371, + "grad_norm": 4.870952129364014, + "learning_rate": 0.00018751901234351773, + "logits/chosen": -0.36845889687538147, + "logits/rejected": -0.4809192717075348, + "logps/chosen": -8.235760688781738, + "logps/rejected": -36.360076904296875, + "loss": 0.8248316645622253, + "memory(GiB)": 39.42, + "nll_loss": 0.6022232174873352, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3026849031448364, + "rewards/margins": 2.4412600994110107, + "rewards/rejected": -2.1385750770568848, + "step": 174, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6108202443280978, + "grad_norm": 2.2720422744750977, + "learning_rate": 0.00018733187952805203, + "logits/chosen": -0.3710225522518158, + "logits/rejected": -0.436270534992218, + "logps/chosen": -6.721004009246826, + "logps/rejected": -34.718502044677734, + "loss": 0.709654688835144, + "memory(GiB)": 39.42, + "nll_loss": 0.452656626701355, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06426668912172318, + "rewards/margins": 2.3792333602905273, + "rewards/rejected": -2.3149666786193848, + "step": 175, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6143106457242583, + "grad_norm": 3.7245795726776123, + "learning_rate": 0.00018714344906489876, + "logits/chosen": -0.4186531901359558, + "logits/rejected": -0.43767866492271423, + "logps/chosen": -9.161328315734863, + "logps/rejected": -32.324134826660156, + "loss": 0.7417446970939636, + "memory(GiB)": 39.42, + "nll_loss": 0.44037187099456787, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13668972253799438, + "rewards/margins": 2.1546599864959717, + "rewards/rejected": -2.017970085144043, + "step": 176, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.6178010471204188, + "grad_norm": 4.3425469398498535, + "learning_rate": 0.00018695372375391014, + "logits/chosen": -0.2733748257160187, + "logits/rejected": -0.44424253702163696, + "logps/chosen": -8.758390426635742, + "logps/rejected": -50.36033630371094, + "loss": 0.8111291527748108, + "memory(GiB)": 39.42, + "nll_loss": 0.4982152581214905, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05674072727560997, + "rewards/margins": 2.9798450469970703, + "rewards/rejected": -3.036585807800293, + "step": 177, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6212914485165794, + "grad_norm": 3.7545742988586426, + "learning_rate": 0.00018676270641417822, + "logits/chosen": -0.2608646750450134, + "logits/rejected": -0.4399104118347168, + "logps/chosen": -9.50798225402832, + "logps/rejected": -46.8238639831543, + "loss": 0.86136394739151, + "memory(GiB)": 39.42, + "nll_loss": 0.6183493137359619, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07049836963415146, + "rewards/margins": 3.0674405097961426, + "rewards/rejected": -2.9969425201416016, + "step": 178, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.62478184991274, + "grad_norm": 2.237074613571167, + "learning_rate": 0.00018657039988399313, + "logits/chosen": -0.29585880041122437, + "logits/rejected": -0.44753074645996094, + "logps/chosen": -11.109559059143066, + "logps/rejected": -39.0783805847168, + "loss": 0.9119770526885986, + "memory(GiB)": 39.42, + "nll_loss": 0.6092584133148193, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.1456558257341385, + "rewards/margins": 2.451812505722046, + "rewards/rejected": -2.306156873703003, + "step": 179, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6282722513089005, + "grad_norm": 2.476689577102661, + "learning_rate": 0.00018637680702080083, + "logits/chosen": -0.3464896082878113, + "logits/rejected": -0.4352068305015564, + "logps/chosen": -8.254376411437988, + "logps/rejected": -35.282325744628906, + "loss": 0.8010640740394592, + "memory(GiB)": 39.42, + "nll_loss": 0.4889823794364929, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.008691787719726562, + "rewards/margins": 2.055556058883667, + "rewards/rejected": -2.0468640327453613, + "step": 180, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.631762652705061, + "grad_norm": 2.8437464237213135, + "learning_rate": 0.0001861819307011606, + "logits/chosen": -0.2394547164440155, + "logits/rejected": -0.4722561538219452, + "logps/chosen": -3.5417962074279785, + "logps/rejected": -64.08223724365234, + "loss": 0.38956594467163086, + "memory(GiB)": 39.42, + "nll_loss": 0.25639209151268005, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2628820538520813, + "rewards/margins": 4.246987819671631, + "rewards/rejected": -3.9841055870056152, + "step": 181, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6352530541012217, + "grad_norm": 2.4585416316986084, + "learning_rate": 0.00018598577382070237, + "logits/chosen": -0.31651991605758667, + "logits/rejected": -0.45492100715637207, + "logps/chosen": -4.335537910461426, + "logps/rejected": -46.55557632446289, + "loss": 0.5344107747077942, + "memory(GiB)": 39.42, + "nll_loss": 0.25841349363327026, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.044346973299980164, + "rewards/margins": 3.0594677925109863, + "rewards/rejected": -3.0151207447052, + "step": 182, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.6387434554973822, + "grad_norm": 2.4348537921905518, + "learning_rate": 0.0001857883392940837, + "logits/chosen": -0.2961544990539551, + "logits/rejected": -0.39695119857788086, + "logps/chosen": -8.935890197753906, + "logps/rejected": -40.11572265625, + "loss": 0.7396747469902039, + "memory(GiB)": 39.42, + "nll_loss": 0.4514305889606476, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.09869125485420227, + "rewards/margins": 2.4125261306762695, + "rewards/rejected": -2.5112171173095703, + "step": 183, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6422338568935427, + "grad_norm": 1.7786816358566284, + "learning_rate": 0.0001855896300549465, + "logits/chosen": -0.2846543490886688, + "logits/rejected": -0.4267013967037201, + "logps/chosen": -9.809104919433594, + "logps/rejected": -42.15507507324219, + "loss": 0.7107999324798584, + "memory(GiB)": 39.42, + "nll_loss": 0.4528127908706665, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12454140186309814, + "rewards/margins": 2.582566976547241, + "rewards/rejected": -2.4580256938934326, + "step": 184, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6457242582897034, + "grad_norm": 2.2489097118377686, + "learning_rate": 0.00018538964905587325, + "logits/chosen": -0.269091933965683, + "logits/rejected": -0.42950770258903503, + "logps/chosen": -5.967182159423828, + "logps/rejected": -50.640193939208984, + "loss": 0.49092888832092285, + "memory(GiB)": 39.42, + "nll_loss": 0.32368528842926025, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08472105860710144, + "rewards/margins": 3.3988823890686035, + "rewards/rejected": -3.3141613006591797, + "step": 185, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6492146596858639, + "grad_norm": 2.9929044246673584, + "learning_rate": 0.0001851883992683434, + "logits/chosen": -0.3572680950164795, + "logits/rejected": -0.4970206022262573, + "logps/chosen": -5.416510105133057, + "logps/rejected": -49.36286163330078, + "loss": 0.6976485252380371, + "memory(GiB)": 39.42, + "nll_loss": 0.423793762922287, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0580061599612236, + "rewards/margins": 3.400108814239502, + "rewards/rejected": -3.3421030044555664, + "step": 186, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6527050610820244, + "grad_norm": 4.574316501617432, + "learning_rate": 0.00018498588368268907, + "logits/chosen": -0.3062989413738251, + "logits/rejected": -0.5117348432540894, + "logps/chosen": -6.240719795227051, + "logps/rejected": -48.549983978271484, + "loss": 0.5926957130432129, + "memory(GiB)": 39.42, + "nll_loss": 0.40774649381637573, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2606131434440613, + "rewards/margins": 3.447434186935425, + "rewards/rejected": -3.1868209838867188, + "step": 187, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6561954624781849, + "grad_norm": 3.87235426902771, + "learning_rate": 0.0001847821053080505, + "logits/chosen": -0.19517812132835388, + "logits/rejected": -0.3854786455631256, + "logps/chosen": -6.145029544830322, + "logps/rejected": -49.55529022216797, + "loss": 0.5662611126899719, + "memory(GiB)": 39.42, + "nll_loss": 0.3513401746749878, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07440924644470215, + "rewards/margins": 3.343357563018799, + "rewards/rejected": -3.268948554992676, + "step": 188, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6596858638743456, + "grad_norm": 2.1893322467803955, + "learning_rate": 0.00018457706717233164, + "logits/chosen": -0.31054210662841797, + "logits/rejected": -0.49141132831573486, + "logps/chosen": -6.330968379974365, + "logps/rejected": -52.3120231628418, + "loss": 0.6129257082939148, + "memory(GiB)": 39.42, + "nll_loss": 0.3919089436531067, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09110119938850403, + "rewards/margins": 3.511691093444824, + "rewards/rejected": -3.4205901622772217, + "step": 189, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6631762652705061, + "grad_norm": 6.626704692840576, + "learning_rate": 0.00018437077232215488, + "logits/chosen": -0.19323404133319855, + "logits/rejected": -0.44291025400161743, + "logps/chosen": -5.826582431793213, + "logps/rejected": -57.65163040161133, + "loss": 0.5744651556015015, + "memory(GiB)": 39.42, + "nll_loss": 0.40510880947113037, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14089857041835785, + "rewards/margins": 4.094103813171387, + "rewards/rejected": -3.953204870223999, + "step": 190, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6666666666666666, + "grad_norm": 3.4911742210388184, + "learning_rate": 0.00018416322382281596, + "logits/chosen": -0.33752942085266113, + "logits/rejected": -0.4428948163986206, + "logps/chosen": -8.038012504577637, + "logps/rejected": -41.6568717956543, + "loss": 0.8088080286979675, + "memory(GiB)": 39.42, + "nll_loss": 0.5819704532623291, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10074374079704285, + "rewards/margins": 3.1307780742645264, + "rewards/rejected": -3.030034065246582, + "step": 191, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6701570680628273, + "grad_norm": 2.2780048847198486, + "learning_rate": 0.0001839544247582383, + "logits/chosen": -0.2744682729244232, + "logits/rejected": -0.3990049362182617, + "logps/chosen": -5.77654504776001, + "logps/rejected": -49.46015167236328, + "loss": 0.5131328105926514, + "memory(GiB)": 39.42, + "nll_loss": 0.3267570436000824, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04739464819431305, + "rewards/margins": 3.7212061882019043, + "rewards/rejected": -3.673811674118042, + "step": 192, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6736474694589878, + "grad_norm": 2.1769142150878906, + "learning_rate": 0.00018374437823092724, + "logits/chosen": -0.22928477823734283, + "logits/rejected": -0.4279863238334656, + "logps/chosen": -4.774344444274902, + "logps/rejected": -50.371253967285156, + "loss": 0.5500450134277344, + "memory(GiB)": 39.42, + "nll_loss": 0.3786742091178894, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1368112862110138, + "rewards/margins": 3.6264164447784424, + "rewards/rejected": -3.48960542678833, + "step": 193, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6771378708551483, + "grad_norm": 2.8660085201263428, + "learning_rate": 0.00018353308736192395, + "logits/chosen": -0.2577519714832306, + "logits/rejected": -0.35627281665802, + "logps/chosen": -8.359132766723633, + "logps/rejected": -42.28339767456055, + "loss": 0.7936825156211853, + "memory(GiB)": 39.42, + "nll_loss": 0.5378628969192505, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02582652121782303, + "rewards/margins": 2.7675435543060303, + "rewards/rejected": -2.7417168617248535, + "step": 194, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.680628272251309, + "grad_norm": 4.319552898406982, + "learning_rate": 0.000183320555290759, + "logits/chosen": -0.2262856811285019, + "logits/rejected": -0.44104352593421936, + "logps/chosen": -5.736568927764893, + "logps/rejected": -54.799659729003906, + "loss": 0.6704956889152527, + "memory(GiB)": 39.42, + "nll_loss": 0.3680565059185028, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.14153015613555908, + "rewards/margins": 3.8039064407348633, + "rewards/rejected": -3.662376642227173, + "step": 195, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6841186736474695, + "grad_norm": 2.4287476539611816, + "learning_rate": 0.0001831067851754058, + "logits/chosen": -0.19324590265750885, + "logits/rejected": -0.3578749895095825, + "logps/chosen": -6.67017936706543, + "logps/rejected": -52.006690979003906, + "loss": 0.7265517115592957, + "memory(GiB)": 39.42, + "nll_loss": 0.4939360022544861, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1414988934993744, + "rewards/margins": 3.7974777221679688, + "rewards/rejected": -3.6559786796569824, + "step": 196, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.68760907504363, + "grad_norm": 4.368680000305176, + "learning_rate": 0.00018289178019223362, + "logits/chosen": -0.35040730237960815, + "logits/rejected": -0.37726685404777527, + "logps/chosen": -6.27101993560791, + "logps/rejected": -35.45961380004883, + "loss": 0.6579899191856384, + "memory(GiB)": 39.42, + "nll_loss": 0.35855987668037415, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.014165613800287247, + "rewards/margins": 2.6007533073425293, + "rewards/rejected": -2.6149189472198486, + "step": 197, + "train_speed(iter/s)": 0.010759 + }, + { + "epoch": 0.6910994764397905, + "grad_norm": 2.2127246856689453, + "learning_rate": 0.00018267554353596025, + "logits/chosen": -0.18989573419094086, + "logits/rejected": -0.3121846318244934, + "logps/chosen": -9.079330444335938, + "logps/rejected": -53.070701599121094, + "loss": 0.6450351476669312, + "memory(GiB)": 39.42, + "nll_loss": 0.4518280029296875, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07810258865356445, + "rewards/margins": 3.928205966949463, + "rewards/rejected": -3.8501036167144775, + "step": 198, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.6945898778359512, + "grad_norm": 6.419010162353516, + "learning_rate": 0.0001824580784196049, + "logits/chosen": -0.15850739181041718, + "logits/rejected": -0.3392256498336792, + "logps/chosen": -4.089353561401367, + "logps/rejected": -61.679054260253906, + "loss": 0.3885970115661621, + "memory(GiB)": 39.42, + "nll_loss": 0.26330527663230896, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06663137674331665, + "rewards/margins": 4.802007675170898, + "rewards/rejected": -4.735375881195068, + "step": 199, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.6980802792321117, + "grad_norm": 24.48959732055664, + "learning_rate": 0.00018223938807444012, + "logits/chosen": -0.15304000675678253, + "logits/rejected": -0.33186089992523193, + "logps/chosen": -8.43168830871582, + "logps/rejected": -54.678863525390625, + "loss": 0.7258750200271606, + "memory(GiB)": 39.42, + "nll_loss": 0.5083752870559692, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.008410019800066948, + "rewards/margins": 3.8021652698516846, + "rewards/rejected": -3.793755292892456, + "step": 200, + "train_speed(iter/s)": 0.01076 + }, + { + "epoch": 0.7015706806282722, + "grad_norm": 16.57341194152832, + "learning_rate": 0.00018201947574994384, + "logits/chosen": -0.036614514887332916, + "logits/rejected": -0.21098592877388, + "logps/chosen": -15.165770530700684, + "logps/rejected": -62.39512252807617, + "loss": 0.9109693765640259, + "memory(GiB)": 39.42, + "nll_loss": 0.5643189549446106, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.21655045449733734, + "rewards/margins": 3.875659465789795, + "rewards/rejected": -4.092210292816162, + "step": 201, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7050610820244329, + "grad_norm": 14.475955963134766, + "learning_rate": 0.00018179834471375125, + "logits/chosen": -0.16321136057376862, + "logits/rejected": -0.27808913588523865, + "logps/chosen": -8.80876350402832, + "logps/rejected": -45.93085479736328, + "loss": 0.8436198830604553, + "memory(GiB)": 39.42, + "nll_loss": 0.47678321599960327, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.02830314263701439, + "rewards/margins": 3.0820868015289307, + "rewards/rejected": -3.053783893585205, + "step": 202, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7085514834205934, + "grad_norm": 3.0808913707733154, + "learning_rate": 0.0001815759982516061, + "logits/chosen": -0.13647180795669556, + "logits/rejected": -0.2876810133457184, + "logps/chosen": -6.556826114654541, + "logps/rejected": -49.313106536865234, + "loss": 0.5729860067367554, + "memory(GiB)": 43.15, + "nll_loss": 0.3778243660926819, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19281527400016785, + "rewards/margins": 3.0519158840179443, + "rewards/rejected": -2.859100580215454, + "step": 203, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7120418848167539, + "grad_norm": 3.0599114894866943, + "learning_rate": 0.00018135243966731194, + "logits/chosen": -0.10809071362018585, + "logits/rejected": -0.29678240418434143, + "logps/chosen": -8.138949394226074, + "logps/rejected": -55.99830627441406, + "loss": 0.6175469160079956, + "memory(GiB)": 43.15, + "nll_loss": 0.4390767514705658, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.19007451832294464, + "rewards/margins": 3.628199338912964, + "rewards/rejected": -3.4381251335144043, + "step": 204, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.7155322862129145, + "grad_norm": 2.4980595111846924, + "learning_rate": 0.00018112767228268295, + "logits/chosen": -0.2528156638145447, + "logits/rejected": -0.35177648067474365, + "logps/chosen": -6.878098011016846, + "logps/rejected": -47.343318939208984, + "loss": 0.49210962653160095, + "memory(GiB)": 43.15, + "nll_loss": 0.3529854416847229, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06216797977685928, + "rewards/margins": 3.3334434032440186, + "rewards/rejected": -3.271275520324707, + "step": 205, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7190226876090751, + "grad_norm": 3.8207039833068848, + "learning_rate": 0.00018090169943749476, + "logits/chosen": -0.2331896275281906, + "logits/rejected": -0.32883408665657043, + "logps/chosen": -7.805197715759277, + "logps/rejected": -35.351806640625, + "loss": 0.8236002922058105, + "memory(GiB)": 43.15, + "nll_loss": 0.5681911110877991, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07105645537376404, + "rewards/margins": 2.3730642795562744, + "rewards/rejected": -2.3020079135894775, + "step": 206, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7225130890052356, + "grad_norm": 2.6041932106018066, + "learning_rate": 0.00018067452448943455, + "logits/chosen": -0.1293148696422577, + "logits/rejected": -0.3031143844127655, + "logps/chosen": -7.286511421203613, + "logps/rejected": -39.59210205078125, + "loss": 0.6952319145202637, + "memory(GiB)": 43.15, + "nll_loss": 0.46597838401794434, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1555314064025879, + "rewards/margins": 2.2323882579803467, + "rewards/rejected": -2.076856851577759, + "step": 207, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7260034904013961, + "grad_norm": 2.2537174224853516, + "learning_rate": 0.00018044615081405153, + "logits/chosen": -0.13209852576255798, + "logits/rejected": -0.32356375455856323, + "logps/chosen": -8.257270812988281, + "logps/rejected": -49.27021789550781, + "loss": 0.7890011668205261, + "memory(GiB)": 43.15, + "nll_loss": 0.5036588311195374, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.13289037346839905, + "rewards/margins": 2.8654632568359375, + "rewards/rejected": -2.7325730323791504, + "step": 208, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.7294938917975567, + "grad_norm": 1.861016869544983, + "learning_rate": 0.0001802165818047063, + "logits/chosen": -0.2812590003013611, + "logits/rejected": -0.4031287133693695, + "logps/chosen": -4.251603126525879, + "logps/rejected": -41.79424285888672, + "loss": 0.47951921820640564, + "memory(GiB)": 43.15, + "nll_loss": 0.2904518246650696, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18484094738960266, + "rewards/margins": 2.8290510177612305, + "rewards/rejected": -2.644209861755371, + "step": 209, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7329842931937173, + "grad_norm": 1.9735944271087646, + "learning_rate": 0.00017998582087252096, + "logits/chosen": -0.26465538144111633, + "logits/rejected": -0.34588366746902466, + "logps/chosen": -8.212122917175293, + "logps/rejected": -48.384342193603516, + "loss": 0.5980856418609619, + "memory(GiB)": 43.15, + "nll_loss": 0.4945921003818512, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.258284330368042, + "rewards/margins": 3.618307113647461, + "rewards/rejected": -3.36002254486084, + "step": 210, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7364746945898778, + "grad_norm": 4.080269813537598, + "learning_rate": 0.000179753871446328, + "logits/chosen": -0.3342297077178955, + "logits/rejected": -0.45482489466667175, + "logps/chosen": -4.805637359619141, + "logps/rejected": -52.97557830810547, + "loss": 0.5000529289245605, + "memory(GiB)": 43.15, + "nll_loss": 0.35857757925987244, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08043038100004196, + "rewards/margins": 4.170455455780029, + "rewards/rejected": -4.090025901794434, + "step": 211, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7399650959860384, + "grad_norm": 2.64082932472229, + "learning_rate": 0.00017952073697261953, + "logits/chosen": -0.271504282951355, + "logits/rejected": -0.3815239667892456, + "logps/chosen": -9.383578300476074, + "logps/rejected": -48.522979736328125, + "loss": 0.7009327411651611, + "memory(GiB)": 43.15, + "nll_loss": 0.5086449384689331, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.2527061700820923, + "rewards/margins": 3.632561683654785, + "rewards/rejected": -3.3798558712005615, + "step": 212, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.743455497382199, + "grad_norm": 3.330623149871826, + "learning_rate": 0.00017928642091549613, + "logits/chosen": -0.242343470454216, + "logits/rejected": -0.423129677772522, + "logps/chosen": -4.796614170074463, + "logps/rejected": -63.50012969970703, + "loss": 0.44938358664512634, + "memory(GiB)": 43.15, + "nll_loss": 0.3825247585773468, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09516870975494385, + "rewards/margins": 4.826410293579102, + "rewards/rejected": -4.731241703033447, + "step": 213, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.7469458987783595, + "grad_norm": 2.482487201690674, + "learning_rate": 0.00017905092675661526, + "logits/chosen": -0.34072667360305786, + "logits/rejected": -0.4692738652229309, + "logps/chosen": -5.442986011505127, + "logps/rejected": -49.02458190917969, + "loss": 0.4906308054924011, + "memory(GiB)": 43.15, + "nll_loss": 0.32957711815834045, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27041277289390564, + "rewards/margins": 3.7490029335021973, + "rewards/rejected": -3.478590488433838, + "step": 214, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7504363001745201, + "grad_norm": 2.0258970260620117, + "learning_rate": 0.00017881425799513956, + "logits/chosen": -0.29332002997398376, + "logits/rejected": -0.4313204884529114, + "logps/chosen": -3.8019495010375977, + "logps/rejected": -48.41462707519531, + "loss": 0.4435490071773529, + "memory(GiB)": 43.15, + "nll_loss": 0.3045477867126465, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21170072257518768, + "rewards/margins": 3.814541816711426, + "rewards/rejected": -3.602841854095459, + "step": 215, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7539267015706806, + "grad_norm": 4.538289546966553, + "learning_rate": 0.00017857641814768486, + "logits/chosen": -0.3277314305305481, + "logits/rejected": -0.4610479176044464, + "logps/chosen": -4.489333152770996, + "logps/rejected": -40.003170013427734, + "loss": 0.6065860986709595, + "memory(GiB)": 43.15, + "nll_loss": 0.3757091760635376, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10844689607620239, + "rewards/margins": 2.677161693572998, + "rewards/rejected": -2.5687148571014404, + "step": 216, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7574171029668412, + "grad_norm": 1.9632673263549805, + "learning_rate": 0.00017833741074826796, + "logits/chosen": -0.3702457547187805, + "logits/rejected": -0.4608480930328369, + "logps/chosen": -6.7205352783203125, + "logps/rejected": -46.00050354003906, + "loss": 0.6259070038795471, + "memory(GiB)": 43.15, + "nll_loss": 0.43828409910202026, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.014492299407720566, + "rewards/margins": 3.3422679901123047, + "rewards/rejected": -3.3277759552001953, + "step": 217, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7609075043630017, + "grad_norm": 4.585688591003418, + "learning_rate": 0.00017809723934825405, + "logits/chosen": -0.30587339401245117, + "logits/rejected": -0.39274415373802185, + "logps/chosen": -7.410132884979248, + "logps/rejected": -39.855010986328125, + "loss": 0.666675865650177, + "memory(GiB)": 43.15, + "nll_loss": 0.44685378670692444, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.244940847158432, + "rewards/margins": 2.851167678833008, + "rewards/rejected": -2.606226921081543, + "step": 218, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7643979057591623, + "grad_norm": 2.123504877090454, + "learning_rate": 0.00017785590751630403, + "logits/chosen": -0.2982403039932251, + "logits/rejected": -0.42090415954589844, + "logps/chosen": -9.233821868896484, + "logps/rejected": -49.84214401245117, + "loss": 0.608307421207428, + "memory(GiB)": 43.15, + "nll_loss": 0.4753749966621399, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.03608651086688042, + "rewards/margins": 3.5284979343414307, + "rewards/rejected": -3.492410898208618, + "step": 219, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7678883071553229, + "grad_norm": 6.390042781829834, + "learning_rate": 0.0001776134188383214, + "logits/chosen": -0.2736659646034241, + "logits/rejected": -0.4519406855106354, + "logps/chosen": -6.503651142120361, + "logps/rejected": -57.02659606933594, + "loss": 0.7497352957725525, + "memory(GiB)": 43.15, + "nll_loss": 0.5775147676467896, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.052752792835235596, + "rewards/margins": 4.1409912109375, + "rewards/rejected": -4.088238716125488, + "step": 220, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.7713787085514834, + "grad_norm": 7.869540691375732, + "learning_rate": 0.00017736977691739904, + "logits/chosen": -0.2893148958683014, + "logits/rejected": -0.45987799763679504, + "logps/chosen": -7.436133861541748, + "logps/rejected": -50.03794860839844, + "loss": 0.7108630537986755, + "memory(GiB)": 43.15, + "nll_loss": 0.5081782341003418, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.02406979724764824, + "rewards/margins": 3.2432000637054443, + "rewards/rejected": -3.219129800796509, + "step": 221, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.774869109947644, + "grad_norm": 2.0641894340515137, + "learning_rate": 0.00017712498537376562, + "logits/chosen": -0.2713238596916199, + "logits/rejected": -0.4085955023765564, + "logps/chosen": -4.430034637451172, + "logps/rejected": -54.22665023803711, + "loss": 0.44029539823532104, + "memory(GiB)": 43.15, + "nll_loss": 0.24963557720184326, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0637066438794136, + "rewards/margins": 4.175436496734619, + "rewards/rejected": -4.111729621887207, + "step": 222, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7783595113438045, + "grad_norm": 4.655381679534912, + "learning_rate": 0.00017687904784473188, + "logits/chosen": -0.32085755467414856, + "logits/rejected": -0.4244834780693054, + "logps/chosen": -9.161126136779785, + "logps/rejected": -49.83354949951172, + "loss": 0.7681727409362793, + "memory(GiB)": 43.15, + "nll_loss": 0.5344542860984802, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11410484462976456, + "rewards/margins": 3.6829593181610107, + "rewards/rejected": -3.568854331970215, + "step": 223, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7818499127399651, + "grad_norm": 4.069851875305176, + "learning_rate": 0.0001766319679846365, + "logits/chosen": -0.34671980142593384, + "logits/rejected": -0.4845532774925232, + "logps/chosen": -5.1330132484436035, + "logps/rejected": -49.80237579345703, + "loss": 0.6742711067199707, + "memory(GiB)": 43.15, + "nll_loss": 0.44481760263442993, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14206397533416748, + "rewards/margins": 3.5617966651916504, + "rewards/rejected": -3.4197328090667725, + "step": 224, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7853403141361257, + "grad_norm": 2.1478774547576904, + "learning_rate": 0.00017638374946479178, + "logits/chosen": -0.19995641708374023, + "logits/rejected": -0.36974671483039856, + "logps/chosen": -11.061859130859375, + "logps/rejected": -55.27647399902344, + "loss": 0.6111169457435608, + "memory(GiB)": 43.15, + "nll_loss": 0.42612090706825256, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12187716364860535, + "rewards/margins": 4.010047435760498, + "rewards/rejected": -3.8881704807281494, + "step": 225, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7888307155322862, + "grad_norm": 2.974621295928955, + "learning_rate": 0.00017613439597342928, + "logits/chosen": -0.3379659056663513, + "logits/rejected": -0.48346391320228577, + "logps/chosen": -5.418050765991211, + "logps/rejected": -52.225372314453125, + "loss": 0.42137280106544495, + "memory(GiB)": 43.15, + "nll_loss": 0.29507750272750854, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3996121883392334, + "rewards/margins": 4.174943447113037, + "rewards/rejected": -3.7753310203552246, + "step": 226, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7923211169284468, + "grad_norm": 2.614269733428955, + "learning_rate": 0.0001758839112156448, + "logits/chosen": -0.235207661986351, + "logits/rejected": -0.4293682277202606, + "logps/chosen": -6.221077919006348, + "logps/rejected": -57.73169708251953, + "loss": 0.5521800518035889, + "memory(GiB)": 43.15, + "nll_loss": 0.409783273935318, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.23889221251010895, + "rewards/margins": 4.359738826751709, + "rewards/rejected": -4.120846748352051, + "step": 227, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7958115183246073, + "grad_norm": 3.5413107872009277, + "learning_rate": 0.00017563229891334338, + "logits/chosen": -0.26919710636138916, + "logits/rejected": -0.45086047053337097, + "logps/chosen": -6.003745079040527, + "logps/rejected": -56.92920684814453, + "loss": 0.45959168672561646, + "memory(GiB)": 43.15, + "nll_loss": 0.3346911668777466, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23589728772640228, + "rewards/margins": 4.437588691711426, + "rewards/rejected": -4.201691627502441, + "step": 228, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.7993019197207679, + "grad_norm": 6.556480407714844, + "learning_rate": 0.00017537956280518416, + "logits/chosen": -0.3422713875770569, + "logits/rejected": -0.451128214597702, + "logps/chosen": -7.599344253540039, + "logps/rejected": -43.70567321777344, + "loss": 0.7094074487686157, + "memory(GiB)": 43.15, + "nll_loss": 0.5201672315597534, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12758715450763702, + "rewards/margins": 3.260258197784424, + "rewards/rejected": -3.1326711177825928, + "step": 229, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8027923211169284, + "grad_norm": 1.586064338684082, + "learning_rate": 0.00017512570664652454, + "logits/chosen": -0.28756070137023926, + "logits/rejected": -0.38268500566482544, + "logps/chosen": -8.602394104003906, + "logps/rejected": -50.081172943115234, + "loss": 0.6170235872268677, + "memory(GiB)": 43.15, + "nll_loss": 0.44623884558677673, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23854351043701172, + "rewards/margins": 3.8032238483428955, + "rewards/rejected": -3.564680337905884, + "step": 230, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.806282722513089, + "grad_norm": 1.6582534313201904, + "learning_rate": 0.00017487073420936465, + "logits/chosen": -0.3135778605937958, + "logits/rejected": -0.46431437134742737, + "logps/chosen": -4.987645626068115, + "logps/rejected": -52.46248245239258, + "loss": 0.4446931779384613, + "memory(GiB)": 43.15, + "nll_loss": 0.30552300810813904, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12139096856117249, + "rewards/margins": 3.983121156692505, + "rewards/rejected": -3.8617305755615234, + "step": 231, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8097731239092496, + "grad_norm": 1.2338401079177856, + "learning_rate": 0.00017461464928229115, + "logits/chosen": -0.379431813955307, + "logits/rejected": -0.48820745944976807, + "logps/chosen": -4.064929485321045, + "logps/rejected": -43.8752555847168, + "loss": 0.4461000859737396, + "memory(GiB)": 43.15, + "nll_loss": 0.31098049879074097, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10920354723930359, + "rewards/margins": 3.3555779457092285, + "rewards/rejected": -3.2463743686676025, + "step": 232, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8132635253054101, + "grad_norm": 3.2604479789733887, + "learning_rate": 0.00017435745567042095, + "logits/chosen": -0.2582184672355652, + "logits/rejected": -0.4310912489891052, + "logps/chosen": -7.691014289855957, + "logps/rejected": -54.24932098388672, + "loss": 0.6151188611984253, + "memory(GiB)": 43.15, + "nll_loss": 0.41653314232826233, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0867326557636261, + "rewards/margins": 3.933887481689453, + "rewards/rejected": -3.8471546173095703, + "step": 233, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8167539267015707, + "grad_norm": 1.4706934690475464, + "learning_rate": 0.0001740991571953447, + "logits/chosen": -0.269050657749176, + "logits/rejected": -0.5030261278152466, + "logps/chosen": -3.481001615524292, + "logps/rejected": -65.2177505493164, + "loss": 0.3229425251483917, + "memory(GiB)": 43.15, + "nll_loss": 0.24098612368106842, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16970407962799072, + "rewards/margins": 4.697527885437012, + "rewards/rejected": -4.527822971343994, + "step": 234, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8202443280977313, + "grad_norm": 3.0432982444763184, + "learning_rate": 0.00017383975769507006, + "logits/chosen": -0.3666076064109802, + "logits/rejected": -0.49905386567115784, + "logps/chosen": -9.495936393737793, + "logps/rejected": -48.320045471191406, + "loss": 0.7838144302368164, + "memory(GiB)": 43.15, + "nll_loss": 0.5246264934539795, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.07730744034051895, + "rewards/margins": 3.346642255783081, + "rewards/rejected": -3.4239492416381836, + "step": 235, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8237347294938918, + "grad_norm": 2.1510214805603027, + "learning_rate": 0.00017357926102396453, + "logits/chosen": -0.3271171748638153, + "logits/rejected": -0.5392114520072937, + "logps/chosen": -6.4805731773376465, + "logps/rejected": -59.16584777832031, + "loss": 0.4900282919406891, + "memory(GiB)": 43.15, + "nll_loss": 0.34938499331474304, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12114590406417847, + "rewards/margins": 4.474804878234863, + "rewards/rejected": -4.353658676147461, + "step": 236, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8272251308900523, + "grad_norm": 3.5285916328430176, + "learning_rate": 0.00017331767105269833, + "logits/chosen": -0.31751590967178345, + "logits/rejected": -0.5495515465736389, + "logps/chosen": -6.772843360900879, + "logps/rejected": -54.508914947509766, + "loss": 0.6024790406227112, + "memory(GiB)": 43.15, + "nll_loss": 0.39046281576156616, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.011396676301956177, + "rewards/margins": 3.849278688430786, + "rewards/rejected": -3.860675096511841, + "step": 237, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8307155322862129, + "grad_norm": 4.1707329750061035, + "learning_rate": 0.0001730549916681868, + "logits/chosen": -0.3934589922428131, + "logits/rejected": -0.5638350248336792, + "logps/chosen": -7.145245552062988, + "logps/rejected": -66.46633911132812, + "loss": 0.5913079380989075, + "memory(GiB)": 43.15, + "nll_loss": 0.42689260840415955, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09644685685634613, + "rewards/margins": 5.219150543212891, + "rewards/rejected": -5.122704982757568, + "step": 238, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8342059336823735, + "grad_norm": 2.7752156257629395, + "learning_rate": 0.00017279122677353262, + "logits/chosen": -0.2619190216064453, + "logits/rejected": -0.569366455078125, + "logps/chosen": -4.285702705383301, + "logps/rejected": -64.56398010253906, + "loss": 0.4372471272945404, + "memory(GiB)": 43.15, + "nll_loss": 0.3511802554130554, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18024781346321106, + "rewards/margins": 4.906952857971191, + "rewards/rejected": -4.726705551147461, + "step": 239, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.837696335078534, + "grad_norm": 12.657561302185059, + "learning_rate": 0.00017252638028796789, + "logits/chosen": -0.4536377787590027, + "logits/rejected": -0.4795590937137604, + "logps/chosen": -10.30350112915039, + "logps/rejected": -50.788246154785156, + "loss": 0.806001603603363, + "memory(GiB)": 43.15, + "nll_loss": 0.533410906791687, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0006981715559959412, + "rewards/margins": 3.909877300262451, + "rewards/rejected": -3.909179210662842, + "step": 240, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.8411867364746946, + "grad_norm": 2.1404943466186523, + "learning_rate": 0.00017226045614679588, + "logits/chosen": -0.47950872778892517, + "logits/rejected": -0.523240864276886, + "logps/chosen": -7.280372619628906, + "logps/rejected": -47.931678771972656, + "loss": 0.5918366312980652, + "memory(GiB)": 43.15, + "nll_loss": 0.4288029670715332, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.12062430381774902, + "rewards/margins": 3.814420461654663, + "rewards/rejected": -3.693796396255493, + "step": 241, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8446771378708552, + "grad_norm": 5.467627048492432, + "learning_rate": 0.00017199345830133252, + "logits/chosen": -0.3690679669380188, + "logits/rejected": -0.5561453700065613, + "logps/chosen": -4.672489166259766, + "logps/rejected": -62.01866912841797, + "loss": 0.6225215196609497, + "memory(GiB)": 43.15, + "nll_loss": 0.43513110280036926, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.028843754902482033, + "rewards/margins": 4.858260154724121, + "rewards/rejected": -4.887104034423828, + "step": 242, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8481675392670157, + "grad_norm": 2.7799084186553955, + "learning_rate": 0.0001717253907188477, + "logits/chosen": -0.3674232065677643, + "logits/rejected": -0.48529595136642456, + "logps/chosen": -11.648327827453613, + "logps/rejected": -51.57639694213867, + "loss": 0.7946433424949646, + "memory(GiB)": 43.15, + "nll_loss": 0.5997886061668396, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04861972853541374, + "rewards/margins": 3.9191184043884277, + "rewards/rejected": -3.8704986572265625, + "step": 243, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8516579406631762, + "grad_norm": 7.648078441619873, + "learning_rate": 0.00017145625738250635, + "logits/chosen": -0.24742871522903442, + "logits/rejected": -0.48127883672714233, + "logps/chosen": -5.881336212158203, + "logps/rejected": -58.709659576416016, + "loss": 0.6269736289978027, + "memory(GiB)": 43.15, + "nll_loss": 0.4364897310733795, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2005631923675537, + "rewards/margins": 4.109006404876709, + "rewards/rejected": -3.9084432125091553, + "step": 244, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8551483420593369, + "grad_norm": 4.572084426879883, + "learning_rate": 0.0001711860622913092, + "logits/chosen": -0.41671591997146606, + "logits/rejected": -0.49388235807418823, + "logps/chosen": -9.111591339111328, + "logps/rejected": -54.797916412353516, + "loss": 0.7940463423728943, + "memory(GiB)": 43.15, + "nll_loss": 0.5814399123191833, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.008319624699652195, + "rewards/margins": 4.386847496032715, + "rewards/rejected": -4.395166873931885, + "step": 245, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8586387434554974, + "grad_norm": 21.202533721923828, + "learning_rate": 0.0001709148094600334, + "logits/chosen": -0.28929275274276733, + "logits/rejected": -0.46102988719940186, + "logps/chosen": -7.337296962738037, + "logps/rejected": -61.80494689941406, + "loss": 0.8849119544029236, + "memory(GiB)": 43.15, + "nll_loss": 0.6312083005905151, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0007691718637943268, + "rewards/margins": 4.356324195861816, + "rewards/rejected": -4.355555057525635, + "step": 246, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8621291448516579, + "grad_norm": 3.0293681621551514, + "learning_rate": 0.00017064250291917295, + "logits/chosen": -0.3044416308403015, + "logits/rejected": -0.45788872241973877, + "logps/chosen": -11.546972274780273, + "logps/rejected": -68.43335723876953, + "loss": 0.4920075833797455, + "memory(GiB)": 43.15, + "nll_loss": 0.39548447728157043, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28147220611572266, + "rewards/margins": 5.060581207275391, + "rewards/rejected": -4.779109001159668, + "step": 247, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8656195462478184, + "grad_norm": 1.1536225080490112, + "learning_rate": 0.00017036914671487852, + "logits/chosen": -0.29598888754844666, + "logits/rejected": -0.5037257075309753, + "logps/chosen": -4.555235862731934, + "logps/rejected": -71.70169830322266, + "loss": 0.29999658465385437, + "memory(GiB)": 43.15, + "nll_loss": 0.22001689672470093, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10969162732362747, + "rewards/margins": 5.70869779586792, + "rewards/rejected": -5.599005699157715, + "step": 248, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8691099476439791, + "grad_norm": 5.403660297393799, + "learning_rate": 0.0001700947449088977, + "logits/chosen": -0.29327866435050964, + "logits/rejected": -0.5409445762634277, + "logps/chosen": -6.3373589515686035, + "logps/rejected": -61.361427307128906, + "loss": 0.7032750248908997, + "memory(GiB)": 43.15, + "nll_loss": 0.49727386236190796, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.12291238456964493, + "rewards/margins": 4.190377712249756, + "rewards/rejected": -4.313290119171143, + "step": 249, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8726003490401396, + "grad_norm": 2.2364845275878906, + "learning_rate": 0.00016981930157851442, + "logits/chosen": -0.4206138253211975, + "logits/rejected": -0.5452399253845215, + "logps/chosen": -6.642912864685059, + "logps/rejected": -55.26310348510742, + "loss": 0.6106983423233032, + "memory(GiB)": 43.15, + "nll_loss": 0.46136975288391113, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1548982709646225, + "rewards/margins": 4.34593391418457, + "rewards/rejected": -4.191035270690918, + "step": 250, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8760907504363001, + "grad_norm": 2.366743326187134, + "learning_rate": 0.0001695428208164884, + "logits/chosen": -0.3339673578739166, + "logits/rejected": -0.46634042263031006, + "logps/chosen": -12.371589660644531, + "logps/rejected": -54.81672668457031, + "loss": 0.7680290937423706, + "memory(GiB)": 43.15, + "nll_loss": 0.5548099279403687, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06331084668636322, + "rewards/margins": 3.747202157974243, + "rewards/rejected": -3.683891773223877, + "step": 251, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.8795811518324608, + "grad_norm": 17.273366928100586, + "learning_rate": 0.00016926530673099443, + "logits/chosen": -0.25144901871681213, + "logits/rejected": -0.5055121779441833, + "logps/chosen": -4.04465389251709, + "logps/rejected": -53.838226318359375, + "loss": 0.5800854563713074, + "memory(GiB)": 43.15, + "nll_loss": 0.43476593494415283, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09720754623413086, + "rewards/margins": 3.768738031387329, + "rewards/rejected": -3.6715304851531982, + "step": 252, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.8830715532286213, + "grad_norm": 3.9538075923919678, + "learning_rate": 0.00016898676344556118, + "logits/chosen": -0.3590712547302246, + "logits/rejected": -0.4731703996658325, + "logps/chosen": -8.093548774719238, + "logps/rejected": -44.39308547973633, + "loss": 0.7438912391662598, + "memory(GiB)": 43.15, + "nll_loss": 0.5026392936706543, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.010781295597553253, + "rewards/margins": 3.2361843585968018, + "rewards/rejected": -3.22540283203125, + "step": 253, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.8865619546247818, + "grad_norm": 4.43087100982666, + "learning_rate": 0.00016870719509901, + "logits/chosen": -0.32536038756370544, + "logits/rejected": -0.5595154166221619, + "logps/chosen": -3.823620319366455, + "logps/rejected": -56.6773681640625, + "loss": 0.545555591583252, + "memory(GiB)": 43.15, + "nll_loss": 0.3203794062137604, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06757771968841553, + "rewards/margins": 3.751739978790283, + "rewards/rejected": -3.684161901473999, + "step": 254, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.8900523560209425, + "grad_norm": 1.5452710390090942, + "learning_rate": 0.00016842660584539352, + "logits/chosen": -0.4577394425868988, + "logits/rejected": -0.5295063853263855, + "logps/chosen": -4.856768608093262, + "logps/rejected": -44.36632537841797, + "loss": 0.5244438052177429, + "memory(GiB)": 43.15, + "nll_loss": 0.3651025891304016, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22487697005271912, + "rewards/margins": 3.4033455848693848, + "rewards/rejected": -3.178468704223633, + "step": 255, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.893542757417103, + "grad_norm": 1.9295581579208374, + "learning_rate": 0.0001681449998539337, + "logits/chosen": -0.3778080642223358, + "logits/rejected": -0.5354687571525574, + "logps/chosen": -7.896451950073242, + "logps/rejected": -60.301177978515625, + "loss": 0.5556483268737793, + "memory(GiB)": 43.15, + "nll_loss": 0.4384987950325012, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.263929545879364, + "rewards/margins": 4.734387397766113, + "rewards/rejected": -4.470458030700684, + "step": 256, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.8970331588132635, + "grad_norm": 2.257227897644043, + "learning_rate": 0.00016786238130896014, + "logits/chosen": -0.4991389214992523, + "logits/rejected": -0.563326895236969, + "logps/chosen": -8.479850769042969, + "logps/rejected": -38.3695182800293, + "loss": 0.6856359243392944, + "memory(GiB)": 43.15, + "nll_loss": 0.49940675497055054, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1654682606458664, + "rewards/margins": 2.8511545658111572, + "rewards/rejected": -2.6856863498687744, + "step": 257, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.900523560209424, + "grad_norm": 19.965892791748047, + "learning_rate": 0.00016757875440984768, + "logits/chosen": -0.416139155626297, + "logits/rejected": -0.4932768940925598, + "logps/chosen": -8.089065551757812, + "logps/rejected": -44.46285629272461, + "loss": 0.870505690574646, + "memory(GiB)": 43.15, + "nll_loss": 0.6033220291137695, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.02359044924378395, + "rewards/margins": 3.3024742603302, + "rewards/rejected": -3.3260645866394043, + "step": 258, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9040139616055847, + "grad_norm": 2.1116161346435547, + "learning_rate": 0.00016729412337095417, + "logits/chosen": -0.34614264965057373, + "logits/rejected": -0.5283275246620178, + "logps/chosen": -8.69775104522705, + "logps/rejected": -58.33458709716797, + "loss": 0.516063392162323, + "memory(GiB)": 43.15, + "nll_loss": 0.3785746097564697, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15219049155712128, + "rewards/margins": 4.3896002769470215, + "rewards/rejected": -4.237410068511963, + "step": 259, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9075043630017452, + "grad_norm": 2.9699761867523193, + "learning_rate": 0.00016700849242155775, + "logits/chosen": -0.2944773733615875, + "logits/rejected": -0.42303967475891113, + "logps/chosen": -7.197884559631348, + "logps/rejected": -58.377899169921875, + "loss": 0.5171555280685425, + "memory(GiB)": 43.15, + "nll_loss": 0.3375796973705292, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.034792426973581314, + "rewards/margins": 4.28358793258667, + "rewards/rejected": -4.318380355834961, + "step": 260, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9109947643979057, + "grad_norm": 8.736255645751953, + "learning_rate": 0.00016672186580579405, + "logits/chosen": -0.3011060357093811, + "logits/rejected": -0.5398942232131958, + "logps/chosen": -5.949331760406494, + "logps/rejected": -64.1772689819336, + "loss": 0.6132771372795105, + "memory(GiB)": 43.15, + "nll_loss": 0.5076794624328613, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.010059421882033348, + "rewards/margins": 4.510530948638916, + "rewards/rejected": -4.500471115112305, + "step": 261, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9144851657940664, + "grad_norm": 5.83949613571167, + "learning_rate": 0.0001664342477825931, + "logits/chosen": -0.44913211464881897, + "logits/rejected": -0.5108562707901001, + "logps/chosen": -12.713695526123047, + "logps/rejected": -51.189388275146484, + "loss": 1.1607731580734253, + "memory(GiB)": 43.15, + "nll_loss": 0.979007363319397, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07644180953502655, + "rewards/margins": 4.067415237426758, + "rewards/rejected": -3.9909729957580566, + "step": 262, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9179755671902269, + "grad_norm": 6.3965911865234375, + "learning_rate": 0.00016614564262561608, + "logits/chosen": -0.3431762456893921, + "logits/rejected": -0.5227538347244263, + "logps/chosen": -8.923004150390625, + "logps/rejected": -66.28221893310547, + "loss": 0.7520352602005005, + "memory(GiB)": 43.15, + "nll_loss": 0.6111706495285034, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0333654060959816, + "rewards/margins": 4.924405574798584, + "rewards/rejected": -4.891040325164795, + "step": 263, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9214659685863874, + "grad_norm": 8.98039722442627, + "learning_rate": 0.0001658560546231918, + "logits/chosen": -0.40294116735458374, + "logits/rejected": -0.5690099000930786, + "logps/chosen": -5.10881233215332, + "logps/rejected": -55.61248779296875, + "loss": 0.469806045293808, + "memory(GiB)": 43.15, + "nll_loss": 0.31188011169433594, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08330924063920975, + "rewards/margins": 4.100750923156738, + "rewards/rejected": -4.017441749572754, + "step": 264, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.924956369982548, + "grad_norm": 2.279346466064453, + "learning_rate": 0.00016556548807825295, + "logits/chosen": -0.2596076726913452, + "logits/rejected": -0.4245615601539612, + "logps/chosen": -8.973737716674805, + "logps/rejected": -63.824710845947266, + "loss": 0.5860002636909485, + "memory(GiB)": 43.15, + "nll_loss": 0.4316725730895996, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03057311289012432, + "rewards/margins": 4.672904014587402, + "rewards/rejected": -4.642330646514893, + "step": 265, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9284467713787086, + "grad_norm": 3.462368965148926, + "learning_rate": 0.00016527394730827229, + "logits/chosen": -0.39172494411468506, + "logits/rejected": -0.5556192994117737, + "logps/chosen": -6.451850891113281, + "logps/rejected": -58.2652473449707, + "loss": 0.6653540134429932, + "memory(GiB)": 43.15, + "nll_loss": 0.5281586647033691, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09578607976436615, + "rewards/margins": 4.3150811195373535, + "rewards/rejected": -4.219295501708984, + "step": 266, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9319371727748691, + "grad_norm": 2.29887056350708, + "learning_rate": 0.00016498143664519824, + "logits/chosen": -0.3378483057022095, + "logits/rejected": -0.5873894095420837, + "logps/chosen": -5.2974629402160645, + "logps/rejected": -76.44007110595703, + "loss": 0.38004374504089355, + "memory(GiB)": 43.15, + "nll_loss": 0.2812269926071167, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.3595108091831207, + "rewards/margins": 5.502060413360596, + "rewards/rejected": -5.14254903793335, + "step": 267, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9354275741710296, + "grad_norm": 8.743305206298828, + "learning_rate": 0.0001646879604353908, + "logits/chosen": -0.3702645003795624, + "logits/rejected": -0.5771125555038452, + "logps/chosen": -9.347705841064453, + "logps/rejected": -49.44694519042969, + "loss": 1.1101744174957275, + "memory(GiB)": 43.15, + "nll_loss": 0.799601137638092, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0540962852537632, + "rewards/margins": 3.277214288711548, + "rewards/rejected": -3.331310749053955, + "step": 268, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9389179755671903, + "grad_norm": 1.31393563747406, + "learning_rate": 0.00016439352303955688, + "logits/chosen": -0.39863336086273193, + "logits/rejected": -0.5482807159423828, + "logps/chosen": -6.748524188995361, + "logps/rejected": -59.980262756347656, + "loss": 0.4353381097316742, + "memory(GiB)": 43.15, + "nll_loss": 0.37013500928878784, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15098029375076294, + "rewards/margins": 4.846394062042236, + "rewards/rejected": -4.695413589477539, + "step": 269, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9424083769633508, + "grad_norm": 5.9643425941467285, + "learning_rate": 0.00016409812883268535, + "logits/chosen": -0.38031327724456787, + "logits/rejected": -0.5494654178619385, + "logps/chosen": -6.423410892486572, + "logps/rejected": -69.69914245605469, + "loss": 0.4006003141403198, + "memory(GiB)": 43.15, + "nll_loss": 0.3364032506942749, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10248701274394989, + "rewards/margins": 5.274303436279297, + "rewards/rejected": -5.376790523529053, + "step": 270, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9458987783595113, + "grad_norm": 7.4491047859191895, + "learning_rate": 0.00016380178220398226, + "logits/chosen": -0.415705144405365, + "logits/rejected": -0.6392954587936401, + "logps/chosen": -6.486244201660156, + "logps/rejected": -56.07293701171875, + "loss": 0.6918466687202454, + "memory(GiB)": 43.15, + "nll_loss": 0.5155448317527771, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09501791000366211, + "rewards/margins": 3.938845634460449, + "rewards/rejected": -3.843827962875366, + "step": 271, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9493891797556719, + "grad_norm": 2.838250160217285, + "learning_rate": 0.0001635044875568054, + "logits/chosen": -0.36552664637565613, + "logits/rejected": -0.4577628970146179, + "logps/chosen": -10.308563232421875, + "logps/rejected": -52.53562927246094, + "loss": 0.4913533926010132, + "memory(GiB)": 43.15, + "nll_loss": 0.34640663862228394, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10055547952651978, + "rewards/margins": 4.087219715118408, + "rewards/rejected": -3.986664056777954, + "step": 272, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9528795811518325, + "grad_norm": 8.961051940917969, + "learning_rate": 0.00016320624930859904, + "logits/chosen": -0.38108423352241516, + "logits/rejected": -0.5300929546356201, + "logps/chosen": -11.11557674407959, + "logps/rejected": -57.43425750732422, + "loss": 0.9454229474067688, + "memory(GiB)": 43.15, + "nll_loss": 0.7644782662391663, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.13230867683887482, + "rewards/margins": 3.9995622634887695, + "rewards/rejected": -4.131870746612549, + "step": 273, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.956369982547993, + "grad_norm": 2.179896116256714, + "learning_rate": 0.0001629070718908283, + "logits/chosen": -0.39322447776794434, + "logits/rejected": -0.5073328614234924, + "logps/chosen": -7.645257949829102, + "logps/rejected": -51.549869537353516, + "loss": 0.5495847463607788, + "memory(GiB)": 43.15, + "nll_loss": 0.3960936665534973, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.038150034844875336, + "rewards/margins": 4.1127800941467285, + "rewards/rejected": -4.074629783630371, + "step": 274, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9598603839441536, + "grad_norm": 2.9412152767181396, + "learning_rate": 0.0001626069597489132, + "logits/chosen": -0.42589738965034485, + "logits/rejected": -0.5628194808959961, + "logps/chosen": -11.427227973937988, + "logps/rejected": -51.27968978881836, + "loss": 0.7387857437133789, + "memory(GiB)": 43.15, + "nll_loss": 0.5841793417930603, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1430855542421341, + "rewards/margins": 3.8389885425567627, + "rewards/rejected": -3.6959028244018555, + "step": 275, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9633507853403142, + "grad_norm": 3.372194528579712, + "learning_rate": 0.0001623059173421625, + "logits/chosen": -0.2240457385778427, + "logits/rejected": -0.5538793802261353, + "logps/chosen": -5.805129051208496, + "logps/rejected": -71.96966552734375, + "loss": 0.5059239268302917, + "memory(GiB)": 43.15, + "nll_loss": 0.42470091581344604, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06452590972185135, + "rewards/margins": 5.022994041442871, + "rewards/rejected": -4.958468437194824, + "step": 276, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9668411867364747, + "grad_norm": 4.933607578277588, + "learning_rate": 0.00016200394914370788, + "logits/chosen": -0.318347692489624, + "logits/rejected": -0.47638392448425293, + "logps/chosen": -8.253751754760742, + "logps/rejected": -57.21156692504883, + "loss": 0.8552437424659729, + "memory(GiB)": 43.15, + "nll_loss": 0.5107535719871521, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.028203103691339493, + "rewards/margins": 3.587186336517334, + "rewards/rejected": -3.615389585494995, + "step": 277, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9703315881326352, + "grad_norm": 1.967063069343567, + "learning_rate": 0.00016170105964043695, + "logits/chosen": -0.46634575724601746, + "logits/rejected": -0.5018796920776367, + "logps/chosen": -8.443771362304688, + "logps/rejected": -36.8387336730957, + "loss": 0.6206079125404358, + "memory(GiB)": 43.15, + "nll_loss": 0.3942241072654724, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14703020453453064, + "rewards/margins": 2.6712229251861572, + "rewards/rejected": -2.5241925716400146, + "step": 278, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9738219895287958, + "grad_norm": 2.5119967460632324, + "learning_rate": 0.0001613972533329269, + "logits/chosen": -0.39050760865211487, + "logits/rejected": -0.5310595035552979, + "logps/chosen": -4.703199863433838, + "logps/rejected": -50.82078552246094, + "loss": 0.4742874801158905, + "memory(GiB)": 43.15, + "nll_loss": 0.2932378947734833, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.04821047931909561, + "rewards/margins": 3.5806527137756348, + "rewards/rejected": -3.5324418544769287, + "step": 279, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9773123909249564, + "grad_norm": 2.238361120223999, + "learning_rate": 0.0001610925347353775, + "logits/chosen": -0.37722715735435486, + "logits/rejected": -0.5656870603561401, + "logps/chosen": -5.8236494064331055, + "logps/rejected": -50.120391845703125, + "loss": 0.5826418995857239, + "memory(GiB)": 43.15, + "nll_loss": 0.4334554672241211, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16360187530517578, + "rewards/margins": 3.4687533378601074, + "rewards/rejected": -3.30515193939209, + "step": 280, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.9808027923211169, + "grad_norm": 2.3494832515716553, + "learning_rate": 0.00016078690837554408, + "logits/chosen": -0.33358269929885864, + "logits/rejected": -0.5591884255409241, + "logps/chosen": -5.116516590118408, + "logps/rejected": -49.51860046386719, + "loss": 0.5960334539413452, + "memory(GiB)": 43.15, + "nll_loss": 0.3720979690551758, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.17815372347831726, + "rewards/margins": 3.0898640155792236, + "rewards/rejected": -2.911710262298584, + "step": 281, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.9842931937172775, + "grad_norm": 3.6189231872558594, + "learning_rate": 0.00016048037879467025, + "logits/chosen": -0.37796086072921753, + "logits/rejected": -0.5117664933204651, + "logps/chosen": -4.1463236808776855, + "logps/rejected": -48.00626754760742, + "loss": 0.5160388946533203, + "memory(GiB)": 43.15, + "nll_loss": 0.31095653772354126, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.034109927713871, + "rewards/margins": 3.225755453109741, + "rewards/rejected": -3.191645383834839, + "step": 282, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.987783595113438, + "grad_norm": 5.189328193664551, + "learning_rate": 0.00016017295054742046, + "logits/chosen": -0.33860644698143005, + "logits/rejected": -0.5117906332015991, + "logps/chosen": -11.231898307800293, + "logps/rejected": -49.39238739013672, + "loss": 0.8505796790122986, + "memory(GiB)": 43.15, + "nll_loss": 0.6683485507965088, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10711870342493057, + "rewards/margins": 3.197012424468994, + "rewards/rejected": -3.089893341064453, + "step": 283, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.9912739965095986, + "grad_norm": 26.61264419555664, + "learning_rate": 0.0001598646282018121, + "logits/chosen": -0.36234819889068604, + "logits/rejected": -0.5464830994606018, + "logps/chosen": -7.227906703948975, + "logps/rejected": -53.168861389160156, + "loss": 0.5734856724739075, + "memory(GiB)": 43.15, + "nll_loss": 0.41981199383735657, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07493679225444794, + "rewards/margins": 3.693037748336792, + "rewards/rejected": -3.618101119995117, + "step": 284, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9947643979057592, + "grad_norm": 5.144956588745117, + "learning_rate": 0.000159555416339148, + "logits/chosen": -0.26111942529678345, + "logits/rejected": -0.5374065041542053, + "logps/chosen": -2.2055845260620117, + "logps/rejected": -68.69379425048828, + "loss": 0.3513576090335846, + "memory(GiB)": 43.15, + "nll_loss": 0.21501211822032928, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09447957575321198, + "rewards/margins": 4.662655830383301, + "rewards/rejected": -4.56817626953125, + "step": 285, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.9982547993019197, + "grad_norm": 2.222433090209961, + "learning_rate": 0.00015924531955394802, + "logits/chosen": -0.3610646724700928, + "logits/rejected": -0.4617045521736145, + "logps/chosen": -7.9295196533203125, + "logps/rejected": -47.70429611206055, + "loss": 0.5501912832260132, + "memory(GiB)": 43.15, + "nll_loss": 0.3641606867313385, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13688504695892334, + "rewards/margins": 3.380563735961914, + "rewards/rejected": -3.243678569793701, + "step": 286, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 1.0034904013961605, + "grad_norm": 2.3679821491241455, + "learning_rate": 0.00015893434245388093, + "logits/chosen": -0.39241814613342285, + "logits/rejected": -0.5621130466461182, + "logps/chosen": -4.523199081420898, + "logps/rejected": -60.3804931640625, + "loss": 0.48575231432914734, + "memory(GiB)": 43.15, + "nll_loss": 0.28081169724464417, + "rewards/accuracies": 0.9411764740943909, + "rewards/chosen": 0.13038994371891022, + "rewards/margins": 4.663900375366211, + "rewards/rejected": -4.533511161804199, + "step": 287, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.006980802792321, + "grad_norm": 1.3687598705291748, + "learning_rate": 0.00015862248965969604, + "logits/chosen": -0.26206377148628235, + "logits/rejected": -0.5492232441902161, + "logps/chosen": -5.55544900894165, + "logps/rejected": -82.16498565673828, + "loss": 0.35956525802612305, + "memory(GiB)": 43.15, + "nll_loss": 0.28186729550361633, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26697322726249695, + "rewards/margins": 6.157236576080322, + "rewards/rejected": -5.89026403427124, + "step": 288, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0104712041884816, + "grad_norm": 4.600061893463135, + "learning_rate": 0.00015830976580515432, + "logits/chosen": -0.3973150849342346, + "logits/rejected": -0.527411937713623, + "logps/chosen": -11.000313758850098, + "logps/rejected": -61.0347785949707, + "loss": 0.7208768129348755, + "memory(GiB)": 43.15, + "nll_loss": 0.5785147547721863, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18078193068504333, + "rewards/margins": 4.359728813171387, + "rewards/rejected": -4.1789469718933105, + "step": 289, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0139616055846423, + "grad_norm": 2.2930972576141357, + "learning_rate": 0.00015799617553695957, + "logits/chosen": -0.4011244773864746, + "logits/rejected": -0.5036134719848633, + "logps/chosen": -6.045261383056641, + "logps/rejected": -57.71175765991211, + "loss": 0.3789103627204895, + "memory(GiB)": 43.15, + "nll_loss": 0.3045409619808197, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3516743779182434, + "rewards/margins": 4.524147033691406, + "rewards/rejected": -4.1724724769592285, + "step": 290, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0174520069808028, + "grad_norm": 1.8977946043014526, + "learning_rate": 0.00015768172351468974, + "logits/chosen": -0.3493969142436981, + "logits/rejected": -0.5791320204734802, + "logps/chosen": -3.5215115547180176, + "logps/rejected": -65.80962371826172, + "loss": 0.36785203218460083, + "memory(GiB)": 43.15, + "nll_loss": 0.26871761679649353, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3286406993865967, + "rewards/margins": 5.31328010559082, + "rewards/rejected": -4.984640121459961, + "step": 291, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0209424083769634, + "grad_norm": 8.48400592803955, + "learning_rate": 0.0001573664144107272, + "logits/chosen": -0.40092405676841736, + "logits/rejected": -0.5814225077629089, + "logps/chosen": -4.21975564956665, + "logps/rejected": -66.2427978515625, + "loss": 0.35437557101249695, + "memory(GiB)": 43.15, + "nll_loss": 0.27522024512290955, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2891547381877899, + "rewards/margins": 4.967032432556152, + "rewards/rejected": -4.677878379821777, + "step": 292, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.024432809773124, + "grad_norm": 2.3877346515655518, + "learning_rate": 0.0001570502529101896, + "logits/chosen": -0.4078039228916168, + "logits/rejected": -0.6340718269348145, + "logps/chosen": -4.7514328956604, + "logps/rejected": -63.62433624267578, + "loss": 0.2756602466106415, + "memory(GiB)": 43.15, + "nll_loss": 0.2308274805545807, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2891400456428528, + "rewards/margins": 5.025572776794434, + "rewards/rejected": -4.736433029174805, + "step": 293, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0279232111692844, + "grad_norm": 1.341036081314087, + "learning_rate": 0.0001567332437108602, + "logits/chosen": -0.4645722806453705, + "logits/rejected": -0.6466395854949951, + "logps/chosen": -4.918186187744141, + "logps/rejected": -55.84804916381836, + "loss": 0.3296290636062622, + "memory(GiB)": 43.15, + "nll_loss": 0.23185840249061584, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28173521161079407, + "rewards/margins": 4.368047714233398, + "rewards/rejected": -4.086312294006348, + "step": 294, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.031413612565445, + "grad_norm": 1.4693691730499268, + "learning_rate": 0.00015641539152311796, + "logits/chosen": -0.42224860191345215, + "logits/rejected": -0.6172808408737183, + "logps/chosen": -5.080268859863281, + "logps/rejected": -57.87543487548828, + "loss": 0.4304788410663605, + "memory(GiB)": 43.15, + "nll_loss": 0.3393770754337311, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2397257685661316, + "rewards/margins": 4.47007942199707, + "rewards/rejected": -4.230354309082031, + "step": 295, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0349040139616057, + "grad_norm": 3.831491231918335, + "learning_rate": 0.00015609670106986776, + "logits/chosen": -0.5365476608276367, + "logits/rejected": -0.6072984337806702, + "logps/chosen": -5.30760383605957, + "logps/rejected": -49.26717758178711, + "loss": 0.36716127395629883, + "memory(GiB)": 43.15, + "nll_loss": 0.2641866207122803, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4461376667022705, + "rewards/margins": 3.9801061153411865, + "rewards/rejected": -3.533968687057495, + "step": 296, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 1.0383944153577662, + "grad_norm": 8.527892112731934, + "learning_rate": 0.0001557771770864701, + "logits/chosen": -0.3701344430446625, + "logits/rejected": -0.5703862309455872, + "logps/chosen": -6.846416473388672, + "logps/rejected": -65.28385925292969, + "loss": 0.47399166226387024, + "memory(GiB)": 43.15, + "nll_loss": 0.32827308773994446, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.45203351974487305, + "rewards/margins": 4.902979850769043, + "rewards/rejected": -4.450946807861328, + "step": 297, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 1.0418848167539267, + "grad_norm": 2.383213520050049, + "learning_rate": 0.00015545682432067067, + "logits/chosen": -0.4192945659160614, + "logits/rejected": -0.663847804069519, + "logps/chosen": -5.736666679382324, + "logps/rejected": -72.8265380859375, + "loss": 0.32853907346725464, + "memory(GiB)": 43.15, + "nll_loss": 0.29026535153388977, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.47007882595062256, + "rewards/margins": 5.804821014404297, + "rewards/rejected": -5.334741592407227, + "step": 298, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 1.0453752181500873, + "grad_norm": 2.514970064163208, + "learning_rate": 0.00015513564753252994, + "logits/chosen": -0.5625315308570862, + "logits/rejected": -0.6955651044845581, + "logps/chosen": -6.750204563140869, + "logps/rejected": -58.264225006103516, + "loss": 0.5013718605041504, + "memory(GiB)": 43.15, + "nll_loss": 0.3770815134048462, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1921541839838028, + "rewards/margins": 4.60443115234375, + "rewards/rejected": -4.4122772216796875, + "step": 299, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 1.0488656195462478, + "grad_norm": 8.446479797363281, + "learning_rate": 0.00015481365149435235, + "logits/chosen": -0.5021696090698242, + "logits/rejected": -0.663796603679657, + "logps/chosen": -6.143691062927246, + "logps/rejected": -53.95307159423828, + "loss": 0.46026065945625305, + "memory(GiB)": 43.15, + "nll_loss": 0.3280302882194519, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2705683708190918, + "rewards/margins": 4.268153190612793, + "rewards/rejected": -3.9975852966308594, + "step": 300, + "train_speed(iter/s)": 0.010751 + } + ], + "logging_steps": 1, + "max_steps": 858, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.637825559532667e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}