{
  "best_global_step": 500,
  "best_metric": 0.49153158,
  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt10_noidk__pari0.8_0602/v0-20250602-171544/checkpoint-500",
  "epoch": 1.7469458987783595,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0034904013961605585,
      "grad_norm": 8.557130813598633,
      "learning_rate": 4.651162790697674e-06,
      "logits/chosen": -0.3853919506072998,
      "logits/rejected": -0.6177393794059753,
      "logps/chosen": -6.8596577644348145,
      "logps/rejected": -24.441532135009766,
      "loss": 1.0831061601638794,
      "memory(GiB)": 34.14,
      "nll_loss": 0.38995903730392456,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.01012
    },
    {
      "epoch": 0.006980802792321117,
      "grad_norm": 4.541833400726318,
      "learning_rate": 9.302325581395349e-06,
      "logits/chosen": -0.4651119112968445,
      "logits/rejected": -0.5818633437156677,
      "logps/chosen": -7.042446136474609,
      "logps/rejected": -15.569698333740234,
      "loss": 1.1563867330551147,
      "memory(GiB)": 34.14,
      "nll_loss": 0.46323955059051514,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.010413
    },
    {
      "epoch": 0.010471204188481676,
      "grad_norm": 5.130979061126709,
      "learning_rate": 1.3953488372093024e-05,
      "logits/chosen": -0.46299099922180176,
      "logits/rejected": -0.584057629108429,
      "logps/chosen": -10.678628921508789,
      "logps/rejected": -13.385712623596191,
      "loss": 1.3326630592346191,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6397495865821838,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.005845161620527506,
      "rewards/margins": 0.0005908504826948047,
      "rewards/rejected": 0.005254311487078667,
      "step": 3,
      "train_speed(iter/s)": 0.010516
    },
    {
      "epoch": 0.013961605584642234,
      "grad_norm": 9.7429780960083,
      "learning_rate": 1.8604651162790697e-05,
      "logits/chosen": -0.5326871871948242,
      "logits/rejected": -0.6262682676315308,
      "logps/chosen": -6.428069114685059,
      "logps/rejected": -15.568026542663574,
      "loss": 1.1956652402877808,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5060467720031738,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.006483433302491903,
      "rewards/margins": 0.0072298417799174786,
      "rewards/rejected": -0.0007464090595021844,
      "step": 4,
      "train_speed(iter/s)": 0.010579
    },
    {
      "epoch": 0.017452006980802792,
      "grad_norm": 5.269836902618408,
      "learning_rate": 2.3255813953488374e-05,
      "logits/chosen": -0.5215513706207275,
      "logits/rejected": -0.6214993000030518,
      "logps/chosen": -6.4991326332092285,
      "logps/rejected": -16.232433319091797,
      "loss": 1.144551396369934,
      "memory(GiB)": 34.14,
      "nll_loss": 0.46053266525268555,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004076660610735416,
      "rewards/margins": 0.018971633166074753,
      "rewards/rejected": -0.014894972555339336,
      "step": 5,
      "train_speed(iter/s)": 0.010622
    },
    {
      "epoch": 0.020942408376963352,
      "grad_norm": 6.636388301849365,
      "learning_rate": 2.7906976744186048e-05,
      "logits/chosen": -0.4555979073047638,
      "logits/rejected": -0.5607460737228394,
      "logps/chosen": -9.820672035217285,
      "logps/rejected": -15.622188568115234,
      "loss": 1.3625051975250244,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6840521693229675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018183868378400803,
      "rewards/margins": 0.03185040131211281,
      "rewards/rejected": -0.01366653386503458,
      "step": 6,
      "train_speed(iter/s)": 0.01064
    },
    {
      "epoch": 0.02443280977312391,
      "grad_norm": 5.103886127471924,
      "learning_rate": 3.2558139534883724e-05,
      "logits/chosen": -0.4968104660511017,
      "logits/rejected": -0.569257378578186,
      "logps/chosen": -10.183512687683105,
      "logps/rejected": -15.70079231262207,
      "loss": 1.4255497455596924,
      "memory(GiB)": 34.14,
      "nll_loss": 0.7536435127258301,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.010219119489192963,
      "rewards/margins": 0.054266996681690216,
      "rewards/rejected": -0.06448611617088318,
      "step": 7,
      "train_speed(iter/s)": 0.010656
    },
    {
      "epoch": 0.027923211169284468,
      "grad_norm": 3.999744176864624,
      "learning_rate": 3.7209302325581394e-05,
      "logits/chosen": -0.4695020914077759,
      "logits/rejected": -0.5785450339317322,
      "logps/chosen": -8.685606002807617,
      "logps/rejected": -19.699125289916992,
      "loss": 1.167594075202942,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5315794944763184,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.033686134964227676,
      "rewards/margins": 0.14157818257808685,
      "rewards/rejected": -0.17526429891586304,
      "step": 8,
      "train_speed(iter/s)": 0.010669
    },
    {
      "epoch": 0.031413612565445025,
      "grad_norm": 7.809542179107666,
      "learning_rate": 4.186046511627907e-05,
      "logits/chosen": -0.5049821138381958,
      "logits/rejected": -0.5677512288093567,
      "logps/chosen": -10.067422866821289,
      "logps/rejected": -13.192615509033203,
      "loss": 1.3344353437423706,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6376084089279175,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1379907727241516,
      "rewards/margins": 0.05565176531672478,
      "rewards/rejected": -0.1936425417661667,
      "step": 9,
      "train_speed(iter/s)": 0.010683
    },
    {
      "epoch": 0.034904013961605584,
      "grad_norm": 10.225089073181152,
      "learning_rate": 4.651162790697675e-05,
      "logits/chosen": -0.5532017350196838,
      "logits/rejected": -0.5782958269119263,
      "logps/chosen": -13.762276649475098,
      "logps/rejected": -13.101286888122559,
      "loss": 1.5571683645248413,
      "memory(GiB)": 34.14,
      "nll_loss": 0.8523505330085754,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.15877792239189148,
      "rewards/margins": 0.03356418013572693,
      "rewards/rejected": -0.19234208762645721,
      "step": 10,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.038394415357766144,
      "grad_norm": 4.407655239105225,
      "learning_rate": 5.1162790697674425e-05,
      "logits/chosen": -0.4217715263366699,
      "logits/rejected": -0.5799385905265808,
      "logps/chosen": -6.071174621582031,
      "logps/rejected": -21.04494857788086,
      "loss": 0.9959170818328857,
      "memory(GiB)": 34.14,
      "nll_loss": 0.4200224280357361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03702637180685997,
      "rewards/margins": 0.3200541138648987,
      "rewards/rejected": -0.35708048939704895,
      "step": 11,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.041884816753926704,
      "grad_norm": 6.385944366455078,
      "learning_rate": 5.5813953488372095e-05,
      "logits/chosen": -0.5289366841316223,
      "logits/rejected": -0.5345426201820374,
      "logps/chosen": -11.089645385742188,
      "logps/rejected": -13.696191787719727,
      "loss": 1.2887790203094482,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5673516392707825,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.10325971990823746,
      "rewards/margins": -0.009972111321985722,
      "rewards/rejected": -0.09328760206699371,
      "step": 12,
      "train_speed(iter/s)": 0.010702
    },
    {
      "epoch": 0.04537521815008726,
      "grad_norm": 4.892186641693115,
      "learning_rate": 6.0465116279069765e-05,
      "logits/chosen": -0.48081111907958984,
      "logits/rejected": -0.5861090421676636,
      "logps/chosen": -7.97854471206665,
      "logps/rejected": -18.214845657348633,
      "loss": 1.23686683177948,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5658432841300964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07524491846561432,
      "rewards/margins": 0.07474859058856964,
      "rewards/rejected": -0.14999350905418396,
      "step": 13,
      "train_speed(iter/s)": 0.010708
    },
    {
      "epoch": 0.04886561954624782,
      "grad_norm": 3.9216458797454834,
      "learning_rate": 6.511627906976745e-05,
      "logits/chosen": -0.4399729073047638,
      "logits/rejected": -0.5742396116256714,
      "logps/chosen": -3.621753692626953,
      "logps/rejected": -17.848834991455078,
      "loss": 0.9254244565963745,
      "memory(GiB)": 34.14,
      "nll_loss": 0.26018136739730835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.069590725004673,
      "rewards/margins": 0.08134818077087402,
      "rewards/rejected": -0.01175746414810419,
      "step": 14,
      "train_speed(iter/s)": 0.010711
    },
    {
      "epoch": 0.05235602094240838,
      "grad_norm": 4.189650058746338,
      "learning_rate": 6.976744186046513e-05,
      "logits/chosen": -0.4310145378112793,
      "logits/rejected": -0.5189388990402222,
      "logps/chosen": -7.985783100128174,
      "logps/rejected": -17.343812942504883,
      "loss": 1.1174938678741455,
      "memory(GiB)": 34.14,
      "nll_loss": 0.4816132187843323,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09267675131559372,
      "rewards/margins": 0.1413428634405136,
      "rewards/rejected": -0.04866611212491989,
      "step": 15,
      "train_speed(iter/s)": 0.010712
    },
    {
      "epoch": 0.055846422338568937,
      "grad_norm": 3.32806658744812,
      "learning_rate": 7.441860465116279e-05,
      "logits/chosen": -0.43775254487991333,
      "logits/rejected": -0.5088982582092285,
      "logps/chosen": -8.117635726928711,
      "logps/rejected": -19.758106231689453,
      "loss": 1.1459051370620728,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5276587605476379,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10978943854570389,
      "rewards/margins": 0.18414708971977234,
      "rewards/rejected": -0.07435765117406845,
      "step": 16,
      "train_speed(iter/s)": 0.010712
    },
    {
      "epoch": 0.059336823734729496,
      "grad_norm": 2.725511312484741,
      "learning_rate": 7.906976744186047e-05,
      "logits/chosen": -0.47936227917671204,
      "logits/rejected": -0.5878744721412659,
      "logps/chosen": -9.313335418701172,
      "logps/rejected": -21.849124908447266,
      "loss": 1.058612585067749,
      "memory(GiB)": 34.14,
      "nll_loss": 0.4116324186325073,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.01833728700876236,
      "rewards/margins": 0.10961442440748215,
      "rewards/rejected": -0.09127713739871979,
      "step": 17,
      "train_speed(iter/s)": 0.010713
    },
    {
      "epoch": 0.06282722513089005,
      "grad_norm": 3.7582316398620605,
      "learning_rate": 8.372093023255814e-05,
      "logits/chosen": -0.476583331823349,
      "logits/rejected": -0.5290005803108215,
      "logps/chosen": -8.529088020324707,
      "logps/rejected": -16.1294002532959,
      "loss": 1.1786390542984009,
      "memory(GiB)": 34.14,
      "nll_loss": 0.578920841217041,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02365059219300747,
      "rewards/margins": 0.23341532051563263,
      "rewards/rejected": -0.25706592202186584,
      "step": 18,
      "train_speed(iter/s)": 0.010716
    },
    {
      "epoch": 0.06631762652705062,
      "grad_norm": 3.4042398929595947,
      "learning_rate": 8.837209302325582e-05,
      "logits/chosen": -0.4703082740306854,
      "logits/rejected": -0.5102555155754089,
      "logps/chosen": -9.428271293640137,
      "logps/rejected": -16.359838485717773,
      "loss": 1.235347867012024,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6085215210914612,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01763956993818283,
      "rewards/margins": 0.1921030879020691,
      "rewards/rejected": -0.17446354031562805,
      "step": 19,
      "train_speed(iter/s)": 0.010716
    },
    {
      "epoch": 0.06980802792321117,
      "grad_norm": 3.1032357215881348,
      "learning_rate": 9.30232558139535e-05,
      "logits/chosen": -0.401233434677124,
      "logits/rejected": -0.494634747505188,
      "logps/chosen": -9.093883514404297,
      "logps/rejected": -18.11814308166504,
      "loss": 1.1991461515426636,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5884966850280762,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02572321705520153,
      "rewards/margins": 0.21222788095474243,
      "rewards/rejected": -0.18650466203689575,
      "step": 20,
      "train_speed(iter/s)": 0.010717
    },
    {
      "epoch": 0.07329842931937172,
      "grad_norm": 3.3876612186431885,
      "learning_rate": 9.767441860465116e-05,
      "logits/chosen": -0.46866974234580994,
      "logits/rejected": -0.530718207359314,
      "logps/chosen": -7.192732334136963,
      "logps/rejected": -14.595564842224121,
      "loss": 1.1225976943969727,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5212462544441223,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1131197139620781,
      "rewards/margins": 0.2251490354537964,
      "rewards/rejected": -0.1120293140411377,
      "step": 21,
      "train_speed(iter/s)": 0.01072
    },
    {
      "epoch": 0.07678883071553229,
      "grad_norm": 2.6822471618652344,
      "learning_rate": 0.00010232558139534885,
      "logits/chosen": -0.5048164129257202,
      "logits/rejected": -0.5435423851013184,
      "logps/chosen": -7.50157356262207,
      "logps/rejected": -14.21214485168457,
      "loss": 1.088458776473999,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5341289043426514,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16505159437656403,
      "rewards/margins": 0.3283952474594116,
      "rewards/rejected": -0.1633436679840088,
      "step": 22,
      "train_speed(iter/s)": 0.010723
    },
    {
      "epoch": 0.08027923211169284,
      "grad_norm": 2.733980417251587,
      "learning_rate": 0.00010697674418604651,
      "logits/chosen": -0.40645402669906616,
      "logits/rejected": -0.5064697861671448,
      "logps/chosen": -8.52833080291748,
      "logps/rejected": -21.627185821533203,
      "loss": 1.119882345199585,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6272884607315063,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21130108833312988,
      "rewards/margins": 0.5197649598121643,
      "rewards/rejected": -0.3084638714790344,
      "step": 23,
      "train_speed(iter/s)": 0.010724
    },
    {
      "epoch": 0.08376963350785341,
      "grad_norm": 3.6145081520080566,
      "learning_rate": 0.00011162790697674419,
      "logits/chosen": -0.40803515911102295,
      "logits/rejected": -0.4656044840812683,
      "logps/chosen": -7.790849208831787,
      "logps/rejected": -19.12059783935547,
      "loss": 1.073976755142212,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5007548332214355,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.021760545670986176,
      "rewards/margins": 0.325797438621521,
      "rewards/rejected": -0.3040368854999542,
      "step": 24,
      "train_speed(iter/s)": 0.010727
    },
    {
      "epoch": 0.08726003490401396,
      "grad_norm": 3.5218026638031006,
      "learning_rate": 0.00011627906976744187,
      "logits/chosen": -0.38492777943611145,
      "logits/rejected": -0.4633912742137909,
      "logps/chosen": -9.945892333984375,
      "logps/rejected": -22.549875259399414,
      "loss": 1.099092960357666,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6007935404777527,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.18332718312740326,
      "rewards/margins": 0.6362053155899048,
      "rewards/rejected": -0.8195324540138245,
      "step": 25,
      "train_speed(iter/s)": 0.010729
    },
    {
      "epoch": 0.09075043630017451,
      "grad_norm": 4.300631523132324,
      "learning_rate": 0.00012093023255813953,
      "logits/chosen": -0.3624393343925476,
      "logits/rejected": -0.40366220474243164,
      "logps/chosen": -13.089804649353027,
      "logps/rejected": -20.56357192993164,
      "loss": 1.241166114807129,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6592291593551636,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10197470337152481,
      "rewards/margins": 0.3783476948738098,
      "rewards/rejected": -0.4803224205970764,
      "step": 26,
      "train_speed(iter/s)": 0.010728
    },
    {
      "epoch": 0.09424083769633508,
      "grad_norm": 6.326510906219482,
      "learning_rate": 0.0001255813953488372,
      "logits/chosen": -0.44795599579811096,
      "logits/rejected": -0.5086410045623779,
      "logps/chosen": -6.848785400390625,
      "logps/rejected": -16.293134689331055,
      "loss": 1.1223118305206299,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5288960933685303,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07493848353624344,
      "rewards/margins": 0.3489125967025757,
      "rewards/rejected": -0.4238511323928833,
      "step": 27,
      "train_speed(iter/s)": 0.010732
    },
    {
      "epoch": 0.09773123909249563,
      "grad_norm": 3.89947509765625,
      "learning_rate": 0.0001302325581395349,
      "logits/chosen": -0.3942677080631256,
      "logits/rejected": -0.44409164786338806,
      "logps/chosen": -12.765388488769531,
      "logps/rejected": -21.65694236755371,
      "loss": 1.2387657165527344,
      "memory(GiB)": 34.14,
      "nll_loss": 0.7279061675071716,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10546489804983139,
      "rewards/margins": 0.5104267001152039,
      "rewards/rejected": -0.6158915758132935,
      "step": 28,
      "train_speed(iter/s)": 0.010734
    },
    {
      "epoch": 0.1012216404886562,
      "grad_norm": 2.5956175327301025,
      "learning_rate": 0.00013488372093023256,
      "logits/chosen": -0.4249032139778137,
      "logits/rejected": -0.47334301471710205,
      "logps/chosen": -8.39433765411377,
      "logps/rejected": -19.74716567993164,
      "loss": 1.050766110420227,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5741204619407654,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.02486160770058632,
      "rewards/margins": 0.6779846549034119,
      "rewards/rejected": -0.6531230211257935,
      "step": 29,
      "train_speed(iter/s)": 0.010736
    },
    {
      "epoch": 0.10471204188481675,
      "grad_norm": 3.123206377029419,
      "learning_rate": 0.00013953488372093025,
      "logits/chosen": -0.3965456485748291,
      "logits/rejected": -0.5027358531951904,
      "logps/chosen": -4.980073928833008,
      "logps/rejected": -25.411418914794922,
      "loss": 0.8653208613395691,
      "memory(GiB)": 34.14,
      "nll_loss": 0.4148715138435364,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.021732095628976822,
      "rewards/margins": 0.7521398663520813,
      "rewards/rejected": -0.73040771484375,
      "step": 30,
      "train_speed(iter/s)": 0.010738
    },
    {
      "epoch": 0.1082024432809773,
      "grad_norm": 3.027268409729004,
      "learning_rate": 0.00014418604651162791,
      "logits/chosen": -0.40849679708480835,
      "logits/rejected": -0.47468990087509155,
      "logps/chosen": -7.037868499755859,
      "logps/rejected": -21.788843154907227,
      "loss": 1.0142453908920288,
      "memory(GiB)": 34.14,
      "nll_loss": 0.48545634746551514,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.026916703209280968,
      "rewards/margins": 0.5216760039329529,
      "rewards/rejected": -0.5485926866531372,
      "step": 31,
      "train_speed(iter/s)": 0.010739
    },
    {
      "epoch": 0.11169284467713787,
      "grad_norm": 4.0368146896362305,
      "learning_rate": 0.00014883720930232558,
      "logits/chosen": -0.3984982967376709,
      "logits/rejected": -0.47634080052375793,
      "logps/chosen": -14.304816246032715,
      "logps/rejected": -21.326597213745117,
      "loss": 1.2883504629135132,
      "memory(GiB)": 34.14,
      "nll_loss": 0.6627980470657349,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.268436074256897,
      "rewards/margins": 0.32418739795684814,
      "rewards/rejected": -0.5926234722137451,
      "step": 32,
      "train_speed(iter/s)": 0.010739
    },
    {
      "epoch": 0.11518324607329843,
      "grad_norm": 2.3341856002807617,
      "learning_rate": 0.00015348837209302327,
      "logits/chosen": -0.34923142194747925,
      "logits/rejected": -0.45246458053588867,
      "logps/chosen": -7.6517744064331055,
      "logps/rejected": -26.005094528198242,
      "loss": 0.8868998885154724,
      "memory(GiB)": 34.14,
      "nll_loss": 0.45535364747047424,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.009979171678423882,
      "rewards/margins": 0.8548348546028137,
      "rewards/rejected": -0.8448556661605835,
      "step": 33,
      "train_speed(iter/s)": 0.010739
    },
    {
      "epoch": 0.11867364746945899,
      "grad_norm": 3.8161845207214355,
      "learning_rate": 0.00015813953488372093,
      "logits/chosen": -0.4433535933494568,
      "logits/rejected": -0.4599112570285797,
      "logps/chosen": -7.665817737579346,
      "logps/rejected": -16.353017807006836,
      "loss": 0.9908475279808044,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5115343332290649,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05993407219648361,
      "rewards/margins": 0.6281672716140747,
      "rewards/rejected": -0.5682331919670105,
      "step": 34,
      "train_speed(iter/s)": 0.010742
    },
    {
      "epoch": 0.12216404886561955,
      "grad_norm": 2.7297890186309814,
      "learning_rate": 0.00016279069767441862,
      "logits/chosen": -0.32466650009155273,
      "logits/rejected": -0.4645735025405884,
      "logps/chosen": -6.396218299865723,
      "logps/rejected": -25.87699317932129,
      "loss": 0.8697510957717896,
      "memory(GiB)": 34.14,
      "nll_loss": 0.38066551089286804,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11624527722597122,
      "rewards/margins": 0.6377441883087158,
      "rewards/rejected": -0.5214988589286804,
      "step": 35,
      "train_speed(iter/s)": 0.010743
    },
    {
      "epoch": 0.1256544502617801,
      "grad_norm": 3.7659413814544678,
      "learning_rate": 0.00016744186046511629,
      "logits/chosen": -0.3423185348510742,
      "logits/rejected": -0.5114607214927673,
      "logps/chosen": -6.822861671447754,
      "logps/rejected": -25.182849884033203,
      "loss": 1.0008474588394165,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5603237152099609,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10806835442781448,
      "rewards/margins": 0.8264971375465393,
      "rewards/rejected": -0.7184286713600159,
      "step": 36,
      "train_speed(iter/s)": 0.010743
    },
    {
      "epoch": 0.12914485165794065,
      "grad_norm": 5.439046382904053,
      "learning_rate": 0.00017209302325581395,
      "logits/chosen": -0.4786157011985779,
      "logits/rejected": -0.4922429919242859,
      "logps/chosen": -6.435626983642578,
      "logps/rejected": -17.2198429107666,
      "loss": 0.9384127259254456,
      "memory(GiB)": 34.14,
      "nll_loss": 0.43419790267944336,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.02293190360069275,
      "rewards/margins": 0.5868319869041443,
      "rewards/rejected": -0.5639001131057739,
      "step": 37,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 0.13263525305410123,
      "grad_norm": 3.3508763313293457,
      "learning_rate": 0.00017674418604651164,
      "logits/chosen": -0.42465993762016296,
      "logits/rejected": -0.4925902783870697,
      "logps/chosen": -8.958169937133789,
      "logps/rejected": -20.73528289794922,
      "loss": 1.111018180847168,
      "memory(GiB)": 34.14,
      "nll_loss": 0.7000492811203003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15357299149036407,
      "rewards/margins": 1.0291643142700195,
      "rewards/rejected": -0.8755912780761719,
      "step": 38,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 0.13612565445026178,
      "grad_norm": 3.021416664123535,
      "learning_rate": 0.0001813953488372093,
      "logits/chosen": -0.4066935181617737,
      "logits/rejected": -0.4705404043197632,
      "logps/chosen": -9.284646987915039,
      "logps/rejected": -20.46940803527832,
      "loss": 0.9931483864784241,
      "memory(GiB)": 34.14,
      "nll_loss": 0.5600231885910034,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11639773845672607,
      "rewards/margins": 0.8292201161384583,
      "rewards/rejected": -0.7128223776817322,
      "step": 39,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.13961605584642234,
      "grad_norm": 3.415731191635132,
      "learning_rate": 0.000186046511627907,
      "logits/chosen": -0.348653107881546,
      "logits/rejected": -0.5043780207633972,
      "logps/chosen": -2.844910144805908,
      "logps/rejected": -24.771116256713867,
      "loss": 0.7038365006446838,
      "memory(GiB)": 34.14,
      "nll_loss": 0.29491642117500305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14402121305465698,
      "rewards/margins": 1.1453313827514648,
      "rewards/rejected": -1.0013102293014526,
      "step": 40,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.1431064572425829,
      "grad_norm": 3.0232815742492676,
      "learning_rate": 0.00019069767441860466,
      "logits/chosen": -0.35887610912323,
      "logits/rejected": -0.38277357816696167,
      "logps/chosen": -10.264820098876953,
      "logps/rejected": -24.83388328552246,
      "loss": 0.8683040738105774,
      "memory(GiB)": 34.14,
      "nll_loss": 0.4005107581615448,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.013488545082509518,
      "rewards/margins": 0.8365979194641113,
      "rewards/rejected": -0.8231093883514404,
      "step": 41,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.14659685863874344,
      "grad_norm": 16.639280319213867,
      "learning_rate": 0.00019534883720930232,
      "logits/chosen": -0.33245426416397095,
      "logits/rejected": -0.4030519127845764,
      "logps/chosen": -10.0272216796875,
      "logps/rejected": -25.483577728271484,
      "loss": 1.21886146068573,
      "memory(GiB)": 34.14,
      "nll_loss": 0.7048445343971252,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.14778879284858704,
      "rewards/margins": 1.201229453086853,
      "rewards/rejected": -1.3490180969238281,
      "step": 42,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.15008726003490402,
      "grad_norm": 6.577022075653076,
      "learning_rate": 0.0002,
      "logits/chosen": -0.29526323080062866,
      "logits/rejected": -0.45897889137268066,
      "logps/chosen": -5.878642559051514,
      "logps/rejected": -39.53807830810547,
      "loss": 0.7673805356025696,
      "memory(GiB)": 35.89,
      "nll_loss": 0.4374830424785614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.023150455206632614,
      "rewards/margins": 1.816420555114746,
      "rewards/rejected": -1.8395708799362183,
      "step": 43,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.15357766143106458,
      "grad_norm": 2.505460262298584,
      "learning_rate": 0.00019999925705956715,
      "logits/chosen": -0.37890657782554626,
      "logits/rejected": -0.44291937351226807,
      "logps/chosen": -8.378349304199219,
      "logps/rejected": -32.722042083740234,
      "loss": 0.8200633525848389,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5234110355377197,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08434319496154785,
      "rewards/margins": 1.911144495010376,
      "rewards/rejected": -1.8268014192581177,
      "step": 44,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.15706806282722513,
      "grad_norm": 6.880366802215576,
      "learning_rate": 0.0001999970282493078,
      "logits/chosen": -0.328549325466156,
      "logits/rejected": -0.4838561713695526,
      "logps/chosen": -6.23681116104126,
      "logps/rejected": -29.523792266845703,
      "loss": 1.0678659677505493,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5795442461967468,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0050928061828017235,
      "rewards/margins": 1.2474907636642456,
      "rewards/rejected": -1.2423979043960571,
      "step": 45,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.16055846422338568,
      "grad_norm": 6.962847709655762,
      "learning_rate": 0.00019999331360233944,
      "logits/chosen": -0.33589401841163635,
      "logits/rejected": -0.45322129130363464,
      "logps/chosen": -6.451783657073975,
      "logps/rejected": -32.003658294677734,
      "loss": 0.9696826338768005,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5726639032363892,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.014406954869627953,
      "rewards/margins": 1.381211757659912,
      "rewards/rejected": -1.3956186771392822,
      "step": 46,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.16404886561954624,
      "grad_norm": 7.938783168792725,
      "learning_rate": 0.00019998811317385729,
      "logits/chosen": -0.3734630346298218,
      "logits/rejected": -0.4592462480068207,
      "logps/chosen": -12.47143268585205,
      "logps/rejected": -26.8325138092041,
      "loss": 1.141100287437439,
      "memory(GiB)": 35.89,
      "nll_loss": 0.6339684724807739,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.027786267921328545,
      "rewards/margins": 0.6427249312400818,
      "rewards/rejected": -0.6149386167526245,
      "step": 47,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.16753926701570682,
      "grad_norm": 2.760986804962158,
      "learning_rate": 0.0001999814270411335,
      "logits/chosen": -0.43143945932388306,
      "logits/rejected": -0.4347304701805115,
      "logps/chosen": -9.348878860473633,
      "logps/rejected": -24.229524612426758,
      "loss": 0.9695644974708557,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5839965343475342,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.007340218871831894,
      "rewards/margins": 1.3898271322250366,
      "rewards/rejected": -1.3824869394302368,
      "step": 48,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.17102966841186737,
      "grad_norm": 11.194586753845215,
      "learning_rate": 0.00019997325530351606,
      "logits/chosen": -0.3548398017883301,
      "logits/rejected": -0.4637337028980255,
      "logps/chosen": -16.913610458374023,
      "logps/rejected": -33.84115982055664,
      "loss": 1.521134614944458,
      "memory(GiB)": 35.89,
      "nll_loss": 1.0307905673980713,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.28235313296318054,
      "rewards/margins": 1.413867712020874,
      "rewards/rejected": -1.6962207555770874,
      "step": 49,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.17452006980802792,
      "grad_norm": 3.4211995601654053,
      "learning_rate": 0.00019996359808242722,
      "logits/chosen": -0.32980048656463623,
      "logits/rejected": -0.5140572190284729,
      "logps/chosen": -4.228909492492676,
      "logps/rejected": -34.62148666381836,
      "loss": 0.6742458343505859,
      "memory(GiB)": 35.89,
      "nll_loss": 0.37631386518478394,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08238644152879715,
      "rewards/margins": 1.9769954681396484,
      "rewards/rejected": -1.894608974456787,
      "step": 50,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.17801047120418848,
      "grad_norm": 4.199666500091553,
      "learning_rate": 0.00019995245552136183,
      "logits/chosen": -0.38337764143943787,
      "logits/rejected": -0.5411950349807739,
      "logps/chosen": -7.726480484008789,
      "logps/rejected": -33.165184020996094,
      "loss": 0.8604421615600586,
      "memory(GiB)": 35.89,
      "nll_loss": 0.4990411400794983,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0083555206656456,
      "rewards/margins": 1.5133056640625,
      "rewards/rejected": -1.5049501657485962,
      "step": 51,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.18150087260034903,
      "grad_norm": 7.909679412841797,
      "learning_rate": 0.00019993982778588506,
      "logits/chosen": -0.42613935470581055,
      "logits/rejected": -0.5141798257827759,
      "logps/chosen": -12.04079818725586,
      "logps/rejected": -32.32722091674805,
      "loss": 1.079092264175415,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5544219613075256,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.21235862374305725,
      "rewards/margins": 1.353042721748352,
      "rewards/rejected": -1.565401315689087,
      "step": 52,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.1849912739965096,
      "grad_norm": 3.7452335357666016,
      "learning_rate": 0.00019992571506363,
      "logits/chosen": -0.3618060052394867,
      "logits/rejected": -0.5190442800521851,
      "logps/chosen": -6.607875347137451,
      "logps/rejected": -25.438873291015625,
      "loss": 1.0128110647201538,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5347008109092712,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0013710036873817444,
      "rewards/margins": 0.7542194724082947,
      "rewards/rejected": -0.7555904388427734,
      "step": 53,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.18848167539267016,
      "grad_norm": 2.7128043174743652,
      "learning_rate": 0.00019991011756429486,
      "logits/chosen": -0.43821805715560913,
      "logits/rejected": -0.48400819301605225,
      "logps/chosen": -8.567977905273438,
      "logps/rejected": -19.301103591918945,
      "loss": 1.0873574018478394,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5480538010597229,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.06769907474517822,
      "rewards/margins": 0.6852700114250183,
      "rewards/rejected": -0.6175709366798401,
      "step": 54,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.19197207678883071,
      "grad_norm": 2.5323894023895264,
      "learning_rate": 0.00019989303551963997,
      "logits/chosen": -0.3141109347343445,
      "logits/rejected": -0.48292648792266846,
      "logps/chosen": -4.789846420288086,
      "logps/rejected": -26.765024185180664,
      "loss": 0.880359411239624,
      "memory(GiB)": 35.89,
      "nll_loss": 0.3599545359611511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09160856902599335,
      "rewards/margins": 0.7724493145942688,
      "rewards/rejected": -0.6808407306671143,
      "step": 55,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.19546247818499127,
      "grad_norm": 2.958927869796753,
      "learning_rate": 0.00019987446918348411,
      "logits/chosen": -0.3392045199871063,
      "logits/rejected": -0.4613609313964844,
      "logps/chosen": -6.773719310760498,
      "logps/rejected": -20.55025291442871,
      "loss": 1.0082640647888184,
      "memory(GiB)": 35.89,
      "nll_loss": 0.44486063718795776,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.06419242918491364,
      "rewards/margins": 0.5171071887016296,
      "rewards/rejected": -0.4529147446155548,
      "step": 56,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.19895287958115182,
      "grad_norm": 3.204970359802246,
      "learning_rate": 0.00019985441883170094,
      "logits/chosen": -0.45556554198265076,
      "logits/rejected": -0.47999584674835205,
      "logps/chosen": -8.982965469360352,
      "logps/rejected": -18.054445266723633,
      "loss": 0.996261477470398,
      "memory(GiB)": 35.89,
      "nll_loss": 0.4766736924648285,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07613325119018555,
      "rewards/margins": 0.774691104888916,
      "rewards/rejected": -0.6985578536987305,
      "step": 57,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.2024432809773124,
      "grad_norm": 3.0290675163269043,
      "learning_rate": 0.0001998328847622148,
      "logits/chosen": -0.43794357776641846,
      "logits/rejected": -0.4973166882991791,
      "logps/chosen": -6.941474437713623,
      "logps/rejected": -21.18113136291504,
      "loss": 0.9271993637084961,
      "memory(GiB)": 35.89,
      "nll_loss": 0.40610748529434204,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.031344618648290634,
      "rewards/margins": 0.7323437929153442,
      "rewards/rejected": -0.7009991407394409,
      "step": 58,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.20593368237347295,
      "grad_norm": 3.3924288749694824,
      "learning_rate": 0.0001998098672949963,
      "logits/chosen": -0.43890056014060974,
      "logits/rejected": -0.47092944383621216,
      "logps/chosen": -6.540399074554443,
      "logps/rejected": -23.68883514404297,
      "loss": 0.8886168003082275,
      "memory(GiB)": 35.89,
      "nll_loss": 0.42720285058021545,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.062318772077560425,
      "rewards/margins": 1.219529151916504,
      "rewards/rejected": -1.1572105884552002,
      "step": 59,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.2094240837696335,
      "grad_norm": 6.436001300811768,
      "learning_rate": 0.00019978536677205753,
      "logits/chosen": -0.3602336645126343,
      "logits/rejected": -0.49971872568130493,
      "logps/chosen": -7.300111770629883,
      "logps/rejected": -35.16114044189453,
      "loss": 0.893642783164978,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5613251328468323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04828762263059616,
      "rewards/margins": 2.0058753490448,
      "rewards/rejected": -2.0541629791259766,
      "step": 60,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.21291448516579406,
      "grad_norm": 9.151269912719727,
      "learning_rate": 0.0001997593835574472,
      "logits/chosen": -0.2997259497642517,
      "logits/rejected": -0.4807735085487366,
      "logps/chosen": -7.510552406311035,
      "logps/rejected": -51.04875946044922,
      "loss": 0.7812849879264832,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5295180082321167,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.27329760789871216,
      "rewards/margins": 3.0174202919006348,
      "rewards/rejected": -3.2907180786132812,
      "step": 61,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2164048865619546,
      "grad_norm": 3.0901706218719482,
      "learning_rate": 0.00019973191803724483,
      "logits/chosen": -0.3464053273200989,
      "logits/rejected": -0.43850764632225037,
      "logps/chosen": -5.784665107727051,
      "logps/rejected": -29.168745040893555,
      "loss": 0.6676730513572693,
      "memory(GiB)": 35.89,
      "nll_loss": 0.2769017219543457,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.09001894295215607,
      "rewards/margins": 1.1760774850845337,
      "rewards/rejected": -1.0860586166381836,
      "step": 62,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2198952879581152,
      "grad_norm": 3.0961596965789795,
      "learning_rate": 0.00019970297061955533,
      "logits/chosen": -0.35332319140434265,
      "logits/rejected": -0.5135117173194885,
      "logps/chosen": -5.485988616943359,
      "logps/rejected": -36.72576141357422,
      "loss": 0.7853246331214905,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5297773480415344,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.02064245566725731,
      "rewards/margins": 2.3967862129211426,
      "rewards/rejected": -2.37614369392395,
      "step": 63,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.22338568935427575,
      "grad_norm": 4.25160026550293,
      "learning_rate": 0.00019967254173450287,
      "logits/chosen": -0.2867816090583801,
      "logits/rejected": -0.44146350026130676,
      "logps/chosen": -5.827723026275635,
      "logps/rejected": -31.073637008666992,
      "loss": 0.7783889770507812,
      "memory(GiB)": 35.89,
      "nll_loss": 0.43023356795310974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09779734909534454,
      "rewards/margins": 1.5616310834884644,
      "rewards/rejected": -1.4638336896896362,
      "step": 64,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.2268760907504363,
      "grad_norm": 2.711167097091675,
      "learning_rate": 0.0001996406318342244,
      "logits/chosen": -0.351021945476532,
      "logits/rejected": -0.4185492694377899,
      "logps/chosen": -6.972557544708252,
      "logps/rejected": -29.75743865966797,
      "loss": 0.7590102553367615,
      "memory(GiB)": 35.89,
      "nll_loss": 0.42238616943359375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07267698645591736,
      "rewards/margins": 1.7358276844024658,
      "rewards/rejected": -1.663150429725647,
      "step": 65,
      "train_speed(iter/s)": 0.010761
    },
    {
      "epoch": 0.23036649214659685,
      "grad_norm": 5.093292236328125,
      "learning_rate": 0.00019960724139286308,
      "logits/chosen": -0.29837629199028015,
      "logits/rejected": -0.38664788007736206,
      "logps/chosen": -11.696621894836426,
      "logps/rejected": -34.02775192260742,
      "loss": 1.0549557209014893,
      "memory(GiB)": 35.89,
      "nll_loss": 0.7019686102867126,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.16252195835113525,
      "rewards/margins": 1.5406640768051147,
      "rewards/rejected": -1.70318603515625,
      "step": 66,
      "train_speed(iter/s)": 0.010761
    },
    {
      "epoch": 0.2338568935427574,
      "grad_norm": 3.672001838684082,
      "learning_rate": 0.00019957237090656102,
      "logits/chosen": -0.2561183571815491,
      "logits/rejected": -0.4472733438014984,
      "logps/chosen": -6.693790912628174,
      "logps/rejected": -52.02687072753906,
      "loss": 0.6644150614738464,
      "memory(GiB)": 35.89,
      "nll_loss": 0.44371137022972107,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.018377352505922318,
      "rewards/margins": 2.62888765335083,
      "rewards/rejected": -2.6472654342651367,
      "step": 67,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.23734729493891799,
      "grad_norm": 8.564802169799805,
      "learning_rate": 0.00019953602089345217,
      "logits/chosen": -0.3133094310760498,
      "logits/rejected": -0.45969486236572266,
      "logps/chosen": -7.815582275390625,
      "logps/rejected": -38.529781341552734,
      "loss": 1.0037181377410889,
      "memory(GiB)": 35.89,
      "nll_loss": 0.6282736659049988,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1540919542312622,
      "rewards/margins": 1.7298449277877808,
      "rewards/rejected": -1.883936882019043,
      "step": 68,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.24083769633507854,
      "grad_norm": 12.39240837097168,
      "learning_rate": 0.00019949819189365433,
      "logits/chosen": -0.3124208152294159,
      "logits/rejected": -0.43993785977363586,
      "logps/chosen": -7.740966796875,
      "logps/rejected": -34.374534606933594,
      "loss": 0.9371472001075745,
      "memory(GiB)": 35.89,
      "nll_loss": 0.46057644486427307,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0446137860417366,
      "rewards/margins": 1.5109288692474365,
      "rewards/rejected": -1.5555427074432373,
      "step": 69,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.2443280977312391,
      "grad_norm": 6.788973331451416,
      "learning_rate": 0.00019945888446926143,
      "logits/chosen": -0.3286086618900299,
      "logits/rejected": -0.40763434767723083,
      "logps/chosen": -10.763023376464844,
      "logps/rejected": -23.91502571105957,
      "loss": 1.182040810585022,
      "memory(GiB)": 35.89,
      "nll_loss": 0.6381036639213562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1615467518568039,
      "rewards/margins": 0.6939660906791687,
      "rewards/rejected": -0.8555128574371338,
      "step": 70,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.24781849912739964,
      "grad_norm": 4.333127975463867,
      "learning_rate": 0.000199418099204335,
      "logits/chosen": -0.3585875630378723,
      "logits/rejected": -0.44930267333984375,
      "logps/chosen": -8.16705322265625,
      "logps/rejected": -22.100000381469727,
      "loss": 0.9709787368774414,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5348560214042664,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04899175092577934,
      "rewards/margins": 0.8827412128448486,
      "rewards/rejected": -0.9317330121994019,
      "step": 71,
      "train_speed(iter/s)": 0.010761
    },
    {
      "epoch": 0.2513089005235602,
      "grad_norm": 3.37380051612854,
      "learning_rate": 0.00019937583670489545,
      "logits/chosen": -0.33419570326805115,
      "logits/rejected": -0.43994009494781494,
      "logps/chosen": -9.082391738891602,
      "logps/rejected": -28.777481079101562,
      "loss": 0.9270099401473999,
      "memory(GiB)": 35.89,
      "nll_loss": 0.48707476258277893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.024601396173238754,
      "rewards/margins": 1.1600151062011719,
      "rewards/rejected": -1.1354137659072876,
      "step": 72,
      "train_speed(iter/s)": 0.010761
    },
    {
      "epoch": 0.2547993019197208,
      "grad_norm": 3.593212366104126,
      "learning_rate": 0.00019933209759891317,
      "logits/chosen": -0.4130091071128845,
      "logits/rejected": -0.4818781018257141,
      "logps/chosen": -10.08952808380127,
      "logps/rejected": -34.58759307861328,
      "loss": 0.9288522601127625,
      "memory(GiB)": 35.89,
      "nll_loss": 0.6054138541221619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11658284813165665,
      "rewards/margins": 2.089298963546753,
      "rewards/rejected": -2.2058815956115723,
      "step": 73,
      "train_speed(iter/s)": 0.010762
    },
    {
      "epoch": 0.2582897033158813,
      "grad_norm": 3.8672237396240234,
      "learning_rate": 0.00019928688253629916,
      "logits/chosen": -0.19415511190891266,
      "logits/rejected": -0.36130261421203613,
      "logps/chosen": -11.009477615356445,
      "logps/rejected": -43.973114013671875,
      "loss": 1.1013773679733276,
      "memory(GiB)": 35.89,
      "nll_loss": 0.6544516086578369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.030637577176094055,
      "rewards/margins": 2.5454745292663574,
      "rewards/rejected": -2.5761120319366455,
      "step": 74,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.2617801047120419,
      "grad_norm": 4.066303730010986,
      "learning_rate": 0.00019924019218889536,
      "logits/chosen": -0.33296871185302734,
      "logits/rejected": -0.4902675747871399,
      "logps/chosen": -9.64806079864502,
      "logps/rejected": -37.88252639770508,
      "loss": 0.8953545093536377,
      "memory(GiB)": 35.89,
      "nll_loss": 0.5640730857849121,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.009408235549926758,
      "rewards/margins": 2.1965668201446533,
      "rewards/rejected": -2.1871585845947266,
      "step": 75,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.26527050610820246,
      "grad_norm": 2.3979082107543945,
      "learning_rate": 0.00019919202725046477,
      "logits/chosen": -0.20567730069160461,
      "logits/rejected": -0.34439828991889954,
      "logps/chosen": -8.100298881530762,
      "logps/rejected": -39.28445816040039,
      "loss": 0.7528076767921448,
      "memory(GiB)": 39.42,
      "nll_loss": 0.39297810196876526,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025398975238204002,
      "rewards/margins": 1.6433254480361938,
      "rewards/rejected": -1.6179265975952148,
      "step": 76,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.268760907504363,
      "grad_norm": 2.7927446365356445,
      "learning_rate": 0.00019914238843668096,
      "logits/chosen": -0.3428923785686493,
      "logits/rejected": -0.4334644675254822,
      "logps/chosen": -6.549041271209717,
      "logps/rejected": -27.63149070739746,
      "loss": 0.796095609664917,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3915262818336487,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06563803553581238,
      "rewards/margins": 1.2631423473358154,
      "rewards/rejected": -1.1975042819976807,
      "step": 77,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.27225130890052357,
      "grad_norm": 4.248587608337402,
      "learning_rate": 0.00019909127648511755,
      "logits/chosen": -0.3395402431488037,
      "logits/rejected": -0.48165830969810486,
      "logps/chosen": -5.904273986816406,
      "logps/rejected": -27.16798973083496,
      "loss": 0.743380606174469,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3838931918144226,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10840368270874023,
      "rewards/margins": 1.24807870388031,
      "rewards/rejected": -1.1396750211715698,
      "step": 78,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2757417102966841,
      "grad_norm": 2.7059414386749268,
      "learning_rate": 0.00019903869215523733,
      "logits/chosen": -0.22887416183948517,
      "logits/rejected": -0.48973891139030457,
      "logps/chosen": -3.493281364440918,
      "logps/rejected": -34.020050048828125,
      "loss": 0.6441786885261536,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3432372510433197,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12880486249923706,
      "rewards/margins": 1.6592087745666504,
      "rewards/rejected": -1.5304038524627686,
      "step": 79,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2792321116928447,
      "grad_norm": 4.634321212768555,
      "learning_rate": 0.0001989846362283807,
      "logits/chosen": -0.29562249779701233,
      "logits/rejected": -0.34773433208465576,
      "logps/chosen": -11.475696563720703,
      "logps/rejected": -27.678905487060547,
      "loss": 1.1467502117156982,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6945275664329529,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10402540862560272,
      "rewards/margins": 1.2328475713729858,
      "rewards/rejected": -1.336873173713684,
      "step": 80,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.28272251308900526,
      "grad_norm": 3.8445465564727783,
      "learning_rate": 0.00019892910950775435,
      "logits/chosen": -0.23248018324375153,
      "logits/rejected": -0.35242336988449097,
      "logps/chosen": -6.628226280212402,
      "logps/rejected": -33.20770263671875,
      "loss": 0.8408610224723816,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5566035509109497,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13027045130729675,
      "rewards/margins": 2.089359998703003,
      "rewards/rejected": -1.9590896368026733,
      "step": 81,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2862129144851658,
      "grad_norm": 2.7230985164642334,
      "learning_rate": 0.00019887211281841924,
      "logits/chosen": -0.2518586218357086,
      "logits/rejected": -0.31852811574935913,
      "logps/chosen": -8.052734375,
      "logps/rejected": -28.10201644897461,
      "loss": 0.7522032856941223,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3988189697265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.162813201546669,
      "rewards/margins": 1.698049783706665,
      "rewards/rejected": -1.5352365970611572,
      "step": 82,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.28970331588132636,
      "grad_norm": 5.669232368469238,
      "learning_rate": 0.00019881364700727823,
      "logits/chosen": -0.13075971603393555,
      "logits/rejected": -0.3868337869644165,
      "logps/chosen": -9.65750503540039,
      "logps/rejected": -36.58469772338867,
      "loss": 1.048365831375122,
      "memory(GiB)": 39.42,
      "nll_loss": 0.7252820730209351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21855562925338745,
      "rewards/margins": 1.9177855253219604,
      "rewards/rejected": -1.6992299556732178,
      "step": 83,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.2931937172774869,
      "grad_norm": 2.559206962585449,
      "learning_rate": 0.00019875371294306366,
      "logits/chosen": -0.07886800169944763,
      "logits/rejected": -0.302437424659729,
      "logps/chosen": -6.699373722076416,
      "logps/rejected": -43.041194915771484,
      "loss": 0.7339796423912048,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4990921914577484,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12242552638053894,
      "rewards/margins": 2.4995968341827393,
      "rewards/rejected": -2.377171277999878,
      "step": 84,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.29668411867364747,
      "grad_norm": 3.492406129837036,
      "learning_rate": 0.0001986923115163243,
      "logits/chosen": -0.13503603637218475,
      "logits/rejected": -0.3026173710823059,
      "logps/chosen": -6.315262317657471,
      "logps/rejected": -38.55803680419922,
      "loss": 0.8880235552787781,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5917512774467468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04879331588745117,
      "rewards/margins": 1.9771116971969604,
      "rewards/rejected": -1.9283183813095093,
      "step": 85,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.30017452006980805,
      "grad_norm": 3.961932897567749,
      "learning_rate": 0.00019862944363941218,
      "logits/chosen": -0.10436159372329712,
      "logits/rejected": -0.3089469075202942,
      "logps/chosen": -6.1503825187683105,
      "logps/rejected": -38.23727035522461,
      "loss": 0.90598064661026,
      "memory(GiB)": 39.42,
      "nll_loss": 0.633453905582428,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14362435042858124,
      "rewards/margins": 2.058913469314575,
      "rewards/rejected": -1.915289044380188,
      "step": 86,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.3036649214659686,
      "grad_norm": 3.4905996322631836,
      "learning_rate": 0.0001985651102464691,
      "logits/chosen": -0.11853908002376556,
      "logits/rejected": -0.24401453137397766,
      "logps/chosen": -7.3659820556640625,
      "logps/rejected": -38.6995849609375,
      "loss": 0.749198853969574,
      "memory(GiB)": 39.42,
      "nll_loss": 0.43377092480659485,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.14165769517421722,
      "rewards/margins": 2.025956630706787,
      "rewards/rejected": -2.167614459991455,
      "step": 87,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.30715532286212915,
      "grad_norm": 3.6831657886505127,
      "learning_rate": 0.00019849931229341258,
      "logits/chosen": -0.1332027167081833,
      "logits/rejected": -0.24646437168121338,
      "logps/chosen": -8.003013610839844,
      "logps/rejected": -37.04756164550781,
      "loss": 0.8140410780906677,
      "memory(GiB)": 39.42,
      "nll_loss": 0.44854581356048584,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.034915290772914886,
      "rewards/margins": 1.382434606552124,
      "rewards/rejected": -1.3475191593170166,
      "step": 88,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.3106457242582897,
      "grad_norm": 2.4958512783050537,
      "learning_rate": 0.00019843205075792186,
      "logits/chosen": -0.11535613983869553,
      "logits/rejected": -0.246973916888237,
      "logps/chosen": -7.1996660232543945,
      "logps/rejected": -32.96755599975586,
      "loss": 0.7998999357223511,
      "memory(GiB)": 39.42,
      "nll_loss": 0.49011173844337463,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08970604836940765,
      "rewards/margins": 1.6430060863494873,
      "rewards/rejected": -1.553299903869629,
      "step": 89,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.31413612565445026,
      "grad_norm": 2.470099687576294,
      "learning_rate": 0.0001983633266394232,
      "logits/chosen": -0.13420692086219788,
      "logits/rejected": -0.3025938868522644,
      "logps/chosen": -10.592232704162598,
      "logps/rejected": -43.20408630371094,
      "loss": 0.8399574160575867,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5707088112831116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.024432912468910217,
      "rewards/margins": 1.8845049142837524,
      "rewards/rejected": -1.8600718975067139,
      "step": 90,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.31762652705061084,
      "grad_norm": 2.7454211711883545,
      "learning_rate": 0.00019829314095907516,
      "logits/chosen": -0.15840838849544525,
      "logits/rejected": -0.29640981554985046,
      "logps/chosen": -7.358951568603516,
      "logps/rejected": -35.788387298583984,
      "loss": 0.7901937961578369,
      "memory(GiB)": 39.42,
      "nll_loss": 0.48892611265182495,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.018669307231903076,
      "rewards/margins": 1.815453290939331,
      "rewards/rejected": -1.7967840433120728,
      "step": 91,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.32111692844677137,
      "grad_norm": 2.443572998046875,
      "learning_rate": 0.0001982214947597533,
      "logits/chosen": -0.13980664312839508,
      "logits/rejected": -0.2785203754901886,
      "logps/chosen": -7.595067024230957,
      "logps/rejected": -29.64536476135254,
      "loss": 0.7760062217712402,
      "memory(GiB)": 39.42,
      "nll_loss": 0.417621374130249,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18094170093536377,
      "rewards/margins": 1.5685675144195557,
      "rewards/rejected": -1.3876259326934814,
      "step": 92,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.32460732984293195,
      "grad_norm": 2.695469856262207,
      "learning_rate": 0.00019814838910603481,
      "logits/chosen": -0.21802689135074615,
      "logits/rejected": -0.30977922677993774,
      "logps/chosen": -8.865757942199707,
      "logps/rejected": -31.998289108276367,
      "loss": 0.8171509504318237,
      "memory(GiB)": 39.42,
      "nll_loss": 0.46762198209762573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08467155694961548,
      "rewards/margins": 1.8997126817703247,
      "rewards/rejected": -1.815041184425354,
      "step": 93,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.32809773123909247,
      "grad_norm": 2.3831567764282227,
      "learning_rate": 0.0001980738250841826,
      "logits/chosen": -0.1939636468887329,
      "logits/rejected": -0.33366823196411133,
      "logps/chosen": -8.547904968261719,
      "logps/rejected": -36.44358444213867,
      "loss": 0.8093112707138062,
      "memory(GiB)": 39.42,
      "nll_loss": 0.46711432933807373,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.023340808227658272,
      "rewards/margins": 1.871730923652649,
      "rewards/rejected": -1.8483901023864746,
      "step": 94,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.33158813263525305,
      "grad_norm": 4.449499130249023,
      "learning_rate": 0.0001979978038021292,
      "logits/chosen": -0.21011316776275635,
      "logits/rejected": -0.3800049126148224,
      "logps/chosen": -6.6607513427734375,
      "logps/rejected": -37.015220642089844,
      "loss": 0.7690098285675049,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3787845969200134,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.022871140390634537,
      "rewards/margins": 1.7818647623062134,
      "rewards/rejected": -1.7589936256408691,
      "step": 95,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.33507853403141363,
      "grad_norm": 8.076894760131836,
      "learning_rate": 0.00019792032638946027,
      "logits/chosen": -0.22930172085762024,
      "logits/rejected": -0.363534152507782,
      "logps/chosen": -9.168333053588867,
      "logps/rejected": -39.97100067138672,
      "loss": 0.999840497970581,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5044428706169128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19479180872440338,
      "rewards/margins": 2.030325174331665,
      "rewards/rejected": -2.225116729736328,
      "step": 96,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.33856893542757416,
      "grad_norm": 2.602322578430176,
      "learning_rate": 0.00019784139399739794,
      "logits/chosen": -0.23653970658779144,
      "logits/rejected": -0.37720808386802673,
      "logps/chosen": -7.70982551574707,
      "logps/rejected": -46.72187423706055,
      "loss": 0.7515106797218323,
      "memory(GiB)": 39.42,
      "nll_loss": 0.49746939539909363,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.16890637576580048,
      "rewards/margins": 2.9459898471832275,
      "rewards/rejected": -3.114896297454834,
      "step": 97,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.34205933682373474,
      "grad_norm": 2.917598009109497,
      "learning_rate": 0.00019776100779878345,
      "logits/chosen": -0.24879562854766846,
      "logits/rejected": -0.4712115526199341,
      "logps/chosen": -4.071171760559082,
      "logps/rejected": -39.8735237121582,
      "loss": 0.6295332908630371,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3750327527523041,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12702509760856628,
      "rewards/margins": 2.281287431716919,
      "rewards/rejected": -2.1542623043060303,
      "step": 98,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.34554973821989526,
      "grad_norm": 3.531543016433716,
      "learning_rate": 0.00019767916898805993,
      "logits/chosen": -0.3587397038936615,
      "logits/rejected": -0.4245099723339081,
      "logps/chosen": -9.423014640808105,
      "logps/rejected": -32.22254943847656,
      "loss": 0.9320547580718994,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5687791109085083,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0431206040084362,
      "rewards/margins": 1.7739802598953247,
      "rewards/rejected": -1.8171007633209229,
      "step": 99,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.34904013961605584,
      "grad_norm": 2.429708242416382,
      "learning_rate": 0.00019759587878125467,
      "logits/chosen": -0.38257211446762085,
      "logits/rejected": -0.46099749207496643,
      "logps/chosen": -6.002010345458984,
      "logps/rejected": -35.495018005371094,
      "loss": 0.7784558534622192,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4933393895626068,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10934662073850632,
      "rewards/margins": 2.241340160369873,
      "rewards/rejected": -2.1319937705993652,
      "step": 100,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.3525305410122164,
      "grad_norm": 2.213499069213867,
      "learning_rate": 0.00019751113841596086,
      "logits/chosen": -0.37704822421073914,
      "logits/rejected": -0.4514527916908264,
      "logps/chosen": -6.885778427124023,
      "logps/rejected": -27.876874923706055,
      "loss": 0.704035758972168,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4308836758136749,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12775495648384094,
      "rewards/margins": 1.5644640922546387,
      "rewards/rejected": -1.4367092847824097,
      "step": 101,
      "train_speed(iter/s)": 0.010742
    },
    {
      "epoch": 0.35602094240837695,
      "grad_norm": 2.3929998874664307,
      "learning_rate": 0.00019742494915131942,
      "logits/chosen": -0.36924856901168823,
      "logits/rejected": -0.41376832127571106,
      "logps/chosen": -5.98624324798584,
      "logps/rejected": -36.84775161743164,
      "loss": 0.5458755493164062,
      "memory(GiB)": 39.42,
      "nll_loss": 0.35526004433631897,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1133466586470604,
      "rewards/margins": 2.4806315898895264,
      "rewards/rejected": -2.3672847747802734,
      "step": 102,
      "train_speed(iter/s)": 0.010743
    },
    {
      "epoch": 0.35951134380453753,
      "grad_norm": 5.081653118133545,
      "learning_rate": 0.00019733731226800015,
      "logits/chosen": -0.2884291112422943,
      "logits/rejected": -0.41856223344802856,
      "logps/chosen": -7.244772911071777,
      "logps/rejected": -43.501136779785156,
      "loss": 1.0223764181137085,
      "memory(GiB)": 39.42,
      "nll_loss": 0.7268551588058472,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.17409339547157288,
      "rewards/margins": 2.4381885528564453,
      "rewards/rejected": -2.6122817993164062,
      "step": 103,
      "train_speed(iter/s)": 0.010743
    },
    {
      "epoch": 0.36300174520069806,
      "grad_norm": 4.38121223449707,
      "learning_rate": 0.00019724822906818265,
      "logits/chosen": -0.3204030692577362,
      "logits/rejected": -0.37165117263793945,
      "logps/chosen": -7.093808650970459,
      "logps/rejected": -36.024147033691406,
      "loss": 0.746134877204895,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4921882450580597,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.053046248853206635,
      "rewards/margins": 2.321000337600708,
      "rewards/rejected": -2.267954111099243,
      "step": 104,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 0.36649214659685864,
      "grad_norm": 7.504276752471924,
      "learning_rate": 0.0001971577008755372,
      "logits/chosen": -0.2759462296962738,
      "logits/rejected": -0.3651443123817444,
      "logps/chosen": -11.585759162902832,
      "logps/rejected": -35.60843276977539,
      "loss": 1.0082931518554688,
      "memory(GiB)": 39.42,
      "nll_loss": 0.7006896734237671,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.08436793088912964,
      "rewards/margins": 2.2114996910095215,
      "rewards/rejected": -2.127131700515747,
      "step": 105,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 0.3699825479930192,
      "grad_norm": 2.724567174911499,
      "learning_rate": 0.00019706572903520494,
      "logits/chosen": -0.3390807509422302,
      "logits/rejected": -0.3694685101509094,
      "logps/chosen": -9.778353691101074,
      "logps/rejected": -30.431264877319336,
      "loss": 0.8204815983772278,
      "memory(GiB)": 39.42,
      "nll_loss": 0.487753689289093,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.11946438252925873,
      "rewards/margins": 1.8712043762207031,
      "rewards/rejected": -1.7517400979995728,
      "step": 106,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 0.37347294938917974,
      "grad_norm": 3.963045358657837,
      "learning_rate": 0.00019697231491377774,
      "logits/chosen": -0.3571932911872864,
      "logits/rejected": -0.41452813148498535,
      "logps/chosen": -6.354501724243164,
      "logps/rejected": -29.302684783935547,
      "loss": 0.8431603908538818,
      "memory(GiB)": 39.42,
      "nll_loss": 0.46558207273483276,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0002909880131483078,
      "rewards/margins": 1.4369157552719116,
      "rewards/rejected": -1.437206745147705,
      "step": 107,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 0.3769633507853403,
      "grad_norm": 4.0586652755737305,
      "learning_rate": 0.00019687745989927823,
      "logits/chosen": -0.3135523796081543,
      "logits/rejected": -0.3737596273422241,
      "logps/chosen": -9.778253555297852,
      "logps/rejected": -28.1949462890625,
      "loss": 0.966033935546875,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6015961170196533,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08021017909049988,
      "rewards/margins": 1.4340676069259644,
      "rewards/rejected": -1.353857398033142,
      "step": 108,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 0.38045375218150085,
      "grad_norm": 5.568571090698242,
      "learning_rate": 0.0001967811654011389,
      "logits/chosen": -0.309251606464386,
      "logits/rejected": -0.3893003463745117,
      "logps/chosen": -7.1915602684021,
      "logps/rejected": -27.5125732421875,
      "loss": 0.9027649164199829,
      "memory(GiB)": 39.42,
      "nll_loss": 0.49153536558151245,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.14030015468597412,
      "rewards/margins": 1.3549611568450928,
      "rewards/rejected": -1.4952614307403564,
      "step": 109,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 0.38394415357766143,
      "grad_norm": 3.212317943572998,
      "learning_rate": 0.00019668343285018125,
      "logits/chosen": -0.30074530839920044,
      "logits/rejected": -0.3879661560058594,
      "logps/chosen": -9.096397399902344,
      "logps/rejected": -42.42276382446289,
      "loss": 0.8074911236763,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5355080962181091,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.040462881326675415,
      "rewards/margins": 2.193300247192383,
      "rewards/rejected": -2.2337629795074463,
      "step": 110,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 0.387434554973822,
      "grad_norm": 4.342100620269775,
      "learning_rate": 0.0001965842636985946,
      "logits/chosen": -0.3064587414264679,
      "logits/rejected": -0.41097527742385864,
      "logps/chosen": -5.184054374694824,
      "logps/rejected": -36.894256591796875,
      "loss": 0.7728255987167358,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5083194971084595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11109408736228943,
      "rewards/margins": 2.293015241622925,
      "rewards/rejected": -2.1819212436676025,
      "step": 111,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.39092495636998253,
      "grad_norm": 9.064605712890625,
      "learning_rate": 0.00019648365941991438,
      "logits/chosen": -0.35194405913352966,
      "logits/rejected": -0.42771002650260925,
      "logps/chosen": -8.304715156555176,
      "logps/rejected": -32.21118927001953,
      "loss": 0.998765766620636,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6066679954528809,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15771695971488953,
      "rewards/margins": 1.7298005819320679,
      "rewards/rejected": -1.572083592414856,
      "step": 112,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.3944153577661431,
      "grad_norm": 1.8839598894119263,
      "learning_rate": 0.00019638162150900027,
      "logits/chosen": -0.2599456310272217,
      "logits/rejected": -0.3942990303039551,
      "logps/chosen": -4.890069484710693,
      "logps/rejected": -48.83489227294922,
      "loss": 0.5059012770652771,
      "memory(GiB)": 39.42,
      "nll_loss": 0.31480100750923157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19593901932239532,
      "rewards/margins": 2.834808588027954,
      "rewards/rejected": -2.638869524002075,
      "step": 113,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.39790575916230364,
      "grad_norm": 2.1122896671295166,
      "learning_rate": 0.00019627815148201417,
      "logits/chosen": -0.35711023211479187,
      "logits/rejected": -0.3968570828437805,
      "logps/chosen": -7.015074729919434,
      "logps/rejected": -31.74431037902832,
      "loss": 0.6316798329353333,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3775865137577057,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09947216510772705,
      "rewards/margins": 2.002511739730835,
      "rewards/rejected": -1.9030394554138184,
      "step": 114,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.4013961605584642,
      "grad_norm": 2.927731990814209,
      "learning_rate": 0.00019617325087639727,
      "logits/chosen": -0.38088345527648926,
      "logits/rejected": -0.435588538646698,
      "logps/chosen": -6.47522497177124,
      "logps/rejected": -24.941457748413086,
      "loss": 0.7609400749206543,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4365846514701843,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15598955750465393,
      "rewards/margins": 1.493629813194275,
      "rewards/rejected": -1.3376405239105225,
      "step": 115,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.4048865619546248,
      "grad_norm": 2.6690480709075928,
      "learning_rate": 0.00019606692125084775,
      "logits/chosen": -0.1995878666639328,
      "logits/rejected": -0.4380982518196106,
      "logps/chosen": -4.782075881958008,
      "logps/rejected": -42.2791862487793,
      "loss": 0.7123544216156006,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4524919390678406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12693020701408386,
      "rewards/margins": 2.1254653930664062,
      "rewards/rejected": -1.9985350370407104,
      "step": 116,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.4083769633507853,
      "grad_norm": 3.349586248397827,
      "learning_rate": 0.00019595916418529706,
      "logits/chosen": -0.3445770740509033,
      "logits/rejected": -0.4120224118232727,
      "logps/chosen": -9.293133735656738,
      "logps/rejected": -35.069541931152344,
      "loss": 0.7675687074661255,
      "memory(GiB)": 39.42,
      "nll_loss": 0.47724154591560364,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06645308434963226,
      "rewards/margins": 1.9597532749176025,
      "rewards/rejected": -1.8933004140853882,
      "step": 117,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.4118673647469459,
      "grad_norm": 3.534466028213501,
      "learning_rate": 0.00019584998128088684,
      "logits/chosen": -0.30230188369750977,
      "logits/rejected": -0.43015873432159424,
      "logps/chosen": -6.687424659729004,
      "logps/rejected": -34.876686096191406,
      "loss": 0.830184817314148,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5149248838424683,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07988367974758148,
      "rewards/margins": 1.8753106594085693,
      "rewards/rejected": -1.795426845550537,
      "step": 118,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.41535776614310643,
      "grad_norm": 2.4526314735412598,
      "learning_rate": 0.00019573937415994506,
      "logits/chosen": -0.3823411464691162,
      "logits/rejected": -0.3723237216472626,
      "logps/chosen": -8.30302906036377,
      "logps/rejected": -28.920549392700195,
      "loss": 0.6885042190551758,
      "memory(GiB)": 39.42,
      "nll_loss": 0.37792226672172546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.050916727632284164,
      "rewards/margins": 1.7539997100830078,
      "rewards/rejected": -1.7030829191207886,
      "step": 119,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.418848167539267,
      "grad_norm": 5.966183185577393,
      "learning_rate": 0.00019562734446596162,
      "logits/chosen": -0.36912596225738525,
      "logits/rejected": -0.4693865478038788,
      "logps/chosen": -7.145996570587158,
      "logps/rejected": -37.43076705932617,
      "loss": 0.9455270767211914,
      "memory(GiB)": 39.42,
      "nll_loss": 0.48743128776550293,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13650617003440857,
      "rewards/margins": 2.0651345252990723,
      "rewards/rejected": -2.2016406059265137,
      "step": 120,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.4223385689354276,
      "grad_norm": 6.053987979888916,
      "learning_rate": 0.00019551389386356443,
      "logits/chosen": -0.35416483879089355,
      "logits/rejected": -0.4060932993888855,
      "logps/chosen": -11.305073738098145,
      "logps/rejected": -39.34588623046875,
      "loss": 1.0363645553588867,
      "memory(GiB)": 39.42,
      "nll_loss": 0.7409172058105469,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3775523602962494,
      "rewards/margins": 2.277557611465454,
      "rewards/rejected": -2.6551101207733154,
      "step": 121,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.4258289703315881,
      "grad_norm": 3.908426523208618,
      "learning_rate": 0.0001953990240384942,
      "logits/chosen": -0.2593086361885071,
      "logits/rejected": -0.45180562138557434,
      "logps/chosen": -7.346467018127441,
      "logps/rejected": -48.82087326049805,
      "loss": 0.7506681680679321,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5086178779602051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.042671557515859604,
      "rewards/margins": 2.6901655197143555,
      "rewards/rejected": -2.647494077682495,
      "step": 122,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.4293193717277487,
      "grad_norm": 10.322138786315918,
      "learning_rate": 0.00019528273669757972,
      "logits/chosen": -0.35214099287986755,
      "logits/rejected": -0.4367164373397827,
      "logps/chosen": -8.127342224121094,
      "logps/rejected": -38.454200744628906,
      "loss": 0.8126936554908752,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5385327935218811,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10964661836624146,
      "rewards/margins": 2.599274158477783,
      "rewards/rejected": -2.4896275997161865,
      "step": 123,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.4328097731239092,
      "grad_norm": 5.38076114654541,
      "learning_rate": 0.00019516503356871234,
      "logits/chosen": -0.2883262634277344,
      "logits/rejected": -0.32226479053497314,
      "logps/chosen": -11.945974349975586,
      "logps/rejected": -43.99768829345703,
      "loss": 0.8885233998298645,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5691980123519897,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.22344030439853668,
      "rewards/margins": 2.4954993724823,
      "rewards/rejected": -2.718939781188965,
      "step": 124,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.4363001745200698,
      "grad_norm": 3.945094347000122,
      "learning_rate": 0.00019504591640082035,
      "logits/chosen": -0.3206433653831482,
      "logits/rejected": -0.3835434913635254,
      "logps/chosen": -9.910886764526367,
      "logps/rejected": -37.540409088134766,
      "loss": 0.8022382855415344,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5557839870452881,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.04477905482053757,
      "rewards/margins": 2.5910792350769043,
      "rewards/rejected": -2.6358585357666016,
      "step": 125,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.4397905759162304,
      "grad_norm": 4.2087531089782715,
      "learning_rate": 0.0001949253869638429,
      "logits/chosen": -0.28855112195014954,
      "logits/rejected": -0.3965805768966675,
      "logps/chosen": -2.8932034969329834,
      "logps/rejected": -33.3571662902832,
      "loss": 0.48199671506881714,
      "memory(GiB)": 39.42,
      "nll_loss": 0.2255885899066925,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10981371998786926,
      "rewards/margins": 2.0570077896118164,
      "rewards/rejected": -1.9471940994262695,
      "step": 126,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.4432809773123909,
      "grad_norm": 3.723099946975708,
      "learning_rate": 0.00019480344704870387,
      "logits/chosen": -0.4086211025714874,
      "logits/rejected": -0.4154241383075714,
      "logps/chosen": -11.32919692993164,
      "logps/rejected": -25.829769134521484,
      "loss": 1.3574812412261963,
      "memory(GiB)": 39.42,
      "nll_loss": 0.8371793627738953,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.18372564017772675,
      "rewards/margins": 1.1997389793395996,
      "rewards/rejected": -1.3834646940231323,
      "step": 127,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.4467713787085515,
      "grad_norm": 3.090599775314331,
      "learning_rate": 0.00019468009846728513,
      "logits/chosen": -0.3228675127029419,
      "logits/rejected": -0.4174826741218567,
      "logps/chosen": -5.812500476837158,
      "logps/rejected": -34.774322509765625,
      "loss": 0.6064198017120361,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3521398603916168,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.049313537776470184,
      "rewards/margins": 2.06463623046875,
      "rewards/rejected": -2.0153229236602783,
      "step": 128,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.450261780104712,
      "grad_norm": 2.82310152053833,
      "learning_rate": 0.00019455534305239965,
      "logits/chosen": -0.3688339293003082,
      "logits/rejected": -0.3691863417625427,
      "logps/chosen": -10.617326736450195,
      "logps/rejected": -23.13429832458496,
      "loss": 0.9776577949523926,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5499588251113892,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03131576254963875,
      "rewards/margins": 1.1372824907302856,
      "rewards/rejected": -1.1059668064117432,
      "step": 129,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.4537521815008726,
      "grad_norm": 2.351433038711548,
      "learning_rate": 0.00019442918265776422,
      "logits/chosen": -0.3329261839389801,
      "logits/rejected": -0.36310911178588867,
      "logps/chosen": -13.333334922790527,
      "logps/rejected": -23.961681365966797,
      "loss": 0.8306537866592407,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5025785565376282,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04980917274951935,
      "rewards/margins": 1.3957810401916504,
      "rewards/rejected": -1.3459720611572266,
      "step": 130,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.4572425828970332,
      "grad_norm": 2.437668561935425,
      "learning_rate": 0.00019430161915797207,
      "logits/chosen": -0.3711564540863037,
      "logits/rejected": -0.40104660391807556,
      "logps/chosen": -4.970005512237549,
      "logps/rejected": -25.718761444091797,
      "loss": 0.6583466529846191,
      "memory(GiB)": 39.42,
      "nll_loss": 0.32833656668663025,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15192265808582306,
      "rewards/margins": 1.5051937103271484,
      "rewards/rejected": -1.3532708883285522,
      "step": 131,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.4607329842931937,
      "grad_norm": 2.258094310760498,
      "learning_rate": 0.00019417265444846476,
      "logits/chosen": -0.2738487422466278,
      "logits/rejected": -0.4087858200073242,
      "logps/chosen": -7.829878330230713,
      "logps/rejected": -34.82905578613281,
      "loss": 0.7665334343910217,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5376078486442566,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25566646456718445,
      "rewards/margins": 2.1234703063964844,
      "rewards/rejected": -1.8678035736083984,
      "step": 132,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.4642233856893543,
      "grad_norm": 2.1597509384155273,
      "learning_rate": 0.00019404229044550433,
      "logits/chosen": -0.33906158804893494,
      "logits/rejected": -0.4303910434246063,
      "logps/chosen": -4.8524885177612305,
      "logps/rejected": -30.576656341552734,
      "loss": 0.7118875980377197,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4106830060482025,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17056593298912048,
      "rewards/margins": 1.768500804901123,
      "rewards/rejected": -1.5979350805282593,
      "step": 133,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4677137870855148,
      "grad_norm": 4.846266269683838,
      "learning_rate": 0.0001939105290861445,
      "logits/chosen": -0.1988743394613266,
      "logits/rejected": -0.41542911529541016,
      "logps/chosen": -7.35241174697876,
      "logps/rejected": -47.594486236572266,
      "loss": 0.8350856304168701,
      "memory(GiB)": 39.42,
      "nll_loss": 0.551296055316925,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09475097805261612,
      "rewards/margins": 2.521266222000122,
      "rewards/rejected": -2.4265151023864746,
      "step": 134,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4712041884816754,
      "grad_norm": 2.220877170562744,
      "learning_rate": 0.00019377737232820208,
      "logits/chosen": -0.2817349433898926,
      "logits/rejected": -0.3702714443206787,
      "logps/chosen": -7.847001552581787,
      "logps/rejected": -44.94153594970703,
      "loss": 0.6441789269447327,
      "memory(GiB)": 39.42,
      "nll_loss": 0.44079264998435974,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16393792629241943,
      "rewards/margins": 2.845656394958496,
      "rewards/rejected": -2.681718587875366,
      "step": 135,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.47469458987783597,
      "grad_norm": 2.1187074184417725,
      "learning_rate": 0.00019364282215022786,
      "logits/chosen": -0.28489771485328674,
      "logits/rejected": -0.37565189599990845,
      "logps/chosen": -5.1259026527404785,
      "logps/rejected": -39.07181930541992,
      "loss": 0.5232971906661987,
      "memory(GiB)": 39.42,
      "nll_loss": 0.31729230284690857,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14290614426136017,
      "rewards/margins": 2.855766773223877,
      "rewards/rejected": -2.712860345840454,
      "step": 136,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4781849912739965,
      "grad_norm": 4.652167320251465,
      "learning_rate": 0.00019350688055147725,
      "logits/chosen": -0.2049810290336609,
      "logits/rejected": -0.35532209277153015,
      "logps/chosen": -8.197148323059082,
      "logps/rejected": -50.812965393066406,
      "loss": 0.5982136130332947,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3783411979675293,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.036504216492176056,
      "rewards/margins": 3.0081937313079834,
      "rewards/rejected": -2.9716897010803223,
      "step": 137,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4816753926701571,
      "grad_norm": 1.6813859939575195,
      "learning_rate": 0.0001933695495518804,
      "logits/chosen": -0.15937744081020355,
      "logits/rejected": -0.36899077892303467,
      "logps/chosen": -5.227466106414795,
      "logps/rejected": -57.80689239501953,
      "loss": 0.4328049421310425,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3204546272754669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08378630876541138,
      "rewards/margins": 3.9659409523010254,
      "rewards/rejected": -3.882154703140259,
      "step": 138,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4851657940663176,
      "grad_norm": 1.7590577602386475,
      "learning_rate": 0.00019323083119201235,
      "logits/chosen": -0.21911956369876862,
      "logits/rejected": -0.3360063135623932,
      "logps/chosen": -10.714500427246094,
      "logps/rejected": -44.478004455566406,
      "loss": 0.6973531246185303,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5391013622283936,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15974897146224976,
      "rewards/margins": 3.1409525871276855,
      "rewards/rejected": -2.981203556060791,
      "step": 139,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.4886561954624782,
      "grad_norm": 4.423391342163086,
      "learning_rate": 0.0001930907275330627,
      "logits/chosen": -0.21830472350120544,
      "logits/rejected": -0.41240787506103516,
      "logps/chosen": -6.044207572937012,
      "logps/rejected": -47.71708679199219,
      "loss": 0.7773061990737915,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4624183773994446,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08216652274131775,
      "rewards/margins": 2.775662422180176,
      "rewards/rejected": -2.8578288555145264,
      "step": 140,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.49214659685863876,
      "grad_norm": 5.617547988891602,
      "learning_rate": 0.00019294924065680488,
      "logits/chosen": -0.3345920145511627,
      "logits/rejected": -0.4564991593360901,
      "logps/chosen": -4.883607387542725,
      "logps/rejected": -50.870941162109375,
      "loss": 0.49507585167884827,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3347318768501282,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0749196857213974,
      "rewards/margins": 3.8949146270751953,
      "rewards/rejected": -3.819995164871216,
      "step": 141,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.4956369982547993,
      "grad_norm": 6.824800491333008,
      "learning_rate": 0.00019280637266556533,
      "logits/chosen": -0.255269318819046,
      "logits/rejected": -0.38520547747612,
      "logps/chosen": -6.951573371887207,
      "logps/rejected": -43.31431579589844,
      "loss": 0.8777123689651489,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6212184429168701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1106327623128891,
      "rewards/margins": 3.119386672973633,
      "rewards/rejected": -3.008753538131714,
      "step": 142,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.49912739965095987,
      "grad_norm": 3.7692058086395264,
      "learning_rate": 0.0001926621256821922,
      "logits/chosen": -0.15499627590179443,
      "logits/rejected": -0.30336061120033264,
      "logps/chosen": -11.738297462463379,
      "logps/rejected": -56.02423095703125,
      "loss": 0.7617011070251465,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5688201189041138,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04947775602340698,
      "rewards/margins": 3.5284841060638428,
      "rewards/rejected": -3.5779623985290527,
      "step": 143,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5026178010471204,
      "grad_norm": 2.1459271907806396,
      "learning_rate": 0.0001925165018500238,
      "logits/chosen": -0.21587318181991577,
      "logits/rejected": -0.3930494487285614,
      "logps/chosen": -9.389811515808105,
      "logps/rejected": -52.38633728027344,
      "loss": 0.6086324453353882,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4416413903236389,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32234904170036316,
      "rewards/margins": 3.8869223594665527,
      "rewards/rejected": -3.5645735263824463,
      "step": 144,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.506108202443281,
      "grad_norm": 5.556336402893066,
      "learning_rate": 0.00019236950333285683,
      "logits/chosen": -0.2211378514766693,
      "logits/rejected": -0.3483867645263672,
      "logps/chosen": -8.280564308166504,
      "logps/rejected": -51.631282806396484,
      "loss": 0.7348828315734863,
      "memory(GiB)": 39.42,
      "nll_loss": 0.48733851313591003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.00019693002104759216,
      "rewards/margins": 3.514716148376465,
      "rewards/rejected": -3.514913320541382,
      "step": 145,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5095986038394416,
      "grad_norm": 2.229085683822632,
      "learning_rate": 0.00019222113231491409,
      "logits/chosen": -0.2527725100517273,
      "logits/rejected": -0.41788095235824585,
      "logps/chosen": -10.011974334716797,
      "logps/rejected": -52.808170318603516,
      "loss": 0.7164273858070374,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5022612810134888,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3206445574760437,
      "rewards/margins": 3.888902425765991,
      "rewards/rejected": -3.568258285522461,
      "step": 146,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5130890052356021,
      "grad_norm": 7.876246452331543,
      "learning_rate": 0.00019207139100081212,
      "logits/chosen": -0.19432660937309265,
      "logits/rejected": -0.33527010679244995,
      "logps/chosen": -7.427047252655029,
      "logps/rejected": -53.75447082519531,
      "loss": 0.760964035987854,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5109995007514954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08073252439498901,
      "rewards/margins": 3.443918228149414,
      "rewards/rejected": -3.524650812149048,
      "step": 147,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5165794066317626,
      "grad_norm": 4.659561634063721,
      "learning_rate": 0.00019192028161552847,
      "logits/chosen": -0.18484185636043549,
      "logits/rejected": -0.39253929257392883,
      "logps/chosen": -8.587300300598145,
      "logps/rejected": -52.95454788208008,
      "loss": 0.7159382104873657,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5496724247932434,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18269562721252441,
      "rewards/margins": 3.7814650535583496,
      "rewards/rejected": -3.598769187927246,
      "step": 148,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5200698080279232,
      "grad_norm": 3.1019811630249023,
      "learning_rate": 0.0001917678064043686,
      "logits/chosen": -0.27576160430908203,
      "logits/rejected": -0.4377831816673279,
      "logps/chosen": -5.784604549407959,
      "logps/rejected": -52.2569580078125,
      "loss": 0.6684166193008423,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4899623990058899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07499371469020844,
      "rewards/margins": 3.7692465782165527,
      "rewards/rejected": -3.6942529678344727,
      "step": 149,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5235602094240838,
      "grad_norm": 5.6313676834106445,
      "learning_rate": 0.00019161396763293253,
      "logits/chosen": -0.24116772413253784,
      "logits/rejected": -0.4378838539123535,
      "logps/chosen": -8.672993659973145,
      "logps/rejected": -52.83271026611328,
      "loss": 0.9051390290260315,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5615367889404297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.022281643003225327,
      "rewards/margins": 3.423705816268921,
      "rewards/rejected": -3.4459874629974365,
      "step": 150,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5270506108202443,
      "grad_norm": 30.42255973815918,
      "learning_rate": 0.00019145876758708106,
      "logits/chosen": -0.23821839690208435,
      "logits/rejected": -0.48250508308410645,
      "logps/chosen": -5.898958683013916,
      "logps/rejected": -51.13921356201172,
      "loss": 1.0185292959213257,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6116878986358643,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1395014375448227,
      "rewards/margins": 3.174698829650879,
      "rewards/rejected": -3.3142004013061523,
      "step": 151,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5305410122164049,
      "grad_norm": 3.989668369293213,
      "learning_rate": 0.00019130220857290202,
      "logits/chosen": -0.2887514531612396,
      "logits/rejected": -0.4765075743198395,
      "logps/chosen": -9.17143726348877,
      "logps/rejected": -41.9118766784668,
      "loss": 1.0394952297210693,
      "memory(GiB)": 39.42,
      "nll_loss": 0.7010207772254944,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1842147558927536,
      "rewards/margins": 2.678420066833496,
      "rewards/rejected": -2.4942049980163574,
      "step": 152,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.5340314136125655,
      "grad_norm": 3.6354000568389893,
      "learning_rate": 0.00019114429291667583,
      "logits/chosen": -0.3704184889793396,
      "logits/rejected": -0.44566863775253296,
      "logps/chosen": -13.917726516723633,
      "logps/rejected": -36.536293029785156,
      "loss": 0.9129284620285034,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6521165370941162,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30949151515960693,
      "rewards/margins": 3.0189402103424072,
      "rewards/rejected": -2.70944881439209,
      "step": 153,
      "train_speed(iter/s)": 0.010757
    },
    {
      "epoch": 0.537521815008726,
      "grad_norm": 3.1834535598754883,
      "learning_rate": 0.000190985022964841,
      "logits/chosen": -0.36771464347839355,
      "logits/rejected": -0.48058295249938965,
      "logps/chosen": -4.574065685272217,
      "logps/rejected": -36.14601135253906,
      "loss": 0.646094024181366,
      "memory(GiB)": 39.42,
      "nll_loss": 0.40117186307907104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.26323747634887695,
      "rewards/margins": 2.6909196376800537,
      "rewards/rejected": -2.4276821613311768,
      "step": 154,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5410122164048866,
      "grad_norm": 4.318117141723633,
      "learning_rate": 0.0001908244010839593,
      "logits/chosen": -0.3432493805885315,
      "logits/rejected": -0.47267085313796997,
      "logps/chosen": -6.45510196685791,
      "logps/rejected": -37.24266815185547,
      "loss": 0.6357433795928955,
      "memory(GiB)": 39.42,
      "nll_loss": 0.35821908712387085,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1637301743030548,
      "rewards/margins": 2.375438690185547,
      "rewards/rejected": -2.2117085456848145,
      "step": 155,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5445026178010471,
      "grad_norm": 3.158153772354126,
      "learning_rate": 0.00019066242966068054,
      "logits/chosen": -0.43242546916007996,
      "logits/rejected": -0.47301432490348816,
      "logps/chosen": -9.04937744140625,
      "logps/rejected": -33.77018356323242,
      "loss": 0.8393766283988953,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5358401536941528,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1126929298043251,
      "rewards/margins": 2.264133930206299,
      "rewards/rejected": -2.1514410972595215,
      "step": 156,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5479930191972077,
      "grad_norm": 4.308590888977051,
      "learning_rate": 0.00019049911110170705,
      "logits/chosen": -0.3886866867542267,
      "logits/rejected": -0.4790132939815521,
      "logps/chosen": -9.111655235290527,
      "logps/rejected": -36.21464538574219,
      "loss": 0.8003115057945251,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4989633858203888,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.07737411558628082,
      "rewards/margins": 2.306735038757324,
      "rewards/rejected": -2.3841090202331543,
      "step": 157,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5514834205933682,
      "grad_norm": 7.6933746337890625,
      "learning_rate": 0.00019033444783375804,
      "logits/chosen": -0.34298592805862427,
      "logits/rejected": -0.4637518525123596,
      "logps/chosen": -8.123557090759277,
      "logps/rejected": -50.31752014160156,
      "loss": 0.6172691583633423,
      "memory(GiB)": 39.42,
      "nll_loss": 0.40251481533050537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.028386808931827545,
      "rewards/margins": 3.503352403640747,
      "rewards/rejected": -3.4749653339385986,
      "step": 158,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5549738219895288,
      "grad_norm": 7.182351112365723,
      "learning_rate": 0.00019016844230353356,
      "logits/chosen": -0.3422991633415222,
      "logits/rejected": -0.42033761739730835,
      "logps/chosen": -8.310185432434082,
      "logps/rejected": -43.29906463623047,
      "loss": 0.7140598893165588,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3994865417480469,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.060791049152612686,
      "rewards/margins": 2.747554302215576,
      "rewards/rejected": -2.808345079421997,
      "step": 159,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5584642233856894,
      "grad_norm": 2.537100076675415,
      "learning_rate": 0.00019000109697767798,
      "logits/chosen": -0.3541107475757599,
      "logits/rejected": -0.44592800736427307,
      "logps/chosen": -7.220765590667725,
      "logps/rejected": -36.86073684692383,
      "loss": 0.6231136918067932,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3817756175994873,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14087501168251038,
      "rewards/margins": 2.464402198791504,
      "rewards/rejected": -2.3235268592834473,
      "step": 160,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5619546247818499,
      "grad_norm": 5.032235145568848,
      "learning_rate": 0.00018983241434274351,
      "logits/chosen": -0.3592570722103119,
      "logits/rejected": -0.4694417417049408,
      "logps/chosen": -8.956731796264648,
      "logps/rejected": -36.53116989135742,
      "loss": 0.8788279294967651,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5014119744300842,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.02851598709821701,
      "rewards/margins": 2.0984349250793457,
      "rewards/rejected": -2.0699188709259033,
      "step": 161,
      "train_speed(iter/s)": 0.010758
    },
    {
      "epoch": 0.5654450261780105,
      "grad_norm": 2.1505980491638184,
      "learning_rate": 0.00018966239690515309,
      "logits/chosen": -0.40191835165023804,
      "logits/rejected": -0.511231541633606,
      "logps/chosen": -8.61328125,
      "logps/rejected": -42.80579376220703,
      "loss": 0.650855302810669,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3947713077068329,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1195174977183342,
      "rewards/margins": 2.9840264320373535,
      "rewards/rejected": -2.864508628845215,
      "step": 162,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.568935427574171,
      "grad_norm": 2.0593886375427246,
      "learning_rate": 0.00018949104719116332,
      "logits/chosen": -0.36368370056152344,
      "logits/rejected": -0.4502822458744049,
      "logps/chosen": -6.056973934173584,
      "logps/rejected": -42.781681060791016,
      "loss": 0.5314807891845703,
      "memory(GiB)": 39.42,
      "nll_loss": 0.31094634532928467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14926032721996307,
      "rewards/margins": 2.994180202484131,
      "rewards/rejected": -2.8449201583862305,
      "step": 163,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5724258289703316,
      "grad_norm": 3.314478635787964,
      "learning_rate": 0.00018931836774682687,
      "logits/chosen": -0.4336795210838318,
      "logits/rejected": -0.4525458812713623,
      "logps/chosen": -7.387899875640869,
      "logps/rejected": -32.55196762084961,
      "loss": 0.7532250285148621,
      "memory(GiB)": 39.42,
      "nll_loss": 0.37785154581069946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06012080982327461,
      "rewards/margins": 2.1696441173553467,
      "rewards/rejected": -2.109523296356201,
      "step": 164,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5759162303664922,
      "grad_norm": 3.4192256927490234,
      "learning_rate": 0.00018914436113795448,
      "logits/chosen": -0.2911241948604584,
      "logits/rejected": -0.4634595513343811,
      "logps/chosen": -7.420339584350586,
      "logps/rejected": -39.048561096191406,
      "loss": 0.8260236978530884,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5035777688026428,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.03999996930360794,
      "rewards/margins": 2.3207309246063232,
      "rewards/rejected": -2.280730724334717,
      "step": 165,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5794066317626527,
      "grad_norm": 4.191897392272949,
      "learning_rate": 0.0001889690299500771,
      "logits/chosen": -0.32450249791145325,
      "logits/rejected": -0.4613388776779175,
      "logps/chosen": -9.48214340209961,
      "logps/rejected": -39.381065368652344,
      "loss": 0.8530201315879822,
      "memory(GiB)": 39.42,
      "nll_loss": 0.59242844581604,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08060712367296219,
      "rewards/margins": 2.242609977722168,
      "rewards/rejected": -2.1620030403137207,
      "step": 166,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5828970331588132,
      "grad_norm": 2.7431366443634033,
      "learning_rate": 0.0001887923767884073,
      "logits/chosen": -0.3079373836517334,
      "logits/rejected": -0.44786494970321655,
      "logps/chosen": -5.737804412841797,
      "logps/rejected": -36.809600830078125,
      "loss": 0.6614618301391602,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4013901352882385,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3159565329551697,
      "rewards/margins": 2.179208755493164,
      "rewards/rejected": -1.8632525205612183,
      "step": 167,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5863874345549738,
      "grad_norm": 3.6514716148376465,
      "learning_rate": 0.0001886144042778006,
      "logits/chosen": -0.39408621191978455,
      "logits/rejected": -0.43902865052223206,
      "logps/chosen": -9.832802772521973,
      "logps/rejected": -41.6961784362793,
      "loss": 0.7166361212730408,
      "memory(GiB)": 39.42,
      "nll_loss": 0.490989625453949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17612463235855103,
      "rewards/margins": 2.8474600315093994,
      "rewards/rejected": -2.671335458755493,
      "step": 168,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5898778359511344,
      "grad_norm": 3.6428065299987793,
      "learning_rate": 0.00018843511506271644,
      "logits/chosen": -0.4216082990169525,
      "logits/rejected": -0.44014686346054077,
      "logps/chosen": -8.206501960754395,
      "logps/rejected": -32.93410873413086,
      "loss": 0.7580087780952454,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4095152020454407,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0056315031833946705,
      "rewards/margins": 2.0248444080352783,
      "rewards/rejected": -2.0192127227783203,
      "step": 169,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5933682373472949,
      "grad_norm": 3.8498518466949463,
      "learning_rate": 0.00018825451180717905,
      "logits/chosen": -0.3026273846626282,
      "logits/rejected": -0.5032517910003662,
      "logps/chosen": -10.3818941116333,
      "logps/rejected": -51.91915512084961,
      "loss": 1.0770937204360962,
      "memory(GiB)": 39.42,
      "nll_loss": 0.713114857673645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18697917461395264,
      "rewards/margins": 2.874116897583008,
      "rewards/rejected": -3.061095952987671,
      "step": 170,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.5968586387434555,
      "grad_norm": 1.5238920450210571,
      "learning_rate": 0.00018807259719473757,
      "logits/chosen": -0.3206470012664795,
      "logits/rejected": -0.46274593472480774,
      "logps/chosen": -4.761229515075684,
      "logps/rejected": -47.91825866699219,
      "loss": 0.5495983362197876,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3658239543437958,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14044716954231262,
      "rewards/margins": 3.341961145401001,
      "rewards/rejected": -3.2015140056610107,
      "step": 171,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6003490401396161,
      "grad_norm": 2.570446252822876,
      "learning_rate": 0.00018788937392842647,
      "logits/chosen": -0.2903798520565033,
      "logits/rejected": -0.4074512720108032,
      "logps/chosen": -5.276430130004883,
      "logps/rejected": -48.5122184753418,
      "loss": 0.5867538452148438,
      "memory(GiB)": 39.42,
      "nll_loss": 0.34603506326675415,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06058134138584137,
      "rewards/margins": 3.133744478225708,
      "rewards/rejected": -3.0731630325317383,
      "step": 172,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6038394415357766,
      "grad_norm": 3.918461799621582,
      "learning_rate": 0.0001877048447307252,
      "logits/chosen": -0.19242779910564423,
      "logits/rejected": -0.38455089926719666,
      "logps/chosen": -5.807734489440918,
      "logps/rejected": -52.73225784301758,
      "loss": 0.6217489838600159,
      "memory(GiB)": 39.42,
      "nll_loss": 0.38360434770584106,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12222804874181747,
      "rewards/margins": 3.3160438537597656,
      "rewards/rejected": -3.1938157081604004,
      "step": 173,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6073298429319371,
      "grad_norm": 4.870952129364014,
      "learning_rate": 0.00018751901234351773,
      "logits/chosen": -0.36845889687538147,
      "logits/rejected": -0.4809192717075348,
      "logps/chosen": -8.235760688781738,
      "logps/rejected": -36.360076904296875,
      "loss": 0.8248316645622253,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6022232174873352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3026849031448364,
      "rewards/margins": 2.4412600994110107,
      "rewards/rejected": -2.1385750770568848,
      "step": 174,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6108202443280978,
      "grad_norm": 2.2720422744750977,
      "learning_rate": 0.00018733187952805203,
      "logits/chosen": -0.3710225522518158,
      "logits/rejected": -0.436270534992218,
      "logps/chosen": -6.721004009246826,
      "logps/rejected": -34.718502044677734,
      "loss": 0.709654688835144,
      "memory(GiB)": 39.42,
      "nll_loss": 0.452656626701355,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06426668912172318,
      "rewards/margins": 2.3792333602905273,
      "rewards/rejected": -2.3149666786193848,
      "step": 175,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6143106457242583,
      "grad_norm": 3.7245795726776123,
      "learning_rate": 0.00018714344906489876,
      "logits/chosen": -0.4186531901359558,
      "logits/rejected": -0.43767866492271423,
      "logps/chosen": -9.161328315734863,
      "logps/rejected": -32.324134826660156,
      "loss": 0.7417446970939636,
      "memory(GiB)": 39.42,
      "nll_loss": 0.44037187099456787,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13668972253799438,
      "rewards/margins": 2.1546599864959717,
      "rewards/rejected": -2.017970085144043,
      "step": 176,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.6178010471204188,
      "grad_norm": 4.3425469398498535,
      "learning_rate": 0.00018695372375391014,
      "logits/chosen": -0.2733748257160187,
      "logits/rejected": -0.44424253702163696,
      "logps/chosen": -8.758390426635742,
      "logps/rejected": -50.36033630371094,
      "loss": 0.8111291527748108,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4982152581214905,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05674072727560997,
      "rewards/margins": 2.9798450469970703,
      "rewards/rejected": -3.036585807800293,
      "step": 177,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6212914485165794,
      "grad_norm": 3.7545742988586426,
      "learning_rate": 0.00018676270641417822,
      "logits/chosen": -0.2608646750450134,
      "logits/rejected": -0.4399104118347168,
      "logps/chosen": -9.50798225402832,
      "logps/rejected": -46.8238639831543,
      "loss": 0.86136394739151,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6183493137359619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07049836963415146,
      "rewards/margins": 3.0674405097961426,
      "rewards/rejected": -2.9969425201416016,
      "step": 178,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.62478184991274,
      "grad_norm": 2.237074613571167,
      "learning_rate": 0.00018657039988399313,
      "logits/chosen": -0.29585880041122437,
      "logits/rejected": -0.44753074645996094,
      "logps/chosen": -11.109559059143066,
      "logps/rejected": -39.0783805847168,
      "loss": 0.9119770526885986,
      "memory(GiB)": 39.42,
      "nll_loss": 0.6092584133148193,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1456558257341385,
      "rewards/margins": 2.451812505722046,
      "rewards/rejected": -2.306156873703003,
      "step": 179,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6282722513089005,
      "grad_norm": 2.476689577102661,
      "learning_rate": 0.00018637680702080083,
      "logits/chosen": -0.3464896082878113,
      "logits/rejected": -0.4352068305015564,
      "logps/chosen": -8.254376411437988,
      "logps/rejected": -35.282325744628906,
      "loss": 0.8010640740394592,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4889823794364929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.008691787719726562,
      "rewards/margins": 2.055556058883667,
      "rewards/rejected": -2.0468640327453613,
      "step": 180,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.631762652705061,
      "grad_norm": 2.8437464237213135,
      "learning_rate": 0.0001861819307011606,
      "logits/chosen": -0.2394547164440155,
      "logits/rejected": -0.4722561538219452,
      "logps/chosen": -3.5417962074279785,
      "logps/rejected": -64.08223724365234,
      "loss": 0.38956594467163086,
      "memory(GiB)": 39.42,
      "nll_loss": 0.25639209151268005,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2628820538520813,
      "rewards/margins": 4.246987819671631,
      "rewards/rejected": -3.9841055870056152,
      "step": 181,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6352530541012217,
      "grad_norm": 2.4585416316986084,
      "learning_rate": 0.00018598577382070237,
      "logits/chosen": -0.31651991605758667,
      "logits/rejected": -0.45492100715637207,
      "logps/chosen": -4.335537910461426,
      "logps/rejected": -46.55557632446289,
      "loss": 0.5344107747077942,
      "memory(GiB)": 39.42,
      "nll_loss": 0.25841349363327026,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.044346973299980164,
      "rewards/margins": 3.0594677925109863,
      "rewards/rejected": -3.0151207447052,
      "step": 182,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.6387434554973822,
      "grad_norm": 2.4348537921905518,
      "learning_rate": 0.0001857883392940837,
      "logits/chosen": -0.2961544990539551,
      "logits/rejected": -0.39695119857788086,
      "logps/chosen": -8.935890197753906,
      "logps/rejected": -40.11572265625,
      "loss": 0.7396747469902039,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4514305889606476,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09869125485420227,
      "rewards/margins": 2.4125261306762695,
      "rewards/rejected": -2.5112171173095703,
      "step": 183,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6422338568935427,
      "grad_norm": 1.7786816358566284,
      "learning_rate": 0.0001855896300549465,
      "logits/chosen": -0.2846543490886688,
      "logits/rejected": -0.4267013967037201,
      "logps/chosen": -9.809104919433594,
      "logps/rejected": -42.15507507324219,
      "loss": 0.7107999324798584,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4528127908706665,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12454140186309814,
      "rewards/margins": 2.582566976547241,
      "rewards/rejected": -2.4580256938934326,
      "step": 184,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6457242582897034,
      "grad_norm": 2.2489097118377686,
      "learning_rate": 0.00018538964905587325,
      "logits/chosen": -0.269091933965683,
      "logits/rejected": -0.42950770258903503,
      "logps/chosen": -5.967182159423828,
      "logps/rejected": -50.640193939208984,
      "loss": 0.49092888832092285,
      "memory(GiB)": 39.42,
      "nll_loss": 0.32368528842926025,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08472105860710144,
      "rewards/margins": 3.3988823890686035,
      "rewards/rejected": -3.3141613006591797,
      "step": 185,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6492146596858639,
      "grad_norm": 2.9929044246673584,
      "learning_rate": 0.0001851883992683434,
      "logits/chosen": -0.3572680950164795,
      "logits/rejected": -0.4970206022262573,
      "logps/chosen": -5.416510105133057,
      "logps/rejected": -49.36286163330078,
      "loss": 0.6976485252380371,
      "memory(GiB)": 39.42,
      "nll_loss": 0.423793762922287,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0580061599612236,
      "rewards/margins": 3.400108814239502,
      "rewards/rejected": -3.3421030044555664,
      "step": 186,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6527050610820244,
      "grad_norm": 4.574316501617432,
      "learning_rate": 0.00018498588368268907,
      "logits/chosen": -0.3062989413738251,
      "logits/rejected": -0.5117348432540894,
      "logps/chosen": -6.240719795227051,
      "logps/rejected": -48.549983978271484,
      "loss": 0.5926957130432129,
      "memory(GiB)": 39.42,
      "nll_loss": 0.40774649381637573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2606131434440613,
      "rewards/margins": 3.447434186935425,
      "rewards/rejected": -3.1868209838867188,
      "step": 187,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6561954624781849,
      "grad_norm": 3.87235426902771,
      "learning_rate": 0.0001847821053080505,
      "logits/chosen": -0.19517812132835388,
      "logits/rejected": -0.3854786455631256,
      "logps/chosen": -6.145029544830322,
      "logps/rejected": -49.55529022216797,
      "loss": 0.5662611126899719,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3513401746749878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07440924644470215,
      "rewards/margins": 3.343357563018799,
      "rewards/rejected": -3.268948554992676,
      "step": 188,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6596858638743456,
      "grad_norm": 2.1893322467803955,
      "learning_rate": 0.00018457706717233164,
      "logits/chosen": -0.31054210662841797,
      "logits/rejected": -0.49141132831573486,
      "logps/chosen": -6.330968379974365,
      "logps/rejected": -52.3120231628418,
      "loss": 0.6129257082939148,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3919089436531067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09110119938850403,
      "rewards/margins": 3.511691093444824,
      "rewards/rejected": -3.4205901622772217,
      "step": 189,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6631762652705061,
      "grad_norm": 6.626704692840576,
      "learning_rate": 0.00018437077232215488,
      "logits/chosen": -0.19323404133319855,
      "logits/rejected": -0.44291025400161743,
      "logps/chosen": -5.826582431793213,
      "logps/rejected": -57.65163040161133,
      "loss": 0.5744651556015015,
      "memory(GiB)": 39.42,
      "nll_loss": 0.40510880947113037,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14089857041835785,
      "rewards/margins": 4.094103813171387,
      "rewards/rejected": -3.953204870223999,
      "step": 190,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 3.4911742210388184,
      "learning_rate": 0.00018416322382281596,
      "logits/chosen": -0.33752942085266113,
      "logits/rejected": -0.4428948163986206,
      "logps/chosen": -8.038012504577637,
      "logps/rejected": -41.6568717956543,
      "loss": 0.8088080286979675,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5819704532623291,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10074374079704285,
      "rewards/margins": 3.1307780742645264,
      "rewards/rejected": -3.030034065246582,
      "step": 191,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6701570680628273,
      "grad_norm": 2.2780048847198486,
      "learning_rate": 0.0001839544247582383,
      "logits/chosen": -0.2744682729244232,
      "logits/rejected": -0.3990049362182617,
      "logps/chosen": -5.77654504776001,
      "logps/rejected": -49.46015167236328,
      "loss": 0.5131328105926514,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3267570436000824,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04739464819431305,
      "rewards/margins": 3.7212061882019043,
      "rewards/rejected": -3.673811674118042,
      "step": 192,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6736474694589878,
      "grad_norm": 2.1769142150878906,
      "learning_rate": 0.00018374437823092724,
      "logits/chosen": -0.22928477823734283,
      "logits/rejected": -0.4279863238334656,
      "logps/chosen": -4.774344444274902,
      "logps/rejected": -50.371253967285156,
      "loss": 0.5500450134277344,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3786742091178894,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1368112862110138,
      "rewards/margins": 3.6264164447784424,
      "rewards/rejected": -3.48960542678833,
      "step": 193,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6771378708551483,
      "grad_norm": 2.8660085201263428,
      "learning_rate": 0.00018353308736192395,
      "logits/chosen": -0.2577519714832306,
      "logits/rejected": -0.35627281665802,
      "logps/chosen": -8.359132766723633,
      "logps/rejected": -42.28339767456055,
      "loss": 0.7936825156211853,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5378628969192505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02582652121782303,
      "rewards/margins": 2.7675435543060303,
      "rewards/rejected": -2.7417168617248535,
      "step": 194,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.680628272251309,
      "grad_norm": 4.319552898406982,
      "learning_rate": 0.000183320555290759,
      "logits/chosen": -0.2262856811285019,
      "logits/rejected": -0.44104352593421936,
      "logps/chosen": -5.736568927764893,
      "logps/rejected": -54.799659729003906,
      "loss": 0.6704956889152527,
      "memory(GiB)": 39.42,
      "nll_loss": 0.3680565059185028,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.14153015613555908,
      "rewards/margins": 3.8039064407348633,
      "rewards/rejected": -3.662376642227173,
      "step": 195,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6841186736474695,
      "grad_norm": 2.4287476539611816,
      "learning_rate": 0.0001831067851754058,
      "logits/chosen": -0.19324590265750885,
      "logits/rejected": -0.3578749895095825,
      "logps/chosen": -6.67017936706543,
      "logps/rejected": -52.006690979003906,
      "loss": 0.7265517115592957,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4939360022544861,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1414988934993744,
      "rewards/margins": 3.7974777221679688,
      "rewards/rejected": -3.6559786796569824,
      "step": 196,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.68760907504363,
      "grad_norm": 4.368680000305176,
      "learning_rate": 0.00018289178019223362,
      "logits/chosen": -0.35040730237960815,
      "logits/rejected": -0.37726685404777527,
      "logps/chosen": -6.27101993560791,
      "logps/rejected": -35.45961380004883,
      "loss": 0.6579899191856384,
      "memory(GiB)": 39.42,
      "nll_loss": 0.35855987668037415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.014165613800287247,
      "rewards/margins": 2.6007533073425293,
      "rewards/rejected": -2.6149189472198486,
      "step": 197,
      "train_speed(iter/s)": 0.010759
    },
    {
      "epoch": 0.6910994764397905,
      "grad_norm": 2.2127246856689453,
      "learning_rate": 0.00018267554353596025,
      "logits/chosen": -0.18989573419094086,
      "logits/rejected": -0.3121846318244934,
      "logps/chosen": -9.079330444335938,
      "logps/rejected": -53.070701599121094,
      "loss": 0.6450351476669312,
      "memory(GiB)": 39.42,
      "nll_loss": 0.4518280029296875,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07810258865356445,
      "rewards/margins": 3.928205966949463,
      "rewards/rejected": -3.8501036167144775,
      "step": 198,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.6945898778359512,
      "grad_norm": 6.419010162353516,
      "learning_rate": 0.0001824580784196049,
      "logits/chosen": -0.15850739181041718,
      "logits/rejected": -0.3392256498336792,
      "logps/chosen": -4.089353561401367,
      "logps/rejected": -61.679054260253906,
      "loss": 0.3885970115661621,
      "memory(GiB)": 39.42,
      "nll_loss": 0.26330527663230896,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06663137674331665,
      "rewards/margins": 4.802007675170898,
      "rewards/rejected": -4.735375881195068,
      "step": 199,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.6980802792321117,
      "grad_norm": 24.48959732055664,
      "learning_rate": 0.00018223938807444012,
      "logits/chosen": -0.15304000675678253,
      "logits/rejected": -0.33186089992523193,
      "logps/chosen": -8.43168830871582,
      "logps/rejected": -54.678863525390625,
      "loss": 0.7258750200271606,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5083752870559692,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.008410019800066948,
      "rewards/margins": 3.8021652698516846,
      "rewards/rejected": -3.793755292892456,
      "step": 200,
      "train_speed(iter/s)": 0.01076
    },
    {
      "epoch": 0.7015706806282722,
      "grad_norm": 16.57341194152832,
      "learning_rate": 0.00018201947574994384,
      "logits/chosen": -0.036614514887332916,
      "logits/rejected": -0.21098592877388,
      "logps/chosen": -15.165770530700684,
      "logps/rejected": -62.39512252807617,
      "loss": 0.9109693765640259,
      "memory(GiB)": 39.42,
      "nll_loss": 0.5643189549446106,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.21655045449733734,
      "rewards/margins": 3.875659465789795,
      "rewards/rejected": -4.092210292816162,
      "step": 201,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7050610820244329,
      "grad_norm": 14.475955963134766,
      "learning_rate": 0.00018179834471375125,
      "logits/chosen": -0.16321136057376862,
      "logits/rejected": -0.27808913588523865,
      "logps/chosen": -8.80876350402832,
      "logps/rejected": -45.93085479736328,
      "loss": 0.8436198830604553,
      "memory(GiB)": 39.42,
      "nll_loss": 0.47678321599960327,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.02830314263701439,
      "rewards/margins": 3.0820868015289307,
      "rewards/rejected": -3.053783893585205,
      "step": 202,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7085514834205934,
      "grad_norm": 3.0808913707733154,
      "learning_rate": 0.0001815759982516061,
      "logits/chosen": -0.13647180795669556,
      "logits/rejected": -0.2876810133457184,
      "logps/chosen": -6.556826114654541,
      "logps/rejected": -49.313106536865234,
      "loss": 0.5729860067367554,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3778243660926819,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19281527400016785,
      "rewards/margins": 3.0519158840179443,
      "rewards/rejected": -2.859100580215454,
      "step": 203,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7120418848167539,
      "grad_norm": 3.0599114894866943,
      "learning_rate": 0.00018135243966731194,
      "logits/chosen": -0.10809071362018585,
      "logits/rejected": -0.29678240418434143,
      "logps/chosen": -8.138949394226074,
      "logps/rejected": -55.99830627441406,
      "loss": 0.6175469160079956,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4390767514705658,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19007451832294464,
      "rewards/margins": 3.628199338912964,
      "rewards/rejected": -3.4381251335144043,
      "step": 204,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.7155322862129145,
      "grad_norm": 2.4980595111846924,
      "learning_rate": 0.00018112767228268295,
      "logits/chosen": -0.2528156638145447,
      "logits/rejected": -0.35177648067474365,
      "logps/chosen": -6.878098011016846,
      "logps/rejected": -47.343318939208984,
      "loss": 0.49210962653160095,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3529854416847229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06216797977685928,
      "rewards/margins": 3.3334434032440186,
      "rewards/rejected": -3.271275520324707,
      "step": 205,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7190226876090751,
      "grad_norm": 3.8207039833068848,
      "learning_rate": 0.00018090169943749476,
      "logits/chosen": -0.2331896275281906,
      "logits/rejected": -0.32883408665657043,
      "logps/chosen": -7.805197715759277,
      "logps/rejected": -35.351806640625,
      "loss": 0.8236002922058105,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5681911110877991,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07105645537376404,
      "rewards/margins": 2.3730642795562744,
      "rewards/rejected": -2.3020079135894775,
      "step": 206,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7225130890052356,
      "grad_norm": 2.6041932106018066,
      "learning_rate": 0.00018067452448943455,
      "logits/chosen": -0.1293148696422577,
      "logits/rejected": -0.3031143844127655,
      "logps/chosen": -7.286511421203613,
      "logps/rejected": -39.59210205078125,
      "loss": 0.6952319145202637,
      "memory(GiB)": 43.15,
      "nll_loss": 0.46597838401794434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1555314064025879,
      "rewards/margins": 2.2323882579803467,
      "rewards/rejected": -2.076856851577759,
      "step": 207,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7260034904013961,
      "grad_norm": 2.2537174224853516,
      "learning_rate": 0.00018044615081405153,
      "logits/chosen": -0.13209852576255798,
      "logits/rejected": -0.32356375455856323,
      "logps/chosen": -8.257270812988281,
      "logps/rejected": -49.27021789550781,
      "loss": 0.7890011668205261,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5036588311195374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13289037346839905,
      "rewards/margins": 2.8654632568359375,
      "rewards/rejected": -2.7325730323791504,
      "step": 208,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 0.7294938917975567,
      "grad_norm": 1.861016869544983,
      "learning_rate": 0.0001802165818047063,
      "logits/chosen": -0.2812590003013611,
      "logits/rejected": -0.4031287133693695,
      "logps/chosen": -4.251603126525879,
      "logps/rejected": -41.79424285888672,
      "loss": 0.47951921820640564,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2904518246650696,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18484094738960266,
      "rewards/margins": 2.8290510177612305,
      "rewards/rejected": -2.644209861755371,
      "step": 209,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7329842931937173,
      "grad_norm": 1.9735944271087646,
      "learning_rate": 0.00017998582087252096,
      "logits/chosen": -0.26465538144111633,
      "logits/rejected": -0.34588366746902466,
      "logps/chosen": -8.212122917175293,
      "logps/rejected": -48.384342193603516,
      "loss": 0.5980856418609619,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4945921003818512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.258284330368042,
      "rewards/margins": 3.618307113647461,
      "rewards/rejected": -3.36002254486084,
      "step": 210,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7364746945898778,
      "grad_norm": 4.080269813537598,
      "learning_rate": 0.000179753871446328,
      "logits/chosen": -0.3342297077178955,
      "logits/rejected": -0.45482489466667175,
      "logps/chosen": -4.805637359619141,
      "logps/rejected": -52.97557830810547,
      "loss": 0.5000529289245605,
      "memory(GiB)": 43.15,
      "nll_loss": 0.35857757925987244,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08043038100004196,
      "rewards/margins": 4.170455455780029,
      "rewards/rejected": -4.090025901794434,
      "step": 211,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7399650959860384,
      "grad_norm": 2.64082932472229,
      "learning_rate": 0.00017952073697261953,
      "logits/chosen": -0.271504282951355,
      "logits/rejected": -0.3815239667892456,
      "logps/chosen": -9.383578300476074,
      "logps/rejected": -48.522979736328125,
      "loss": 0.7009327411651611,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5086449384689331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2527061700820923,
      "rewards/margins": 3.632561683654785,
      "rewards/rejected": -3.3798558712005615,
      "step": 212,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.743455497382199,
      "grad_norm": 3.330623149871826,
      "learning_rate": 0.00017928642091549613,
      "logits/chosen": -0.242343470454216,
      "logits/rejected": -0.423129677772522,
      "logps/chosen": -4.796614170074463,
      "logps/rejected": -63.50012969970703,
      "loss": 0.44938358664512634,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3825247585773468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09516870975494385,
      "rewards/margins": 4.826410293579102,
      "rewards/rejected": -4.731241703033447,
      "step": 213,
      "train_speed(iter/s)": 0.01075
    },
    {
      "epoch": 0.7469458987783595,
      "grad_norm": 2.482487201690674,
      "learning_rate": 0.00017905092675661526,
      "logits/chosen": -0.34072667360305786,
      "logits/rejected": -0.4692738652229309,
      "logps/chosen": -5.442986011505127,
      "logps/rejected": -49.02458190917969,
      "loss": 0.4906308054924011,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32957711815834045,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27041277289390564,
      "rewards/margins": 3.7490029335021973,
      "rewards/rejected": -3.478590488433838,
      "step": 214,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7504363001745201,
      "grad_norm": 2.0258970260620117,
      "learning_rate": 0.00017881425799513956,
      "logits/chosen": -0.29332002997398376,
      "logits/rejected": -0.4313204884529114,
      "logps/chosen": -3.8019495010375977,
      "logps/rejected": -48.41462707519531,
      "loss": 0.4435490071773529,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3045477867126465,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21170072257518768,
      "rewards/margins": 3.814541816711426,
      "rewards/rejected": -3.602841854095459,
      "step": 215,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7539267015706806,
      "grad_norm": 4.538289546966553,
      "learning_rate": 0.00017857641814768486,
      "logits/chosen": -0.3277314305305481,
      "logits/rejected": -0.4610479176044464,
      "logps/chosen": -4.489333152770996,
      "logps/rejected": -40.003170013427734,
      "loss": 0.6065860986709595,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3757091760635376,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10844689607620239,
      "rewards/margins": 2.677161693572998,
      "rewards/rejected": -2.5687148571014404,
      "step": 216,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7574171029668412,
      "grad_norm": 1.9632673263549805,
      "learning_rate": 0.00017833741074826796,
      "logits/chosen": -0.3702457547187805,
      "logits/rejected": -0.4608480930328369,
      "logps/chosen": -6.7205352783203125,
      "logps/rejected": -46.00050354003906,
      "loss": 0.6259070038795471,
      "memory(GiB)": 43.15,
      "nll_loss": 0.43828409910202026,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.014492299407720566,
      "rewards/margins": 3.3422679901123047,
      "rewards/rejected": -3.3277759552001953,
      "step": 217,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7609075043630017,
      "grad_norm": 4.585688591003418,
      "learning_rate": 0.00017809723934825405,
      "logits/chosen": -0.30587339401245117,
      "logits/rejected": -0.39274415373802185,
      "logps/chosen": -7.410132884979248,
      "logps/rejected": -39.855010986328125,
      "loss": 0.666675865650177,
      "memory(GiB)": 43.15,
      "nll_loss": 0.44685378670692444,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.244940847158432,
      "rewards/margins": 2.851167678833008,
      "rewards/rejected": -2.606226921081543,
      "step": 218,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7643979057591623,
      "grad_norm": 2.123504877090454,
      "learning_rate": 0.00017785590751630403,
      "logits/chosen": -0.2982403039932251,
      "logits/rejected": -0.42090415954589844,
      "logps/chosen": -9.233821868896484,
      "logps/rejected": -49.84214401245117,
      "loss": 0.608307421207428,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4753749966621399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03608651086688042,
      "rewards/margins": 3.5284979343414307,
      "rewards/rejected": -3.492410898208618,
      "step": 219,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7678883071553229,
      "grad_norm": 6.390042781829834,
      "learning_rate": 0.0001776134188383214,
      "logits/chosen": -0.2736659646034241,
      "logits/rejected": -0.4519406855106354,
      "logps/chosen": -6.503651142120361,
      "logps/rejected": -57.02659606933594,
      "loss": 0.7497352957725525,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5775147676467896,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.052752792835235596,
      "rewards/margins": 4.1409912109375,
      "rewards/rejected": -4.088238716125488,
      "step": 220,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 0.7713787085514834,
      "grad_norm": 7.869540691375732,
      "learning_rate": 0.00017736977691739904,
      "logits/chosen": -0.2893148958683014,
      "logits/rejected": -0.45987799763679504,
      "logps/chosen": -7.436133861541748,
      "logps/rejected": -50.03794860839844,
      "loss": 0.7108630537986755,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5081782341003418,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.02406979724764824,
      "rewards/margins": 3.2432000637054443,
      "rewards/rejected": -3.219129800796509,
      "step": 221,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.774869109947644,
      "grad_norm": 2.0641894340515137,
      "learning_rate": 0.00017712498537376562,
      "logits/chosen": -0.2713238596916199,
      "logits/rejected": -0.4085955023765564,
      "logps/chosen": -4.430034637451172,
      "logps/rejected": -54.22665023803711,
      "loss": 0.44029539823532104,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24963557720184326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0637066438794136,
      "rewards/margins": 4.175436496734619,
      "rewards/rejected": -4.111729621887207,
      "step": 222,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7783595113438045,
      "grad_norm": 4.655381679534912,
      "learning_rate": 0.00017687904784473188,
      "logits/chosen": -0.32085755467414856,
      "logits/rejected": -0.4244834780693054,
      "logps/chosen": -9.161126136779785,
      "logps/rejected": -49.83354949951172,
      "loss": 0.7681727409362793,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5344542860984802,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11410484462976456,
      "rewards/margins": 3.6829593181610107,
      "rewards/rejected": -3.568854331970215,
      "step": 223,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7818499127399651,
      "grad_norm": 4.069851875305176,
      "learning_rate": 0.0001766319679846365,
      "logits/chosen": -0.34671980142593384,
      "logits/rejected": -0.4845532774925232,
      "logps/chosen": -5.1330132484436035,
      "logps/rejected": -49.80237579345703,
      "loss": 0.6742711067199707,
      "memory(GiB)": 43.15,
      "nll_loss": 0.44481760263442993,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14206397533416748,
      "rewards/margins": 3.5617966651916504,
      "rewards/rejected": -3.4197328090667725,
      "step": 224,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7853403141361257,
      "grad_norm": 2.1478774547576904,
      "learning_rate": 0.00017638374946479178,
      "logits/chosen": -0.19995641708374023,
      "logits/rejected": -0.36974671483039856,
      "logps/chosen": -11.061859130859375,
      "logps/rejected": -55.27647399902344,
      "loss": 0.6111169457435608,
      "memory(GiB)": 43.15,
      "nll_loss": 0.42612090706825256,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12187716364860535,
      "rewards/margins": 4.010047435760498,
      "rewards/rejected": -3.8881704807281494,
      "step": 225,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7888307155322862,
      "grad_norm": 2.974621295928955,
      "learning_rate": 0.00017613439597342928,
      "logits/chosen": -0.3379659056663513,
      "logits/rejected": -0.48346391320228577,
      "logps/chosen": -5.418050765991211,
      "logps/rejected": -52.225372314453125,
      "loss": 0.42137280106544495,
      "memory(GiB)": 43.15,
      "nll_loss": 0.29507750272750854,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3996121883392334,
      "rewards/margins": 4.174943447113037,
      "rewards/rejected": -3.7753310203552246,
      "step": 226,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7923211169284468,
      "grad_norm": 2.614269733428955,
      "learning_rate": 0.0001758839112156448,
      "logits/chosen": -0.235207661986351,
      "logits/rejected": -0.4293682277202606,
      "logps/chosen": -6.221077919006348,
      "logps/rejected": -57.73169708251953,
      "loss": 0.5521800518035889,
      "memory(GiB)": 43.15,
      "nll_loss": 0.409783273935318,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23889221251010895,
      "rewards/margins": 4.359738826751709,
      "rewards/rejected": -4.120846748352051,
      "step": 227,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7958115183246073,
      "grad_norm": 3.5413107872009277,
      "learning_rate": 0.00017563229891334338,
      "logits/chosen": -0.26919710636138916,
      "logits/rejected": -0.45086047053337097,
      "logps/chosen": -6.003745079040527,
      "logps/rejected": -56.92920684814453,
      "loss": 0.45959168672561646,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3346911668777466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23589728772640228,
      "rewards/margins": 4.437588691711426,
      "rewards/rejected": -4.201691627502441,
      "step": 228,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 0.7993019197207679,
      "grad_norm": 6.556480407714844,
      "learning_rate": 0.00017537956280518416,
      "logits/chosen": -0.3422713875770569,
      "logits/rejected": -0.451128214597702,
      "logps/chosen": -7.599344253540039,
      "logps/rejected": -43.70567321777344,
      "loss": 0.7094074487686157,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5201672315597534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12758715450763702,
      "rewards/margins": 3.260258197784424,
      "rewards/rejected": -3.1326711177825928,
      "step": 229,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8027923211169284,
      "grad_norm": 1.586064338684082,
      "learning_rate": 0.00017512570664652454,
      "logits/chosen": -0.28756070137023926,
      "logits/rejected": -0.38268500566482544,
      "logps/chosen": -8.602394104003906,
      "logps/rejected": -50.081172943115234,
      "loss": 0.6170235872268677,
      "memory(GiB)": 43.15,
      "nll_loss": 0.44623884558677673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23854351043701172,
      "rewards/margins": 3.8032238483428955,
      "rewards/rejected": -3.564680337905884,
      "step": 230,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.806282722513089,
      "grad_norm": 1.6582534313201904,
      "learning_rate": 0.00017487073420936465,
      "logits/chosen": -0.3135778605937958,
      "logits/rejected": -0.46431437134742737,
      "logps/chosen": -4.987645626068115,
      "logps/rejected": -52.46248245239258,
      "loss": 0.4446931779384613,
      "memory(GiB)": 43.15,
      "nll_loss": 0.30552300810813904,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12139096856117249,
      "rewards/margins": 3.983121156692505,
      "rewards/rejected": -3.8617305755615234,
      "step": 231,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8097731239092496,
      "grad_norm": 1.2338401079177856,
      "learning_rate": 0.00017461464928229115,
      "logits/chosen": -0.379431813955307,
      "logits/rejected": -0.48820745944976807,
      "logps/chosen": -4.064929485321045,
      "logps/rejected": -43.8752555847168,
      "loss": 0.4461000859737396,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31098049879074097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10920354723930359,
      "rewards/margins": 3.3555779457092285,
      "rewards/rejected": -3.2463743686676025,
      "step": 232,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8132635253054101,
      "grad_norm": 3.2604479789733887,
      "learning_rate": 0.00017435745567042095,
      "logits/chosen": -0.2582184672355652,
      "logits/rejected": -0.4310912489891052,
      "logps/chosen": -7.691014289855957,
      "logps/rejected": -54.24932098388672,
      "loss": 0.6151188611984253,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41653314232826233,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0867326557636261,
      "rewards/margins": 3.933887481689453,
      "rewards/rejected": -3.8471546173095703,
      "step": 233,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8167539267015707,
      "grad_norm": 1.4706934690475464,
      "learning_rate": 0.0001740991571953447,
      "logits/chosen": -0.269050657749176,
      "logits/rejected": -0.5030261278152466,
      "logps/chosen": -3.481001615524292,
      "logps/rejected": -65.2177505493164,
      "loss": 0.3229425251483917,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24098612368106842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16970407962799072,
      "rewards/margins": 4.697527885437012,
      "rewards/rejected": -4.527822971343994,
      "step": 234,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8202443280977313,
      "grad_norm": 3.0432982444763184,
      "learning_rate": 0.00017383975769507006,
      "logits/chosen": -0.3666076064109802,
      "logits/rejected": -0.49905386567115784,
      "logps/chosen": -9.495936393737793,
      "logps/rejected": -48.320045471191406,
      "loss": 0.7838144302368164,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5246264934539795,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07730744034051895,
      "rewards/margins": 3.346642255783081,
      "rewards/rejected": -3.4239492416381836,
      "step": 235,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8237347294938918,
      "grad_norm": 2.1510214805603027,
      "learning_rate": 0.00017357926102396453,
      "logits/chosen": -0.3271171748638153,
      "logits/rejected": -0.5392114520072937,
      "logps/chosen": -6.4805731773376465,
      "logps/rejected": -59.16584777832031,
      "loss": 0.4900282919406891,
      "memory(GiB)": 43.15,
      "nll_loss": 0.34938499331474304,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12114590406417847,
      "rewards/margins": 4.474804878234863,
      "rewards/rejected": -4.353658676147461,
      "step": 236,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8272251308900523,
      "grad_norm": 3.5285916328430176,
      "learning_rate": 0.00017331767105269833,
      "logits/chosen": -0.31751590967178345,
      "logits/rejected": -0.5495515465736389,
      "logps/chosen": -6.772843360900879,
      "logps/rejected": -54.508914947509766,
      "loss": 0.6024790406227112,
      "memory(GiB)": 43.15,
      "nll_loss": 0.39046281576156616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.011396676301956177,
      "rewards/margins": 3.849278688430786,
      "rewards/rejected": -3.860675096511841,
      "step": 237,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8307155322862129,
      "grad_norm": 4.1707329750061035,
      "learning_rate": 0.0001730549916681868,
      "logits/chosen": -0.3934589922428131,
      "logits/rejected": -0.5638350248336792,
      "logps/chosen": -7.145245552062988,
      "logps/rejected": -66.46633911132812,
      "loss": 0.5913079380989075,
      "memory(GiB)": 43.15,
      "nll_loss": 0.42689260840415955,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09644685685634613,
      "rewards/margins": 5.219150543212891,
      "rewards/rejected": -5.122704982757568,
      "step": 238,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8342059336823735,
      "grad_norm": 2.7752156257629395,
      "learning_rate": 0.00017279122677353262,
      "logits/chosen": -0.2619190216064453,
      "logits/rejected": -0.569366455078125,
      "logps/chosen": -4.285702705383301,
      "logps/rejected": -64.56398010253906,
      "loss": 0.4372471272945404,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3511802554130554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18024781346321106,
      "rewards/margins": 4.906952857971191,
      "rewards/rejected": -4.726705551147461,
      "step": 239,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.837696335078534,
      "grad_norm": 12.657561302185059,
      "learning_rate": 0.00017252638028796789,
      "logits/chosen": -0.4536377787590027,
      "logits/rejected": -0.4795590937137604,
      "logps/chosen": -10.30350112915039,
      "logps/rejected": -50.788246154785156,
      "loss": 0.806001603603363,
      "memory(GiB)": 43.15,
      "nll_loss": 0.533410906791687,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0006981715559959412,
      "rewards/margins": 3.909877300262451,
      "rewards/rejected": -3.909179210662842,
      "step": 240,
      "train_speed(iter/s)": 0.010753
    },
    {
      "epoch": 0.8411867364746946,
      "grad_norm": 2.1404943466186523,
      "learning_rate": 0.00017226045614679588,
      "logits/chosen": -0.47950872778892517,
      "logits/rejected": -0.523240864276886,
      "logps/chosen": -7.280372619628906,
      "logps/rejected": -47.931678771972656,
      "loss": 0.5918366312980652,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4288029670715332,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12062430381774902,
      "rewards/margins": 3.814420461654663,
      "rewards/rejected": -3.693796396255493,
      "step": 241,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8446771378708552,
      "grad_norm": 5.467627048492432,
      "learning_rate": 0.00017199345830133252,
      "logits/chosen": -0.3690679669380188,
      "logits/rejected": -0.5561453700065613,
      "logps/chosen": -4.672489166259766,
      "logps/rejected": -62.01866912841797,
      "loss": 0.6225215196609497,
      "memory(GiB)": 43.15,
      "nll_loss": 0.43513110280036926,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.028843754902482033,
      "rewards/margins": 4.858260154724121,
      "rewards/rejected": -4.887104034423828,
      "step": 242,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8481675392670157,
      "grad_norm": 2.7799084186553955,
      "learning_rate": 0.0001717253907188477,
      "logits/chosen": -0.3674232065677643,
      "logits/rejected": -0.48529595136642456,
      "logps/chosen": -11.648327827453613,
      "logps/rejected": -51.57639694213867,
      "loss": 0.7946433424949646,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5997886061668396,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04861972853541374,
      "rewards/margins": 3.9191184043884277,
      "rewards/rejected": -3.8704986572265625,
      "step": 243,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8516579406631762,
      "grad_norm": 7.648078441619873,
      "learning_rate": 0.00017145625738250635,
      "logits/chosen": -0.24742871522903442,
      "logits/rejected": -0.48127883672714233,
      "logps/chosen": -5.881336212158203,
      "logps/rejected": -58.709659576416016,
      "loss": 0.6269736289978027,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4364897310733795,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2005631923675537,
      "rewards/margins": 4.109006404876709,
      "rewards/rejected": -3.9084432125091553,
      "step": 244,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8551483420593369,
      "grad_norm": 4.572084426879883,
      "learning_rate": 0.0001711860622913092,
      "logits/chosen": -0.41671591997146606,
      "logits/rejected": -0.49388235807418823,
      "logps/chosen": -9.111591339111328,
      "logps/rejected": -54.797916412353516,
      "loss": 0.7940463423728943,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5814399123191833,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008319624699652195,
      "rewards/margins": 4.386847496032715,
      "rewards/rejected": -4.395166873931885,
      "step": 245,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8586387434554974,
      "grad_norm": 21.202533721923828,
      "learning_rate": 0.0001709148094600334,
      "logits/chosen": -0.28929275274276733,
      "logits/rejected": -0.46102988719940186,
      "logps/chosen": -7.337296962738037,
      "logps/rejected": -61.80494689941406,
      "loss": 0.8849119544029236,
      "memory(GiB)": 43.15,
      "nll_loss": 0.6312083005905151,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0007691718637943268,
      "rewards/margins": 4.356324195861816,
      "rewards/rejected": -4.355555057525635,
      "step": 246,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8621291448516579,
      "grad_norm": 3.0293681621551514,
      "learning_rate": 0.00017064250291917295,
      "logits/chosen": -0.3044416308403015,
      "logits/rejected": -0.45788872241973877,
      "logps/chosen": -11.546972274780273,
      "logps/rejected": -68.43335723876953,
      "loss": 0.4920075833797455,
      "memory(GiB)": 43.15,
      "nll_loss": 0.39548447728157043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28147220611572266,
      "rewards/margins": 5.060581207275391,
      "rewards/rejected": -4.779109001159668,
      "step": 247,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8656195462478184,
      "grad_norm": 1.1536225080490112,
      "learning_rate": 0.00017036914671487852,
      "logits/chosen": -0.29598888754844666,
      "logits/rejected": -0.5037257075309753,
      "logps/chosen": -4.555235862731934,
      "logps/rejected": -71.70169830322266,
      "loss": 0.29999658465385437,
      "memory(GiB)": 43.15,
      "nll_loss": 0.22001689672470093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10969162732362747,
      "rewards/margins": 5.70869779586792,
      "rewards/rejected": -5.599005699157715,
      "step": 248,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8691099476439791,
      "grad_norm": 5.403660297393799,
      "learning_rate": 0.0001700947449088977,
      "logits/chosen": -0.29327866435050964,
      "logits/rejected": -0.5409445762634277,
      "logps/chosen": -6.3373589515686035,
      "logps/rejected": -61.361427307128906,
      "loss": 0.7032750248908997,
      "memory(GiB)": 43.15,
      "nll_loss": 0.49727386236190796,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.12291238456964493,
      "rewards/margins": 4.190377712249756,
      "rewards/rejected": -4.313290119171143,
      "step": 249,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8726003490401396,
      "grad_norm": 2.2364845275878906,
      "learning_rate": 0.00016981930157851442,
      "logits/chosen": -0.4206138253211975,
      "logits/rejected": -0.5452399253845215,
      "logps/chosen": -6.642912864685059,
      "logps/rejected": -55.26310348510742,
      "loss": 0.6106983423233032,
      "memory(GiB)": 43.15,
      "nll_loss": 0.46136975288391113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1548982709646225,
      "rewards/margins": 4.34593391418457,
      "rewards/rejected": -4.191035270690918,
      "step": 250,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8760907504363001,
      "grad_norm": 2.366743326187134,
      "learning_rate": 0.0001695428208164884,
      "logits/chosen": -0.3339673578739166,
      "logits/rejected": -0.46634042263031006,
      "logps/chosen": -12.371589660644531,
      "logps/rejected": -54.81672668457031,
      "loss": 0.7680290937423706,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5548099279403687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06331084668636322,
      "rewards/margins": 3.747202157974243,
      "rewards/rejected": -3.683891773223877,
      "step": 251,
      "train_speed(iter/s)": 0.010754
    },
    {
      "epoch": 0.8795811518324608,
      "grad_norm": 17.273366928100586,
      "learning_rate": 0.00016926530673099443,
      "logits/chosen": -0.25144901871681213,
      "logits/rejected": -0.5055121779441833,
      "logps/chosen": -4.04465389251709,
      "logps/rejected": -53.838226318359375,
      "loss": 0.5800854563713074,
      "memory(GiB)": 43.15,
      "nll_loss": 0.43476593494415283,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09720754623413086,
      "rewards/margins": 3.768738031387329,
      "rewards/rejected": -3.6715304851531982,
      "step": 252,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.8830715532286213,
      "grad_norm": 3.9538075923919678,
      "learning_rate": 0.00016898676344556118,
      "logits/chosen": -0.3590712547302246,
      "logits/rejected": -0.4731703996658325,
      "logps/chosen": -8.093548774719238,
      "logps/rejected": -44.39308547973633,
      "loss": 0.7438912391662598,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5026392936706543,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.010781295597553253,
      "rewards/margins": 3.2361843585968018,
      "rewards/rejected": -3.22540283203125,
      "step": 253,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.8865619546247818,
      "grad_norm": 4.43087100982666,
      "learning_rate": 0.00016870719509901,
      "logits/chosen": -0.32536038756370544,
      "logits/rejected": -0.5595154166221619,
      "logps/chosen": -3.823620319366455,
      "logps/rejected": -56.6773681640625,
      "loss": 0.545555591583252,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3203794062137604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06757771968841553,
      "rewards/margins": 3.751739978790283,
      "rewards/rejected": -3.684161901473999,
      "step": 254,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.8900523560209425,
      "grad_norm": 1.5452710390090942,
      "learning_rate": 0.00016842660584539352,
      "logits/chosen": -0.4577394425868988,
      "logits/rejected": -0.5295063853263855,
      "logps/chosen": -4.856768608093262,
      "logps/rejected": -44.36632537841797,
      "loss": 0.5244438052177429,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3651025891304016,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22487697005271912,
      "rewards/margins": 3.4033455848693848,
      "rewards/rejected": -3.178468704223633,
      "step": 255,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.893542757417103,
      "grad_norm": 1.9295581579208374,
      "learning_rate": 0.0001681449998539337,
      "logits/chosen": -0.3778080642223358,
      "logits/rejected": -0.5354687571525574,
      "logps/chosen": -7.896451950073242,
      "logps/rejected": -60.301177978515625,
      "loss": 0.5556483268737793,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4384987950325012,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.263929545879364,
      "rewards/margins": 4.734387397766113,
      "rewards/rejected": -4.470458030700684,
      "step": 256,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.8970331588132635,
      "grad_norm": 2.257227897644043,
      "learning_rate": 0.00016786238130896014,
      "logits/chosen": -0.4991389214992523,
      "logits/rejected": -0.563326895236969,
      "logps/chosen": -8.479850769042969,
      "logps/rejected": -38.3695182800293,
      "loss": 0.6856359243392944,
      "memory(GiB)": 43.15,
      "nll_loss": 0.49940675497055054,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1654682606458664,
      "rewards/margins": 2.8511545658111572,
      "rewards/rejected": -2.6856863498687744,
      "step": 257,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.900523560209424,
      "grad_norm": 19.965892791748047,
      "learning_rate": 0.00016757875440984768,
      "logits/chosen": -0.416139155626297,
      "logits/rejected": -0.4932768940925598,
      "logps/chosen": -8.089065551757812,
      "logps/rejected": -44.46285629272461,
      "loss": 0.870505690574646,
      "memory(GiB)": 43.15,
      "nll_loss": 0.6033220291137695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02359044924378395,
      "rewards/margins": 3.3024742603302,
      "rewards/rejected": -3.3260645866394043,
      "step": 258,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9040139616055847,
      "grad_norm": 2.1116161346435547,
      "learning_rate": 0.00016729412337095417,
      "logits/chosen": -0.34614264965057373,
      "logits/rejected": -0.5283275246620178,
      "logps/chosen": -8.69775104522705,
      "logps/rejected": -58.33458709716797,
      "loss": 0.516063392162323,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3785746097564697,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15219049155712128,
      "rewards/margins": 4.3896002769470215,
      "rewards/rejected": -4.237410068511963,
      "step": 259,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9075043630017452,
      "grad_norm": 2.9699761867523193,
      "learning_rate": 0.00016700849242155775,
      "logits/chosen": -0.2944773733615875,
      "logits/rejected": -0.42303967475891113,
      "logps/chosen": -7.197884559631348,
      "logps/rejected": -58.377899169921875,
      "loss": 0.5171555280685425,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3375796973705292,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.034792426973581314,
      "rewards/margins": 4.28358793258667,
      "rewards/rejected": -4.318380355834961,
      "step": 260,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9109947643979057,
      "grad_norm": 8.736255645751953,
      "learning_rate": 0.00016672186580579405,
      "logits/chosen": -0.3011060357093811,
      "logits/rejected": -0.5398942232131958,
      "logps/chosen": -5.949331760406494,
      "logps/rejected": -64.1772689819336,
      "loss": 0.6132771372795105,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5076794624328613,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.010059421882033348,
      "rewards/margins": 4.510530948638916,
      "rewards/rejected": -4.500471115112305,
      "step": 261,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9144851657940664,
      "grad_norm": 5.83949613571167,
      "learning_rate": 0.0001664342477825931,
      "logits/chosen": -0.44913211464881897,
      "logits/rejected": -0.5108562707901001,
      "logps/chosen": -12.713695526123047,
      "logps/rejected": -51.189388275146484,
      "loss": 1.1607731580734253,
      "memory(GiB)": 43.15,
      "nll_loss": 0.979007363319397,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07644180953502655,
      "rewards/margins": 4.067415237426758,
      "rewards/rejected": -3.9909729957580566,
      "step": 262,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9179755671902269,
      "grad_norm": 6.3965911865234375,
      "learning_rate": 0.00016614564262561608,
      "logits/chosen": -0.3431762456893921,
      "logits/rejected": -0.5227538347244263,
      "logps/chosen": -8.923004150390625,
      "logps/rejected": -66.28221893310547,
      "loss": 0.7520352602005005,
      "memory(GiB)": 43.15,
      "nll_loss": 0.6111706495285034,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0333654060959816,
      "rewards/margins": 4.924405574798584,
      "rewards/rejected": -4.891040325164795,
      "step": 263,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9214659685863874,
      "grad_norm": 8.98039722442627,
      "learning_rate": 0.0001658560546231918,
      "logits/chosen": -0.40294116735458374,
      "logits/rejected": -0.5690099000930786,
      "logps/chosen": -5.10881233215332,
      "logps/rejected": -55.61248779296875,
      "loss": 0.469806045293808,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31188011169433594,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08330924063920975,
      "rewards/margins": 4.100750923156738,
      "rewards/rejected": -4.017441749572754,
      "step": 264,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.924956369982548,
      "grad_norm": 2.279346466064453,
      "learning_rate": 0.00016556548807825295,
      "logits/chosen": -0.2596076726913452,
      "logits/rejected": -0.4245615601539612,
      "logps/chosen": -8.973737716674805,
      "logps/rejected": -63.824710845947266,
      "loss": 0.5860002636909485,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4316725730895996,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03057311289012432,
      "rewards/margins": 4.672904014587402,
      "rewards/rejected": -4.642330646514893,
      "step": 265,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9284467713787086,
      "grad_norm": 3.462368965148926,
      "learning_rate": 0.00016527394730827229,
      "logits/chosen": -0.39172494411468506,
      "logits/rejected": -0.5556192994117737,
      "logps/chosen": -6.451850891113281,
      "logps/rejected": -58.2652473449707,
      "loss": 0.6653540134429932,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5281586647033691,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09578607976436615,
      "rewards/margins": 4.3150811195373535,
      "rewards/rejected": -4.219295501708984,
      "step": 266,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9319371727748691,
      "grad_norm": 2.29887056350708,
      "learning_rate": 0.00016498143664519824,
      "logits/chosen": -0.3378483057022095,
      "logits/rejected": -0.5873894095420837,
      "logps/chosen": -5.2974629402160645,
      "logps/rejected": -76.44007110595703,
      "loss": 0.38004374504089355,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2812269926071167,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3595108091831207,
      "rewards/margins": 5.502060413360596,
      "rewards/rejected": -5.14254903793335,
      "step": 267,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9354275741710296,
      "grad_norm": 8.743305206298828,
      "learning_rate": 0.0001646879604353908,
      "logits/chosen": -0.3702645003795624,
      "logits/rejected": -0.5771125555038452,
      "logps/chosen": -9.347705841064453,
      "logps/rejected": -49.44694519042969,
      "loss": 1.1101744174957275,
      "memory(GiB)": 43.15,
      "nll_loss": 0.799601137638092,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0540962852537632,
      "rewards/margins": 3.277214288711548,
      "rewards/rejected": -3.331310749053955,
      "step": 268,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9389179755671903,
      "grad_norm": 1.31393563747406,
      "learning_rate": 0.00016439352303955688,
      "logits/chosen": -0.39863336086273193,
      "logits/rejected": -0.5482807159423828,
      "logps/chosen": -6.748524188995361,
      "logps/rejected": -59.980262756347656,
      "loss": 0.4353381097316742,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37013500928878784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15098029375076294,
      "rewards/margins": 4.846394062042236,
      "rewards/rejected": -4.695413589477539,
      "step": 269,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9424083769633508,
      "grad_norm": 5.9643425941467285,
      "learning_rate": 0.00016409812883268535,
      "logits/chosen": -0.38031327724456787,
      "logits/rejected": -0.5494654178619385,
      "logps/chosen": -6.423410892486572,
      "logps/rejected": -69.69914245605469,
      "loss": 0.4006003141403198,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3364032506942749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10248701274394989,
      "rewards/margins": 5.274303436279297,
      "rewards/rejected": -5.376790523529053,
      "step": 270,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9458987783595113,
      "grad_norm": 7.4491047859191895,
      "learning_rate": 0.00016380178220398226,
      "logits/chosen": -0.415705144405365,
      "logits/rejected": -0.6392954587936401,
      "logps/chosen": -6.486244201660156,
      "logps/rejected": -56.07293701171875,
      "loss": 0.6918466687202454,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5155448317527771,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09501791000366211,
      "rewards/margins": 3.938845634460449,
      "rewards/rejected": -3.843827962875366,
      "step": 271,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9493891797556719,
      "grad_norm": 2.838250160217285,
      "learning_rate": 0.0001635044875568054,
      "logits/chosen": -0.36552664637565613,
      "logits/rejected": -0.4577628970146179,
      "logps/chosen": -10.308563232421875,
      "logps/rejected": -52.53562927246094,
      "loss": 0.4913533926010132,
      "memory(GiB)": 43.15,
      "nll_loss": 0.34640663862228394,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10055547952651978,
      "rewards/margins": 4.087219715118408,
      "rewards/rejected": -3.986664056777954,
      "step": 272,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9528795811518325,
      "grad_norm": 8.961051940917969,
      "learning_rate": 0.00016320624930859904,
      "logits/chosen": -0.38108423352241516,
      "logits/rejected": -0.5300929546356201,
      "logps/chosen": -11.11557674407959,
      "logps/rejected": -57.43425750732422,
      "loss": 0.9454229474067688,
      "memory(GiB)": 43.15,
      "nll_loss": 0.7644782662391663,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.13230867683887482,
      "rewards/margins": 3.9995622634887695,
      "rewards/rejected": -4.131870746612549,
      "step": 273,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.956369982547993,
      "grad_norm": 2.179896116256714,
      "learning_rate": 0.0001629070718908283,
      "logits/chosen": -0.39322447776794434,
      "logits/rejected": -0.5073328614234924,
      "logps/chosen": -7.645257949829102,
      "logps/rejected": -51.549869537353516,
      "loss": 0.5495847463607788,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3960936665534973,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.038150034844875336,
      "rewards/margins": 4.1127800941467285,
      "rewards/rejected": -4.074629783630371,
      "step": 274,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9598603839441536,
      "grad_norm": 2.9412152767181396,
      "learning_rate": 0.0001626069597489132,
      "logits/chosen": -0.42589738965034485,
      "logits/rejected": -0.5628194808959961,
      "logps/chosen": -11.427227973937988,
      "logps/rejected": -51.27968978881836,
      "loss": 0.7387857437133789,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5841793417930603,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1430855542421341,
      "rewards/margins": 3.8389885425567627,
      "rewards/rejected": -3.6959028244018555,
      "step": 275,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9633507853403142,
      "grad_norm": 3.372194528579712,
      "learning_rate": 0.0001623059173421625,
      "logits/chosen": -0.2240457385778427,
      "logits/rejected": -0.5538793802261353,
      "logps/chosen": -5.805129051208496,
      "logps/rejected": -71.96966552734375,
      "loss": 0.5059239268302917,
      "memory(GiB)": 43.15,
      "nll_loss": 0.42470091581344604,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06452590972185135,
      "rewards/margins": 5.022994041442871,
      "rewards/rejected": -4.958468437194824,
      "step": 276,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9668411867364747,
      "grad_norm": 4.933607578277588,
      "learning_rate": 0.00016200394914370788,
      "logits/chosen": -0.318347692489624,
      "logits/rejected": -0.47638392448425293,
      "logps/chosen": -8.253751754760742,
      "logps/rejected": -57.21156692504883,
      "loss": 0.8552437424659729,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5107535719871521,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.028203103691339493,
      "rewards/margins": 3.587186336517334,
      "rewards/rejected": -3.615389585494995,
      "step": 277,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9703315881326352,
      "grad_norm": 1.967063069343567,
      "learning_rate": 0.00016170105964043695,
      "logits/chosen": -0.46634575724601746,
      "logits/rejected": -0.5018796920776367,
      "logps/chosen": -8.443771362304688,
      "logps/rejected": -36.8387336730957,
      "loss": 0.6206079125404358,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3942241072654724,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14703020453453064,
      "rewards/margins": 2.6712229251861572,
      "rewards/rejected": -2.5241925716400146,
      "step": 278,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9738219895287958,
      "grad_norm": 2.5119967460632324,
      "learning_rate": 0.0001613972533329269,
      "logits/chosen": -0.39050760865211487,
      "logits/rejected": -0.5310595035552979,
      "logps/chosen": -4.703199863433838,
      "logps/rejected": -50.82078552246094,
      "loss": 0.4742874801158905,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2932378947734833,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04821047931909561,
      "rewards/margins": 3.5806527137756348,
      "rewards/rejected": -3.5324418544769287,
      "step": 279,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9773123909249564,
      "grad_norm": 2.238361120223999,
      "learning_rate": 0.0001610925347353775,
      "logits/chosen": -0.37722715735435486,
      "logits/rejected": -0.5656870603561401,
      "logps/chosen": -5.8236494064331055,
      "logps/rejected": -50.120391845703125,
      "loss": 0.5826418995857239,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4334554672241211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16360187530517578,
      "rewards/margins": 3.4687533378601074,
      "rewards/rejected": -3.30515193939209,
      "step": 280,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.9808027923211169,
      "grad_norm": 2.3494832515716553,
      "learning_rate": 0.00016078690837554408,
      "logits/chosen": -0.33358269929885864,
      "logits/rejected": -0.5591884255409241,
      "logps/chosen": -5.116516590118408,
      "logps/rejected": -49.51860046386719,
      "loss": 0.5960334539413452,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3720979690551758,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17815372347831726,
      "rewards/margins": 3.0898640155792236,
      "rewards/rejected": -2.911710262298584,
      "step": 281,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.9842931937172775,
      "grad_norm": 3.6189231872558594,
      "learning_rate": 0.00016048037879467025,
      "logits/chosen": -0.37796086072921753,
      "logits/rejected": -0.5117664933204651,
      "logps/chosen": -4.1463236808776855,
      "logps/rejected": -48.00626754760742,
      "loss": 0.5160388946533203,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31095653772354126,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.034109927713871,
      "rewards/margins": 3.225755453109741,
      "rewards/rejected": -3.191645383834839,
      "step": 282,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.987783595113438,
      "grad_norm": 5.189328193664551,
      "learning_rate": 0.00016017295054742046,
      "logits/chosen": -0.33860644698143005,
      "logits/rejected": -0.5117906332015991,
      "logps/chosen": -11.231898307800293,
      "logps/rejected": -49.39238739013672,
      "loss": 0.8505796790122986,
      "memory(GiB)": 43.15,
      "nll_loss": 0.6683485507965088,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10711870342493057,
      "rewards/margins": 3.197012424468994,
      "rewards/rejected": -3.089893341064453,
      "step": 283,
      "train_speed(iter/s)": 0.010756
    },
    {
      "epoch": 0.9912739965095986,
      "grad_norm": 26.61264419555664,
      "learning_rate": 0.0001598646282018121,
      "logits/chosen": -0.36234819889068604,
      "logits/rejected": -0.5464830994606018,
      "logps/chosen": -7.227906703948975,
      "logps/rejected": -53.168861389160156,
      "loss": 0.5734856724739075,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41981199383735657,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07493679225444794,
      "rewards/margins": 3.693037748336792,
      "rewards/rejected": -3.618101119995117,
      "step": 284,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9947643979057592,
      "grad_norm": 5.144956588745117,
      "learning_rate": 0.000159555416339148,
      "logits/chosen": -0.26111942529678345,
      "logits/rejected": -0.5374065041542053,
      "logps/chosen": -2.2055845260620117,
      "logps/rejected": -68.69379425048828,
      "loss": 0.3513576090335846,
      "memory(GiB)": 43.15,
      "nll_loss": 0.21501211822032928,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09447957575321198,
      "rewards/margins": 4.662655830383301,
      "rewards/rejected": -4.56817626953125,
      "step": 285,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 0.9982547993019197,
      "grad_norm": 2.222433090209961,
      "learning_rate": 0.00015924531955394802,
      "logits/chosen": -0.3610646724700928,
      "logits/rejected": -0.4617045521736145,
      "logps/chosen": -7.9295196533203125,
      "logps/rejected": -47.70429611206055,
      "loss": 0.5501912832260132,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3641606867313385,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13688504695892334,
      "rewards/margins": 3.380563735961914,
      "rewards/rejected": -3.243678569793701,
      "step": 286,
      "train_speed(iter/s)": 0.010755
    },
    {
      "epoch": 1.0034904013961605,
      "grad_norm": 2.3679821491241455,
      "learning_rate": 0.00015893434245388093,
      "logits/chosen": -0.39241814613342285,
      "logits/rejected": -0.5621130466461182,
      "logps/chosen": -4.523199081420898,
      "logps/rejected": -60.3804931640625,
      "loss": 0.48575231432914734,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28081169724464417,
      "rewards/accuracies": 0.9411764740943909,
      "rewards/chosen": 0.13038994371891022,
      "rewards/margins": 4.663900375366211,
      "rewards/rejected": -4.533511161804199,
      "step": 287,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.006980802792321,
      "grad_norm": 1.3687598705291748,
      "learning_rate": 0.00015862248965969604,
      "logits/chosen": -0.26206377148628235,
      "logits/rejected": -0.5492232441902161,
      "logps/chosen": -5.55544900894165,
      "logps/rejected": -82.16498565673828,
      "loss": 0.35956525802612305,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28186729550361633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26697322726249695,
      "rewards/margins": 6.157236576080322,
      "rewards/rejected": -5.89026403427124,
      "step": 288,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0104712041884816,
      "grad_norm": 4.600061893463135,
      "learning_rate": 0.00015830976580515432,
      "logits/chosen": -0.3973150849342346,
      "logits/rejected": -0.527411937713623,
      "logps/chosen": -11.000313758850098,
      "logps/rejected": -61.0347785949707,
      "loss": 0.7208768129348755,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5785147547721863,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18078193068504333,
      "rewards/margins": 4.359728813171387,
      "rewards/rejected": -4.1789469718933105,
      "step": 289,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0139616055846423,
      "grad_norm": 2.2930972576141357,
      "learning_rate": 0.00015799617553695957,
      "logits/chosen": -0.4011244773864746,
      "logits/rejected": -0.5036134719848633,
      "logps/chosen": -6.045261383056641,
      "logps/rejected": -57.71175765991211,
      "loss": 0.3789103627204895,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3045409619808197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3516743779182434,
      "rewards/margins": 4.524147033691406,
      "rewards/rejected": -4.1724724769592285,
      "step": 290,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0174520069808028,
      "grad_norm": 1.8977946043014526,
      "learning_rate": 0.00015768172351468974,
      "logits/chosen": -0.3493969142436981,
      "logits/rejected": -0.5791320204734802,
      "logps/chosen": -3.5215115547180176,
      "logps/rejected": -65.80962371826172,
      "loss": 0.36785203218460083,
      "memory(GiB)": 43.15,
      "nll_loss": 0.26871761679649353,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3286406993865967,
      "rewards/margins": 5.31328010559082,
      "rewards/rejected": -4.984640121459961,
      "step": 291,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0209424083769634,
      "grad_norm": 8.48400592803955,
      "learning_rate": 0.0001573664144107272,
      "logits/chosen": -0.40092405676841736,
      "logits/rejected": -0.5814225077629089,
      "logps/chosen": -4.21975564956665,
      "logps/rejected": -66.2427978515625,
      "loss": 0.35437557101249695,
      "memory(GiB)": 43.15,
      "nll_loss": 0.27522024512290955,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2891547381877899,
      "rewards/margins": 4.967032432556152,
      "rewards/rejected": -4.677878379821777,
      "step": 292,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.024432809773124,
      "grad_norm": 2.3877346515655518,
      "learning_rate": 0.0001570502529101896,
      "logits/chosen": -0.4078039228916168,
      "logits/rejected": -0.6340718269348145,
      "logps/chosen": -4.7514328956604,
      "logps/rejected": -63.62433624267578,
      "loss": 0.2756602466106415,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2308274805545807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2891400456428528,
      "rewards/margins": 5.025572776794434,
      "rewards/rejected": -4.736433029174805,
      "step": 293,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0279232111692844,
      "grad_norm": 1.341036081314087,
      "learning_rate": 0.0001567332437108602,
      "logits/chosen": -0.4645722806453705,
      "logits/rejected": -0.6466395854949951,
      "logps/chosen": -4.918186187744141,
      "logps/rejected": -55.84804916381836,
      "loss": 0.3296290636062622,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23185840249061584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28173521161079407,
      "rewards/margins": 4.368047714233398,
      "rewards/rejected": -4.086312294006348,
      "step": 294,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.031413612565445,
      "grad_norm": 1.4693691730499268,
      "learning_rate": 0.00015641539152311796,
      "logits/chosen": -0.42224860191345215,
      "logits/rejected": -0.6172808408737183,
      "logps/chosen": -5.080268859863281,
      "logps/rejected": -57.87543487548828,
      "loss": 0.4304788410663605,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3393770754337311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2397257685661316,
      "rewards/margins": 4.47007942199707,
      "rewards/rejected": -4.230354309082031,
      "step": 295,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0349040139616057,
      "grad_norm": 3.831491231918335,
      "learning_rate": 0.00015609670106986776,
      "logits/chosen": -0.5365476608276367,
      "logits/rejected": -0.6072984337806702,
      "logps/chosen": -5.30760383605957,
      "logps/rejected": -49.26717758178711,
      "loss": 0.36716127395629883,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2641866207122803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4461376667022705,
      "rewards/margins": 3.9801061153411865,
      "rewards/rejected": -3.533968687057495,
      "step": 296,
      "train_speed(iter/s)": 0.010752
    },
    {
      "epoch": 1.0383944153577662,
      "grad_norm": 8.527892112731934,
      "learning_rate": 0.0001557771770864701,
      "logits/chosen": -0.3701344430446625,
      "logits/rejected": -0.5703862309455872,
      "logps/chosen": -6.846416473388672,
      "logps/rejected": -65.28385925292969,
      "loss": 0.47399166226387024,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32827308773994446,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.45203351974487305,
      "rewards/margins": 4.902979850769043,
      "rewards/rejected": -4.450946807861328,
      "step": 297,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 1.0418848167539267,
      "grad_norm": 2.383213520050049,
      "learning_rate": 0.00015545682432067067,
      "logits/chosen": -0.4192945659160614,
      "logits/rejected": -0.663847804069519,
      "logps/chosen": -5.736666679382324,
      "logps/rejected": -72.8265380859375,
      "loss": 0.32853907346725464,
      "memory(GiB)": 43.15,
      "nll_loss": 0.29026535153388977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47007882595062256,
      "rewards/margins": 5.804821014404297,
      "rewards/rejected": -5.334741592407227,
      "step": 298,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 1.0453752181500873,
      "grad_norm": 2.514970064163208,
      "learning_rate": 0.00015513564753252994,
      "logits/chosen": -0.5625315308570862,
      "logits/rejected": -0.6955651044845581,
      "logps/chosen": -6.750204563140869,
      "logps/rejected": -58.264225006103516,
      "loss": 0.5013718605041504,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3770815134048462,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1921541839838028,
      "rewards/margins": 4.60443115234375,
      "rewards/rejected": -4.4122772216796875,
      "step": 299,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 1.0488656195462478,
      "grad_norm": 8.446479797363281,
      "learning_rate": 0.00015481365149435235,
      "logits/chosen": -0.5021696090698242,
      "logits/rejected": -0.663796603679657,
      "logps/chosen": -6.143691062927246,
      "logps/rejected": -53.95307159423828,
      "loss": 0.46026065945625305,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3280302882194519,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2705683708190918,
      "rewards/margins": 4.268153190612793,
      "rewards/rejected": -3.9975852966308594,
      "step": 300,
      "train_speed(iter/s)": 0.010751
    },
    {
      "epoch": 1.0523560209424083,
      "grad_norm": 1.4564425945281982,
      "learning_rate": 0.00015449084099061538,
      "logits/chosen": -0.5470643043518066,
      "logits/rejected": -0.5662429332733154,
      "logps/chosen": -8.135005950927734,
      "logps/rejected": -61.87958908081055,
      "loss": 0.3989267945289612,
      "memory(GiB)": 43.15,
      "nll_loss": 0.35136380791664124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.475752592086792,
      "rewards/margins": 5.381492614746094,
      "rewards/rejected": -4.905740737915039,
      "step": 301,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.0558464223385688,
      "grad_norm": 2.5607101917266846,
      "learning_rate": 0.00015416722081789864,
      "logits/chosen": -0.4304802715778351,
      "logits/rejected": -0.6808840036392212,
      "logps/chosen": -3.730553388595581,
      "logps/rejected": -77.83374786376953,
      "loss": 0.31356939673423767,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23746523261070251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32321637868881226,
      "rewards/margins": 5.76716423034668,
      "rewards/rejected": -5.443948745727539,
      "step": 302,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.0593368237347296,
      "grad_norm": 3.4172539710998535,
      "learning_rate": 0.00015384279578481221,
      "logits/chosen": -0.5958032608032227,
      "logits/rejected": -0.6914587020874023,
      "logps/chosen": -11.875475883483887,
      "logps/rejected": -56.1957893371582,
      "loss": 0.6093863844871521,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5033939480781555,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5004825592041016,
      "rewards/margins": 4.409386157989502,
      "rewards/rejected": -3.9089038372039795,
      "step": 303,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0628272251308901,
      "grad_norm": 4.900504112243652,
      "learning_rate": 0.00015351757071192573,
      "logits/chosen": -0.36820149421691895,
      "logits/rejected": -0.689315676689148,
      "logps/chosen": -3.029294729232788,
      "logps/rejected": -76.44140625,
      "loss": 0.3941238224506378,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2878090739250183,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1027887836098671,
      "rewards/margins": 5.742969989776611,
      "rewards/rejected": -5.640181541442871,
      "step": 304,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0663176265270506,
      "grad_norm": 2.218637466430664,
      "learning_rate": 0.00015319155043169618,
      "logits/chosen": -0.5266806483268738,
      "logits/rejected": -0.7329103946685791,
      "logps/chosen": -6.292713642120361,
      "logps/rejected": -63.29831314086914,
      "loss": 0.409374862909317,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3364257216453552,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.40051528811454773,
      "rewards/margins": 5.179149150848389,
      "rewards/rejected": -4.778634548187256,
      "step": 305,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0698080279232112,
      "grad_norm": 8.687586784362793,
      "learning_rate": 0.00015286473978839662,
      "logits/chosen": -0.6478016376495361,
      "logits/rejected": -0.7238842248916626,
      "logps/chosen": -5.924484729766846,
      "logps/rejected": -61.01313400268555,
      "loss": 0.4875742495059967,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3984666168689728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39782842993736267,
      "rewards/margins": 5.429813861846924,
      "rewards/rejected": -5.031985282897949,
      "step": 306,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0732984293193717,
      "grad_norm": 5.456116199493408,
      "learning_rate": 0.00015253714363804388,
      "logits/chosen": -0.5222834944725037,
      "logits/rejected": -0.7372316718101501,
      "logps/chosen": -5.703394889831543,
      "logps/rejected": -82.65278625488281,
      "loss": 0.42027172446250916,
      "memory(GiB)": 43.15,
      "nll_loss": 0.30709269642829895,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19577693939208984,
      "rewards/margins": 6.722568988800049,
      "rewards/rejected": -6.526791572570801,
      "step": 307,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0767888307155322,
      "grad_norm": 1.5310981273651123,
      "learning_rate": 0.00015220876684832638,
      "logits/chosen": -0.6310667395591736,
      "logits/rejected": -0.7239534854888916,
      "logps/chosen": -5.980924606323242,
      "logps/rejected": -68.00843811035156,
      "loss": 0.34952861070632935,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3003528416156769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32078802585601807,
      "rewards/margins": 5.430644512176514,
      "rewards/rejected": -5.109856605529785,
      "step": 308,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0802792321116927,
      "grad_norm": 4.554830074310303,
      "learning_rate": 0.0001518796142985321,
      "logits/chosen": -0.6367871761322021,
      "logits/rejected": -0.7653591632843018,
      "logps/chosen": -6.020457744598389,
      "logps/rejected": -65.98897552490234,
      "loss": 0.4749387502670288,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37060117721557617,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19638437032699585,
      "rewards/margins": 5.541833877563477,
      "rewards/rejected": -5.345449447631836,
      "step": 309,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0837696335078535,
      "grad_norm": 1.6975951194763184,
      "learning_rate": 0.00015154969087947573,
      "logits/chosen": -0.48107388615608215,
      "logits/rejected": -0.7438764572143555,
      "logps/chosen": -3.089120864868164,
      "logps/rejected": -69.92082214355469,
      "loss": 0.2973018288612366,
      "memory(GiB)": 43.15,
      "nll_loss": 0.22607487440109253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3830302953720093,
      "rewards/margins": 5.329667091369629,
      "rewards/rejected": -4.94663667678833,
      "step": 310,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.087260034904014,
      "grad_norm": 2.6259241104125977,
      "learning_rate": 0.0001512190014934263,
      "logits/chosen": -0.5431985259056091,
      "logits/rejected": -0.7063261866569519,
      "logps/chosen": -5.224724769592285,
      "logps/rejected": -71.81791687011719,
      "loss": 0.3114906847476959,
      "memory(GiB)": 43.15,
      "nll_loss": 0.238429993391037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20617195963859558,
      "rewards/margins": 5.925920009613037,
      "rewards/rejected": -5.719748497009277,
      "step": 311,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0907504363001745,
      "grad_norm": 1.2943837642669678,
      "learning_rate": 0.00015088755105403405,
      "logits/chosen": -0.5590757131576538,
      "logits/rejected": -0.8148021697998047,
      "logps/chosen": -4.819448947906494,
      "logps/rejected": -70.58955383300781,
      "loss": 0.34402620792388916,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28382259607315063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2690161168575287,
      "rewards/margins": 5.675660133361816,
      "rewards/rejected": -5.406644344329834,
      "step": 312,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.094240837696335,
      "grad_norm": 1.4312578439712524,
      "learning_rate": 0.00015055534448625766,
      "logits/chosen": -0.49336162209510803,
      "logits/rejected": -0.7816673517227173,
      "logps/chosen": -6.685326099395752,
      "logps/rejected": -80.9956283569336,
      "loss": 0.4569896161556244,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4009275436401367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3557565212249756,
      "rewards/margins": 6.279770374298096,
      "rewards/rejected": -5.924014091491699,
      "step": 313,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.0977312390924956,
      "grad_norm": 1.5266355276107788,
      "learning_rate": 0.00015022238672629095,
      "logits/chosen": -0.5756551623344421,
      "logits/rejected": -0.735444962978363,
      "logps/chosen": -5.3377766609191895,
      "logps/rejected": -67.37596130371094,
      "loss": 0.3693499267101288,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2790605127811432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3705032169818878,
      "rewards/margins": 5.493788242340088,
      "rewards/rejected": -5.123284816741943,
      "step": 314,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.1012216404886561,
      "grad_norm": 1.9074639081954956,
      "learning_rate": 0.0001498886827214896,
      "logits/chosen": -0.6456474661827087,
      "logits/rejected": -0.7724269032478333,
      "logps/chosen": -5.277746677398682,
      "logps/rejected": -53.66168212890625,
      "loss": 0.44892674684524536,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37271296977996826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25592944025993347,
      "rewards/margins": 4.511635780334473,
      "rewards/rejected": -4.255705833435059,
      "step": 315,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.1047120418848166,
      "grad_norm": 1.9207693338394165,
      "learning_rate": 0.0001495542374302975,
      "logits/chosen": -0.5007003545761108,
      "logits/rejected": -0.650519609451294,
      "logps/chosen": -5.601229190826416,
      "logps/rejected": -65.04399108886719,
      "loss": 0.418306440114975,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32126715779304504,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3169766068458557,
      "rewards/margins": 5.420646667480469,
      "rewards/rejected": -5.103670597076416,
      "step": 316,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.1082024432809774,
      "grad_norm": 2.1684069633483887,
      "learning_rate": 0.00014921905582217332,
      "logits/chosen": -0.3752322793006897,
      "logits/rejected": -0.6496881246566772,
      "logps/chosen": -4.458385944366455,
      "logps/rejected": -83.53166198730469,
      "loss": 0.37263908982276917,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3141525983810425,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30656152963638306,
      "rewards/margins": 5.798944473266602,
      "rewards/rejected": -5.492383003234863,
      "step": 317,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.111692844677138,
      "grad_norm": 1.412676453590393,
      "learning_rate": 0.0001488831428775164,
      "logits/chosen": -0.5132664442062378,
      "logits/rejected": -0.7257671356201172,
      "logps/chosen": -3.272702693939209,
      "logps/rejected": -57.20233917236328,
      "loss": 0.3777000308036804,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31745412945747375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22153383493423462,
      "rewards/margins": 4.813529014587402,
      "rewards/rejected": -4.5919952392578125,
      "step": 318,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.1151832460732984,
      "grad_norm": 2.84291410446167,
      "learning_rate": 0.0001485465035875929,
      "logits/chosen": -0.5710486173629761,
      "logits/rejected": -0.6527326107025146,
      "logps/chosen": -7.473095417022705,
      "logps/rejected": -60.35542297363281,
      "loss": 0.5568766593933105,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4240596890449524,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12074887752532959,
      "rewards/margins": 4.750669002532959,
      "rewards/rejected": -4.62992000579834,
      "step": 319,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.118673647469459,
      "grad_norm": 10.697195053100586,
      "learning_rate": 0.00014820914295446164,
      "logits/chosen": -0.48244622349739075,
      "logits/rejected": -0.6707563400268555,
      "logps/chosen": -4.583101272583008,
      "logps/rejected": -70.41757202148438,
      "loss": 0.45662960410118103,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3494943082332611,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22834914922714233,
      "rewards/margins": 5.394464492797852,
      "rewards/rejected": -5.166115760803223,
      "step": 320,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.1221640488656195,
      "grad_norm": 7.470077037811279,
      "learning_rate": 0.00014787106599089967,
      "logits/chosen": -0.4395173192024231,
      "logits/rejected": -0.7748122215270996,
      "logps/chosen": -4.163114070892334,
      "logps/rejected": -87.97148132324219,
      "loss": 0.42231428623199463,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3452557325363159,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.40086498856544495,
      "rewards/margins": 7.3803606033325195,
      "rewards/rejected": -6.979496002197266,
      "step": 321,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.12565445026178,
      "grad_norm": 6.354022979736328,
      "learning_rate": 0.00014753227772032793,
      "logits/chosen": -0.4927988350391388,
      "logits/rejected": -0.6983435153961182,
      "logps/chosen": -9.013876914978027,
      "logps/rejected": -81.17129516601562,
      "loss": 0.9018564224243164,
      "memory(GiB)": 43.15,
      "nll_loss": 0.7067390084266663,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.17468257248401642,
      "rewards/margins": 6.47627067565918,
      "rewards/rejected": -6.650952339172363,
      "step": 322,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1291448516579408,
      "grad_norm": 6.593186378479004,
      "learning_rate": 0.00014719278317673655,
      "logits/chosen": -0.5038294196128845,
      "logits/rejected": -0.6811163425445557,
      "logps/chosen": -5.416735649108887,
      "logps/rejected": -68.10504150390625,
      "loss": 0.31916317343711853,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2741475999355316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33685386180877686,
      "rewards/margins": 5.806307792663574,
      "rewards/rejected": -5.469453811645508,
      "step": 323,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1326352530541013,
      "grad_norm": 1.6076407432556152,
      "learning_rate": 0.00014685258740460995,
      "logits/chosen": -0.4740574359893799,
      "logits/rejected": -0.6336863040924072,
      "logps/chosen": -5.115401268005371,
      "logps/rejected": -71.67650604248047,
      "loss": 0.42807137966156006,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3246609568595886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2911081910133362,
      "rewards/margins": 5.678893089294434,
      "rewards/rejected": -5.387784957885742,
      "step": 324,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1361256544502618,
      "grad_norm": 4.390758991241455,
      "learning_rate": 0.00014651169545885199,
      "logits/chosen": -0.4668413996696472,
      "logits/rejected": -0.6320906281471252,
      "logps/chosen": -5.329308032989502,
      "logps/rejected": -63.19139862060547,
      "loss": 0.45732176303863525,
      "memory(GiB)": 43.15,
      "nll_loss": 0.39123204350471497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4576411843299866,
      "rewards/margins": 5.102199554443359,
      "rewards/rejected": -4.644558429718018,
      "step": 325,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1396160558464223,
      "grad_norm": 2.024564743041992,
      "learning_rate": 0.00014617011240471094,
      "logits/chosen": -0.5206573009490967,
      "logits/rejected": -0.6565874218940735,
      "logps/chosen": -5.336911678314209,
      "logps/rejected": -56.41254806518555,
      "loss": 0.46840786933898926,
      "memory(GiB)": 43.15,
      "nll_loss": 0.35700729489326477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21654362976551056,
      "rewards/margins": 4.641784191131592,
      "rewards/rejected": -4.425240516662598,
      "step": 326,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1431064572425829,
      "grad_norm": 2.056162118911743,
      "learning_rate": 0.00014582784331770393,
      "logits/chosen": -0.5073651671409607,
      "logits/rejected": -0.659345269203186,
      "logps/chosen": -5.8628034591674805,
      "logps/rejected": -49.929962158203125,
      "loss": 0.5279353857040405,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41238051652908325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2424943745136261,
      "rewards/margins": 3.679898500442505,
      "rewards/rejected": -3.4374046325683594,
      "step": 327,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1465968586387434,
      "grad_norm": 2.5892443656921387,
      "learning_rate": 0.00014548489328354195,
      "logits/chosen": -0.4672063887119293,
      "logits/rejected": -0.6109826564788818,
      "logps/chosen": -5.497149467468262,
      "logps/rejected": -60.077178955078125,
      "loss": 0.4822987914085388,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37062495946884155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.36120548844337463,
      "rewards/margins": 5.0480451583862305,
      "rewards/rejected": -4.686839580535889,
      "step": 328,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.150087260034904,
      "grad_norm": 1.3888144493103027,
      "learning_rate": 0.00014514126739805387,
      "logits/chosen": -0.5015818476676941,
      "logits/rejected": -0.6728599071502686,
      "logps/chosen": -3.3580546379089355,
      "logps/rejected": -54.8983268737793,
      "loss": 0.370685338973999,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2433609664440155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18075354397296906,
      "rewards/margins": 4.068675994873047,
      "rewards/rejected": -3.8879222869873047,
      "step": 329,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.1535776614310647,
      "grad_norm": 1.22072172164917,
      "learning_rate": 0.00014479697076711097,
      "logits/chosen": -0.533420741558075,
      "logits/rejected": -0.5922872424125671,
      "logps/chosen": -4.107625961303711,
      "logps/rejected": -53.41065979003906,
      "loss": 0.26737067103385925,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2190181165933609,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.47103744745254517,
      "rewards/margins": 4.485008716583252,
      "rewards/rejected": -4.013971328735352,
      "step": 330,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.1570680628272252,
      "grad_norm": 1.1153243780136108,
      "learning_rate": 0.00014445200850655106,
      "logits/chosen": -0.5124134421348572,
      "logits/rejected": -0.5720605254173279,
      "logps/chosen": -4.25541877746582,
      "logps/rejected": -47.59088897705078,
      "loss": 0.4234582781791687,
      "memory(GiB)": 43.15,
      "nll_loss": 0.29615476727485657,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3857133984565735,
      "rewards/margins": 4.029388427734375,
      "rewards/rejected": -3.643674850463867,
      "step": 331,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.1605584642233857,
      "grad_norm": 1.839998722076416,
      "learning_rate": 0.00014410638574210231,
      "logits/chosen": -0.5487766265869141,
      "logits/rejected": -0.6048561930656433,
      "logps/chosen": -2.9526216983795166,
      "logps/rejected": -49.01722717285156,
      "loss": 0.33782559633255005,
      "memory(GiB)": 43.15,
      "nll_loss": 0.25286024808883667,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19235680997371674,
      "rewards/margins": 4.1119232177734375,
      "rewards/rejected": -3.9195663928985596,
      "step": 332,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.1640488656195462,
      "grad_norm": 0.9550164937973022,
      "learning_rate": 0.00014376010760930728,
      "logits/chosen": -0.48927396535873413,
      "logits/rejected": -0.6853697299957275,
      "logps/chosen": -2.1129355430603027,
      "logps/rejected": -59.902103424072266,
      "loss": 0.2407364398241043,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2040787637233734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18717700242996216,
      "rewards/margins": 4.609613418579102,
      "rewards/rejected": -4.422436237335205,
      "step": 333,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.1675392670157068,
      "grad_norm": 1.5491670370101929,
      "learning_rate": 0.00014341317925344645,
      "logits/chosen": -0.5509553551673889,
      "logits/rejected": -0.5921827554702759,
      "logps/chosen": -8.330162048339844,
      "logps/rejected": -54.603511810302734,
      "loss": 0.46315959095954895,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37922072410583496,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3218461573123932,
      "rewards/margins": 4.3676910400390625,
      "rewards/rejected": -4.045845031738281,
      "step": 334,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.1710296684118673,
      "grad_norm": 3.619370460510254,
      "learning_rate": 0.00014306560582946193,
      "logits/chosen": -0.584208607673645,
      "logits/rejected": -0.665631890296936,
      "logps/chosen": -2.970817804336548,
      "logps/rejected": -52.78791046142578,
      "loss": 0.32715079188346863,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23035843670368195,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0841044932603836,
      "rewards/margins": 4.168006420135498,
      "rewards/rejected": -4.083901405334473,
      "step": 335,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.174520069808028,
      "grad_norm": 1.3068387508392334,
      "learning_rate": 0.0001427173925018807,
      "logits/chosen": -0.459281861782074,
      "logits/rejected": -0.6941739320755005,
      "logps/chosen": -3.556865930557251,
      "logps/rejected": -60.05839538574219,
      "loss": 0.2930494248867035,
      "memory(GiB)": 43.15,
      "nll_loss": 0.21146121621131897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2878793478012085,
      "rewards/margins": 4.445915699005127,
      "rewards/rejected": -4.158036231994629,
      "step": 336,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1780104712041886,
      "grad_norm": 1.542277216911316,
      "learning_rate": 0.00014236854444473794,
      "logits/chosen": -0.45352160930633545,
      "logits/rejected": -0.5974130034446716,
      "logps/chosen": -7.562567710876465,
      "logps/rejected": -66.69306945800781,
      "loss": 0.40887123346328735,
      "memory(GiB)": 43.15,
      "nll_loss": 0.38999485969543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5074263215065002,
      "rewards/margins": 5.6102190017700195,
      "rewards/rejected": -5.102793216705322,
      "step": 337,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.181500872600349,
      "grad_norm": 1.908805012702942,
      "learning_rate": 0.0001420190668415002,
      "logits/chosen": -0.5282233953475952,
      "logits/rejected": -0.66242516040802,
      "logps/chosen": -8.06371021270752,
      "logps/rejected": -59.486412048339844,
      "loss": 0.48714980483055115,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4481129050254822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2379024177789688,
      "rewards/margins": 4.8613810539245605,
      "rewards/rejected": -4.623478889465332,
      "step": 338,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1849912739965096,
      "grad_norm": 1.9679303169250488,
      "learning_rate": 0.0001416689648849883,
      "logits/chosen": -0.5385857820510864,
      "logits/rejected": -0.7091735601425171,
      "logps/chosen": -4.285862922668457,
      "logps/rejected": -55.13109588623047,
      "loss": 0.3885069191455841,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3152792453765869,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4284796416759491,
      "rewards/margins": 4.7022929191589355,
      "rewards/rejected": -4.273813247680664,
      "step": 339,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1884816753926701,
      "grad_norm": 6.159523010253906,
      "learning_rate": 0.00014131824377730024,
      "logits/chosen": -0.6195085048675537,
      "logits/rejected": -0.63163161277771,
      "logps/chosen": -5.583183288574219,
      "logps/rejected": -50.4035758972168,
      "loss": 0.4245375990867615,
      "memory(GiB)": 43.15,
      "nll_loss": 0.33572766184806824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2565540373325348,
      "rewards/margins": 4.001959323883057,
      "rewards/rejected": -3.745405673980713,
      "step": 340,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1919720767888307,
      "grad_norm": 4.105616092681885,
      "learning_rate": 0.00014096690872973387,
      "logits/chosen": -0.5850836634635925,
      "logits/rejected": -0.6230161786079407,
      "logps/chosen": -6.9361677169799805,
      "logps/rejected": -55.36536407470703,
      "loss": 0.594155490398407,
      "memory(GiB)": 43.15,
      "nll_loss": 0.49159759283065796,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.17695164680480957,
      "rewards/margins": 4.677291393280029,
      "rewards/rejected": -4.500339984893799,
      "step": 341,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1954624781849912,
      "grad_norm": 1.673965334892273,
      "learning_rate": 0.00014061496496270943,
      "logits/chosen": -0.5429196357727051,
      "logits/rejected": -0.7085021734237671,
      "logps/chosen": -4.494854927062988,
      "logps/rejected": -58.405250549316406,
      "loss": 0.3368845283985138,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24193470180034637,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.33928442001342773,
      "rewards/margins": 4.766523361206055,
      "rewards/rejected": -4.427238941192627,
      "step": 342,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.1989528795811517,
      "grad_norm": 1.5858604907989502,
      "learning_rate": 0.00014026241770569197,
      "logits/chosen": -0.4735041856765747,
      "logits/rejected": -0.5973408222198486,
      "logps/chosen": -5.155520915985107,
      "logps/rejected": -61.884307861328125,
      "loss": 0.380929172039032,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2979738116264343,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3542560338973999,
      "rewards/margins": 5.134637832641602,
      "rewards/rejected": -4.78038215637207,
      "step": 343,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.2024432809773125,
      "grad_norm": 1.420960783958435,
      "learning_rate": 0.00013990927219711377,
      "logits/chosen": -0.390002578496933,
      "logits/rejected": -0.7470687627792358,
      "logps/chosen": -3.7865405082702637,
      "logps/rejected": -72.54590606689453,
      "loss": 0.33121392130851746,
      "memory(GiB)": 43.15,
      "nll_loss": 0.285677433013916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38604891300201416,
      "rewards/margins": 5.873488426208496,
      "rewards/rejected": -5.4874396324157715,
      "step": 344,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.205933682373473,
      "grad_norm": 1.665773868560791,
      "learning_rate": 0.0001395555336842963,
      "logits/chosen": -0.4295946955680847,
      "logits/rejected": -0.6522460579872131,
      "logps/chosen": -4.432346820831299,
      "logps/rejected": -72.02410888671875,
      "loss": 0.31357499957084656,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2627772390842438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3896270990371704,
      "rewards/margins": 5.963184833526611,
      "rewards/rejected": -5.5735578536987305,
      "step": 345,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.2094240837696335,
      "grad_norm": 1.5276520252227783,
      "learning_rate": 0.00013920120742337254,
      "logits/chosen": -0.4331398904323578,
      "logits/rejected": -0.6702686548233032,
      "logps/chosen": -5.334844589233398,
      "logps/rejected": -66.99110412597656,
      "loss": 0.41836920380592346,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3846779465675354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30076733231544495,
      "rewards/margins": 5.455785274505615,
      "rewards/rejected": -5.155017852783203,
      "step": 346,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.212914485165794,
      "grad_norm": 2.6530966758728027,
      "learning_rate": 0.00013884629867920854,
      "logits/chosen": -0.42600876092910767,
      "logits/rejected": -0.6288998126983643,
      "logps/chosen": -4.370656967163086,
      "logps/rejected": -60.58926010131836,
      "loss": 0.5389068126678467,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4162063002586365,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2829643189907074,
      "rewards/margins": 4.741201400756836,
      "rewards/rejected": -4.4582366943359375,
      "step": 347,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2164048865619546,
      "grad_norm": 2.025510311126709,
      "learning_rate": 0.00013849081272532544,
      "logits/chosen": -0.42667314410209656,
      "logits/rejected": -0.6648786067962646,
      "logps/chosen": -5.017510414123535,
      "logps/rejected": -76.23703002929688,
      "loss": 0.4896831512451172,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3755706548690796,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2456466108560562,
      "rewards/margins": 6.204758167266846,
      "rewards/rejected": -5.95911169052124,
      "step": 348,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2198952879581153,
      "grad_norm": 1.5598872900009155,
      "learning_rate": 0.000138134754843821,
      "logits/chosen": -0.46219947934150696,
      "logits/rejected": -0.6632512211799622,
      "logps/chosen": -3.0970990657806396,
      "logps/rejected": -67.87566375732422,
      "loss": 0.24084413051605225,
      "memory(GiB)": 43.15,
      "nll_loss": 0.20238317549228668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20403513312339783,
      "rewards/margins": 5.524411678314209,
      "rewards/rejected": -5.320376396179199,
      "step": 349,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2233856893542758,
      "grad_norm": 4.688317775726318,
      "learning_rate": 0.00013777813032529116,
      "logits/chosen": -0.5623309016227722,
      "logits/rejected": -0.6962246298789978,
      "logps/chosen": -4.781303405761719,
      "logps/rejected": -58.7125244140625,
      "loss": 0.4752863645553589,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36557623744010925,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3706919252872467,
      "rewards/margins": 4.9188079833984375,
      "rewards/rejected": -4.548116207122803,
      "step": 350,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2268760907504364,
      "grad_norm": 3.3078691959381104,
      "learning_rate": 0.0001374209444687514,
      "logits/chosen": -0.40793412923812866,
      "logits/rejected": -0.674126923084259,
      "logps/chosen": -4.038498401641846,
      "logps/rejected": -79.37727355957031,
      "loss": 0.3362622857093811,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2403348684310913,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1741873025894165,
      "rewards/margins": 6.036406517028809,
      "rewards/rejected": -5.862218856811523,
      "step": 351,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2303664921465969,
      "grad_norm": 26.792097091674805,
      "learning_rate": 0.000137063202581558,
      "logits/chosen": -0.5794625878334045,
      "logits/rejected": -0.5993282198905945,
      "logps/chosen": -6.951697826385498,
      "logps/rejected": -56.99049377441406,
      "loss": 0.5700662732124329,
      "memory(GiB)": 43.15,
      "nll_loss": 0.47176504135131836,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23041146993637085,
      "rewards/margins": 4.97003698348999,
      "rewards/rejected": -4.739625453948975,
      "step": 352,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2338568935427574,
      "grad_norm": 1.5458157062530518,
      "learning_rate": 0.00013670490997932922,
      "logits/chosen": -0.4155159592628479,
      "logits/rejected": -0.6365483403205872,
      "logps/chosen": -5.431699752807617,
      "logps/rejected": -76.18798065185547,
      "loss": 0.401727557182312,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3681918978691101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21467794477939606,
      "rewards/margins": 6.059256076812744,
      "rewards/rejected": -5.844577312469482,
      "step": 353,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.237347294938918,
      "grad_norm": 2.289004325866699,
      "learning_rate": 0.0001363460719858663,
      "logits/chosen": -0.4773619472980499,
      "logits/rejected": -0.6628973484039307,
      "logps/chosen": -4.388545513153076,
      "logps/rejected": -64.09728240966797,
      "loss": 0.4164501428604126,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32804638147354126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1376553624868393,
      "rewards/margins": 5.028504371643066,
      "rewards/rejected": -4.8908491134643555,
      "step": 354,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2408376963350785,
      "grad_norm": 4.971522808074951,
      "learning_rate": 0.00013598669393307427,
      "logits/chosen": -0.597011387348175,
      "logits/rejected": -0.6669636368751526,
      "logps/chosen": -6.635625839233398,
      "logps/rejected": -56.963565826416016,
      "loss": 0.6429786086082458,
      "memory(GiB)": 43.15,
      "nll_loss": 0.42030149698257446,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.17075896263122559,
      "rewards/margins": 4.277583122253418,
      "rewards/rejected": -4.448342323303223,
      "step": 355,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.244328097731239,
      "grad_norm": 2.402831792831421,
      "learning_rate": 0.00013562678116088293,
      "logits/chosen": -0.5042529702186584,
      "logits/rejected": -0.6746811866760254,
      "logps/chosen": -4.892055988311768,
      "logps/rejected": -72.40818786621094,
      "loss": 0.4584486782550812,
      "memory(GiB)": 43.15,
      "nll_loss": 0.39387038350105286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2380715012550354,
      "rewards/margins": 6.017812252044678,
      "rewards/rejected": -5.779740333557129,
      "step": 356,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2478184991273997,
      "grad_norm": 4.964642524719238,
      "learning_rate": 0.00013526633901716725,
      "logits/chosen": -0.4217373728752136,
      "logits/rejected": -0.5746545791625977,
      "logps/chosen": -4.907092571258545,
      "logps/rejected": -62.99272155761719,
      "loss": 0.42838555574417114,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31198155879974365,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18360355496406555,
      "rewards/margins": 5.089962482452393,
      "rewards/rejected": -4.90635871887207,
      "step": 357,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2513089005235603,
      "grad_norm": 1.2928236722946167,
      "learning_rate": 0.00013490537285766808,
      "logits/chosen": -0.5333009958267212,
      "logits/rejected": -0.7148969173431396,
      "logps/chosen": -4.048029899597168,
      "logps/rejected": -66.88418579101562,
      "loss": 0.32720184326171875,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2798851728439331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3548097014427185,
      "rewards/margins": 5.343298435211182,
      "rewards/rejected": -4.988488674163818,
      "step": 358,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2547993019197208,
      "grad_norm": 1.6423141956329346,
      "learning_rate": 0.00013454388804591254,
      "logits/chosen": -0.5600902438163757,
      "logits/rejected": -0.644845187664032,
      "logps/chosen": -5.385828018188477,
      "logps/rejected": -59.31862258911133,
      "loss": 0.45650359988212585,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3494071960449219,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23318687081336975,
      "rewards/margins": 4.714806079864502,
      "rewards/rejected": -4.481619358062744,
      "step": 359,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2582897033158813,
      "grad_norm": 1.255145788192749,
      "learning_rate": 0.00013418188995313423,
      "logits/chosen": -0.34982648491859436,
      "logits/rejected": -0.6173741221427917,
      "logps/chosen": -3.911236524581909,
      "logps/rejected": -74.90790557861328,
      "loss": 0.3336854577064514,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2689821124076843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27013736963272095,
      "rewards/margins": 5.215266227722168,
      "rewards/rejected": -4.945127487182617,
      "step": 360,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2617801047120418,
      "grad_norm": 1.7591581344604492,
      "learning_rate": 0.00013381938395819353,
      "logits/chosen": -0.40532568097114563,
      "logits/rejected": -0.597572922706604,
      "logps/chosen": -5.498960971832275,
      "logps/rejected": -64.85173034667969,
      "loss": 0.38682490587234497,
      "memory(GiB)": 43.15,
      "nll_loss": 0.29091084003448486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25826895236968994,
      "rewards/margins": 4.7590532302856445,
      "rewards/rejected": -4.500784397125244,
      "step": 361,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2652705061082026,
      "grad_norm": 1.7910423278808594,
      "learning_rate": 0.00013345637544749775,
      "logits/chosen": -0.4055846035480499,
      "logits/rejected": -0.6238888502120972,
      "logps/chosen": -7.300588130950928,
      "logps/rejected": -61.26845169067383,
      "loss": 0.4961000680923462,
      "memory(GiB)": 43.15,
      "nll_loss": 0.42045268416404724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22531506419181824,
      "rewards/margins": 4.594475269317627,
      "rewards/rejected": -4.369160175323486,
      "step": 362,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.268760907504363,
      "grad_norm": 1.6129978895187378,
      "learning_rate": 0.00013309286981492085,
      "logits/chosen": -0.39129945635795593,
      "logits/rejected": -0.6439290046691895,
      "logps/chosen": -3.721993923187256,
      "logps/rejected": -69.40048217773438,
      "loss": 0.3446144163608551,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2827191948890686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15239138901233673,
      "rewards/margins": 4.774838447570801,
      "rewards/rejected": -4.622446537017822,
      "step": 363,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2722513089005236,
      "grad_norm": 1.268687129020691,
      "learning_rate": 0.00013272887246172343,
      "logits/chosen": -0.5232235193252563,
      "logits/rejected": -0.7373345494270325,
      "logps/chosen": -2.54833722114563,
      "logps/rejected": -60.53718566894531,
      "loss": 0.2926982343196869,
      "memory(GiB)": 43.15,
      "nll_loss": 0.229936420917511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21947187185287476,
      "rewards/margins": 4.873050689697266,
      "rewards/rejected": -4.653579235076904,
      "step": 364,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2757417102966842,
      "grad_norm": 1.1374390125274658,
      "learning_rate": 0.00013236438879647255,
      "logits/chosen": -0.4346693158149719,
      "logits/rejected": -0.6651886105537415,
      "logps/chosen": -3.06109619140625,
      "logps/rejected": -58.315589904785156,
      "loss": 0.31402793526649475,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23573513329029083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2153843492269516,
      "rewards/margins": 4.50330924987793,
      "rewards/rejected": -4.2879252433776855,
      "step": 365,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2792321116928447,
      "grad_norm": 1.2531336545944214,
      "learning_rate": 0.00013199942423496123,
      "logits/chosen": -0.5495594143867493,
      "logits/rejected": -0.6571058034896851,
      "logps/chosen": -3.8754944801330566,
      "logps/rejected": -57.72053146362305,
      "loss": 0.37907201051712036,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3117825984954834,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25686168670654297,
      "rewards/margins": 4.914424896240234,
      "rewards/rejected": -4.657562732696533,
      "step": 366,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2827225130890052,
      "grad_norm": 1.3153668642044067,
      "learning_rate": 0.00013163398420012808,
      "logits/chosen": -0.4384489357471466,
      "logits/rejected": -0.603872537612915,
      "logps/chosen": -5.088449478149414,
      "logps/rejected": -58.85956573486328,
      "loss": 0.4146476984024048,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3133094012737274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34978798031806946,
      "rewards/margins": 4.592877388000488,
      "rewards/rejected": -4.243089199066162,
      "step": 367,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2862129144851657,
      "grad_norm": 1.0968199968338013,
      "learning_rate": 0.00013126807412197665,
      "logits/chosen": -0.5985041260719299,
      "logits/rejected": -0.6937984824180603,
      "logps/chosen": -3.8719797134399414,
      "logps/rejected": -63.86729049682617,
      "loss": 0.3036377727985382,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2704383432865143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30202141404151917,
      "rewards/margins": 5.531611442565918,
      "rewards/rejected": -5.229589462280273,
      "step": 368,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2897033158813263,
      "grad_norm": 1.6636030673980713,
      "learning_rate": 0.00013090169943749476,
      "logits/chosen": -0.35927969217300415,
      "logits/rejected": -0.6646055579185486,
      "logps/chosen": -2.9964470863342285,
      "logps/rejected": -71.79264831542969,
      "loss": 0.39267948269844055,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3159026503562927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28754928708076477,
      "rewards/margins": 5.463479995727539,
      "rewards/rejected": -5.175930976867676,
      "step": 369,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2931937172774868,
      "grad_norm": 5.024345397949219,
      "learning_rate": 0.00013053486559057374,
      "logits/chosen": -0.5237628221511841,
      "logits/rejected": -0.716973066329956,
      "logps/chosen": -2.4937894344329834,
      "logps/rejected": -60.08405303955078,
      "loss": 0.30608034133911133,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2122155725955963,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21323460340499878,
      "rewards/margins": 4.852639675140381,
      "rewards/rejected": -4.639405250549316,
      "step": 370,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.2966841186736475,
      "grad_norm": 1.1299628019332886,
      "learning_rate": 0.00013016757803192757,
      "logits/chosen": -0.5134186744689941,
      "logits/rejected": -0.6998711228370667,
      "logps/chosen": -3.4617767333984375,
      "logps/rejected": -75.16548919677734,
      "loss": 0.2562916576862335,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23842087388038635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31686967611312866,
      "rewards/margins": 6.462481498718262,
      "rewards/rejected": -6.145611763000488,
      "step": 371,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.300174520069808,
      "grad_norm": 1.3862718343734741,
      "learning_rate": 0.00012979984221901173,
      "logits/chosen": -0.5728235840797424,
      "logits/rejected": -0.7681739330291748,
      "logps/chosen": -3.975404739379883,
      "logps/rejected": -64.77377319335938,
      "loss": 0.37514734268188477,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28942835330963135,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19743427634239197,
      "rewards/margins": 5.269680976867676,
      "rewards/rejected": -5.072246551513672,
      "step": 372,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.3036649214659686,
      "grad_norm": 1.4730603694915771,
      "learning_rate": 0.00012943166361594242,
      "logits/chosen": -0.5595740079879761,
      "logits/rejected": -0.703965425491333,
      "logps/chosen": -4.846633434295654,
      "logps/rejected": -65.64612579345703,
      "loss": 0.3534131348133087,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3020310401916504,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15922296047210693,
      "rewards/margins": 5.3241071701049805,
      "rewards/rejected": -5.164884567260742,
      "step": 373,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.307155322862129,
      "grad_norm": 1.5823194980621338,
      "learning_rate": 0.00012906304769341493,
      "logits/chosen": -0.5636857151985168,
      "logits/rejected": -0.7057007551193237,
      "logps/chosen": -7.8343048095703125,
      "logps/rejected": -75.39463806152344,
      "loss": 0.4537757635116577,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36747270822525024,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.44301411509513855,
      "rewards/margins": 6.434762477874756,
      "rewards/rejected": -5.991747856140137,
      "step": 374,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.3106457242582896,
      "grad_norm": 2.39945650100708,
      "learning_rate": 0.0001286939999286228,
      "logits/chosen": -0.6491515636444092,
      "logits/rejected": -0.7588983774185181,
      "logps/chosen": -7.3910675048828125,
      "logps/rejected": -66.71189880371094,
      "loss": 0.46566852927207947,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3986610472202301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3218778073787689,
      "rewards/margins": 5.600278377532959,
      "rewards/rejected": -5.278400421142578,
      "step": 375,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.3141361256544504,
      "grad_norm": 12.878860473632812,
      "learning_rate": 0.0001283245258051761,
      "logits/chosen": -0.6149054765701294,
      "logits/rejected": -0.7632960081100464,
      "logps/chosen": -3.6257922649383545,
      "logps/rejected": -65.39382934570312,
      "loss": 0.36752089858055115,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2868227958679199,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.34518730640411377,
      "rewards/margins": 5.607741832733154,
      "rewards/rejected": -5.26255464553833,
      "step": 376,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.317626527050611,
      "grad_norm": 4.349419593811035,
      "learning_rate": 0.00012795463081302018,
      "logits/chosen": -0.48424577713012695,
      "logits/rejected": -0.7138010263442993,
      "logps/chosen": -5.860461711883545,
      "logps/rejected": -69.88520812988281,
      "loss": 0.5185278654098511,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3800834119319916,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3379773497581482,
      "rewards/margins": 5.424846172332764,
      "rewards/rejected": -5.086869716644287,
      "step": 377,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3211169284467714,
      "grad_norm": 2.1673505306243896,
      "learning_rate": 0.00012758432044835392,
      "logits/chosen": -0.5925784111022949,
      "logits/rejected": -0.7586888670921326,
      "logps/chosen": -4.270134925842285,
      "logps/rejected": -70.65003967285156,
      "loss": 0.4049075245857239,
      "memory(GiB)": 43.15,
      "nll_loss": 0.33120524883270264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20168441534042358,
      "rewards/margins": 5.87272834777832,
      "rewards/rejected": -5.67104434967041,
      "step": 378,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.324607329842932,
      "grad_norm": 10.76476001739502,
      "learning_rate": 0.00012721360021354818,
      "logits/chosen": -0.5449612736701965,
      "logits/rejected": -0.7324461936950684,
      "logps/chosen": -8.914838790893555,
      "logps/rejected": -71.4675521850586,
      "loss": 0.5083584189414978,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4726629853248596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3555769622325897,
      "rewards/margins": 6.001497745513916,
      "rewards/rejected": -5.645921230316162,
      "step": 379,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3280977312390925,
      "grad_norm": 1.7700749635696411,
      "learning_rate": 0.00012684247561706402,
      "logits/chosen": -0.5105177760124207,
      "logits/rejected": -0.7782354950904846,
      "logps/chosen": -3.1164560317993164,
      "logps/rejected": -69.61066436767578,
      "loss": 0.40009862184524536,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32328617572784424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27115288376808167,
      "rewards/margins": 5.493119716644287,
      "rewards/rejected": -5.221966743469238,
      "step": 380,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.331588132635253,
      "grad_norm": 1.5235404968261719,
      "learning_rate": 0.00012647095217337078,
      "logits/chosen": -0.576638400554657,
      "logits/rejected": -0.7515760660171509,
      "logps/chosen": -4.216984748840332,
      "logps/rejected": -76.12001037597656,
      "loss": 0.3426749110221863,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28955399990081787,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24891535937786102,
      "rewards/margins": 6.022099494934082,
      "rewards/rejected": -5.773184776306152,
      "step": 381,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3350785340314135,
      "grad_norm": 1.1489766836166382,
      "learning_rate": 0.00012609903540286422,
      "logits/chosen": -0.5197568535804749,
      "logits/rejected": -0.6939013004302979,
      "logps/chosen": -7.216485977172852,
      "logps/rejected": -77.32923889160156,
      "loss": 0.4592680335044861,
      "memory(GiB)": 43.15,
      "nll_loss": 0.39243242144584656,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27840492129325867,
      "rewards/margins": 6.491175651550293,
      "rewards/rejected": -6.212770938873291,
      "step": 382,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.338568935427574,
      "grad_norm": 1.7705833911895752,
      "learning_rate": 0.0001257267308317845,
      "logits/chosen": -0.49760669469833374,
      "logits/rejected": -0.7222346067428589,
      "logps/chosen": -5.577725410461426,
      "logps/rejected": -75.5153579711914,
      "loss": 0.39679276943206787,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3286840319633484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4885926842689514,
      "rewards/margins": 6.211091041564941,
      "rewards/rejected": -5.722498416900635,
      "step": 383,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3420593368237348,
      "grad_norm": 4.093616008758545,
      "learning_rate": 0.00012535404399213393,
      "logits/chosen": -0.6001275181770325,
      "logits/rejected": -0.7799440622329712,
      "logps/chosen": -4.971902847290039,
      "logps/rejected": -74.22907257080078,
      "loss": 0.40544986724853516,
      "memory(GiB)": 43.15,
      "nll_loss": 0.30647265911102295,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.222365140914917,
      "rewards/margins": 5.5865654945373535,
      "rewards/rejected": -5.364200592041016,
      "step": 384,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3455497382198953,
      "grad_norm": 1.860032081604004,
      "learning_rate": 0.000124980980421595,
      "logits/chosen": -0.49115800857543945,
      "logits/rejected": -0.70196932554245,
      "logps/chosen": -7.895517826080322,
      "logps/rejected": -78.90962982177734,
      "loss": 0.5182665586471558,
      "memory(GiB)": 43.15,
      "nll_loss": 0.46628567576408386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32257279753685,
      "rewards/margins": 6.41339111328125,
      "rewards/rejected": -6.090818405151367,
      "step": 385,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3490401396160558,
      "grad_norm": 1.2258514165878296,
      "learning_rate": 0.00012460754566344796,
      "logits/chosen": -0.5766517519950867,
      "logits/rejected": -0.6811379194259644,
      "logps/chosen": -4.616692066192627,
      "logps/rejected": -62.37327194213867,
      "loss": 0.3498663902282715,
      "memory(GiB)": 43.15,
      "nll_loss": 0.26951494812965393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3747017979621887,
      "rewards/margins": 5.300102710723877,
      "rewards/rejected": -4.925401210784912,
      "step": 386,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3525305410122164,
      "grad_norm": 2.228578805923462,
      "learning_rate": 0.0001242337452664884,
      "logits/chosen": -0.5187143683433533,
      "logits/rejected": -0.6932430863380432,
      "logps/chosen": -4.746977806091309,
      "logps/rejected": -69.02983856201172,
      "loss": 0.48509079217910767,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41189730167388916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26210126280784607,
      "rewards/margins": 5.590664863586426,
      "rewards/rejected": -5.328563213348389,
      "step": 387,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.356020942408377,
      "grad_norm": 1.4744908809661865,
      "learning_rate": 0.00012385958478494487,
      "logits/chosen": -0.45504745841026306,
      "logits/rejected": -0.7032338976860046,
      "logps/chosen": -3.22519588470459,
      "logps/rejected": -84.13624572753906,
      "loss": 0.2042679786682129,
      "memory(GiB)": 43.15,
      "nll_loss": 0.17380760610103607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3250097632408142,
      "rewards/margins": 6.8548583984375,
      "rewards/rejected": -6.529849529266357,
      "step": 388,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3595113438045376,
      "grad_norm": 1.0986099243164062,
      "learning_rate": 0.0001234850697783964,
      "logits/chosen": -0.537411093711853,
      "logits/rejected": -0.7160115242004395,
      "logps/chosen": -5.000178337097168,
      "logps/rejected": -64.82563781738281,
      "loss": 0.3837279975414276,
      "memory(GiB)": 43.15,
      "nll_loss": 0.324247807264328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3417741656303406,
      "rewards/margins": 5.4918107986450195,
      "rewards/rejected": -5.150036811828613,
      "step": 389,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3630017452006982,
      "grad_norm": 1.2075557708740234,
      "learning_rate": 0.00012311020581168972,
      "logits/chosen": -0.550103485584259,
      "logits/rejected": -0.6733621954917908,
      "logps/chosen": -7.16984748840332,
      "logps/rejected": -66.13978576660156,
      "loss": 0.363157719373703,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3398098349571228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5980087518692017,
      "rewards/margins": 5.761294364929199,
      "rewards/rejected": -5.163285255432129,
      "step": 390,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3664921465968587,
      "grad_norm": 2.592634677886963,
      "learning_rate": 0.00012273499845485695,
      "logits/chosen": -0.5785650014877319,
      "logits/rejected": -0.754797637462616,
      "logps/chosen": -5.5793585777282715,
      "logps/rejected": -64.19160461425781,
      "loss": 0.3668617010116577,
      "memory(GiB)": 43.15,
      "nll_loss": 0.33123254776000977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2795164883136749,
      "rewards/margins": 5.247326850891113,
      "rewards/rejected": -4.967810153961182,
      "step": 391,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3699825479930192,
      "grad_norm": 2.1149039268493652,
      "learning_rate": 0.00012235945328303226,
      "logits/chosen": -0.40138736367225647,
      "logits/rejected": -0.6702361702919006,
      "logps/chosen": -3.509735584259033,
      "logps/rejected": -76.15532684326172,
      "loss": 0.28672605752944946,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2373819351196289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31093913316726685,
      "rewards/margins": 5.402565002441406,
      "rewards/rejected": -5.091626167297363,
      "step": 392,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3734729493891797,
      "grad_norm": 0.9537525177001953,
      "learning_rate": 0.00012198357587636957,
      "logits/chosen": -0.5130572319030762,
      "logits/rejected": -0.7259546518325806,
      "logps/chosen": -4.134064674377441,
      "logps/rejected": -73.6041030883789,
      "loss": 0.34708356857299805,
      "memory(GiB)": 43.15,
      "nll_loss": 0.305166631937027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39239123463630676,
      "rewards/margins": 6.155148506164551,
      "rewards/rejected": -5.762757301330566,
      "step": 393,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3769633507853403,
      "grad_norm": 2.37007999420166,
      "learning_rate": 0.00012160737181995936,
      "logits/chosen": -0.491438627243042,
      "logits/rejected": -0.6527938842773438,
      "logps/chosen": -7.896700382232666,
      "logps/rejected": -81.0208969116211,
      "loss": 0.3957880139350891,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36261922121047974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7101905941963196,
      "rewards/margins": 6.88217306137085,
      "rewards/rejected": -6.1719818115234375,
      "step": 394,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3804537521815008,
      "grad_norm": 1.6053402423858643,
      "learning_rate": 0.0001212308467037457,
      "logits/chosen": -0.5596390962600708,
      "logits/rejected": -0.6721497178077698,
      "logps/chosen": -5.880399703979492,
      "logps/rejected": -72.99723052978516,
      "loss": 0.3911302089691162,
      "memory(GiB)": 43.15,
      "nll_loss": 0.30207929015159607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3339996039867401,
      "rewards/margins": 6.267764568328857,
      "rewards/rejected": -5.933764934539795,
      "step": 395,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3839441535776613,
      "grad_norm": 1.745980978012085,
      "learning_rate": 0.00012085400612244322,
      "logits/chosen": -0.4846367835998535,
      "logits/rejected": -0.6683922410011292,
      "logps/chosen": -6.298645496368408,
      "logps/rejected": -79.30961608886719,
      "loss": 0.43555381894111633,
      "memory(GiB)": 43.15,
      "nll_loss": 0.33482685685157776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17846114933490753,
      "rewards/margins": 6.246786594390869,
      "rewards/rejected": -6.068325519561768,
      "step": 396,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.387434554973822,
      "grad_norm": 4.111983776092529,
      "learning_rate": 0.00012047685567545405,
      "logits/chosen": -0.3804283142089844,
      "logits/rejected": -0.7110333442687988,
      "logps/chosen": -5.558120250701904,
      "logps/rejected": -88.71563720703125,
      "loss": 0.5450111031532288,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41072627902030945,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2601194977760315,
      "rewards/margins": 6.857279300689697,
      "rewards/rejected": -6.597159385681152,
      "step": 397,
      "train_speed(iter/s)": 0.010749
    },
    {
      "epoch": 1.3909249563699826,
      "grad_norm": 1.6386064291000366,
      "learning_rate": 0.00012009940096678452,
      "logits/chosen": -0.44062259793281555,
      "logits/rejected": -0.7049256563186646,
      "logps/chosen": -5.1494059562683105,
      "logps/rejected": -77.1871337890625,
      "loss": 0.32311907410621643,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24005520343780518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36464688181877136,
      "rewards/margins": 6.342651844024658,
      "rewards/rejected": -5.978005409240723,
      "step": 398,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.3944153577661431,
      "grad_norm": 1.3504530191421509,
      "learning_rate": 0.00011972164760496187,
      "logits/chosen": -0.42509207129478455,
      "logits/rejected": -0.7086035013198853,
      "logps/chosen": -3.266116142272949,
      "logps/rejected": -78.83551788330078,
      "loss": 0.3055873513221741,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23884357511997223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2924925684928894,
      "rewards/margins": 6.441683292388916,
      "rewards/rejected": -6.149190425872803,
      "step": 399,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.3979057591623036,
      "grad_norm": 1.7327511310577393,
      "learning_rate": 0.00011934360120295106,
      "logits/chosen": -0.4947013556957245,
      "logits/rejected": -0.7843842506408691,
      "logps/chosen": -3.727259635925293,
      "logps/rejected": -82.5062255859375,
      "loss": 0.3797696828842163,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3138769268989563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26138922572135925,
      "rewards/margins": 6.876194000244141,
      "rewards/rejected": -6.614805221557617,
      "step": 400,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.4013961605584642,
      "grad_norm": 2.4903643131256104,
      "learning_rate": 0.00011896526737807123,
      "logits/chosen": -0.4609818756580353,
      "logits/rejected": -0.7451502084732056,
      "logps/chosen": -5.295093536376953,
      "logps/rejected": -84.0618667602539,
      "loss": 0.4176844656467438,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37628936767578125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4193187952041626,
      "rewards/margins": 6.849295616149902,
      "rewards/rejected": -6.429976463317871,
      "step": 401,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.404886561954625,
      "grad_norm": 4.363712787628174,
      "learning_rate": 0.00011858665175191232,
      "logits/chosen": -0.5121644139289856,
      "logits/rejected": -0.7446450591087341,
      "logps/chosen": -6.169421195983887,
      "logps/rejected": -78.18269348144531,
      "loss": 0.528075635433197,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4192356467247009,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1313595473766327,
      "rewards/margins": 6.1796555519104,
      "rewards/rejected": -6.0482964515686035,
      "step": 402,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4083769633507854,
      "grad_norm": 10.279874801635742,
      "learning_rate": 0.00011820775995025147,
      "logits/chosen": -0.5245956182479858,
      "logits/rejected": -0.701598584651947,
      "logps/chosen": -7.414410591125488,
      "logps/rejected": -72.86483764648438,
      "loss": 0.7285939455032349,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5291787981987,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1376880556344986,
      "rewards/margins": 6.163814544677734,
      "rewards/rejected": -6.026126861572266,
      "step": 403,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.411867364746946,
      "grad_norm": 13.068238258361816,
      "learning_rate": 0.00011782859760296943,
      "logits/chosen": -0.5409221053123474,
      "logits/rejected": -0.7858689427375793,
      "logps/chosen": -7.152325630187988,
      "logps/rejected": -79.44561767578125,
      "loss": 0.878388524055481,
      "memory(GiB)": 43.15,
      "nll_loss": 0.7040680050849915,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03869117796421051,
      "rewards/margins": 6.490888595581055,
      "rewards/rejected": -6.45219612121582,
      "step": 404,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4153577661431065,
      "grad_norm": 1.4294313192367554,
      "learning_rate": 0.00011744917034396697,
      "logits/chosen": -0.5631940960884094,
      "logits/rejected": -0.664790153503418,
      "logps/chosen": -5.726756572723389,
      "logps/rejected": -69.50979614257812,
      "loss": 0.40249183773994446,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32488980889320374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3584996461868286,
      "rewards/margins": 5.730526447296143,
      "rewards/rejected": -5.372026443481445,
      "step": 405,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.418848167539267,
      "grad_norm": 1.297458529472351,
      "learning_rate": 0.00011706948381108116,
      "logits/chosen": -0.49395444989204407,
      "logits/rejected": -0.7352310419082642,
      "logps/chosen": -3.01485276222229,
      "logps/rejected": -72.34119415283203,
      "loss": 0.2647218108177185,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2067299783229828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1923447847366333,
      "rewards/margins": 5.942482948303223,
      "rewards/rejected": -5.750137805938721,
      "step": 406,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4223385689354275,
      "grad_norm": 1.5999650955200195,
      "learning_rate": 0.00011668954364600153,
      "logits/chosen": -0.456596702337265,
      "logits/rejected": -0.6609106063842773,
      "logps/chosen": -3.8558003902435303,
      "logps/rejected": -69.09425354003906,
      "loss": 0.32530221343040466,
      "memory(GiB)": 43.15,
      "nll_loss": 0.280635803937912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20687787234783173,
      "rewards/margins": 5.761228084564209,
      "rewards/rejected": -5.554349899291992,
      "step": 407,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.425828970331588,
      "grad_norm": 11.606874465942383,
      "learning_rate": 0.00011630935549418627,
      "logits/chosen": -0.562304675579071,
      "logits/rejected": -0.732158899307251,
      "logps/chosen": -4.860799312591553,
      "logps/rejected": -73.3448715209961,
      "loss": 0.412598192691803,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2766556143760681,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25393882393836975,
      "rewards/margins": 5.896204471588135,
      "rewards/rejected": -5.642266273498535,
      "step": 408,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4293193717277486,
      "grad_norm": 4.748804092407227,
      "learning_rate": 0.00011592892500477843,
      "logits/chosen": -0.5332602858543396,
      "logits/rejected": -0.7645926475524902,
      "logps/chosen": -6.279226303100586,
      "logps/rejected": -75.35002899169922,
      "loss": 0.6248916983604431,
      "memory(GiB)": 43.15,
      "nll_loss": 0.46002817153930664,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.025768298655748367,
      "rewards/margins": 5.945999622344971,
      "rewards/rejected": -5.920231342315674,
      "step": 409,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4328097731239091,
      "grad_norm": 1.5880932807922363,
      "learning_rate": 0.00011554825783052181,
      "logits/chosen": -0.5537830591201782,
      "logits/rejected": -0.7294723391532898,
      "logps/chosen": -7.196538925170898,
      "logps/rejected": -64.66232299804688,
      "loss": 0.4965304136276245,
      "memory(GiB)": 43.15,
      "nll_loss": 0.41626840829849243,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30123329162597656,
      "rewards/margins": 5.4694390296936035,
      "rewards/rejected": -5.168205261230469,
      "step": 410,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4363001745200699,
      "grad_norm": 1.2536892890930176,
      "learning_rate": 0.00011516735962767716,
      "logits/chosen": -0.399604856967926,
      "logits/rejected": -0.6857424974441528,
      "logps/chosen": -5.57369327545166,
      "logps/rejected": -88.55608367919922,
      "loss": 0.3149430751800537,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2797389030456543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31008225679397583,
      "rewards/margins": 6.985775947570801,
      "rewards/rejected": -6.675694465637207,
      "step": 411,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4397905759162304,
      "grad_norm": 1.6588139533996582,
      "learning_rate": 0.00011478623605593795,
      "logits/chosen": -0.6225078105926514,
      "logits/rejected": -0.6670627593994141,
      "logps/chosen": -6.920505523681641,
      "logps/rejected": -61.97294235229492,
      "loss": 0.4648480713367462,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3973652720451355,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32238057255744934,
      "rewards/margins": 5.21965217590332,
      "rewards/rejected": -4.897271633148193,
      "step": 412,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.443280977312391,
      "grad_norm": 2.0025949478149414,
      "learning_rate": 0.00011440489277834645,
      "logits/chosen": -0.5606848001480103,
      "logits/rejected": -0.669112503528595,
      "logps/chosen": -6.905231952667236,
      "logps/rejected": -62.44579315185547,
      "loss": 0.5008029341697693,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3874211609363556,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2375316321849823,
      "rewards/margins": 5.009645938873291,
      "rewards/rejected": -4.772113800048828,
      "step": 413,
      "train_speed(iter/s)": 0.010744
    },
    {
      "epoch": 1.4467713787085514,
      "grad_norm": 3.3872854709625244,
      "learning_rate": 0.0001140233354612094,
      "logits/chosen": -0.5606234669685364,
      "logits/rejected": -0.6871129870414734,
      "logps/chosen": -5.88873291015625,
      "logps/rejected": -76.19691467285156,
      "loss": 0.34879976511001587,
      "memory(GiB)": 43.15,
      "nll_loss": 0.296891450881958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3382805585861206,
      "rewards/margins": 6.477003574371338,
      "rewards/rejected": -6.138722896575928,
      "step": 414,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.450261780104712,
      "grad_norm": 1.3678333759307861,
      "learning_rate": 0.00011364156977401404,
      "logits/chosen": -0.5135593414306641,
      "logits/rejected": -0.7239964008331299,
      "logps/chosen": -5.929028511047363,
      "logps/rejected": -76.85517120361328,
      "loss": 0.43436723947525024,
      "memory(GiB)": 43.15,
      "nll_loss": 0.37630510330200195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3310508131980896,
      "rewards/margins": 6.641447067260742,
      "rewards/rejected": -6.310396671295166,
      "step": 415,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4537521815008727,
      "grad_norm": 1.838230848312378,
      "learning_rate": 0.00011325960138934359,
      "logits/chosen": -0.4536757171154022,
      "logits/rejected": -0.6283993721008301,
      "logps/chosen": -6.089097023010254,
      "logps/rejected": -69.50761413574219,
      "loss": 0.3671635389328003,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3000299036502838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2476513832807541,
      "rewards/margins": 5.625397682189941,
      "rewards/rejected": -5.37774658203125,
      "step": 416,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4572425828970332,
      "grad_norm": 1.2322739362716675,
      "learning_rate": 0.00011287743598279324,
      "logits/chosen": -0.5539292097091675,
      "logits/rejected": -0.6913775205612183,
      "logps/chosen": -4.643549919128418,
      "logps/rejected": -69.86929321289062,
      "loss": 0.34559106826782227,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2662113904953003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18328435719013214,
      "rewards/margins": 5.924973964691162,
      "rewards/rejected": -5.741689205169678,
      "step": 417,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4607329842931938,
      "grad_norm": 1.800267219543457,
      "learning_rate": 0.00011249507923288562,
      "logits/chosen": -0.4190499186515808,
      "logits/rejected": -0.5892344117164612,
      "logps/chosen": -8.646174430847168,
      "logps/rejected": -73.5853042602539,
      "loss": 0.5353580713272095,
      "memory(GiB)": 43.15,
      "nll_loss": 0.45828354358673096,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5416325926780701,
      "rewards/margins": 5.931480407714844,
      "rewards/rejected": -5.389847755432129,
      "step": 418,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4642233856893543,
      "grad_norm": 3.372116804122925,
      "learning_rate": 0.00011211253682098652,
      "logits/chosen": -0.5501023530960083,
      "logits/rejected": -0.7180389761924744,
      "logps/chosen": -3.604302167892456,
      "logps/rejected": -72.84112548828125,
      "loss": 0.3503636419773102,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32656538486480713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43345749378204346,
      "rewards/margins": 6.499542713165283,
      "rewards/rejected": -6.066085338592529,
      "step": 419,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4677137870855148,
      "grad_norm": 1.3007837533950806,
      "learning_rate": 0.00011172981443122048,
      "logits/chosen": -0.6193907260894775,
      "logits/rejected": -0.6977078914642334,
      "logps/chosen": -5.653091907501221,
      "logps/rejected": -58.53229522705078,
      "loss": 0.3499506115913391,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2857433557510376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31003448367118835,
      "rewards/margins": 5.088003635406494,
      "rewards/rejected": -4.777968883514404,
      "step": 420,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4712041884816753,
      "grad_norm": 2.268998384475708,
      "learning_rate": 0.0001113469177503862,
      "logits/chosen": -0.548383355140686,
      "logits/rejected": -0.7274916172027588,
      "logps/chosen": -4.028204441070557,
      "logps/rejected": -74.34265899658203,
      "loss": 0.3686060309410095,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3145729899406433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2974086403846741,
      "rewards/margins": 6.226130485534668,
      "rewards/rejected": -5.928721904754639,
      "step": 421,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4746945898778359,
      "grad_norm": 2.849327325820923,
      "learning_rate": 0.00011096385246787224,
      "logits/chosen": -0.4342499077320099,
      "logits/rejected": -0.58034747838974,
      "logps/chosen": -4.600721836090088,
      "logps/rejected": -74.7936782836914,
      "loss": 0.31432202458381653,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2648724913597107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46156781911849976,
      "rewards/margins": 6.070746898651123,
      "rewards/rejected": -5.6091790199279785,
      "step": 422,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4781849912739964,
      "grad_norm": 1.789528727531433,
      "learning_rate": 0.00011058062427557229,
      "logits/chosen": -0.5314887762069702,
      "logits/rejected": -0.6687928438186646,
      "logps/chosen": -3.7092649936676025,
      "logps/rejected": -70.43209838867188,
      "loss": 0.41692402958869934,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3454267978668213,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4879876375198364,
      "rewards/margins": 6.042082786560059,
      "rewards/rejected": -5.554095268249512,
      "step": 423,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4816753926701571,
      "grad_norm": 1.2130359411239624,
      "learning_rate": 0.00011019723886780076,
      "logits/chosen": -0.5408445596694946,
      "logits/rejected": -0.6576545238494873,
      "logps/chosen": -4.23127555847168,
      "logps/rejected": -73.83074951171875,
      "loss": 0.32007429003715515,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23954784870147705,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3880947232246399,
      "rewards/margins": 6.281963348388672,
      "rewards/rejected": -5.893868446350098,
      "step": 424,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4851657940663177,
      "grad_norm": 3.3867220878601074,
      "learning_rate": 0.00010981370194120808,
      "logits/chosen": -0.5246488451957703,
      "logits/rejected": -0.6161497235298157,
      "logps/chosen": -6.636181831359863,
      "logps/rejected": -54.938148498535156,
      "loss": 0.5445500612258911,
      "memory(GiB)": 43.15,
      "nll_loss": 0.40025097131729126,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3486095070838928,
      "rewards/margins": 4.349201202392578,
      "rewards/rejected": -4.000591278076172,
      "step": 425,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4886561954624782,
      "grad_norm": 1.267012357711792,
      "learning_rate": 0.00010943001919469602,
      "logits/chosen": -0.5046868920326233,
      "logits/rejected": -0.6956534385681152,
      "logps/chosen": -3.8510453701019287,
      "logps/rejected": -72.2486343383789,
      "loss": 0.3183547258377075,
      "memory(GiB)": 43.15,
      "nll_loss": 0.26646360754966736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24279548227787018,
      "rewards/margins": 6.0074357986450195,
      "rewards/rejected": -5.7646403312683105,
      "step": 426,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4921465968586387,
      "grad_norm": 5.630609035491943,
      "learning_rate": 0.00010904619632933313,
      "logits/chosen": -0.45180657505989075,
      "logits/rejected": -0.7082334756851196,
      "logps/chosen": -4.919317245483398,
      "logps/rejected": -92.1916275024414,
      "loss": 0.4040582478046417,
      "memory(GiB)": 43.15,
      "nll_loss": 0.35989558696746826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1976865828037262,
      "rewards/margins": 7.597080230712891,
      "rewards/rejected": -7.399393558502197,
      "step": 427,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.4956369982547992,
      "grad_norm": 1.2298264503479004,
      "learning_rate": 0.0001086622390482699,
      "logits/chosen": -0.4164445996284485,
      "logits/rejected": -0.6739879250526428,
      "logps/chosen": -3.47589373588562,
      "logps/rejected": -90.46715545654297,
      "loss": 0.2605077922344208,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24074657261371613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42819085717201233,
      "rewards/margins": 7.35953426361084,
      "rewards/rejected": -6.931342601776123,
      "step": 428,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.49912739965096,
      "grad_norm": 7.767935752868652,
      "learning_rate": 0.00010827815305665412,
      "logits/chosen": -0.5705397129058838,
      "logits/rejected": -0.7187249660491943,
      "logps/chosen": -8.886819839477539,
      "logps/rejected": -78.2231674194336,
      "loss": 0.6680692434310913,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5623351335525513,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.25776392221450806,
      "rewards/margins": 6.651043891906738,
      "rewards/rejected": -6.393280029296875,
      "step": 429,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5026178010471205,
      "grad_norm": 2.1218976974487305,
      "learning_rate": 0.00010789394406154602,
      "logits/chosen": -0.4624744653701782,
      "logits/rejected": -0.7085034251213074,
      "logps/chosen": -3.271778106689453,
      "logps/rejected": -77.90242004394531,
      "loss": 0.2982616424560547,
      "memory(GiB)": 43.15,
      "nll_loss": 0.237554132938385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1897493451833725,
      "rewards/margins": 6.301713943481445,
      "rewards/rejected": -6.111965179443359,
      "step": 430,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.506108202443281,
      "grad_norm": 1.2916324138641357,
      "learning_rate": 0.00010750961777183357,
      "logits/chosen": -0.4626178741455078,
      "logits/rejected": -0.6473190784454346,
      "logps/chosen": -6.6744914054870605,
      "logps/rejected": -85.55329895019531,
      "loss": 0.40405890345573425,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31866455078125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4000608026981354,
      "rewards/margins": 6.864266395568848,
      "rewards/rejected": -6.464205265045166,
      "step": 431,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5095986038394416,
      "grad_norm": 1.857893943786621,
      "learning_rate": 0.00010712517989814754,
      "logits/chosen": -0.49479395151138306,
      "logits/rejected": -0.6366540193557739,
      "logps/chosen": -5.596293926239014,
      "logps/rejected": -71.23036193847656,
      "loss": 0.45099005103111267,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31333470344543457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2475249469280243,
      "rewards/margins": 5.814399242401123,
      "rewards/rejected": -5.5668745040893555,
      "step": 432,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.513089005235602,
      "grad_norm": 1.5123834609985352,
      "learning_rate": 0.0001067406361527768,
      "logits/chosen": -0.5022237300872803,
      "logits/rejected": -0.6664936542510986,
      "logps/chosen": -5.3614912033081055,
      "logps/rejected": -72.45608520507812,
      "loss": 0.3943566679954529,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3421926498413086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20720945298671722,
      "rewards/margins": 5.731356143951416,
      "rewards/rejected": -5.524147033691406,
      "step": 433,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5165794066317626,
      "grad_norm": 1.4642670154571533,
      "learning_rate": 0.00010635599224958322,
      "logits/chosen": -0.5665416717529297,
      "logits/rejected": -0.5965166091918945,
      "logps/chosen": -7.792428970336914,
      "logps/rejected": -61.219356536865234,
      "loss": 0.4489312767982483,
      "memory(GiB)": 43.15,
      "nll_loss": 0.34731173515319824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2655758857727051,
      "rewards/margins": 5.149145126342773,
      "rewards/rejected": -4.883568286895752,
      "step": 434,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5200698080279231,
      "grad_norm": 2.060508966445923,
      "learning_rate": 0.00010597125390391698,
      "logits/chosen": -0.4818264842033386,
      "logits/rejected": -0.6956694722175598,
      "logps/chosen": -5.463670253753662,
      "logps/rejected": -71.13404846191406,
      "loss": 0.3084399402141571,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2919044494628906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44975191354751587,
      "rewards/margins": 5.932075500488281,
      "rewards/rejected": -5.48232364654541,
      "step": 435,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5235602094240837,
      "grad_norm": 1.344682216644287,
      "learning_rate": 0.00010558642683253152,
      "logits/chosen": -0.4367799162864685,
      "logits/rejected": -0.6908420920372009,
      "logps/chosen": -5.97914981842041,
      "logps/rejected": -91.7225112915039,
      "loss": 0.38166770339012146,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3533383011817932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25942543148994446,
      "rewards/margins": 7.6511406898498535,
      "rewards/rejected": -7.3917155265808105,
      "step": 436,
      "train_speed(iter/s)": 0.010745
    },
    {
      "epoch": 1.5270506108202442,
      "grad_norm": 1.8352774381637573,
      "learning_rate": 0.00010520151675349871,
      "logits/chosen": -0.5730445384979248,
      "logits/rejected": -0.646553635597229,
      "logps/chosen": -5.68608283996582,
      "logps/rejected": -63.03091049194336,
      "loss": 0.4128214418888092,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36257404088974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5576317310333252,
      "rewards/margins": 5.7393364906311035,
      "rewards/rejected": -5.181704998016357,
      "step": 437,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.530541012216405,
      "grad_norm": 1.5020852088928223,
      "learning_rate": 0.00010481652938612374,
      "logits/chosen": -0.5322219133377075,
      "logits/rejected": -0.6736618876457214,
      "logps/chosen": -5.650023460388184,
      "logps/rejected": -84.75536346435547,
      "loss": 0.3760172128677368,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31763553619384766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2672083079814911,
      "rewards/margins": 7.1481733322143555,
      "rewards/rejected": -6.880964756011963,
      "step": 438,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5340314136125655,
      "grad_norm": 1.2762045860290527,
      "learning_rate": 0.00010443147045086017,
      "logits/chosen": -0.5660447478294373,
      "logits/rejected": -0.7661846876144409,
      "logps/chosen": -3.1776599884033203,
      "logps/rejected": -77.935302734375,
      "loss": 0.3217516243457794,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2736022472381592,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1498115211725235,
      "rewards/margins": 6.538293361663818,
      "rewards/rejected": -6.388482093811035,
      "step": 439,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.537521815008726,
      "grad_norm": 4.351047039031982,
      "learning_rate": 0.00010404634566922515,
      "logits/chosen": -0.4415951669216156,
      "logits/rejected": -0.6990665793418884,
      "logps/chosen": -3.981513023376465,
      "logps/rejected": -75.63274383544922,
      "loss": 0.35725584626197815,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3062575161457062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19793441891670227,
      "rewards/margins": 6.056670665740967,
      "rewards/rejected": -5.858736515045166,
      "step": 440,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5410122164048867,
      "grad_norm": 1.1987534761428833,
      "learning_rate": 0.00010366116076371406,
      "logits/chosen": -0.5242297649383545,
      "logits/rejected": -0.7449771165847778,
      "logps/chosen": -2.1849770545959473,
      "logps/rejected": -82.05471801757812,
      "loss": 0.2391047477722168,
      "memory(GiB)": 43.15,
      "nll_loss": 0.19235552847385406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2233942449092865,
      "rewards/margins": 6.825995922088623,
      "rewards/rejected": -6.602601528167725,
      "step": 441,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5445026178010473,
      "grad_norm": 1.0310100317001343,
      "learning_rate": 0.00010327592145771574,
      "logits/chosen": -0.5226780772209167,
      "logits/rejected": -0.6522566676139832,
      "logps/chosen": -4.976744651794434,
      "logps/rejected": -77.1199722290039,
      "loss": 0.28156208992004395,
      "memory(GiB)": 43.15,
      "nll_loss": 0.25276780128479004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3434697985649109,
      "rewards/margins": 6.673407077789307,
      "rewards/rejected": -6.32993745803833,
      "step": 442,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5479930191972078,
      "grad_norm": 1.6905871629714966,
      "learning_rate": 0.00010289063347542726,
      "logits/chosen": -0.49878814816474915,
      "logits/rejected": -0.7566611170768738,
      "logps/chosen": -5.842344284057617,
      "logps/rejected": -90.62516021728516,
      "loss": 0.3308601379394531,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2835053503513336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48695820569992065,
      "rewards/margins": 7.632462024688721,
      "rewards/rejected": -7.145503997802734,
      "step": 443,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5514834205933683,
      "grad_norm": 1.446353554725647,
      "learning_rate": 0.00010250530254176914,
      "logits/chosen": -0.5367974042892456,
      "logits/rejected": -0.6674500107765198,
      "logps/chosen": -7.052587032318115,
      "logps/rejected": -69.16866302490234,
      "loss": 0.48406222462654114,
      "memory(GiB)": 43.15,
      "nll_loss": 0.416780948638916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4342191219329834,
      "rewards/margins": 5.5024871826171875,
      "rewards/rejected": -5.068268299102783,
      "step": 444,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5549738219895288,
      "grad_norm": 1.2833924293518066,
      "learning_rate": 0.00010211993438229985,
      "logits/chosen": -0.4966129958629608,
      "logits/rejected": -0.7367646098136902,
      "logps/chosen": -2.2900989055633545,
      "logps/rejected": -92.28775787353516,
      "loss": 0.23101164400577545,
      "memory(GiB)": 43.15,
      "nll_loss": 0.18072807788848877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3229841887950897,
      "rewards/margins": 7.81334114074707,
      "rewards/rejected": -7.490356922149658,
      "step": 445,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5584642233856894,
      "grad_norm": 2.64135479927063,
      "learning_rate": 0.00010173453472313126,
      "logits/chosen": -0.43859654664993286,
      "logits/rejected": -0.72212153673172,
      "logps/chosen": -3.855778455734253,
      "logps/rejected": -102.45668029785156,
      "loss": 0.33878108859062195,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3061646819114685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28175267577171326,
      "rewards/margins": 8.548420906066895,
      "rewards/rejected": -8.266668319702148,
      "step": 446,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5619546247818499,
      "grad_norm": 0.9250645041465759,
      "learning_rate": 0.00010134910929084307,
      "logits/chosen": -0.49251896142959595,
      "logits/rejected": -0.7127062082290649,
      "logps/chosen": -3.829066276550293,
      "logps/rejected": -73.96434020996094,
      "loss": 0.30373579263687134,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23478108644485474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4689459204673767,
      "rewards/margins": 6.388603210449219,
      "rewards/rejected": -5.919657230377197,
      "step": 447,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5654450261780104,
      "grad_norm": 1.2933753728866577,
      "learning_rate": 0.00010096366381239808,
      "logits/chosen": -0.5456343293190002,
      "logits/rejected": -0.7084980607032776,
      "logps/chosen": -5.273447036743164,
      "logps/rejected": -76.9305191040039,
      "loss": 0.38520318269729614,
      "memory(GiB)": 43.15,
      "nll_loss": 0.321531742811203,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4554443657398224,
      "rewards/margins": 6.646027565002441,
      "rewards/rejected": -6.190584182739258,
      "step": 448,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.568935427574171,
      "grad_norm": 1.217418909072876,
      "learning_rate": 0.00010057820401505691,
      "logits/chosen": -0.49013572931289673,
      "logits/rejected": -0.7416660785675049,
      "logps/chosen": -4.202518939971924,
      "logps/rejected": -81.60181427001953,
      "loss": 0.3519480526447296,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2851409614086151,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4234984517097473,
      "rewards/margins": 6.911059379577637,
      "rewards/rejected": -6.487560272216797,
      "step": 449,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5724258289703315,
      "grad_norm": 2.0490660667419434,
      "learning_rate": 0.0001001927356262929,
      "logits/chosen": -0.4484134316444397,
      "logits/rejected": -0.6950099468231201,
      "logps/chosen": -4.823796272277832,
      "logps/rejected": -94.5858154296875,
      "loss": 0.31138429045677185,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23047974705696106,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11626379191875458,
      "rewards/margins": 7.952792167663574,
      "rewards/rejected": -7.836528301239014,
      "step": 450,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5759162303664922,
      "grad_norm": 1.7198628187179565,
      "learning_rate": 9.980726437370712e-05,
      "logits/chosen": -0.45464956760406494,
      "logits/rejected": -0.7056002616882324,
      "logps/chosen": -5.082573890686035,
      "logps/rejected": -101.8458251953125,
      "loss": 0.31865912675857544,
      "memory(GiB)": 43.15,
      "nll_loss": 0.26458409428596497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.31848958134651184,
      "rewards/margins": 8.600041389465332,
      "rewards/rejected": -8.281551361083984,
      "step": 451,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5794066317626527,
      "grad_norm": 6.508523941040039,
      "learning_rate": 9.94217959849431e-05,
      "logits/chosen": -0.4674018621444702,
      "logits/rejected": -0.7319305539131165,
      "logps/chosen": -6.289431095123291,
      "logps/rejected": -95.78480529785156,
      "loss": 0.6865589618682861,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5592325329780579,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.015579154714941978,
      "rewards/margins": 8.017131805419922,
      "rewards/rejected": -8.001551628112793,
      "step": 452,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5828970331588132,
      "grad_norm": 13.3508939743042,
      "learning_rate": 9.903633618760195e-05,
      "logits/chosen": -0.4849949777126312,
      "logits/rejected": -0.7907320261001587,
      "logps/chosen": -4.951356410980225,
      "logps/rejected": -82.5472640991211,
      "loss": 0.8948383927345276,
      "memory(GiB)": 43.15,
      "nll_loss": 0.7533862590789795,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.12311647832393646,
      "rewards/margins": 6.550403118133545,
      "rewards/rejected": -6.673520088195801,
      "step": 453,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5863874345549738,
      "grad_norm": 6.573890209197998,
      "learning_rate": 9.865089070915696e-05,
      "logits/chosen": -0.4060344994068146,
      "logits/rejected": -0.7550903558731079,
      "logps/chosen": -5.221435070037842,
      "logps/rejected": -96.10535430908203,
      "loss": 0.5043012499809265,
      "memory(GiB)": 43.15,
      "nll_loss": 0.44070732593536377,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2424362450838089,
      "rewards/margins": 7.383746147155762,
      "rewards/rejected": -7.14130973815918,
      "step": 454,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5898778359511345,
      "grad_norm": 9.605191230773926,
      "learning_rate": 9.826546527686879e-05,
      "logits/chosen": -0.5186147093772888,
      "logits/rejected": -0.6258412599563599,
      "logps/chosen": -7.3261566162109375,
      "logps/rejected": -69.6639633178711,
      "loss": 0.5112545490264893,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4276677370071411,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.42547550797462463,
      "rewards/margins": 6.084844589233398,
      "rewards/rejected": -5.659369468688965,
      "step": 455,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.593368237347295,
      "grad_norm": 26.42751121520996,
      "learning_rate": 9.788006561770016e-05,
      "logits/chosen": -0.49290916323661804,
      "logits/rejected": -0.6933097243309021,
      "logps/chosen": -3.184025764465332,
      "logps/rejected": -82.68756103515625,
      "loss": 0.26833221316337585,
      "memory(GiB)": 43.15,
      "nll_loss": 0.19985711574554443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19965597987174988,
      "rewards/margins": 6.889904022216797,
      "rewards/rejected": -6.690248012542725,
      "step": 456,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.5968586387434556,
      "grad_norm": 6.064898490905762,
      "learning_rate": 9.749469745823092e-05,
      "logits/chosen": -0.575084924697876,
      "logits/rejected": -0.6971481442451477,
      "logps/chosen": -8.84771728515625,
      "logps/rejected": -77.52954864501953,
      "loss": 0.7120727300643921,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5085330009460449,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2835085093975067,
      "rewards/margins": 7.017773151397705,
      "rewards/rejected": -6.734263896942139,
      "step": 457,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.600349040139616,
      "grad_norm": 1.13936448097229,
      "learning_rate": 9.710936652457276e-05,
      "logits/chosen": -0.5578764081001282,
      "logits/rejected": -0.724193811416626,
      "logps/chosen": -4.292195796966553,
      "logps/rejected": -87.24616241455078,
      "loss": 0.2597983181476593,
      "memory(GiB)": 43.15,
      "nll_loss": 0.21350032091140747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28079283237457275,
      "rewards/margins": 7.473378658294678,
      "rewards/rejected": -7.192586898803711,
      "step": 458,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6038394415357766,
      "grad_norm": 1.4284478425979614,
      "learning_rate": 9.672407854228427e-05,
      "logits/chosen": -0.5638883113861084,
      "logits/rejected": -0.6472236514091492,
      "logps/chosen": -7.458891868591309,
      "logps/rejected": -88.3903579711914,
      "loss": 0.33047589659690857,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32156452536582947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09782328456640244,
      "rewards/margins": 7.796804904937744,
      "rewards/rejected": -7.698981285095215,
      "step": 459,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6073298429319371,
      "grad_norm": 1.8427724838256836,
      "learning_rate": 9.633883923628593e-05,
      "logits/chosen": -0.5136711597442627,
      "logits/rejected": -0.670426070690155,
      "logps/chosen": -5.641696453094482,
      "logps/rejected": -68.54437255859375,
      "loss": 0.43776947259902954,
      "memory(GiB)": 43.15,
      "nll_loss": 0.32517340779304504,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2341962605714798,
      "rewards/margins": 5.578415870666504,
      "rewards/rejected": -5.344220161437988,
      "step": 460,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6108202443280977,
      "grad_norm": 1.3355876207351685,
      "learning_rate": 9.595365433077483e-05,
      "logits/chosen": -0.5112326145172119,
      "logits/rejected": -0.6481039524078369,
      "logps/chosen": -5.295035362243652,
      "logps/rejected": -69.49890899658203,
      "loss": 0.3256393074989319,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2783932089805603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5631068348884583,
      "rewards/margins": 6.177706718444824,
      "rewards/rejected": -5.61460018157959,
      "step": 461,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6143106457242582,
      "grad_norm": 1.4627971649169922,
      "learning_rate": 9.556852954913981e-05,
      "logits/chosen": -0.5415772199630737,
      "logits/rejected": -0.6977207660675049,
      "logps/chosen": -3.7730510234832764,
      "logps/rejected": -70.53782653808594,
      "loss": 0.3772314786911011,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3154931366443634,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.28429800271987915,
      "rewards/margins": 6.2479248046875,
      "rewards/rejected": -5.963626384735107,
      "step": 462,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6178010471204187,
      "grad_norm": 1.4400521516799927,
      "learning_rate": 9.518347061387628e-05,
      "logits/chosen": -0.5067111253738403,
      "logits/rejected": -0.7407639622688293,
      "logps/chosen": -3.358861207962036,
      "logps/rejected": -75.34605407714844,
      "loss": 0.31904613971710205,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24776507914066315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2687406539916992,
      "rewards/margins": 6.2818603515625,
      "rewards/rejected": -6.013119697570801,
      "step": 463,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6212914485165792,
      "grad_norm": 1.1085314750671387,
      "learning_rate": 9.47984832465013e-05,
      "logits/chosen": -0.5132672786712646,
      "logits/rejected": -0.7135886549949646,
      "logps/chosen": -3.035696506500244,
      "logps/rejected": -77.64856719970703,
      "loss": 0.2410428375005722,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2050429880619049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5391538143157959,
      "rewards/margins": 6.414307117462158,
      "rewards/rejected": -5.875153064727783,
      "step": 464,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.62478184991274,
      "grad_norm": 1.2350654602050781,
      "learning_rate": 9.441357316746849e-05,
      "logits/chosen": -0.4304356575012207,
      "logits/rejected": -0.647516131401062,
      "logps/chosen": -3.670703887939453,
      "logps/rejected": -66.64935302734375,
      "loss": 0.290671706199646,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2181183397769928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40155574679374695,
      "rewards/margins": 5.363650798797607,
      "rewards/rejected": -4.962095260620117,
      "step": 465,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.6282722513089005,
      "grad_norm": 1.0405222177505493,
      "learning_rate": 9.402874609608304e-05,
      "logits/chosen": -0.5338455438613892,
      "logits/rejected": -0.6881372928619385,
      "logps/chosen": -3.2838728427886963,
      "logps/rejected": -70.93064880371094,
      "loss": 0.26523253321647644,
      "memory(GiB)": 43.15,
      "nll_loss": 0.21268227696418762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3254692554473877,
      "rewards/margins": 6.0326128005981445,
      "rewards/rejected": -5.707143783569336,
      "step": 466,
      "train_speed(iter/s)": 0.010746
    },
    {
      "epoch": 1.631762652705061,
      "grad_norm": 1.730974555015564,
      "learning_rate": 9.36440077504168e-05,
      "logits/chosen": -0.6126997470855713,
      "logits/rejected": -0.7266684770584106,
      "logps/chosen": -3.05517840385437,
      "logps/rejected": -80.148681640625,
      "loss": 0.2501164376735687,
      "memory(GiB)": 43.15,
      "nll_loss": 0.21693116426467896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44437089562416077,
      "rewards/margins": 7.217838287353516,
      "rewards/rejected": -6.773467540740967,
      "step": 467,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6352530541012218,
      "grad_norm": 3.8707399368286133,
      "learning_rate": 9.325936384722321e-05,
      "logits/chosen": -0.49259650707244873,
      "logits/rejected": -0.6852439045906067,
      "logps/chosen": -5.644749641418457,
      "logps/rejected": -75.47988891601562,
      "loss": 0.401599258184433,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31637659668922424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2561740279197693,
      "rewards/margins": 5.929844379425049,
      "rewards/rejected": -5.673670768737793,
      "step": 468,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6387434554973823,
      "grad_norm": 1.9370460510253906,
      "learning_rate": 9.287482010185247e-05,
      "logits/chosen": -0.5154401659965515,
      "logits/rejected": -0.723540723323822,
      "logps/chosen": -6.370390892028809,
      "logps/rejected": -75.91340637207031,
      "loss": 0.4852099120616913,
      "memory(GiB)": 43.15,
      "nll_loss": 0.40603795647621155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24762868881225586,
      "rewards/margins": 6.241936206817627,
      "rewards/rejected": -5.994307994842529,
      "step": 469,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6422338568935428,
      "grad_norm": 1.6384847164154053,
      "learning_rate": 9.249038222816645e-05,
      "logits/chosen": -0.4204860329627991,
      "logits/rejected": -0.7506822943687439,
      "logps/chosen": -3.80747127532959,
      "logps/rejected": -93.09049224853516,
      "loss": 0.3156612813472748,
      "memory(GiB)": 43.15,
      "nll_loss": 0.25148946046829224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34722334146499634,
      "rewards/margins": 7.566956520080566,
      "rewards/rejected": -7.219733715057373,
      "step": 470,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6457242582897034,
      "grad_norm": 2.041177272796631,
      "learning_rate": 9.210605593845401e-05,
      "logits/chosen": -0.6100115180015564,
      "logits/rejected": -0.7477937340736389,
      "logps/chosen": -6.873973846435547,
      "logps/rejected": -66.74372863769531,
      "loss": 0.5695972442626953,
      "memory(GiB)": 43.15,
      "nll_loss": 0.44725123047828674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3558385372161865,
      "rewards/margins": 5.833300590515137,
      "rewards/rejected": -5.4774627685546875,
      "step": 471,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.649214659685864,
      "grad_norm": 2.2523133754730225,
      "learning_rate": 9.172184694334591e-05,
      "logits/chosen": -0.5835016965866089,
      "logits/rejected": -0.7850641012191772,
      "logps/chosen": -4.768137454986572,
      "logps/rejected": -88.08627319335938,
      "loss": 0.3464867174625397,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3102959096431732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4118156433105469,
      "rewards/margins": 7.500361442565918,
      "rewards/rejected": -7.088545799255371,
      "step": 472,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6527050610820244,
      "grad_norm": 2.1749207973480225,
      "learning_rate": 9.133776095173015e-05,
      "logits/chosen": -0.6270173788070679,
      "logits/rejected": -0.7678170204162598,
      "logps/chosen": -5.546754837036133,
      "logps/rejected": -79.69754028320312,
      "loss": 0.3659020662307739,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31625014543533325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21115486323833466,
      "rewards/margins": 6.8044962882995605,
      "rewards/rejected": -6.593341827392578,
      "step": 473,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.656195462478185,
      "grad_norm": 2.448395252227783,
      "learning_rate": 9.09538036706669e-05,
      "logits/chosen": -0.481596440076828,
      "logits/rejected": -0.7727686166763306,
      "logps/chosen": -9.757466316223145,
      "logps/rejected": -99.3200912475586,
      "loss": 0.6309171319007874,
      "memory(GiB)": 43.15,
      "nll_loss": 0.541617751121521,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1632377803325653,
      "rewards/margins": 8.147398948669434,
      "rewards/rejected": -7.984160900115967,
      "step": 474,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6596858638743455,
      "grad_norm": 9.251994132995605,
      "learning_rate": 9.056998080530398e-05,
      "logits/chosen": -0.562772274017334,
      "logits/rejected": -0.8359721302986145,
      "logps/chosen": -6.0720930099487305,
      "logps/rejected": -94.93759155273438,
      "loss": 0.4291512966156006,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3865175247192383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4334118664264679,
      "rewards/margins": 8.281164169311523,
      "rewards/rejected": -7.847752571105957,
      "step": 475,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.663176265270506,
      "grad_norm": 2.4467082023620605,
      "learning_rate": 9.018629805879194e-05,
      "logits/chosen": -0.4421442449092865,
      "logits/rejected": -0.6901416182518005,
      "logps/chosen": -7.532122611999512,
      "logps/rejected": -91.17680358886719,
      "loss": 0.38504907488822937,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36676025390625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3057595491409302,
      "rewards/margins": 7.532265663146973,
      "rewards/rejected": -7.22650671005249,
      "step": 476,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 1.2942126989364624,
      "learning_rate": 8.980276113219924e-05,
      "logits/chosen": -0.5596615076065063,
      "logits/rejected": -0.8118699789047241,
      "logps/chosen": -3.7042813301086426,
      "logps/rejected": -86.83468627929688,
      "loss": 0.3292457163333893,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2878093123435974,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22699333727359772,
      "rewards/margins": 7.463640213012695,
      "rewards/rejected": -7.236647605895996,
      "step": 477,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6701570680628273,
      "grad_norm": 1.8572261333465576,
      "learning_rate": 8.941937572442773e-05,
      "logits/chosen": -0.4306020736694336,
      "logits/rejected": -0.7685049772262573,
      "logps/chosen": -5.437053203582764,
      "logps/rejected": -89.96847534179688,
      "loss": 0.36804744601249695,
      "memory(GiB)": 43.15,
      "nll_loss": 0.29312145709991455,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30451783537864685,
      "rewards/margins": 7.180737495422363,
      "rewards/rejected": -6.876219749450684,
      "step": 478,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6736474694589878,
      "grad_norm": 1.1792480945587158,
      "learning_rate": 8.903614753212778e-05,
      "logits/chosen": -0.5531143546104431,
      "logits/rejected": -0.7622645497322083,
      "logps/chosen": -5.353999614715576,
      "logps/rejected": -92.50546264648438,
      "loss": 0.3255807161331177,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28069090843200684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28517261147499084,
      "rewards/margins": 8.108088493347168,
      "rewards/rejected": -7.822916030883789,
      "step": 479,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6771378708551483,
      "grad_norm": 1.6261513233184814,
      "learning_rate": 8.865308224961381e-05,
      "logits/chosen": -0.5815747380256653,
      "logits/rejected": -0.7042577266693115,
      "logps/chosen": -6.8108344078063965,
      "logps/rejected": -81.57181549072266,
      "loss": 0.3821701407432556,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3587985038757324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39591410756111145,
      "rewards/margins": 7.037505626678467,
      "rewards/rejected": -6.641592025756836,
      "step": 480,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.680628272251309,
      "grad_norm": 1.7716622352600098,
      "learning_rate": 8.827018556877953e-05,
      "logits/chosen": -0.6016179323196411,
      "logits/rejected": -0.831665575504303,
      "logps/chosen": -4.813789367675781,
      "logps/rejected": -88.77471923828125,
      "loss": 0.40825486183166504,
      "memory(GiB)": 43.15,
      "nll_loss": 0.36596742272377014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3279199004173279,
      "rewards/margins": 7.504549026489258,
      "rewards/rejected": -7.176629066467285,
      "step": 481,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6841186736474696,
      "grad_norm": 1.6834444999694824,
      "learning_rate": 8.788746317901349e-05,
      "logits/chosen": -0.5991214513778687,
      "logits/rejected": -0.7489640116691589,
      "logps/chosen": -5.925536632537842,
      "logps/rejected": -72.82659149169922,
      "loss": 0.46086686849594116,
      "memory(GiB)": 43.15,
      "nll_loss": 0.4042823314666748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39119282364845276,
      "rewards/margins": 6.154544353485107,
      "rewards/rejected": -5.7633514404296875,
      "step": 482,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6876090750436301,
      "grad_norm": 1.5792211294174194,
      "learning_rate": 8.750492076711439e-05,
      "logits/chosen": -0.589457094669342,
      "logits/rejected": -0.7296780943870544,
      "logps/chosen": -4.901717185974121,
      "logps/rejected": -64.11253356933594,
      "loss": 0.4066055417060852,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3368377983570099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30969735980033875,
      "rewards/margins": 5.3579888343811035,
      "rewards/rejected": -5.0482916831970215,
      "step": 483,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6910994764397906,
      "grad_norm": 14.861246109008789,
      "learning_rate": 8.71225640172068e-05,
      "logits/chosen": -0.5961053967475891,
      "logits/rejected": -0.7103674411773682,
      "logps/chosen": -8.013514518737793,
      "logps/rejected": -73.48511505126953,
      "loss": 0.403678297996521,
      "memory(GiB)": 43.15,
      "nll_loss": 0.31652554869651794,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20527437329292297,
      "rewards/margins": 6.156968593597412,
      "rewards/rejected": -5.951694011688232,
      "step": 484,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6945898778359512,
      "grad_norm": 4.586897850036621,
      "learning_rate": 8.674039861065643e-05,
      "logits/chosen": -0.5703863501548767,
      "logits/rejected": -0.7276860475540161,
      "logps/chosen": -9.062816619873047,
      "logps/rejected": -71.1295394897461,
      "loss": 0.9288193583488464,
      "memory(GiB)": 43.15,
      "nll_loss": 0.7453719973564148,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12258605659008026,
      "rewards/margins": 5.861667156219482,
      "rewards/rejected": -5.739081382751465,
      "step": 485,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.6980802792321117,
      "grad_norm": 4.373604774475098,
      "learning_rate": 8.635843022598601e-05,
      "logits/chosen": -0.5944116711616516,
      "logits/rejected": -0.7287894487380981,
      "logps/chosen": -8.35546875,
      "logps/rejected": -74.72799682617188,
      "loss": 0.7051505446434021,
      "memory(GiB)": 43.15,
      "nll_loss": 0.6106510162353516,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24222475290298462,
      "rewards/margins": 6.460598945617676,
      "rewards/rejected": -6.218373775482178,
      "step": 486,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7015706806282722,
      "grad_norm": 2.3533213138580322,
      "learning_rate": 8.597666453879062e-05,
      "logits/chosen": -0.5893316864967346,
      "logits/rejected": -0.722305178642273,
      "logps/chosen": -3.870330333709717,
      "logps/rejected": -72.59976959228516,
      "loss": 0.23593875765800476,
      "memory(GiB)": 43.15,
      "nll_loss": 0.19806373119354248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5103552937507629,
      "rewards/margins": 6.509716033935547,
      "rewards/rejected": -5.999361038208008,
      "step": 487,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7050610820244327,
      "grad_norm": 1.3914954662322998,
      "learning_rate": 8.55951072216536e-05,
      "logits/chosen": -0.5020650029182434,
      "logits/rejected": -0.7016733884811401,
      "logps/chosen": -6.336236953735352,
      "logps/rejected": -77.70530700683594,
      "loss": 0.41162416338920593,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2839837670326233,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.31479066610336304,
      "rewards/margins": 6.008511543273926,
      "rewards/rejected": -5.693720817565918,
      "step": 488,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7085514834205933,
      "grad_norm": 1.889309287071228,
      "learning_rate": 8.52137639440621e-05,
      "logits/chosen": -0.6282503008842468,
      "logits/rejected": -0.8060559630393982,
      "logps/chosen": -6.973021984100342,
      "logps/rejected": -75.95047760009766,
      "loss": 0.3657294809818268,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28552117943763733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3438401520252228,
      "rewards/margins": 6.749382972717285,
      "rewards/rejected": -6.405542373657227,
      "step": 489,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7120418848167538,
      "grad_norm": 3.3332746028900146,
      "learning_rate": 8.483264037232284e-05,
      "logits/chosen": -0.5885301828384399,
      "logits/rejected": -0.7886412739753723,
      "logps/chosen": -7.287195682525635,
      "logps/rejected": -83.81373596191406,
      "loss": 0.5585455298423767,
      "memory(GiB)": 43.15,
      "nll_loss": 0.45055481791496277,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.00887676328420639,
      "rewards/margins": 6.767753601074219,
      "rewards/rejected": -6.776630401611328,
      "step": 490,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7155322862129145,
      "grad_norm": 1.113408088684082,
      "learning_rate": 8.445174216947819e-05,
      "logits/chosen": -0.5470391511917114,
      "logits/rejected": -0.7462581396102905,
      "logps/chosen": -2.588343381881714,
      "logps/rejected": -71.17115020751953,
      "loss": 0.2312757968902588,
      "memory(GiB)": 43.15,
      "nll_loss": 0.17606161534786224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41102471947669983,
      "rewards/margins": 6.223390102386475,
      "rewards/rejected": -5.8123650550842285,
      "step": 491,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.719022687609075,
      "grad_norm": 1.0025933980941772,
      "learning_rate": 8.407107499522158e-05,
      "logits/chosen": -0.4948168992996216,
      "logits/rejected": -0.7623763084411621,
      "logps/chosen": -4.8139729499816895,
      "logps/rejected": -95.55278015136719,
      "loss": 0.27136656641960144,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2624860107898712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4436112642288208,
      "rewards/margins": 8.318868637084961,
      "rewards/rejected": -7.8752570152282715,
      "step": 492,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7225130890052356,
      "grad_norm": 1.701587200164795,
      "learning_rate": 8.369064450581373e-05,
      "logits/chosen": -0.612269937992096,
      "logits/rejected": -0.7896562218666077,
      "logps/chosen": -5.041234970092773,
      "logps/rejected": -84.5491943359375,
      "loss": 0.40640538930892944,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3292236924171448,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43859872221946716,
      "rewards/margins": 7.184287071228027,
      "rewards/rejected": -6.7456889152526855,
      "step": 493,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7260034904013961,
      "grad_norm": 1.9424786567687988,
      "learning_rate": 8.33104563539985e-05,
      "logits/chosen": -0.5213719010353088,
      "logits/rejected": -0.7520919442176819,
      "logps/chosen": -4.295230865478516,
      "logps/rejected": -97.60242462158203,
      "loss": 0.34295564889907837,
      "memory(GiB)": 43.15,
      "nll_loss": 0.23993462324142456,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24691414833068848,
      "rewards/margins": 8.112930297851562,
      "rewards/rejected": -7.866016387939453,
      "step": 494,
      "train_speed(iter/s)": 0.010747
    },
    {
      "epoch": 1.7294938917975569,
      "grad_norm": 1.8167755603790283,
      "learning_rate": 8.293051618891885e-05,
      "logits/chosen": -0.5651763677597046,
      "logits/rejected": -0.7933282852172852,
      "logps/chosen": -3.743563175201416,
      "logps/rejected": -84.2485580444336,
      "loss": 0.34840941429138184,
      "memory(GiB)": 43.15,
      "nll_loss": 0.24304822087287903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3500049412250519,
      "rewards/margins": 7.271448135375977,
      "rewards/rejected": -6.92144250869751,
      "step": 495,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.7329842931937174,
      "grad_norm": 1.4177367687225342,
      "learning_rate": 8.255082965603306e-05,
      "logits/chosen": -0.4521150290966034,
      "logits/rejected": -0.7816924452781677,
      "logps/chosen": -2.747921943664551,
      "logps/rejected": -99.33414459228516,
      "loss": 0.26944684982299805,
      "memory(GiB)": 43.15,
      "nll_loss": 0.2259138524532318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1833350956439972,
      "rewards/margins": 8.444989204406738,
      "rewards/rejected": -8.261652946472168,
      "step": 496,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.736474694589878,
      "grad_norm": 1.7345248460769653,
      "learning_rate": 8.21714023970306e-05,
      "logits/chosen": -0.6219483017921448,
      "logits/rejected": -0.8654438853263855,
      "logps/chosen": -4.519089221954346,
      "logps/rejected": -79.08432006835938,
      "loss": 0.37975239753723145,
      "memory(GiB)": 43.15,
      "nll_loss": 0.3202629089355469,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.31086960434913635,
      "rewards/margins": 6.751467704772949,
      "rewards/rejected": -6.440598011016846,
      "step": 497,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.7399650959860384,
      "grad_norm": 1.3441096544265747,
      "learning_rate": 8.179224004974857e-05,
      "logits/chosen": -0.7073596715927124,
      "logits/rejected": -0.8711289763450623,
      "logps/chosen": -4.434149265289307,
      "logps/rejected": -81.61371612548828,
      "loss": 0.3156903088092804,
      "memory(GiB)": 43.15,
      "nll_loss": 0.28625619411468506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3435167074203491,
      "rewards/margins": 7.131415367126465,
      "rewards/rejected": -6.787898540496826,
      "step": 498,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.743455497382199,
      "grad_norm": 1.561885952949524,
      "learning_rate": 8.141334824808769e-05,
      "logits/chosen": -0.5434170365333557,
      "logits/rejected": -0.8003084659576416,
      "logps/chosen": -4.422205924987793,
      "logps/rejected": -72.26570129394531,
      "loss": 0.35296329855918884,
      "memory(GiB)": 43.15,
      "nll_loss": 0.25030094385147095,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3269076645374298,
      "rewards/margins": 5.8413004875183105,
      "rewards/rejected": -5.514392852783203,
      "step": 499,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.7469458987783595,
      "grad_norm": 2.3898956775665283,
      "learning_rate": 8.103473262192879e-05,
      "logits/chosen": -0.4674520492553711,
      "logits/rejected": -0.6909574866294861,
      "logps/chosen": -8.937088966369629,
      "logps/rejected": -93.10710144042969,
      "loss": 0.7829589247703552,
      "memory(GiB)": 43.15,
      "nll_loss": 0.5921401977539062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20197468996047974,
      "rewards/margins": 7.561078071594238,
      "rewards/rejected": -7.359103202819824,
      "step": 500,
      "train_speed(iter/s)": 0.010748
    },
    {
      "epoch": 1.7469458987783595,
      "eval_logits/chosen": -0.5579695105552673,
      "eval_logits/rejected": -0.7757669687271118,
      "eval_logps/chosen": -7.580451488494873,
      "eval_logps/rejected": -75.10726928710938,
      "eval_loss": 0.4915315806865692,
      "eval_nll_loss": 0.4280147850513458,
      "eval_rewards/accuracies": 0.967391312122345,
      "eval_rewards/chosen": 0.42943841218948364,
      "eval_rewards/margins": 6.220393180847168,
      "eval_rewards/rejected": -5.79095458984375,
      "eval_runtime": 105.0113,
      "eval_samples_per_second": 0.876,
      "eval_steps_per_second": 0.438,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 858,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 2.7276809364884685e+17,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}