{
  "best_global_step": 600,
  "best_metric": 0.26477173,
  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task3_dpo_no_task2_gt60.5+task3gt5/v0-20250609-005919/checkpoint-600",
  "epoch": 1.685491723466407,
  "eval_steps": 300,
  "global_step": 650,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002596559558584875,
      "grad_norm": 4.769598007202148,
      "learning_rate": 5.128205128205128e-06,
      "logits/chosen": -0.5944205522537231,
      "logits/rejected": -0.5917393565177917,
      "logps/chosen": -11.821638107299805,
      "logps/rejected": -10.731678009033203,
      "loss": 1.193467140197754,
      "memory(GiB)": 46.82,
      "nll_loss": 0.50031977891922,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.004987
    },
    {
      "epoch": 0.00519311911716975,
      "grad_norm": 7.638933181762695,
      "learning_rate": 1.0256410256410256e-05,
      "logits/chosen": -0.6539207696914673,
      "logits/rejected": -0.6607442498207092,
      "logps/chosen": -9.628549575805664,
      "logps/rejected": -15.572840690612793,
      "loss": 1.1913397312164307,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4981924891471863,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005274
    },
    {
      "epoch": 0.007789678675754625,
      "grad_norm": 8.172294616699219,
      "learning_rate": 1.5384615384615387e-05,
      "logits/chosen": -0.6215255260467529,
      "logits/rejected": -0.6243188381195068,
      "logps/chosen": -10.225044250488281,
      "logps/rejected": -12.433060646057129,
      "loss": 1.3010810613632202,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6032217741012573,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.002520320238545537,
      "rewards/margins": -0.00917358510196209,
      "rewards/rejected": 0.006653264630585909,
      "step": 3,
      "train_speed(iter/s)": 0.005242
    },
    {
      "epoch": 0.0103862382343395,
      "grad_norm": 7.314579010009766,
      "learning_rate": 2.0512820512820512e-05,
      "logits/chosen": -0.6062605381011963,
      "logits/rejected": -0.6072395443916321,
      "logps/chosen": -9.956047058105469,
      "logps/rejected": -11.640246391296387,
      "loss": 1.2542610168457031,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5589770674705505,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.007420660927891731,
      "rewards/margins": -0.004003261215984821,
      "rewards/rejected": 0.011423922143876553,
      "step": 4,
      "train_speed(iter/s)": 0.005268
    },
    {
      "epoch": 0.012982797792924375,
      "grad_norm": 3.7953975200653076,
      "learning_rate": 2.564102564102564e-05,
      "logits/chosen": -0.5976298451423645,
      "logits/rejected": -0.5966740846633911,
      "logps/chosen": -13.070164680480957,
      "logps/rejected": -9.661067008972168,
      "loss": 1.215171456336975,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5106430649757385,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00016488437540829182,
      "rewards/margins": -0.020555444061756134,
      "rewards/rejected": 0.02072032354772091,
      "step": 5,
      "train_speed(iter/s)": 0.00529
    },
    {
      "epoch": 0.01557935735150925,
      "grad_norm": 7.202587604522705,
      "learning_rate": 3.0769230769230774e-05,
      "logits/chosen": -0.6029255390167236,
      "logits/rejected": -0.6082680225372314,
      "logps/chosen": -6.68419075012207,
      "logps/rejected": -14.836040496826172,
      "loss": 1.1022435426712036,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4213921129703522,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.05949082225561142,
      "rewards/margins": 0.03669985011219978,
      "rewards/rejected": 0.02279096469283104,
      "step": 6,
      "train_speed(iter/s)": 0.00532
    },
    {
      "epoch": 0.018175916910094125,
      "grad_norm": 4.17504358291626,
      "learning_rate": 3.58974358974359e-05,
      "logits/chosen": -0.5737150311470032,
      "logits/rejected": -0.5751169919967651,
      "logps/chosen": -10.05550479888916,
      "logps/rejected": -10.229061126708984,
      "loss": 1.1968086957931519,
      "memory(GiB)": 46.82,
      "nll_loss": 0.451814740896225,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.018731240183115005,
      "rewards/margins": -0.06073049083352089,
      "rewards/rejected": 0.0794617310166359,
      "step": 7,
      "train_speed(iter/s)": 0.005319
    },
    {
      "epoch": 0.020772476468679,
      "grad_norm": 4.966729164123535,
      "learning_rate": 4.1025641025641023e-05,
      "logits/chosen": -0.6149957180023193,
      "logits/rejected": -0.619285523891449,
      "logps/chosen": -10.129886627197266,
      "logps/rejected": -14.14559268951416,
      "loss": 1.1496262550354004,
      "memory(GiB)": 46.82,
      "nll_loss": 0.45597678422927856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.048677604645490646,
      "rewards/margins": 0.07307372987270355,
      "rewards/rejected": -0.1217513307929039,
      "step": 8,
      "train_speed(iter/s)": 0.005322
    },
    {
      "epoch": 0.023369036027263874,
      "grad_norm": 4.258759021759033,
      "learning_rate": 4.615384615384616e-05,
      "logits/chosen": -0.6464206576347351,
      "logits/rejected": -0.649814248085022,
      "logps/chosen": -6.2765960693359375,
      "logps/rejected": -14.259992599487305,
      "loss": 0.9683928489685059,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27603328227996826,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.028385277837514877,
      "rewards/margins": 0.1275366246700287,
      "rewards/rejected": -0.15592190623283386,
      "step": 9,
      "train_speed(iter/s)": 0.005346
    },
    {
      "epoch": 0.02596559558584875,
      "grad_norm": 3.5765655040740967,
      "learning_rate": 5.128205128205128e-05,
      "logits/chosen": -0.5905835628509521,
      "logits/rejected": -0.5969584584236145,
      "logps/chosen": -14.795049667358398,
      "logps/rejected": -15.656811714172363,
      "loss": 1.238275408744812,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5923190116882324,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012541369535028934,
      "rewards/margins": 0.1883796602487564,
      "rewards/rejected": -0.20092105865478516,
      "step": 10,
      "train_speed(iter/s)": 0.005347
    },
    {
      "epoch": 0.028562155144433627,
      "grad_norm": 5.3241682052612305,
      "learning_rate": 5.6410256410256414e-05,
      "logits/chosen": -0.5469571948051453,
      "logits/rejected": -0.5478999614715576,
      "logps/chosen": -11.43381118774414,
      "logps/rejected": -13.521588325500488,
      "loss": 1.2181414365768433,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4799095690250397,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11080781370401382,
      "rewards/margins": -0.000970199704170227,
      "rewards/rejected": -0.109837606549263,
      "step": 11,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.0311587147030185,
      "grad_norm": 3.739349126815796,
      "learning_rate": 6.153846153846155e-05,
      "logits/chosen": -0.5663439631462097,
      "logits/rejected": -0.5685043931007385,
      "logps/chosen": -9.113786697387695,
      "logps/rejected": -12.898387908935547,
      "loss": 1.1865863800048828,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5080631971359253,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.07351399958133698,
      "rewards/margins": 0.09553620219230652,
      "rewards/rejected": -0.022022202610969543,
      "step": 12,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.03375527426160337,
      "grad_norm": 2.732536792755127,
      "learning_rate": 6.666666666666667e-05,
      "logits/chosen": -0.6258959770202637,
      "logits/rejected": -0.628243088722229,
      "logps/chosen": -11.908061027526855,
      "logps/rejected": -14.742002487182617,
      "loss": 1.0720946788787842,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44474831223487854,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0872192531824112,
      "rewards/margins": 0.16859112679958344,
      "rewards/rejected": -0.08137187361717224,
      "step": 13,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.03635183382018825,
      "grad_norm": 2.286402702331543,
      "learning_rate": 7.17948717948718e-05,
      "logits/chosen": -0.5518996119499207,
      "logits/rejected": -0.5536460280418396,
      "logps/chosen": -7.985256195068359,
      "logps/rejected": -9.997514724731445,
      "loss": 1.0465917587280273,
      "memory(GiB)": 46.82,
      "nll_loss": 0.37517642974853516,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.09782104939222336,
      "rewards/margins": 0.08964069932699203,
      "rewards/rejected": 0.008180351927876472,
      "step": 14,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.03894839337877313,
      "grad_norm": 2.8793153762817383,
      "learning_rate": 7.692307692307693e-05,
      "logits/chosen": -0.5976672172546387,
      "logits/rejected": -0.5990516543388367,
      "logps/chosen": -11.681562423706055,
      "logps/rejected": -15.269111633300781,
      "loss": 1.0413833856582642,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44159939885139465,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.18112002313137054,
      "rewards/margins": 0.246211975812912,
      "rewards/rejected": -0.06509192287921906,
      "step": 15,
      "train_speed(iter/s)": 0.005369
    },
    {
      "epoch": 0.041544952937358,
      "grad_norm": 3.362415313720703,
      "learning_rate": 8.205128205128205e-05,
      "logits/chosen": -0.6091838479042053,
      "logits/rejected": -0.6097983717918396,
      "logps/chosen": -6.9782915115356445,
      "logps/rejected": -9.985608100891113,
      "loss": 1.0336495637893677,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3420756161212921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05590391904115677,
      "rewards/margins": 0.03598181903362274,
      "rewards/rejected": 0.019922103732824326,
      "step": 16,
      "train_speed(iter/s)": 0.005368
    },
    {
      "epoch": 0.04414151249594288,
      "grad_norm": 2.314833641052246,
      "learning_rate": 8.717948717948718e-05,
      "logits/chosen": -0.6001038551330566,
      "logits/rejected": -0.6009984612464905,
      "logps/chosen": -8.711943626403809,
      "logps/rejected": -8.347241401672363,
      "loss": 1.0018643140792847,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3586280345916748,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.17141370475292206,
      "rewards/margins": 0.15122823417186737,
      "rewards/rejected": 0.020185478031635284,
      "step": 17,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.04673807205452775,
      "grad_norm": 2.6080470085144043,
      "learning_rate": 9.230769230769232e-05,
      "logits/chosen": -0.6079075932502747,
      "logits/rejected": -0.6091002225875854,
      "logps/chosen": -9.866250991821289,
      "logps/rejected": -9.885502815246582,
      "loss": 1.0599387884140015,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3953135013580322,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1653863787651062,
      "rewards/margins": 0.13114988803863525,
      "rewards/rejected": 0.03423647955060005,
      "step": 18,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.04933463161311263,
      "grad_norm": 2.5750620365142822,
      "learning_rate": 9.743589743589744e-05,
      "logits/chosen": -0.5754107236862183,
      "logits/rejected": -0.5755229592323303,
      "logps/chosen": -9.663117408752441,
      "logps/rejected": -14.233282089233398,
      "loss": 0.9537867307662964,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35284289717674255,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.12623246014118195,
      "rewards/margins": 0.24378177523612976,
      "rewards/rejected": -0.11754930019378662,
      "step": 19,
      "train_speed(iter/s)": 0.00537
    },
    {
      "epoch": 0.0519311911716975,
      "grad_norm": 4.289522171020508,
      "learning_rate": 0.00010256410256410256,
      "logits/chosen": -0.5981144905090332,
      "logits/rejected": -0.6002673506736755,
      "logps/chosen": -8.273541450500488,
      "logps/rejected": -12.811166763305664,
      "loss": 1.422603964805603,
      "memory(GiB)": 46.82,
      "nll_loss": 0.8183298707008362,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.21172407269477844,
      "rewards/margins": 0.2575765550136566,
      "rewards/rejected": -0.04585248976945877,
      "step": 20,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.054527750730282376,
      "grad_norm": 2.396873712539673,
      "learning_rate": 0.0001076923076923077,
      "logits/chosen": -0.5655781030654907,
      "logits/rejected": -0.5672769546508789,
      "logps/chosen": -7.642134189605713,
      "logps/rejected": -16.37198257446289,
      "loss": 0.8944156765937805,
      "memory(GiB)": 46.82,
      "nll_loss": 0.28976452350616455,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2191261202096939,
      "rewards/margins": 0.2641218602657318,
      "rewards/rejected": -0.04499572888016701,
      "step": 21,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.057124310288867254,
      "grad_norm": 3.0281121730804443,
      "learning_rate": 0.00011282051282051283,
      "logits/chosen": -0.5596702694892883,
      "logits/rejected": -0.5609368681907654,
      "logps/chosen": -7.446313381195068,
      "logps/rejected": -12.651912689208984,
      "loss": 0.9526722431182861,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33721715211868286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15709233283996582,
      "rewards/margins": 0.2245473712682724,
      "rewards/rejected": -0.06745504587888718,
      "step": 22,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.059720869847452125,
      "grad_norm": 2.6955020427703857,
      "learning_rate": 0.00011794871794871796,
      "logits/chosen": -0.5400959849357605,
      "logits/rejected": -0.5422949194908142,
      "logps/chosen": -6.002541542053223,
      "logps/rejected": -12.28770637512207,
      "loss": 0.9044801592826843,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2814594805240631,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11246027797460556,
      "rewards/margins": 0.21622154116630554,
      "rewards/rejected": -0.10376127809286118,
      "step": 23,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.062317429406037,
      "grad_norm": 3.1954290866851807,
      "learning_rate": 0.0001230769230769231,
      "logits/chosen": -0.5763551592826843,
      "logits/rejected": -0.5829795598983765,
      "logps/chosen": -8.22636890411377,
      "logps/rejected": -12.94092845916748,
      "loss": 0.9596253037452698,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3679075837135315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17844173312187195,
      "rewards/margins": 0.2646990418434143,
      "rewards/rejected": -0.08625732362270355,
      "step": 24,
      "train_speed(iter/s)": 0.005367
    },
    {
      "epoch": 0.06491398896462187,
      "grad_norm": 3.440917730331421,
      "learning_rate": 0.00012820512820512823,
      "logits/chosen": -0.5423401594161987,
      "logits/rejected": -0.5454680919647217,
      "logps/chosen": -6.751742362976074,
      "logps/rejected": -14.677633285522461,
      "loss": 0.8222842216491699,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2530384063720703,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.20278047025203705,
      "rewards/margins": 0.37300848960876465,
      "rewards/rejected": -0.1702280342578888,
      "step": 25,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.06751054852320675,
      "grad_norm": 7.046718120574951,
      "learning_rate": 0.00013333333333333334,
      "logits/chosen": -0.531994640827179,
      "logits/rejected": -0.5324733257293701,
      "logps/chosen": -7.691443920135498,
      "logps/rejected": -12.912195205688477,
      "loss": 0.8851491212844849,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2668984532356262,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03996456041932106,
      "rewards/margins": 0.30043962597846985,
      "rewards/rejected": -0.2604750692844391,
      "step": 26,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.07010710808179163,
      "grad_norm": 3.7837774753570557,
      "learning_rate": 0.00013846153846153847,
      "logits/chosen": -0.5356835722923279,
      "logits/rejected": -0.535167932510376,
      "logps/chosen": -7.836236476898193,
      "logps/rejected": -12.5257568359375,
      "loss": 0.9785585999488831,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39958691596984863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11726226657629013,
      "rewards/margins": 0.30467987060546875,
      "rewards/rejected": -0.18741759657859802,
      "step": 27,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 0.0727036676403765,
      "grad_norm": 2.986128568649292,
      "learning_rate": 0.0001435897435897436,
      "logits/chosen": -0.5374022126197815,
      "logits/rejected": -0.5330510139465332,
      "logps/chosen": -10.774028778076172,
      "logps/rejected": -10.21718692779541,
      "loss": 1.0210673809051514,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44256913661956787,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.08123424649238586,
      "rewards/margins": 0.32150229811668396,
      "rewards/rejected": -0.2402680516242981,
      "step": 28,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.07530022719896137,
      "grad_norm": 8.227151870727539,
      "learning_rate": 0.00014871794871794872,
      "logits/chosen": -0.555951714515686,
      "logits/rejected": -0.5587239265441895,
      "logps/chosen": -7.656264781951904,
      "logps/rejected": -16.2135066986084,
      "loss": 1.024498701095581,
      "memory(GiB)": 46.82,
      "nll_loss": 0.467337429523468,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1679726243019104,
      "rewards/margins": 0.43920189142227173,
      "rewards/rejected": -0.2712292969226837,
      "step": 29,
      "train_speed(iter/s)": 0.005367
    },
    {
      "epoch": 0.07789678675754626,
      "grad_norm": 3.866678476333618,
      "learning_rate": 0.00015384615384615385,
      "logits/chosen": -0.5329037308692932,
      "logits/rejected": -0.5308660864830017,
      "logps/chosen": -13.327455520629883,
      "logps/rejected": -12.309192657470703,
      "loss": 1.1753352880477905,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5743139386177063,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.10232796519994736,
      "rewards/margins": 0.27346518635749817,
      "rewards/rejected": -0.1711372286081314,
      "step": 30,
      "train_speed(iter/s)": 0.00537
    },
    {
      "epoch": 0.08049334631613113,
      "grad_norm": 2.7835240364074707,
      "learning_rate": 0.00015897435897435896,
      "logits/chosen": -0.5286478996276855,
      "logits/rejected": -0.5297562479972839,
      "logps/chosen": -8.105016708374023,
      "logps/rejected": -18.577655792236328,
      "loss": 0.8083048462867737,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3598409593105316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3429361879825592,
      "rewards/margins": 0.6762320399284363,
      "rewards/rejected": -0.3332958221435547,
      "step": 31,
      "train_speed(iter/s)": 0.005368
    },
    {
      "epoch": 0.083089905874716,
      "grad_norm": 2.621320962905884,
      "learning_rate": 0.0001641025641025641,
      "logits/chosen": -0.5535257458686829,
      "logits/rejected": -0.5577558279037476,
      "logps/chosen": -7.381773948669434,
      "logps/rejected": -20.8293514251709,
      "loss": 0.7857053279876709,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3121376037597656,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11771110445261002,
      "rewards/margins": 0.6395547389984131,
      "rewards/rejected": -0.5218436121940613,
      "step": 32,
      "train_speed(iter/s)": 0.005373
    },
    {
      "epoch": 0.08568646543330087,
      "grad_norm": 5.273679256439209,
      "learning_rate": 0.00016923076923076923,
      "logits/chosen": -0.5348193645477295,
      "logits/rejected": -0.5335426330566406,
      "logps/chosen": -16.012008666992188,
      "logps/rejected": -16.090946197509766,
      "loss": 1.150058388710022,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5311561822891235,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09080329537391663,
      "rewards/margins": 0.3698381185531616,
      "rewards/rejected": -0.46064135432243347,
      "step": 33,
      "train_speed(iter/s)": 0.005375
    },
    {
      "epoch": 0.08828302499188576,
      "grad_norm": 3.7380588054656982,
      "learning_rate": 0.00017435897435897436,
      "logits/chosen": -0.526316225528717,
      "logits/rejected": -0.5241590738296509,
      "logps/chosen": -13.188718795776367,
      "logps/rejected": -18.877933502197266,
      "loss": 0.8901183009147644,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4010193347930908,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13551224768161774,
      "rewards/margins": 0.7849555015563965,
      "rewards/rejected": -0.6494433283805847,
      "step": 34,
      "train_speed(iter/s)": 0.005376
    },
    {
      "epoch": 0.09087958455047063,
      "grad_norm": 3.2903590202331543,
      "learning_rate": 0.0001794871794871795,
      "logits/chosen": -0.5694751739501953,
      "logits/rejected": -0.5714199542999268,
      "logps/chosen": -13.836735725402832,
      "logps/rejected": -17.397571563720703,
      "loss": 0.9489783644676208,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4817684292793274,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.17933052778244019,
      "rewards/margins": 0.7199034690856934,
      "rewards/rejected": -0.5405729413032532,
      "step": 35,
      "train_speed(iter/s)": 0.005374
    },
    {
      "epoch": 0.0934761441090555,
      "grad_norm": 7.443658351898193,
      "learning_rate": 0.00018461538461538463,
      "logits/chosen": -0.5579850077629089,
      "logits/rejected": -0.5598161220550537,
      "logps/chosen": -9.724953651428223,
      "logps/rejected": -19.001192092895508,
      "loss": 1.025741696357727,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4933169484138489,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09439410269260406,
      "rewards/margins": 0.531223714351654,
      "rewards/rejected": -0.4368295967578888,
      "step": 36,
      "train_speed(iter/s)": 0.005367
    },
    {
      "epoch": 0.09607270366764038,
      "grad_norm": 4.221776962280273,
      "learning_rate": 0.00018974358974358974,
      "logits/chosen": -0.5673465728759766,
      "logits/rejected": -0.568681001663208,
      "logps/chosen": -7.996758937835693,
      "logps/rejected": -16.828737258911133,
      "loss": 0.768932044506073,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3632465600967407,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.20153376460075378,
      "rewards/margins": 0.8081123232841492,
      "rewards/rejected": -0.6065785884857178,
      "step": 37,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.09866926322622525,
      "grad_norm": 4.438755989074707,
      "learning_rate": 0.00019487179487179487,
      "logits/chosen": -0.5432108640670776,
      "logits/rejected": -0.5440645217895508,
      "logps/chosen": -11.19113826751709,
      "logps/rejected": -16.801233291625977,
      "loss": 0.9643179178237915,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5330875515937805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.212477445602417,
      "rewards/margins": 0.7622401714324951,
      "rewards/rejected": -0.5497627258300781,
      "step": 38,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.10126582278481013,
      "grad_norm": 3.844766855239868,
      "learning_rate": 0.0002,
      "logits/chosen": -0.5104550719261169,
      "logits/rejected": -0.5119606852531433,
      "logps/chosen": -11.040297508239746,
      "logps/rejected": -18.91219711303711,
      "loss": 0.9704711437225342,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5230136513710022,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.14157143235206604,
      "rewards/margins": 0.7000695466995239,
      "rewards/rejected": -0.5584981441497803,
      "step": 39,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.103862382343395,
      "grad_norm": 5.2911272048950195,
      "learning_rate": 0.00019999907650547008,
      "logits/chosen": -0.563255250453949,
      "logits/rejected": -0.5627263784408569,
      "logps/chosen": -8.962218284606934,
      "logps/rejected": -20.214658737182617,
      "loss": 0.9730861783027649,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4494069516658783,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03735905885696411,
      "rewards/margins": 0.7039426565170288,
      "rewards/rejected": -0.6665836572647095,
      "step": 40,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.10645894190197988,
      "grad_norm": 3.2932822704315186,
      "learning_rate": 0.0001999963060389371,
      "logits/chosen": -0.49407708644866943,
      "logits/rejected": -0.4951040744781494,
      "logps/chosen": -4.336781978607178,
      "logps/rejected": -23.352622985839844,
      "loss": 0.5769761800765991,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21373578906059265,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2602457106113434,
      "rewards/margins": 1.0540037155151367,
      "rewards/rejected": -0.7937580347061157,
      "step": 41,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.10905550146056475,
      "grad_norm": 3.550082206726074,
      "learning_rate": 0.00019999168865157137,
      "logits/chosen": -0.5316025018692017,
      "logits/rejected": -0.5340486764907837,
      "logps/chosen": -8.441543579101562,
      "logps/rejected": -18.124588012695312,
      "loss": 0.9663518071174622,
      "memory(GiB)": 46.82,
      "nll_loss": 0.47912853956222534,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.14834752678871155,
      "rewards/margins": 0.7632903456687927,
      "rewards/rejected": -0.6149427890777588,
      "step": 42,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.11165206101914962,
      "grad_norm": 3.8798763751983643,
      "learning_rate": 0.0001999852244286554,
      "logits/chosen": -0.5125827193260193,
      "logits/rejected": -0.5118388533592224,
      "logps/chosen": -8.259347915649414,
      "logps/rejected": -18.246912002563477,
      "loss": 0.8706857562065125,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3688449263572693,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1189112439751625,
      "rewards/margins": 0.5402826070785522,
      "rewards/rejected": -0.4213714003562927,
      "step": 43,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.11424862057773451,
      "grad_norm": 5.702610015869141,
      "learning_rate": 0.0001999769134895828,
      "logits/chosen": -0.5676945447921753,
      "logits/rejected": -0.5712423324584961,
      "logps/chosen": -7.507511138916016,
      "logps/rejected": -17.306344985961914,
      "loss": 0.9447178840637207,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5068771839141846,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1572760045528412,
      "rewards/margins": 0.8183934688568115,
      "rewards/rejected": -0.6611174941062927,
      "step": 44,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.11684518013631938,
      "grad_norm": 5.846091270446777,
      "learning_rate": 0.0001999667559878556,
      "logits/chosen": -0.5530126094818115,
      "logits/rejected": -0.5544271469116211,
      "logps/chosen": -9.34422779083252,
      "logps/rejected": -21.99274444580078,
      "loss": 1.0033745765686035,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5510709285736084,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06781984120607376,
      "rewards/margins": 0.8660807609558105,
      "rewards/rejected": -0.7982609868049622,
      "step": 45,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.11944173969490425,
      "grad_norm": 3.4377248287200928,
      "learning_rate": 0.00019995475211108185,
      "logits/chosen": -0.5516988039016724,
      "logits/rejected": -0.5513719916343689,
      "logps/chosen": -9.321004867553711,
      "logps/rejected": -18.94799041748047,
      "loss": 0.838591992855072,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44976377487182617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15304125845432281,
      "rewards/margins": 0.9354310631752014,
      "rewards/rejected": -0.782389760017395,
      "step": 46,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.12203829925348912,
      "grad_norm": 3.021584987640381,
      "learning_rate": 0.00019994090208097176,
      "logits/chosen": -0.5453197956085205,
      "logits/rejected": -0.5457130670547485,
      "logps/chosen": -7.513545036315918,
      "logps/rejected": -19.929990768432617,
      "loss": 0.7878033518791199,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2906043827533722,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.04635176807641983,
      "rewards/margins": 0.8055768609046936,
      "rewards/rejected": -0.759225070476532,
      "step": 47,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.124634858812074,
      "grad_norm": 2.4632484912872314,
      "learning_rate": 0.00019992520615333393,
      "logits/chosen": -0.5288453102111816,
      "logits/rejected": -0.5294773578643799,
      "logps/chosen": -4.316083908081055,
      "logps/rejected": -16.75088119506836,
      "loss": 0.6369743347167969,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19174346327781677,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1844159960746765,
      "rewards/margins": 0.7728991508483887,
      "rewards/rejected": -0.5884830951690674,
      "step": 48,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.12723141837065888,
      "grad_norm": 2.225041627883911,
      "learning_rate": 0.00019990766461807039,
      "logits/chosen": -0.5786827802658081,
      "logits/rejected": -0.5775803923606873,
      "logps/chosen": -7.1691179275512695,
      "logps/rejected": -19.747949600219727,
      "loss": 0.611931562423706,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26027703285217285,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32533061504364014,
      "rewards/margins": 1.0796949863433838,
      "rewards/rejected": -0.7543643712997437,
      "step": 49,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.12982797792924375,
      "grad_norm": 3.631110429763794,
      "learning_rate": 0.00019988827779917137,
      "logits/chosen": -0.5347975492477417,
      "logits/rejected": -0.5404946804046631,
      "logps/chosen": -8.115504264831543,
      "logps/rejected": -22.913742065429688,
      "loss": 0.8714032769203186,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4530017375946045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22861376404762268,
      "rewards/margins": 0.9870915412902832,
      "rewards/rejected": -0.7584777474403381,
      "step": 50,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.13242453748782862,
      "grad_norm": 4.098914623260498,
      "learning_rate": 0.0001998670460547093,
      "logits/chosen": -0.5628920197486877,
      "logits/rejected": -0.5603100657463074,
      "logps/chosen": -12.636977195739746,
      "logps/rejected": -17.273649215698242,
      "loss": 1.033919095993042,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4824763834476471,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011750273406505585,
      "rewards/margins": 0.7105109095573425,
      "rewards/rejected": -0.6987606287002563,
      "step": 51,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 0.1350210970464135,
      "grad_norm": 2.982447385787964,
      "learning_rate": 0.00019984396977683222,
      "logits/chosen": -0.5565685629844666,
      "logits/rejected": -0.5531243085861206,
      "logps/chosen": -9.393210411071777,
      "logps/rejected": -17.276996612548828,
      "loss": 0.7912466526031494,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44081103801727295,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22525233030319214,
      "rewards/margins": 1.060104250907898,
      "rewards/rejected": -0.8348519206047058,
      "step": 52,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.1376176566049984,
      "grad_norm": 5.296391010284424,
      "learning_rate": 0.0001998190493917564,
      "logits/chosen": -0.5625238418579102,
      "logits/rejected": -0.5669078826904297,
      "logps/chosen": -9.36229133605957,
      "logps/rejected": -21.19735336303711,
      "loss": 0.9399389028549194,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5554667115211487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25273460149765015,
      "rewards/margins": 1.137141227722168,
      "rewards/rejected": -0.8844066858291626,
      "step": 53,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.14021421616358326,
      "grad_norm": 5.798766136169434,
      "learning_rate": 0.00019979228535975865,
      "logits/chosen": -0.5416492223739624,
      "logits/rejected": -0.5440787076950073,
      "logps/chosen": -9.865373611450195,
      "logps/rejected": -29.639345169067383,
      "loss": 0.7824708223342896,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4418797492980957,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05873842537403107,
      "rewards/margins": 1.351473093032837,
      "rewards/rejected": -1.2927345037460327,
      "step": 54,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.14281077572216813,
      "grad_norm": 2.68666934967041,
      "learning_rate": 0.00019976367817516773,
      "logits/chosen": -0.5077602863311768,
      "logits/rejected": -0.5117127299308777,
      "logps/chosen": -6.119892120361328,
      "logps/rejected": -24.990623474121094,
      "loss": 0.7171817421913147,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3807484805583954,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23153242468833923,
      "rewards/margins": 1.3034626245498657,
      "rewards/rejected": -1.0719302892684937,
      "step": 55,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.145407335280753,
      "grad_norm": 12.753750801086426,
      "learning_rate": 0.00019973322836635518,
      "logits/chosen": -0.5142076015472412,
      "logits/rejected": -0.5133773684501648,
      "logps/chosen": -10.577905654907227,
      "logps/rejected": -17.775619506835938,
      "loss": 1.0084953308105469,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5207356214523315,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.055092498660087585,
      "rewards/margins": 0.8445413708686829,
      "rewards/rejected": -0.7894489169120789,
      "step": 56,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.14800389483933787,
      "grad_norm": 4.176927089691162,
      "learning_rate": 0.00019970093649572564,
      "logits/chosen": -0.4988924562931061,
      "logits/rejected": -0.5031363368034363,
      "logps/chosen": -5.868337631225586,
      "logps/rejected": -32.978477478027344,
      "loss": 0.5518871545791626,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23210853338241577,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1517554670572281,
      "rewards/margins": 1.8292872905731201,
      "rewards/rejected": -1.6775319576263428,
      "step": 57,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.15060045439792275,
      "grad_norm": 5.881600856781006,
      "learning_rate": 0.00019966680315970648,
      "logits/chosen": -0.5035607218742371,
      "logits/rejected": -0.5062836408615112,
      "logps/chosen": -12.07951831817627,
      "logps/rejected": -29.57831573486328,
      "loss": 1.039928674697876,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4533731937408447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19116882979869843,
      "rewards/margins": 1.3504120111465454,
      "rewards/rejected": -1.5415807962417603,
      "step": 58,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.15319701395650762,
      "grad_norm": 5.895066738128662,
      "learning_rate": 0.0001996308289887366,
      "logits/chosen": -0.504879355430603,
      "logits/rejected": -0.5091466903686523,
      "logps/chosen": -8.11007022857666,
      "logps/rejected": -27.097564697265625,
      "loss": 0.9131656289100647,
      "memory(GiB)": 46.82,
      "nll_loss": 0.47925126552581787,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04523879289627075,
      "rewards/margins": 1.2242006063461304,
      "rewards/rejected": -1.178961992263794,
      "step": 59,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.15579357351509251,
      "grad_norm": 2.7233798503875732,
      "learning_rate": 0.00019959301464725506,
      "logits/chosen": -0.5388925671577454,
      "logits/rejected": -0.5401982665061951,
      "logps/chosen": -5.20983362197876,
      "logps/rejected": -22.794513702392578,
      "loss": 0.6133744716644287,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2746528089046478,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06251248717308044,
      "rewards/margins": 1.3268343210220337,
      "rewards/rejected": -1.2643219232559204,
      "step": 60,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.15839013307367739,
      "grad_norm": 5.102992057800293,
      "learning_rate": 0.00019955336083368858,
      "logits/chosen": -0.5972931385040283,
      "logits/rejected": -0.601641058921814,
      "logps/chosen": -8.611083984375,
      "logps/rejected": -22.611942291259766,
      "loss": 1.1286181211471558,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6720573902130127,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.05290444567799568,
      "rewards/margins": 0.9988237619400024,
      "rewards/rejected": -0.9459192752838135,
      "step": 61,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.16098669263226226,
      "grad_norm": 6.744442462921143,
      "learning_rate": 0.00019951186828043877,
      "logits/chosen": -0.5748434662818909,
      "logits/rejected": -0.5786728858947754,
      "logps/chosen": -7.9780778884887695,
      "logps/rejected": -19.627988815307617,
      "loss": 1.0838773250579834,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5846377611160278,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.011326288804411888,
      "rewards/margins": 0.7546138167381287,
      "rewards/rejected": -0.7659401297569275,
      "step": 62,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.16358325219084713,
      "grad_norm": 3.6435351371765137,
      "learning_rate": 0.00019946853775386857,
      "logits/chosen": -0.6392778158187866,
      "logits/rejected": -0.6405251622200012,
      "logps/chosen": -7.981713771820068,
      "logps/rejected": -18.500165939331055,
      "loss": 0.8841695189476013,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4594660997390747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1688350886106491,
      "rewards/margins": 0.8609640598297119,
      "rewards/rejected": -0.6921288967132568,
      "step": 63,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.166179811749432,
      "grad_norm": 2.4889438152313232,
      "learning_rate": 0.00019942337005428803,
      "logits/chosen": -0.614960253238678,
      "logits/rejected": -0.6170456409454346,
      "logps/chosen": -6.154093265533447,
      "logps/rejected": -14.32476806640625,
      "loss": 0.8417397737503052,
      "memory(GiB)": 46.82,
      "nll_loss": 0.42383700609207153,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22809524834156036,
      "rewards/margins": 0.7583045959472656,
      "rewards/rejected": -0.5302093625068665,
      "step": 64,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.16877637130801687,
      "grad_norm": 3.13456654548645,
      "learning_rate": 0.00019937636601593964,
      "logits/chosen": -0.6562027335166931,
      "logits/rejected": -0.657011091709137,
      "logps/chosen": -11.849466323852539,
      "logps/rejected": -21.00613021850586,
      "loss": 0.8801897168159485,
      "memory(GiB)": 46.82,
      "nll_loss": 0.417907178401947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.217839315533638,
      "rewards/margins": 0.9292322397232056,
      "rewards/rejected": -0.711392879486084,
      "step": 65,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.17137293086660174,
      "grad_norm": 2.492602586746216,
      "learning_rate": 0.00019932752650698285,
      "logits/chosen": -0.6384371519088745,
      "logits/rejected": -0.639873743057251,
      "logps/chosen": -7.750038146972656,
      "logps/rejected": -19.651975631713867,
      "loss": 0.7662312984466553,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34055230021476746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.30775657296180725,
      "rewards/margins": 1.0670785903930664,
      "rewards/rejected": -0.7593219876289368,
      "step": 66,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.17396949042518664,
      "grad_norm": 3.4701662063598633,
      "learning_rate": 0.00019927685242947805,
      "logits/chosen": -0.6081066131591797,
      "logits/rejected": -0.60552579164505,
      "logps/chosen": -8.996465682983398,
      "logps/rejected": -16.082275390625,
      "loss": 0.8045691251754761,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3208611011505127,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.17242535948753357,
      "rewards/margins": 0.7575393915176392,
      "rewards/rejected": -0.5851140022277832,
      "step": 67,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.1765660499837715,
      "grad_norm": 2.7060115337371826,
      "learning_rate": 0.00019922434471936987,
      "logits/chosen": -0.6539565324783325,
      "logits/rejected": -0.6539203524589539,
      "logps/chosen": -11.010636329650879,
      "logps/rejected": -20.040176391601562,
      "loss": 0.8421358466148376,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4449455142021179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2390671670436859,
      "rewards/margins": 0.9861317873001099,
      "rewards/rejected": -0.7470647096633911,
      "step": 68,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.17916260954235638,
      "grad_norm": 3.1315884590148926,
      "learning_rate": 0.00019917000434647,
      "logits/chosen": -0.6536306738853455,
      "logits/rejected": -0.6518079042434692,
      "logps/chosen": -8.555349349975586,
      "logps/rejected": -16.54492950439453,
      "loss": 0.8154255151748657,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3061010241508484,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.10620275139808655,
      "rewards/margins": 0.7189066410064697,
      "rewards/rejected": -0.6127039194107056,
      "step": 69,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.18175916910094125,
      "grad_norm": 4.3252716064453125,
      "learning_rate": 0.0001991138323144392,
      "logits/chosen": -0.6485374569892883,
      "logits/rejected": -0.6516393423080444,
      "logps/chosen": -8.738260269165039,
      "logps/rejected": -25.283050537109375,
      "loss": 0.7622725963592529,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4182221591472626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32638847827911377,
      "rewards/margins": 1.3203245401382446,
      "rewards/rejected": -0.9939361810684204,
      "step": 70,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.18435572865952612,
      "grad_norm": 2.5611085891723633,
      "learning_rate": 0.0001990558296607687,
      "logits/chosen": -0.6623813509941101,
      "logits/rejected": -0.6669056415557861,
      "logps/chosen": -6.732373237609863,
      "logps/rejected": -28.474903106689453,
      "loss": 0.6175573468208313,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30857983231544495,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.274377703666687,
      "rewards/margins": 1.688110589981079,
      "rewards/rejected": -1.4137327671051025,
      "step": 71,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.186952288218111,
      "grad_norm": 4.000219821929932,
      "learning_rate": 0.00019899599745676122,
      "logits/chosen": -0.6445091366767883,
      "logits/rejected": -0.6437230706214905,
      "logps/chosen": -9.047367095947266,
      "logps/rejected": -24.770275115966797,
      "loss": 0.6452784538269043,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3213377594947815,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2230340838432312,
      "rewards/margins": 1.4539806842803955,
      "rewards/rejected": -1.230946660041809,
      "step": 72,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.18954884777669587,
      "grad_norm": 6.319863319396973,
      "learning_rate": 0.00019893433680751103,
      "logits/chosen": -0.6865643262863159,
      "logits/rejected": -0.6869932413101196,
      "logps/chosen": -7.858439922332764,
      "logps/rejected": -23.94890785217285,
      "loss": 0.686578631401062,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3744764029979706,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1327737271785736,
      "rewards/margins": 1.4861775636672974,
      "rewards/rejected": -1.3534038066864014,
      "step": 73,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.19214540733528077,
      "grad_norm": 6.422192573547363,
      "learning_rate": 0.00019887084885188353,
      "logits/chosen": -0.6767789125442505,
      "logits/rejected": -0.6818562746047974,
      "logps/chosen": -10.235507011413574,
      "logps/rejected": -27.55042266845703,
      "loss": 1.0105760097503662,
      "memory(GiB)": 46.82,
      "nll_loss": 0.45665743947029114,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17956775426864624,
      "rewards/margins": 1.3902134895324707,
      "rewards/rejected": -1.5697813034057617,
      "step": 74,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.19474196689386564,
      "grad_norm": 5.481616973876953,
      "learning_rate": 0.00019880553476249436,
      "logits/chosen": -0.7030400037765503,
      "logits/rejected": -0.7033373713493347,
      "logps/chosen": -10.782912254333496,
      "logps/rejected": -32.654239654541016,
      "loss": 0.9248250722885132,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4116487205028534,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2691394090652466,
      "rewards/margins": 1.6453478336334229,
      "rewards/rejected": -1.914487361907959,
      "step": 75,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 0.1973385264524505,
      "grad_norm": 4.9708380699157715,
      "learning_rate": 0.00019873839574568756,
      "logits/chosen": -0.7089657783508301,
      "logits/rejected": -0.7070764303207397,
      "logps/chosen": -15.426520347595215,
      "logps/rejected": -19.238183975219727,
      "loss": 0.9280877113342285,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4586380124092102,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.10893960297107697,
      "rewards/margins": 1.0797489881515503,
      "rewards/rejected": -0.9708094000816345,
      "step": 76,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 0.19993508601103538,
      "grad_norm": 6.939232349395752,
      "learning_rate": 0.00019866943304151343,
      "logits/chosen": -0.7235709428787231,
      "logits/rejected": -0.7286774516105652,
      "logps/chosen": -8.378010749816895,
      "logps/rejected": -22.562332153320312,
      "loss": 0.7349967956542969,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4009379744529724,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21920469403266907,
      "rewards/margins": 1.1362016201019287,
      "rewards/rejected": -0.916996955871582,
      "step": 77,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 0.20253164556962025,
      "grad_norm": 7.251801013946533,
      "learning_rate": 0.00019859864792370563,
      "logits/chosen": -0.7075158357620239,
      "logits/rejected": -0.7119045257568359,
      "logps/chosen": -6.7388834953308105,
      "logps/rejected": -26.275516510009766,
      "loss": 0.6517897844314575,
      "memory(GiB)": 46.82,
      "nll_loss": 0.36059334874153137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1867286115884781,
      "rewards/margins": 1.5948593616485596,
      "rewards/rejected": -1.4081308841705322,
      "step": 78,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 0.20512820512820512,
      "grad_norm": 2.740595579147339,
      "learning_rate": 0.00019852604169965748,
      "logits/chosen": -0.7263121604919434,
      "logits/rejected": -0.7323212027549744,
      "logps/chosen": -8.532756805419922,
      "logps/rejected": -26.737321853637695,
      "loss": 0.7062958478927612,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3304542601108551,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15770669281482697,
      "rewards/margins": 1.20403254032135,
      "rewards/rejected": -1.0463258028030396,
      "step": 79,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 0.20772476468679,
      "grad_norm": 8.650463104248047,
      "learning_rate": 0.00019845161571039802,
      "logits/chosen": -0.665086567401886,
      "logits/rejected": -0.6679493188858032,
      "logps/chosen": -6.538414001464844,
      "logps/rejected": -23.43639373779297,
      "loss": 0.6757080554962158,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3169613480567932,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1747138351202011,
      "rewards/margins": 1.2467294931411743,
      "rewards/rejected": -1.072015643119812,
      "step": 80,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.2103213242453749,
      "grad_norm": 2.5581393241882324,
      "learning_rate": 0.00019837537133056716,
      "logits/chosen": -0.7091171145439148,
      "logits/rejected": -0.7068539261817932,
      "logps/chosen": -7.670237064361572,
      "logps/rejected": -22.62818717956543,
      "loss": 0.6010414958000183,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2785060405731201,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23199363052845,
      "rewards/margins": 1.2859355211257935,
      "rewards/rejected": -1.0539417266845703,
      "step": 81,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 0.21291788380395976,
      "grad_norm": 5.669743537902832,
      "learning_rate": 0.0001982973099683902,
      "logits/chosen": -0.6808027029037476,
      "logits/rejected": -0.6830285787582397,
      "logps/chosen": -9.610679626464844,
      "logps/rejected": -27.96251678466797,
      "loss": 0.6995903849601746,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4465019702911377,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23076733946800232,
      "rewards/margins": 1.6701395511627197,
      "rewards/rejected": -1.4393723011016846,
      "step": 82,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.21551444336254463,
      "grad_norm": 3.1134045124053955,
      "learning_rate": 0.00019821743306565196,
      "logits/chosen": -0.6749523878097534,
      "logits/rejected": -0.6780697107315063,
      "logps/chosen": -7.1323065757751465,
      "logps/rejected": -20.9448299407959,
      "loss": 0.8713948726654053,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39334219694137573,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03965923935174942,
      "rewards/margins": 1.1248116493225098,
      "rewards/rejected": -1.0851523876190186,
      "step": 83,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.2181110029211295,
      "grad_norm": 4.337428569793701,
      "learning_rate": 0.00019813574209767012,
      "logits/chosen": -0.7203540205955505,
      "logits/rejected": -0.7236320972442627,
      "logps/chosen": -5.232553482055664,
      "logps/rejected": -20.862529754638672,
      "loss": 0.766181230545044,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35961851477622986,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18721464276313782,
      "rewards/margins": 1.240044116973877,
      "rewards/rejected": -1.052829384803772,
      "step": 84,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.22070756247971438,
      "grad_norm": 3.9627957344055176,
      "learning_rate": 0.00019805223857326793,
      "logits/chosen": -0.6975981593132019,
      "logits/rejected": -0.69466632604599,
      "logps/chosen": -10.297499656677246,
      "logps/rejected": -18.96347999572754,
      "loss": 0.791930615901947,
      "memory(GiB)": 46.82,
      "nll_loss": 0.43207138776779175,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.31037044525146484,
      "rewards/margins": 1.2147060632705688,
      "rewards/rejected": -0.904335618019104,
      "step": 85,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.22330412203829925,
      "grad_norm": 5.315914630889893,
      "learning_rate": 0.00019796692403474634,
      "logits/chosen": -0.7207286953926086,
      "logits/rejected": -0.7248555421829224,
      "logps/chosen": -7.064785480499268,
      "logps/rejected": -26.38395881652832,
      "loss": 0.6355230212211609,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31730470061302185,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.19611045718193054,
      "rewards/margins": 1.5834972858428955,
      "rewards/rejected": -1.3873869180679321,
      "step": 86,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.22590068159688412,
      "grad_norm": 3.8507533073425293,
      "learning_rate": 0.0001978798000578555,
      "logits/chosen": -0.6815958619117737,
      "logits/rejected": -0.6819433569908142,
      "logps/chosen": -10.517496109008789,
      "logps/rejected": -21.736141204833984,
      "loss": 0.7764264941215515,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3441038131713867,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.11584789305925369,
      "rewards/margins": 1.0969102382659912,
      "rewards/rejected": -0.9810622930526733,
      "step": 87,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.22849724115546902,
      "grad_norm": 2.613917350769043,
      "learning_rate": 0.0001977908682517658,
      "logits/chosen": -0.7139868140220642,
      "logits/rejected": -0.7127493023872375,
      "logps/chosen": -7.027731418609619,
      "logps/rejected": -21.87887954711914,
      "loss": 0.5793716907501221,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24400269985198975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.159662663936615,
      "rewards/margins": 1.3076614141464233,
      "rewards/rejected": -1.1479988098144531,
      "step": 88,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.2310938007140539,
      "grad_norm": 2.5475218296051025,
      "learning_rate": 0.00019770013025903794,
      "logits/chosen": -0.7572659254074097,
      "logits/rejected": -0.7608864307403564,
      "logps/chosen": -7.705483436584473,
      "logps/rejected": -23.772804260253906,
      "loss": 0.7914519906044006,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4494747221469879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.029678428545594215,
      "rewards/margins": 1.3612412214279175,
      "rewards/rejected": -1.3315627574920654,
      "step": 89,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.23369036027263876,
      "grad_norm": 2.815131902694702,
      "learning_rate": 0.00019760758775559274,
      "logits/chosen": -0.7308779954910278,
      "logits/rejected": -0.7346111536026001,
      "logps/chosen": -6.071584224700928,
      "logps/rejected": -31.994752883911133,
      "loss": 0.45671066641807556,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21047784388065338,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14397624135017395,
      "rewards/margins": 1.9211211204528809,
      "rewards/rejected": -1.7771449089050293,
      "step": 90,
      "train_speed(iter/s)": 0.005366
    },
    {
      "epoch": 0.23628691983122363,
      "grad_norm": 2.638777494430542,
      "learning_rate": 0.00019751324245068005,
      "logits/chosen": -0.7884888648986816,
      "logits/rejected": -0.7910454273223877,
      "logps/chosen": -7.571388244628906,
      "logps/rejected": -31.14786720275879,
      "loss": 0.47864896059036255,
      "memory(GiB)": 46.82,
      "nll_loss": 0.28661027550697327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23762081563472748,
      "rewards/margins": 2.1247057914733887,
      "rewards/rejected": -1.887085199356079,
      "step": 91,
      "train_speed(iter/s)": 0.005367
    },
    {
      "epoch": 0.2388834793898085,
      "grad_norm": 2.7935073375701904,
      "learning_rate": 0.0001974170960868474,
      "logits/chosen": -0.7978686094284058,
      "logits/rejected": -0.7982107996940613,
      "logps/chosen": -9.373262405395508,
      "logps/rejected": -31.14629364013672,
      "loss": 0.7341030240058899,
      "memory(GiB)": 46.82,
      "nll_loss": 0.45608797669410706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1958312690258026,
      "rewards/margins": 2.0244781970977783,
      "rewards/rejected": -1.8286471366882324,
      "step": 92,
      "train_speed(iter/s)": 0.005366
    },
    {
      "epoch": 0.24148003894839337,
      "grad_norm": 9.233238220214844,
      "learning_rate": 0.00019731915043990757,
      "logits/chosen": -0.8195321559906006,
      "logits/rejected": -0.8183485269546509,
      "logps/chosen": -9.839788436889648,
      "logps/rejected": -29.437850952148438,
      "loss": 0.6806643009185791,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4058592915534973,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22106225788593292,
      "rewards/margins": 1.8086858987808228,
      "rewards/rejected": -1.5876235961914062,
      "step": 93,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.24407659850697824,
      "grad_norm": 16.396970748901367,
      "learning_rate": 0.00019721940731890598,
      "logits/chosen": -0.8492038249969482,
      "logits/rejected": -0.854621946811676,
      "logps/chosen": -10.201713562011719,
      "logps/rejected": -26.62791633605957,
      "loss": 0.8960828185081482,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5508711338043213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06682707369327545,
      "rewards/margins": 1.7468764781951904,
      "rewards/rejected": -1.6800494194030762,
      "step": 94,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.24667315806556314,
      "grad_norm": 4.18104887008667,
      "learning_rate": 0.00019711786856608713,
      "logits/chosen": -0.8644828796386719,
      "logits/rejected": -0.8670830726623535,
      "logps/chosen": -8.474161148071289,
      "logps/rejected": -27.058048248291016,
      "loss": 0.9486290216445923,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6114800572395325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1142926812171936,
      "rewards/margins": 1.905853509902954,
      "rewards/rejected": -1.7915611267089844,
      "step": 95,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.249269717624148,
      "grad_norm": 4.6679511070251465,
      "learning_rate": 0.00019701453605686068,
      "logits/chosen": -0.807258129119873,
      "logits/rejected": -0.8097864985466003,
      "logps/chosen": -12.544672966003418,
      "logps/rejected": -33.301963806152344,
      "loss": 0.7105246186256409,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4519771933555603,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13032159209251404,
      "rewards/margins": 2.0599286556243896,
      "rewards/rejected": -1.9296070337295532,
      "step": 96,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.25186627718273286,
      "grad_norm": 9.922234535217285,
      "learning_rate": 0.0001969094116997668,
      "logits/chosen": -0.82038813829422,
      "logits/rejected": -0.8280550837516785,
      "logps/chosen": -6.2751688957214355,
      "logps/rejected": -35.381473541259766,
      "loss": 0.6427486538887024,
      "memory(GiB)": 46.82,
      "nll_loss": 0.40357428789138794,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10702433437108994,
      "rewards/margins": 2.1959128379821777,
      "rewards/rejected": -2.088888168334961,
      "step": 97,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.25446283674131775,
      "grad_norm": 2.88350248336792,
      "learning_rate": 0.00019680249743644082,
      "logits/chosen": -0.8155514001846313,
      "logits/rejected": -0.8185899257659912,
      "logps/chosen": -8.072955131530762,
      "logps/rejected": -31.97516632080078,
      "loss": 0.4997653067111969,
      "memory(GiB)": 46.82,
      "nll_loss": 0.29022216796875,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18802499771118164,
      "rewards/margins": 2.076292037963867,
      "rewards/rejected": -1.8882670402526855,
      "step": 98,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.25705939629990265,
      "grad_norm": 6.562138080596924,
      "learning_rate": 0.00019669379524157754,
      "logits/chosen": -0.7990139722824097,
      "logits/rejected": -0.7997216582298279,
      "logps/chosen": -10.97663688659668,
      "logps/rejected": -38.73711395263672,
      "loss": 1.0108985900878906,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6511343121528625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.15600137412548065,
      "rewards/margins": 2.344217538833618,
      "rewards/rejected": -2.5002188682556152,
      "step": 99,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.2596559558584875,
      "grad_norm": 3.38523530960083,
      "learning_rate": 0.00019658330712289454,
      "logits/chosen": -0.7976465225219727,
      "logits/rejected": -0.8036509156227112,
      "logps/chosen": -5.74189567565918,
      "logps/rejected": -38.44095993041992,
      "loss": 0.44277071952819824,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23434144258499146,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08411058783531189,
      "rewards/margins": 2.847978115081787,
      "rewards/rejected": -2.7638673782348633,
      "step": 100,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.2622525154170724,
      "grad_norm": 4.052488327026367,
      "learning_rate": 0.00019647103512109536,
      "logits/chosen": -0.7472208738327026,
      "logits/rejected": -0.7484402656555176,
      "logps/chosen": -8.85651683807373,
      "logps/rejected": -31.95272445678711,
      "loss": 0.7574372291564941,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4765327274799347,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06969474256038666,
      "rewards/margins": 2.256247043609619,
      "rewards/rejected": -2.1865522861480713,
      "step": 101,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.26484907497565724,
      "grad_norm": 9.65314769744873,
      "learning_rate": 0.0001963569813098315,
      "logits/chosen": -0.7638405561447144,
      "logits/rejected": -0.7673444747924805,
      "logps/chosen": -9.536478042602539,
      "logps/rejected": -24.767440795898438,
      "loss": 0.9202646017074585,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5910370945930481,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0962737649679184,
      "rewards/margins": 1.5450704097747803,
      "rewards/rejected": -1.4487966299057007,
      "step": 102,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.26744563453424214,
      "grad_norm": 9.249479293823242,
      "learning_rate": 0.00019624114779566448,
      "logits/chosen": -0.7637047171592712,
      "logits/rejected": -0.7682798504829407,
      "logps/chosen": -9.765649795532227,
      "logps/rejected": -26.720335006713867,
      "loss": 0.7715259790420532,
      "memory(GiB)": 46.82,
      "nll_loss": 0.40688371658325195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.025217050686478615,
      "rewards/margins": 1.340709924697876,
      "rewards/rejected": -1.36592698097229,
      "step": 103,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.270042194092827,
      "grad_norm": 18.381378173828125,
      "learning_rate": 0.00019612353671802656,
      "logits/chosen": -0.7441118955612183,
      "logits/rejected": -0.7465165853500366,
      "logps/chosen": -10.135462760925293,
      "logps/rejected": -24.317302703857422,
      "loss": 1.0861011743545532,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6500208377838135,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0288698673248291,
      "rewards/margins": 1.2047085762023926,
      "rewards/rejected": -1.2335785627365112,
      "step": 104,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.2726387536514119,
      "grad_norm": 3.6603286266326904,
      "learning_rate": 0.0001960041502491815,
      "logits/chosen": -0.7493617534637451,
      "logits/rejected": -0.7547396421432495,
      "logps/chosen": -9.255151748657227,
      "logps/rejected": -26.553424835205078,
      "loss": 0.7831539511680603,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4715706706047058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23407554626464844,
      "rewards/margins": 1.5187476873397827,
      "rewards/rejected": -1.2846721410751343,
      "step": 105,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.2752353132099968,
      "grad_norm": 3.8713457584381104,
      "learning_rate": 0.00019588299059418432,
      "logits/chosen": -0.7057076692581177,
      "logits/rejected": -0.7100006341934204,
      "logps/chosen": -7.245676517486572,
      "logps/rejected": -30.364728927612305,
      "loss": 0.47607186436653137,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2242860198020935,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16006791591644287,
      "rewards/margins": 1.8761022090911865,
      "rewards/rejected": -1.7160344123840332,
      "step": 106,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.2778318727685816,
      "grad_norm": 4.534320831298828,
      "learning_rate": 0.0001957600599908406,
      "logits/chosen": -0.7508159875869751,
      "logits/rejected": -0.7570351958274841,
      "logps/chosen": -7.806947708129883,
      "logps/rejected": -35.8264274597168,
      "loss": 0.7150964140892029,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3946303129196167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08480332791805267,
      "rewards/margins": 2.440999984741211,
      "rewards/rejected": -2.356196880340576,
      "step": 107,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.2804284323271665,
      "grad_norm": 4.433022975921631,
      "learning_rate": 0.00019563536070966512,
      "logits/chosen": -0.7469314932823181,
      "logits/rejected": -0.7510149478912354,
      "logps/chosen": -6.266570568084717,
      "logps/rejected": -27.143407821655273,
      "loss": 0.7763616442680359,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3898002505302429,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.2665005028247833,
      "rewards/margins": 1.8339991569519043,
      "rewards/rejected": -1.5674986839294434,
      "step": 108,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.28302499188575136,
      "grad_norm": 3.777085065841675,
      "learning_rate": 0.00019550889505383996,
      "logits/chosen": -0.7664754986763,
      "logits/rejected": -0.7695226669311523,
      "logps/chosen": -8.639087677001953,
      "logps/rejected": -32.59357833862305,
      "loss": 0.6963664889335632,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3630713224411011,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09401198476552963,
      "rewards/margins": 2.053591728210449,
      "rewards/rejected": -1.9595798254013062,
      "step": 109,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.28562155144433626,
      "grad_norm": 3.1223268508911133,
      "learning_rate": 0.00019538066535917196,
      "logits/chosen": -0.7881599068641663,
      "logits/rejected": -0.7903666496276855,
      "logps/chosen": -10.761030197143555,
      "logps/rejected": -24.686668395996094,
      "loss": 0.812722384929657,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4688267707824707,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07528804242610931,
      "rewards/margins": 1.5661133527755737,
      "rewards/rejected": -1.4908254146575928,
      "step": 110,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.2882181110029211,
      "grad_norm": 3.300981283187866,
      "learning_rate": 0.00019525067399404957,
      "logits/chosen": -0.7968149185180664,
      "logits/rejected": -0.7997515797615051,
      "logps/chosen": -6.535473346710205,
      "logps/rejected": -19.67113494873047,
      "loss": 0.577447772026062,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27950558066368103,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3308606743812561,
      "rewards/margins": 1.4230268001556396,
      "rewards/rejected": -1.0921661853790283,
      "step": 111,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.290814670561506,
      "grad_norm": 4.260409832000732,
      "learning_rate": 0.00019511892335939904,
      "logits/chosen": -0.7704994678497314,
      "logits/rejected": -0.773585855960846,
      "logps/chosen": -7.999111175537109,
      "logps/rejected": -23.012435913085938,
      "loss": 0.8953735828399658,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38217711448669434,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.008818171918392181,
      "rewards/margins": 1.0762327909469604,
      "rewards/rejected": -1.0850509405136108,
      "step": 112,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.2934112301200909,
      "grad_norm": 2.6586623191833496,
      "learning_rate": 0.0001949854158886402,
      "logits/chosen": -0.8259446024894714,
      "logits/rejected": -0.8321948647499084,
      "logps/chosen": -5.621776580810547,
      "logps/rejected": -21.676912307739258,
      "loss": 0.5901638269424438,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2607543170452118,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.231726735830307,
      "rewards/margins": 1.2758175134658813,
      "rewards/rejected": -1.0440906286239624,
      "step": 113,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.29600778967867575,
      "grad_norm": 2.47329044342041,
      "learning_rate": 0.00019485015404764142,
      "logits/chosen": -0.8323794007301331,
      "logits/rejected": -0.8363099098205566,
      "logps/chosen": -7.094597339630127,
      "logps/rejected": -23.47966766357422,
      "loss": 0.5581099987030029,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2674333155155182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24452531337738037,
      "rewards/margins": 1.409566879272461,
      "rewards/rejected": -1.1650415658950806,
      "step": 114,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.29860434923726065,
      "grad_norm": 2.6978213787078857,
      "learning_rate": 0.00019471314033467412,
      "logits/chosen": -0.8472287654876709,
      "logits/rejected": -0.8518344163894653,
      "logps/chosen": -6.279523849487305,
      "logps/rejected": -26.379831314086914,
      "loss": 0.6252153515815735,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3001650869846344,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2660066485404968,
      "rewards/margins": 1.4429866075515747,
      "rewards/rejected": -1.1769800186157227,
      "step": 115,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3012009087958455,
      "grad_norm": 3.7869536876678467,
      "learning_rate": 0.00019457437728036657,
      "logits/chosen": -0.9077713489532471,
      "logits/rejected": -0.9146382212638855,
      "logps/chosen": -7.037418842315674,
      "logps/rejected": -28.10509490966797,
      "loss": 0.6573376059532166,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39039283990859985,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2199217826128006,
      "rewards/margins": 1.892009973526001,
      "rewards/rejected": -1.6720881462097168,
      "step": 116,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.3037974683544304,
      "grad_norm": 7.053493022918701,
      "learning_rate": 0.00019443386744765723,
      "logits/chosen": -0.9531128406524658,
      "logits/rejected": -0.960386335849762,
      "logps/chosen": -6.833982467651367,
      "logps/rejected": -32.22799301147461,
      "loss": 0.5475011467933655,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30003488063812256,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2115585058927536,
      "rewards/margins": 1.97544527053833,
      "rewards/rejected": -1.7638866901397705,
      "step": 117,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.30639402791301523,
      "grad_norm": 10.012398719787598,
      "learning_rate": 0.00019429161343174732,
      "logits/chosen": -0.9923131465911865,
      "logits/rejected": -0.9990757703781128,
      "logps/chosen": -6.25333309173584,
      "logps/rejected": -37.52185821533203,
      "loss": 0.47200581431388855,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2796837091445923,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15778988599777222,
      "rewards/margins": 2.6613681316375732,
      "rewards/rejected": -2.5035781860351562,
      "step": 118,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.30899058747160013,
      "grad_norm": 6.096463203430176,
      "learning_rate": 0.00019414761786005293,
      "logits/chosen": -1.0189846754074097,
      "logits/rejected": -1.0254305601119995,
      "logps/chosen": -10.205361366271973,
      "logps/rejected": -36.81409454345703,
      "loss": 0.8465672731399536,
      "memory(GiB)": 46.82,
      "nll_loss": 0.41437357664108276,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.21620658040046692,
      "rewards/margins": 2.7219061851501465,
      "rewards/rejected": -2.505699634552002,
      "step": 119,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.31158714703018503,
      "grad_norm": 5.313393592834473,
      "learning_rate": 0.00019400188339215656,
      "logits/chosen": -1.046160340309143,
      "logits/rejected": -1.0573420524597168,
      "logps/chosen": -7.700190544128418,
      "logps/rejected": -47.04288101196289,
      "loss": 0.6054658889770508,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3044295608997345,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0029486743733286858,
      "rewards/margins": 3.3137736320495605,
      "rewards/rejected": -3.3108251094818115,
      "step": 120,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3141837065887699,
      "grad_norm": 4.291571140289307,
      "learning_rate": 0.00019385441271975787,
      "logits/chosen": -0.9954756498336792,
      "logits/rejected": -1.0022181272506714,
      "logps/chosen": -8.529633522033691,
      "logps/rejected": -45.85521697998047,
      "loss": 0.5717029571533203,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26478123664855957,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10705923289060593,
      "rewards/margins": 3.2083029747009277,
      "rewards/rejected": -3.101243734359741,
      "step": 121,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.31678026614735477,
      "grad_norm": 4.251352310180664,
      "learning_rate": 0.00019370520856662403,
      "logits/chosen": -0.9976125359535217,
      "logits/rejected": -1.0026432275772095,
      "logps/chosen": -8.120194435119629,
      "logps/rejected": -29.240516662597656,
      "loss": 0.5614541172981262,
      "memory(GiB)": 46.82,
      "nll_loss": 0.28819626569747925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25042724609375,
      "rewards/margins": 2.2021260261535645,
      "rewards/rejected": -1.9516987800598145,
      "step": 122,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3193768257059396,
      "grad_norm": 3.4627809524536133,
      "learning_rate": 0.00019355427368853944,
      "logits/chosen": -0.9717710614204407,
      "logits/rejected": -0.9764275550842285,
      "logps/chosen": -6.992214679718018,
      "logps/rejected": -39.05671691894531,
      "loss": 0.4260398745536804,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22311873733997345,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1755235493183136,
      "rewards/margins": 2.8104753494262695,
      "rewards/rejected": -2.6349518299102783,
      "step": 123,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.3219733852645245,
      "grad_norm": 5.5872344970703125,
      "learning_rate": 0.0001934016108732548,
      "logits/chosen": -0.9085801243782043,
      "logits/rejected": -0.9201411008834839,
      "logps/chosen": -6.697179794311523,
      "logps/rejected": -39.4910774230957,
      "loss": 0.5134581327438354,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22149759531021118,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12765586376190186,
      "rewards/margins": 2.5553700923919678,
      "rewards/rejected": -2.4277141094207764,
      "step": 124,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.32456994482310936,
      "grad_norm": 4.304178237915039,
      "learning_rate": 0.00019324722294043558,
      "logits/chosen": -0.9116522073745728,
      "logits/rejected": -0.9168075323104858,
      "logps/chosen": -11.646254539489746,
      "logps/rejected": -32.823944091796875,
      "loss": 0.9349632859230042,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6308795213699341,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.16005653142929077,
      "rewards/margins": 1.716849684715271,
      "rewards/rejected": -1.8769065141677856,
      "step": 125,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.32716650438169426,
      "grad_norm": 8.176533699035645,
      "learning_rate": 0.00019309111274161005,
      "logits/chosen": -0.9096921682357788,
      "logits/rejected": -0.9123918414115906,
      "logps/chosen": -7.077070236206055,
      "logps/rejected": -30.848291397094727,
      "loss": 0.7031348943710327,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4654442369937897,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14152386784553528,
      "rewards/margins": 1.975406527519226,
      "rewards/rejected": -1.8338826894760132,
      "step": 126,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.32976306394027916,
      "grad_norm": 2.428119659423828,
      "learning_rate": 0.00019293328316011643,
      "logits/chosen": -0.8453407287597656,
      "logits/rejected": -0.8446547985076904,
      "logps/chosen": -5.936784744262695,
      "logps/rejected": -24.515966415405273,
      "loss": 0.5038060545921326,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2542741894721985,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.29046115279197693,
      "rewards/margins": 1.757112741470337,
      "rewards/rejected": -1.4666515588760376,
      "step": 127,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.332359623498864,
      "grad_norm": 6.065800666809082,
      "learning_rate": 0.00019277373711104988,
      "logits/chosen": -0.79842609167099,
      "logits/rejected": -0.8017737865447998,
      "logps/chosen": -11.66524887084961,
      "logps/rejected": -29.264678955078125,
      "loss": 0.9522978067398071,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6585720777511597,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.18028652667999268,
      "rewards/margins": 1.8596545457839966,
      "rewards/rejected": -1.679368257522583,
      "step": 128,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.3349561830574489,
      "grad_norm": 1.9705984592437744,
      "learning_rate": 0.00019261247754120845,
      "logits/chosen": -0.8193144202232361,
      "logits/rejected": -0.8230156302452087,
      "logps/chosen": -6.537097454071045,
      "logps/rejected": -34.18002700805664,
      "loss": 0.45630478858947754,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26018983125686646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31401899456977844,
      "rewards/margins": 2.220799446105957,
      "rewards/rejected": -1.906780481338501,
      "step": 129,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.33755274261603374,
      "grad_norm": 8.319106101989746,
      "learning_rate": 0.00019244950742903878,
      "logits/chosen": -0.8408774137496948,
      "logits/rejected": -0.8418459892272949,
      "logps/chosen": -11.665637969970703,
      "logps/rejected": -31.675600051879883,
      "loss": 0.8428304195404053,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5734348297119141,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2836587727069855,
      "rewards/margins": 2.027439832687378,
      "rewards/rejected": -1.7437809705734253,
      "step": 130,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.34014930217461864,
      "grad_norm": 41.45463943481445,
      "learning_rate": 0.00019228482978458098,
      "logits/chosen": -0.838470995426178,
      "logits/rejected": -0.8445388078689575,
      "logps/chosen": -9.106077194213867,
      "logps/rejected": -35.46880340576172,
      "loss": 0.5834946036338806,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3602459132671356,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17075614631175995,
      "rewards/margins": 2.3818886280059814,
      "rewards/rejected": -2.211132526397705,
      "step": 131,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.3427458617332035,
      "grad_norm": 3.643427848815918,
      "learning_rate": 0.00019211844764941316,
      "logits/chosen": -0.8022149801254272,
      "logits/rejected": -0.8042948842048645,
      "logps/chosen": -6.459863662719727,
      "logps/rejected": -25.415029525756836,
      "loss": 0.6114150285720825,
      "memory(GiB)": 46.82,
      "nll_loss": 0.275367796421051,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15174095332622528,
      "rewards/margins": 1.8365050554275513,
      "rewards/rejected": -1.68476402759552,
      "step": 132,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.3453424212917884,
      "grad_norm": 5.966490268707275,
      "learning_rate": 0.0001919503640965951,
      "logits/chosen": -0.8290812969207764,
      "logits/rejected": -0.8256518840789795,
      "logps/chosen": -11.998124122619629,
      "logps/rejected": -33.305782318115234,
      "loss": 0.8110654950141907,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4670514762401581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07125922292470932,
      "rewards/margins": 2.1010446548461914,
      "rewards/rejected": -2.1723036766052246,
      "step": 133,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.3479389808503733,
      "grad_norm": 7.3433074951171875,
      "learning_rate": 0.00019178058223061172,
      "logits/chosen": -0.8187044262886047,
      "logits/rejected": -0.8188722133636475,
      "logps/chosen": -10.026674270629883,
      "logps/rejected": -34.666481018066406,
      "loss": 0.714328944683075,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3863736093044281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03853297978639603,
      "rewards/margins": 2.199920415878296,
      "rewards/rejected": -2.2384533882141113,
      "step": 134,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.3505355404089581,
      "grad_norm": 3.3124072551727295,
      "learning_rate": 0.0001916091051873154,
      "logits/chosen": -0.8322088122367859,
      "logits/rejected": -0.8396530151367188,
      "logps/chosen": -7.216854095458984,
      "logps/rejected": -37.36469650268555,
      "loss": 0.6344051361083984,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33177444338798523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11395619809627533,
      "rewards/margins": 2.6887590885162354,
      "rewards/rejected": -2.574802875518799,
      "step": 135,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.353132099967543,
      "grad_norm": 5.023634910583496,
      "learning_rate": 0.00019143593613386845,
      "logits/chosen": -0.8675904870033264,
      "logits/rejected": -0.8706792593002319,
      "logps/chosen": -11.020098686218262,
      "logps/rejected": -27.420391082763672,
      "loss": 0.9489123821258545,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5339450836181641,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.009774968028068542,
      "rewards/margins": 1.8276335000991821,
      "rewards/rejected": -1.837408423423767,
      "step": 136,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.35572865952612787,
      "grad_norm": 7.146136283874512,
      "learning_rate": 0.00019126107826868434,
      "logits/chosen": -0.8088135123252869,
      "logits/rejected": -0.8119392395019531,
      "logps/chosen": -6.920078277587891,
      "logps/rejected": -27.85674285888672,
      "loss": 0.5413922071456909,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3426899015903473,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3540826737880707,
      "rewards/margins": 2.053499698638916,
      "rewards/rejected": -1.6994171142578125,
      "step": 137,
      "train_speed(iter/s)": 0.005363
    },
    {
      "epoch": 0.35832521908471276,
      "grad_norm": 3.10225772857666,
      "learning_rate": 0.00019108453482136864,
      "logits/chosen": -0.797788679599762,
      "logits/rejected": -0.801692008972168,
      "logps/chosen": -7.574698448181152,
      "logps/rejected": -27.11581039428711,
      "loss": 0.5337320566177368,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2580120861530304,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1669113039970398,
      "rewards/margins": 1.648089051246643,
      "rewards/rejected": -1.4811776876449585,
      "step": 138,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.3609217786432976,
      "grad_norm": 4.059778213500977,
      "learning_rate": 0.00019090630905265962,
      "logits/chosen": -0.7970037460327148,
      "logits/rejected": -0.8018133640289307,
      "logps/chosen": -6.6308183670043945,
      "logps/rejected": -31.498334884643555,
      "loss": 0.526771068572998,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2799426317214966,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1864967793226242,
      "rewards/margins": 1.9347224235534668,
      "rewards/rejected": -1.7482255697250366,
      "step": 139,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.3635183382018825,
      "grad_norm": 3.430341958999634,
      "learning_rate": 0.00019072640425436764,
      "logits/chosen": -0.7270718216896057,
      "logits/rejected": -0.7312384843826294,
      "logps/chosen": -8.781835556030273,
      "logps/rejected": -27.820384979248047,
      "loss": 0.9166272878646851,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6030000448226929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1951862871646881,
      "rewards/margins": 1.9607356786727905,
      "rewards/rejected": -1.7655493021011353,
      "step": 140,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.3661148977604674,
      "grad_norm": 7.683961391448975,
      "learning_rate": 0.00019054482374931467,
      "logits/chosen": -0.7383142709732056,
      "logits/rejected": -0.7448272705078125,
      "logps/chosen": -8.916434288024902,
      "logps/rejected": -25.045127868652344,
      "loss": 0.7173936367034912,
      "memory(GiB)": 46.82,
      "nll_loss": 0.37474605441093445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10682700574398041,
      "rewards/margins": 1.3628969192504883,
      "rewards/rejected": -1.2560698986053467,
      "step": 141,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.36871145731905225,
      "grad_norm": 1.7709827423095703,
      "learning_rate": 0.0001903615708912728,
      "logits/chosen": -0.7578890919685364,
      "logits/rejected": -0.7647184729576111,
      "logps/chosen": -7.2641496658325195,
      "logps/rejected": -36.448509216308594,
      "loss": 0.4749196469783783,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3057454228401184,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32155346870422363,
      "rewards/margins": 2.4896819591522217,
      "rewards/rejected": -2.168128728866577,
      "step": 142,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.37130801687763715,
      "grad_norm": 3.9916441440582275,
      "learning_rate": 0.00019017664906490218,
      "logits/chosen": -0.725705087184906,
      "logits/rejected": -0.7305294275283813,
      "logps/chosen": -7.681970119476318,
      "logps/rejected": -25.85850715637207,
      "loss": 0.720486581325531,
      "memory(GiB)": 46.82,
      "nll_loss": 0.41218873858451843,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3683932423591614,
      "rewards/margins": 1.7790489196777344,
      "rewards/rejected": -1.4106559753417969,
      "step": 143,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.373904576436222,
      "grad_norm": 3.9804799556732178,
      "learning_rate": 0.00018999006168568883,
      "logits/chosen": -0.7427334785461426,
      "logits/rejected": -0.7495901584625244,
      "logps/chosen": -9.262022018432617,
      "logps/rejected": -25.315187454223633,
      "loss": 0.6805542707443237,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2929586172103882,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05294320732355118,
      "rewards/margins": 1.5572216510772705,
      "rewards/rejected": -1.5042784214019775,
      "step": 144,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3765011359948069,
      "grad_norm": 2.4131383895874023,
      "learning_rate": 0.00018980181219988116,
      "logits/chosen": -0.7519474029541016,
      "logits/rejected": -0.7524614930152893,
      "logps/chosen": -7.246754169464111,
      "logps/rejected": -24.415956497192383,
      "loss": 0.6144455075263977,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33591488003730774,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.36078375577926636,
      "rewards/margins": 1.827178716659546,
      "rewards/rejected": -1.4663947820663452,
      "step": 145,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.37909769555339173,
      "grad_norm": 3.9731338024139404,
      "learning_rate": 0.00018961190408442658,
      "logits/chosen": -0.7098850011825562,
      "logits/rejected": -0.7081133127212524,
      "logps/chosen": -11.87010669708252,
      "logps/rejected": -28.748165130615234,
      "loss": 0.7033959627151489,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38679245114326477,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.13101431727409363,
      "rewards/margins": 1.9088966846466064,
      "rewards/rejected": -1.7778825759887695,
      "step": 146,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.38169425511197663,
      "grad_norm": 5.180086135864258,
      "learning_rate": 0.00018942034084690724,
      "logits/chosen": -0.7729518413543701,
      "logits/rejected": -0.7725076675415039,
      "logps/chosen": -10.054743766784668,
      "logps/rejected": -27.725801467895508,
      "loss": 0.7672705054283142,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32955068349838257,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.007290400564670563,
      "rewards/margins": 1.9963154792785645,
      "rewards/rejected": -2.003605842590332,
      "step": 147,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.38429081467056153,
      "grad_norm": 3.313408374786377,
      "learning_rate": 0.00018922712602547517,
      "logits/chosen": -0.7976462244987488,
      "logits/rejected": -0.7998054623603821,
      "logps/chosen": -9.814254760742188,
      "logps/rejected": -36.34145736694336,
      "loss": 0.648523211479187,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38048407435417175,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21935109794139862,
      "rewards/margins": 2.4820148944854736,
      "rewards/rejected": -2.2626638412475586,
      "step": 148,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3868873742291464,
      "grad_norm": 4.517300128936768,
      "learning_rate": 0.00018903226318878698,
      "logits/chosen": -0.8082268238067627,
      "logits/rejected": -0.8085505962371826,
      "logps/chosen": -7.290218830108643,
      "logps/rejected": -28.533199310302734,
      "loss": 0.49404019117355347,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3020762801170349,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1652073860168457,
      "rewards/margins": 1.9900455474853516,
      "rewards/rejected": -1.8248380422592163,
      "step": 149,
      "train_speed(iter/s)": 0.005365
    },
    {
      "epoch": 0.3894839337877313,
      "grad_norm": 23.563270568847656,
      "learning_rate": 0.00018883575593593791,
      "logits/chosen": -0.7831783294677734,
      "logits/rejected": -0.7918115854263306,
      "logps/chosen": -5.868438243865967,
      "logps/rejected": -36.0763053894043,
      "loss": 0.833998441696167,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5523523688316345,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07082141935825348,
      "rewards/margins": 2.5503041744232178,
      "rewards/rejected": -2.479482889175415,
      "step": 150,
      "train_speed(iter/s)": 0.005364
    },
    {
      "epoch": 0.3920804933463161,
      "grad_norm": 2.2661221027374268,
      "learning_rate": 0.00018863760789639548,
      "logits/chosen": -0.8205257058143616,
      "logits/rejected": -0.8205671310424805,
      "logps/chosen": -8.58309268951416,
      "logps/rejected": -31.51813507080078,
      "loss": 0.5230035781860352,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34633222222328186,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2879600524902344,
      "rewards/margins": 2.3763887882232666,
      "rewards/rejected": -2.0884287357330322,
      "step": 151,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.394677052904901,
      "grad_norm": 3.6260762214660645,
      "learning_rate": 0.00018843782272993224,
      "logits/chosen": -0.7732735872268677,
      "logits/rejected": -0.7779878973960876,
      "logps/chosen": -7.900063514709473,
      "logps/rejected": -36.353912353515625,
      "loss": 1.1941097974777222,
      "memory(GiB)": 46.82,
      "nll_loss": 0.7860652208328247,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1569097638130188,
      "rewards/margins": 2.229072093963623,
      "rewards/rejected": -2.385981798171997,
      "step": 152,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.39727361246348586,
      "grad_norm": 3.8922035694122314,
      "learning_rate": 0.00018823640412655844,
      "logits/chosen": -0.7862505316734314,
      "logits/rejected": -0.7926680445671082,
      "logps/chosen": -7.828291893005371,
      "logps/rejected": -39.66058349609375,
      "loss": 0.6880181431770325,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5066484808921814,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14748194813728333,
      "rewards/margins": 3.0227272510528564,
      "rewards/rejected": -2.8752455711364746,
      "step": 153,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.39987017202207076,
      "grad_norm": 7.4526047706604,
      "learning_rate": 0.00018803335580645358,
      "logits/chosen": -0.7698212265968323,
      "logits/rejected": -0.7689813375473022,
      "logps/chosen": -13.24219036102295,
      "logps/rejected": -25.66214942932129,
      "loss": 0.9967556595802307,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6198260188102722,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07408954948186874,
      "rewards/margins": 1.6275382041931152,
      "rewards/rejected": -1.5534486770629883,
      "step": 154,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.40246673158065566,
      "grad_norm": 4.633146286010742,
      "learning_rate": 0.0001878286815198979,
      "logits/chosen": -0.7589452266693115,
      "logits/rejected": -0.7617721557617188,
      "logps/chosen": -7.7529802322387695,
      "logps/rejected": -27.287784576416016,
      "loss": 0.6172698140144348,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3214649558067322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12731438875198364,
      "rewards/margins": 1.5837674140930176,
      "rewards/rejected": -1.4564530849456787,
      "step": 155,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.4050632911392405,
      "grad_norm": 3.162893772125244,
      "learning_rate": 0.0001876223850472032,
      "logits/chosen": -0.7637428045272827,
      "logits/rejected": -0.7626054286956787,
      "logps/chosen": -8.649560928344727,
      "logps/rejected": -27.37163543701172,
      "loss": 0.616457462310791,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3673415780067444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23885931074619293,
      "rewards/margins": 1.9427287578582764,
      "rewards/rejected": -1.7038694620132446,
      "step": 156,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.4076598506978254,
      "grad_norm": 2.893603563308716,
      "learning_rate": 0.0001874144701986426,
      "logits/chosen": -0.7310610413551331,
      "logits/rejected": -0.7361200451850891,
      "logps/chosen": -6.76317024230957,
      "logps/rejected": -33.79893112182617,
      "loss": 0.4578598737716675,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24759513139724731,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.255088746547699,
      "rewards/margins": 2.5995802879333496,
      "rewards/rejected": -2.344491720199585,
      "step": 157,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.41025641025641024,
      "grad_norm": 2.6806418895721436,
      "learning_rate": 0.00018720494081438078,
      "logits/chosen": -0.7193287014961243,
      "logits/rejected": -0.7268579006195068,
      "logps/chosen": -6.555389404296875,
      "logps/rejected": -33.83311462402344,
      "loss": 0.5056754350662231,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2390821874141693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20247599482536316,
      "rewards/margins": 2.335620403289795,
      "rewards/rejected": -2.1331448554992676,
      "step": 158,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.41285296981499514,
      "grad_norm": 6.877208232879639,
      "learning_rate": 0.0001869938007644024,
      "logits/chosen": -0.774702787399292,
      "logits/rejected": -0.7733670473098755,
      "logps/chosen": -12.015567779541016,
      "logps/rejected": -39.611549377441406,
      "loss": 0.8152862191200256,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35584431886672974,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06409724056720734,
      "rewards/margins": 2.519212007522583,
      "rewards/rejected": -2.5833089351654053,
      "step": 159,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.41544952937358,
      "grad_norm": 4.4329938888549805,
      "learning_rate": 0.00018678105394844113,
      "logits/chosen": -0.705638587474823,
      "logits/rejected": -0.7075241208076477,
      "logps/chosen": -8.828238487243652,
      "logps/rejected": -34.07341766357422,
      "loss": 0.7131232023239136,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38522323966026306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08116397261619568,
      "rewards/margins": 2.2081894874572754,
      "rewards/rejected": -2.1270253658294678,
      "step": 160,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4180460889321649,
      "grad_norm": 3.8587117195129395,
      "learning_rate": 0.00018656670429590744,
      "logits/chosen": -0.7865703105926514,
      "logits/rejected": -0.7856872081756592,
      "logps/chosen": -7.486833572387695,
      "logps/rejected": -26.6343936920166,
      "loss": 0.6464986801147461,
      "memory(GiB)": 46.82,
      "nll_loss": 0.320234477519989,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1649102419614792,
      "rewards/margins": 1.9207243919372559,
      "rewards/rejected": -1.7558141946792603,
      "step": 161,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.4206426484907498,
      "grad_norm": 5.098015308380127,
      "learning_rate": 0.00018635075576581587,
      "logits/chosen": -0.815811276435852,
      "logits/rejected": -0.8179509043693542,
      "logps/chosen": -8.045232772827148,
      "logps/rejected": -26.606388092041016,
      "loss": 0.6883640289306641,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3271428346633911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21010762453079224,
      "rewards/margins": 1.64859139919281,
      "rewards/rejected": -1.438483715057373,
      "step": 162,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.4232392080493346,
      "grad_norm": 5.79577112197876,
      "learning_rate": 0.0001861332123467122,
      "logits/chosen": -0.7825253009796143,
      "logits/rejected": -0.7867128252983093,
      "logps/chosen": -7.512404441833496,
      "logps/rejected": -33.15559387207031,
      "loss": 0.4946397542953491,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26739412546157837,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2013475000858307,
      "rewards/margins": 2.0465731620788574,
      "rewards/rejected": -1.84522545337677,
      "step": 163,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.4258357676079195,
      "grad_norm": 3.521145820617676,
      "learning_rate": 0.0001859140780565996,
      "logits/chosen": -0.7922295331954956,
      "logits/rejected": -0.7915840148925781,
      "logps/chosen": -8.717174530029297,
      "logps/rejected": -24.929059982299805,
      "loss": 0.6819888949394226,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4021107256412506,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1168108582496643,
      "rewards/margins": 1.8219399452209473,
      "rewards/rejected": -1.7051292657852173,
      "step": 164,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.42843232716650437,
      "grad_norm": 17.069190979003906,
      "learning_rate": 0.0001856933569428644,
      "logits/chosen": -0.7880842089653015,
      "logits/rejected": -0.7944654822349548,
      "logps/chosen": -10.384450912475586,
      "logps/rejected": -25.57999610900879,
      "loss": 1.0119094848632812,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6152634024620056,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.14946657419204712,
      "rewards/margins": 1.2663317918777466,
      "rewards/rejected": -1.1168652772903442,
      "step": 165,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.43102888672508927,
      "grad_norm": 3.2614221572875977,
      "learning_rate": 0.0001854710530822014,
      "logits/chosen": -0.7307618856430054,
      "logits/rejected": -0.7359931468963623,
      "logps/chosen": -6.218672752380371,
      "logps/rejected": -25.049577713012695,
      "loss": 0.6519892811775208,
      "memory(GiB)": 46.82,
      "nll_loss": 0.363025426864624,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21699494123458862,
      "rewards/margins": 1.6366937160491943,
      "rewards/rejected": -1.4196988344192505,
      "step": 166,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.4336254462836741,
      "grad_norm": 3.8368124961853027,
      "learning_rate": 0.00018524717058053866,
      "logits/chosen": -0.741214394569397,
      "logits/rejected": -0.7512940764427185,
      "logps/chosen": -6.5773725509643555,
      "logps/rejected": -29.348596572875977,
      "loss": 0.8446623682975769,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5003358721733093,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.029447460547089577,
      "rewards/margins": 1.5836502313613892,
      "rewards/rejected": -1.554202914237976,
      "step": 167,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.436222005842259,
      "grad_norm": 1.9537255764007568,
      "learning_rate": 0.00018502171357296144,
      "logits/chosen": -0.7404088377952576,
      "logits/rejected": -0.7386180758476257,
      "logps/chosen": -8.013400077819824,
      "logps/rejected": -28.649869918823242,
      "loss": 0.5149758458137512,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32292458415031433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2221800982952118,
      "rewards/margins": 1.952845811843872,
      "rewards/rejected": -1.730665683746338,
      "step": 168,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4388185654008439,
      "grad_norm": 3.6709702014923096,
      "learning_rate": 0.000184794686223636,
      "logits/chosen": -0.7925949096679688,
      "logits/rejected": -0.7922758460044861,
      "logps/chosen": -8.695777893066406,
      "logps/rejected": -24.725303649902344,
      "loss": 0.5783385634422302,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30575335025787354,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.208786278963089,
      "rewards/margins": 1.6382977962493896,
      "rewards/rejected": -1.429511308670044,
      "step": 169,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.44141512495942875,
      "grad_norm": 6.263500690460205,
      "learning_rate": 0.00018456609272573266,
      "logits/chosen": -0.7436801791191101,
      "logits/rejected": -0.7518426179885864,
      "logps/chosen": -5.066699981689453,
      "logps/rejected": -31.86433982849121,
      "loss": 0.5026156306266785,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3272435665130615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1566862165927887,
      "rewards/margins": 2.2096331119537354,
      "rewards/rejected": -2.0529470443725586,
      "step": 170,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.44401168451801365,
      "grad_norm": 2.6846933364868164,
      "learning_rate": 0.00018433593730134831,
      "logits/chosen": -0.8177323937416077,
      "logits/rejected": -0.8159899711608887,
      "logps/chosen": -10.043192863464355,
      "logps/rejected": -36.874168395996094,
      "loss": 0.5169069766998291,
      "memory(GiB)": 46.82,
      "nll_loss": 0.280762255191803,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15715357661247253,
      "rewards/margins": 2.441157341003418,
      "rewards/rejected": -2.284003973007202,
      "step": 171,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4466082440765985,
      "grad_norm": 3.354492664337158,
      "learning_rate": 0.0001841042242014285,
      "logits/chosen": -0.7943728566169739,
      "logits/rejected": -0.8022831082344055,
      "logps/chosen": -4.931389808654785,
      "logps/rejected": -46.30964279174805,
      "loss": 0.3581126630306244,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17072941362857819,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2874028980731964,
      "rewards/margins": 3.4458858966827393,
      "rewards/rejected": -3.1584832668304443,
      "step": 172,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4492048036351834,
      "grad_norm": 2.2436320781707764,
      "learning_rate": 0.00018387095770568877,
      "logits/chosen": -0.8137360215187073,
      "logits/rejected": -0.8178499937057495,
      "logps/chosen": -8.257625579833984,
      "logps/rejected": -41.017391204833984,
      "loss": 0.5087756514549255,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34518077969551086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.40273773670196533,
      "rewards/margins": 3.349743127822876,
      "rewards/rejected": -2.9470055103302,
      "step": 173,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.45180136319376824,
      "grad_norm": 3.1421475410461426,
      "learning_rate": 0.00018363614212253584,
      "logits/chosen": -0.8107144236564636,
      "logits/rejected": -0.8169907331466675,
      "logps/chosen": -6.6368021965026855,
      "logps/rejected": -47.06969451904297,
      "loss": 0.4038211703300476,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22018837928771973,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21348311007022858,
      "rewards/margins": 3.67350697517395,
      "rewards/rejected": -3.4600236415863037,
      "step": 174,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.45439792275235313,
      "grad_norm": 4.707547664642334,
      "learning_rate": 0.0001833997817889878,
      "logits/chosen": -0.7821629047393799,
      "logits/rejected": -0.7885245084762573,
      "logps/chosen": -6.257838249206543,
      "logps/rejected": -47.79384231567383,
      "loss": 0.43321770429611206,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3060261607170105,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2357151210308075,
      "rewards/margins": 3.7243151664733887,
      "rewards/rejected": -3.4885997772216797,
      "step": 175,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.45699448231093803,
      "grad_norm": 6.834311008453369,
      "learning_rate": 0.00018316188107059419,
      "logits/chosen": -0.8164551854133606,
      "logits/rejected": -0.8227632641792297,
      "logps/chosen": -8.357691764831543,
      "logps/rejected": -53.20665740966797,
      "loss": 0.5088233351707458,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34228751063346863,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1479293704032898,
      "rewards/margins": 4.200577735900879,
      "rewards/rejected": -4.052648544311523,
      "step": 176,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4595910418695229,
      "grad_norm": 1.9933769702911377,
      "learning_rate": 0.00018292244436135516,
      "logits/chosen": -0.8226829767227173,
      "logits/rejected": -0.8278478384017944,
      "logps/chosen": -8.886636734008789,
      "logps/rejected": -43.808494567871094,
      "loss": 0.4773145914077759,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2843259274959564,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21271556615829468,
      "rewards/margins": 3.3535256385803223,
      "rewards/rejected": -3.140810489654541,
      "step": 177,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4621876014281078,
      "grad_norm": 2.1277883052825928,
      "learning_rate": 0.00018268147608364066,
      "logits/chosen": -0.8168373107910156,
      "logits/rejected": -0.8192884922027588,
      "logps/chosen": -10.113336563110352,
      "logps/rejected": -44.43722152709961,
      "loss": 0.47269877791404724,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33138713240623474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20839698612689972,
      "rewards/margins": 3.481367349624634,
      "rewards/rejected": -3.272970199584961,
      "step": 178,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.4647841609866926,
      "grad_norm": 3.652038335800171,
      "learning_rate": 0.00018243898068810835,
      "logits/chosen": -0.8304356932640076,
      "logits/rejected": -0.8354857563972473,
      "logps/chosen": -7.381147384643555,
      "logps/rejected": -37.62067794799805,
      "loss": 0.598408579826355,
      "memory(GiB)": 46.82,
      "nll_loss": 0.41189002990722656,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20993797481060028,
      "rewards/margins": 2.9361226558685303,
      "rewards/rejected": -2.726184844970703,
      "step": 179,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4673807205452775,
      "grad_norm": 5.588886737823486,
      "learning_rate": 0.00018219496265362164,
      "logits/chosen": -0.7966144680976868,
      "logits/rejected": -0.7976107597351074,
      "logps/chosen": -5.64756441116333,
      "logps/rejected": -41.18315124511719,
      "loss": 0.4210509657859802,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3009505271911621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2384023666381836,
      "rewards/margins": 3.277693033218384,
      "rewards/rejected": -3.0392909049987793,
      "step": 180,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.46997728010386236,
      "grad_norm": 3.0966458320617676,
      "learning_rate": 0.00018194942648716697,
      "logits/chosen": -0.8336725234985352,
      "logits/rejected": -0.8417295813560486,
      "logps/chosen": -6.757675647735596,
      "logps/rejected": -42.556461334228516,
      "loss": 0.46501806378364563,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27556124329566956,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1294286698102951,
      "rewards/margins": 3.264049530029297,
      "rewards/rejected": -3.1346213817596436,
      "step": 181,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.47257383966244726,
      "grad_norm": 1.5267800092697144,
      "learning_rate": 0.00018170237672377043,
      "logits/chosen": -0.8151792883872986,
      "logits/rejected": -0.8204985857009888,
      "logps/chosen": -6.037264347076416,
      "logps/rejected": -40.91991424560547,
      "loss": 0.30481547117233276,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22177544236183167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28671082854270935,
      "rewards/margins": 3.2523105144500732,
      "rewards/rejected": -2.965599775314331,
      "step": 182,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.47517039922103216,
      "grad_norm": 5.700167655944824,
      "learning_rate": 0.0001814538179264142,
      "logits/chosen": -0.8828625082969666,
      "logits/rejected": -0.8847114443778992,
      "logps/chosen": -11.471461296081543,
      "logps/rejected": -36.56057357788086,
      "loss": 0.9585452675819397,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6207931041717529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10233151167631149,
      "rewards/margins": 2.520526885986328,
      "rewards/rejected": -2.4181957244873047,
      "step": 183,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.477766958779617,
      "grad_norm": 4.820940971374512,
      "learning_rate": 0.00018120375468595198,
      "logits/chosen": -0.8398681282997131,
      "logits/rejected": -0.843747615814209,
      "logps/chosen": -9.928145408630371,
      "logps/rejected": -35.531009674072266,
      "loss": 0.7689952254295349,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5200174450874329,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15448038280010223,
      "rewards/margins": 2.677025318145752,
      "rewards/rejected": -2.522545099258423,
      "step": 184,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.4803635183382019,
      "grad_norm": 2.308289051055908,
      "learning_rate": 0.00018095219162102453,
      "logits/chosen": -0.8587791919708252,
      "logits/rejected": -0.8568933606147766,
      "logps/chosen": -6.138177394866943,
      "logps/rejected": -36.87736129760742,
      "loss": 0.5506963133811951,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35889768600463867,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09419973194599152,
      "rewards/margins": 2.7206168174743652,
      "rewards/rejected": -2.6264166831970215,
      "step": 185,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.48296007789678674,
      "grad_norm": 3.0462734699249268,
      "learning_rate": 0.00018069913337797412,
      "logits/chosen": -0.82170170545578,
      "logits/rejected": -0.8261471390724182,
      "logps/chosen": -9.066842079162598,
      "logps/rejected": -47.4820671081543,
      "loss": 0.6803156137466431,
      "memory(GiB)": 46.82,
      "nll_loss": 0.46885693073272705,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17911966145038605,
      "rewards/margins": 3.736632823944092,
      "rewards/rejected": -3.5575132369995117,
      "step": 186,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.48555663745537164,
      "grad_norm": 12.903093338012695,
      "learning_rate": 0.0001804445846307588,
      "logits/chosen": -0.8383846879005432,
      "logits/rejected": -0.8428988456726074,
      "logps/chosen": -13.801538467407227,
      "logps/rejected": -41.331783294677734,
      "loss": 1.4305500984191895,
      "memory(GiB)": 46.82,
      "nll_loss": 1.0338698625564575,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.2808900773525238,
      "rewards/margins": 2.8232994079589844,
      "rewards/rejected": -3.104188919067383,
      "step": 187,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4881531970139565,
      "grad_norm": 1.0728437900543213,
      "learning_rate": 0.0001801885500808661,
      "logits/chosen": -0.7999005317687988,
      "logits/rejected": -0.8075475096702576,
      "logps/chosen": -6.161966323852539,
      "logps/rejected": -52.183937072753906,
      "loss": 0.32457199692726135,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26306816935539246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30446696281433105,
      "rewards/margins": 4.325974464416504,
      "rewards/rejected": -4.0215067863464355,
      "step": 188,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4907497565725414,
      "grad_norm": 2.3573195934295654,
      "learning_rate": 0.00017993103445722614,
      "logits/chosen": -0.8147333860397339,
      "logits/rejected": -0.8255075216293335,
      "logps/chosen": -5.808372497558594,
      "logps/rejected": -48.91783142089844,
      "loss": 0.39948803186416626,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22272005677223206,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18873995542526245,
      "rewards/margins": 3.7226080894470215,
      "rewards/rejected": -3.533867835998535,
      "step": 189,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4933463161311263,
      "grad_norm": 2.2260046005249023,
      "learning_rate": 0.00017967204251612433,
      "logits/chosen": -0.7621344923973083,
      "logits/rejected": -0.7712112069129944,
      "logps/chosen": -5.210122108459473,
      "logps/rejected": -57.8108024597168,
      "loss": 0.4442843496799469,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2754063904285431,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19802872836589813,
      "rewards/margins": 4.561690807342529,
      "rewards/rejected": -4.363662242889404,
      "step": 190,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.4959428756897111,
      "grad_norm": 5.498357772827148,
      "learning_rate": 0.00017941157904111346,
      "logits/chosen": -0.7840267419815063,
      "logits/rejected": -0.7963895797729492,
      "logps/chosen": -6.320296764373779,
      "logps/rejected": -63.66170120239258,
      "loss": 0.4151027798652649,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2456730306148529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1802489459514618,
      "rewards/margins": 5.30258846282959,
      "rewards/rejected": -5.122340202331543,
      "step": 191,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.498539435248296,
      "grad_norm": 2.877337694168091,
      "learning_rate": 0.00017914964884292544,
      "logits/chosen": -0.8350490927696228,
      "logits/rejected": -0.8606675863265991,
      "logps/chosen": -8.253477096557617,
      "logps/rejected": -83.04051971435547,
      "loss": 0.4113784730434418,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2746655344963074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2550317049026489,
      "rewards/margins": 6.853097438812256,
      "rewards/rejected": -6.598065376281738,
      "step": 192,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.5011359948068809,
      "grad_norm": 9.82312297821045,
      "learning_rate": 0.00017888625675938235,
      "logits/chosen": -0.8031945824623108,
      "logits/rejected": -0.814129650592804,
      "logps/chosen": -7.723809719085693,
      "logps/rejected": -71.692626953125,
      "loss": 0.5333746671676636,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2982180714607239,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03098137304186821,
      "rewards/margins": 5.821050643920898,
      "rewards/rejected": -5.85203218460083,
      "step": 193,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.5037325543654657,
      "grad_norm": 1.5864787101745605,
      "learning_rate": 0.00017862140765530717,
      "logits/chosen": -0.7884975671768188,
      "logits/rejected": -0.8070309162139893,
      "logps/chosen": -6.657387733459473,
      "logps/rejected": -88.2061538696289,
      "loss": 0.3408563733100891,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23068124055862427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25283610820770264,
      "rewards/margins": 7.557528972625732,
      "rewards/rejected": -7.304693222045898,
      "step": 194,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 2.8981215953826904,
      "learning_rate": 0.0001783551064224339,
      "logits/chosen": -0.8138140439987183,
      "logits/rejected": -0.8286132216453552,
      "logps/chosen": -5.021820545196533,
      "logps/rejected": -76.1704330444336,
      "loss": 0.43742382526397705,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2779857814311981,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15874776244163513,
      "rewards/margins": 6.478954792022705,
      "rewards/rejected": -6.320207118988037,
      "step": 195,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.5089256734826355,
      "grad_norm": 3.9564876556396484,
      "learning_rate": 0.00017808735797931715,
      "logits/chosen": -0.8671743869781494,
      "logits/rejected": -0.8750807046890259,
      "logps/chosen": -7.3802947998046875,
      "logps/rejected": -63.22589874267578,
      "loss": 0.5510232448577881,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3880046606063843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3043110966682434,
      "rewards/margins": 5.6318769454956055,
      "rewards/rejected": -5.327565670013428,
      "step": 196,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.5115222330412204,
      "grad_norm": 7.476530075073242,
      "learning_rate": 0.0001778181672712414,
      "logits/chosen": -0.8401522636413574,
      "logits/rejected": -0.8446535468101501,
      "logps/chosen": -10.472354888916016,
      "logps/rejected": -55.56855010986328,
      "loss": 0.5609198212623596,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4021284878253937,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.33400610089302063,
      "rewards/margins": 4.6901750564575195,
      "rewards/rejected": -4.356168746948242,
      "step": 197,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.5141187925998053,
      "grad_norm": 1.5591124296188354,
      "learning_rate": 0.00017754753927012954,
      "logits/chosen": -0.881521463394165,
      "logits/rejected": -0.8908553123474121,
      "logps/chosen": -7.769586086273193,
      "logps/rejected": -70.43656158447266,
      "loss": 0.40073180198669434,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2971632182598114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23546172678470612,
      "rewards/margins": 6.18820858001709,
      "rewards/rejected": -5.952746868133545,
      "step": 198,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.5167153521583902,
      "grad_norm": 3.914903163909912,
      "learning_rate": 0.00017727547897445118,
      "logits/chosen": -0.8886235952377319,
      "logits/rejected": -0.8989691138267517,
      "logps/chosen": -4.683451175689697,
      "logps/rejected": -60.072357177734375,
      "loss": 0.3925155997276306,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2971319854259491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33483725786209106,
      "rewards/margins": 5.336654186248779,
      "rewards/rejected": -5.001816749572754,
      "step": 199,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.519311911716975,
      "grad_norm": 10.00102424621582,
      "learning_rate": 0.00017700199140913017,
      "logits/chosen": -0.9024062752723694,
      "logits/rejected": -0.8995949029922485,
      "logps/chosen": -13.132458686828613,
      "logps/rejected": -58.003902435302734,
      "loss": 0.9102693200111389,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6192734241485596,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10886114090681076,
      "rewards/margins": 4.763448238372803,
      "rewards/rejected": -4.872309684753418,
      "step": 200,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.5219084712755598,
      "grad_norm": 12.833686828613281,
      "learning_rate": 0.000176727081625452,
      "logits/chosen": -0.8882104158401489,
      "logits/rejected": -0.9030141830444336,
      "logps/chosen": -7.329769134521484,
      "logps/rejected": -84.83427429199219,
      "loss": 0.6302378177642822,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4732781648635864,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08130178600549698,
      "rewards/margins": 7.068781852722168,
      "rewards/rejected": -6.987480640411377,
      "step": 201,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5245050308341448,
      "grad_norm": 27.52665138244629,
      "learning_rate": 0.00017645075470097024,
      "logits/chosen": -0.9375100135803223,
      "logits/rejected": -0.9529730677604675,
      "logps/chosen": -6.13963508605957,
      "logps/rejected": -66.99122619628906,
      "loss": 0.5372448563575745,
      "memory(GiB)": 46.82,
      "nll_loss": 0.37428009510040283,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11455152183771133,
      "rewards/margins": 5.315877914428711,
      "rewards/rejected": -5.2013258934021,
      "step": 202,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5271015903927296,
      "grad_norm": 2.7620508670806885,
      "learning_rate": 0.00017617301573941296,
      "logits/chosen": -0.8983043432235718,
      "logits/rejected": -0.8981289863586426,
      "logps/chosen": -11.069366455078125,
      "logps/rejected": -39.12937927246094,
      "loss": 0.6441234350204468,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4368978440761566,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21257682144641876,
      "rewards/margins": 3.110691785812378,
      "rewards/rejected": -2.8981149196624756,
      "step": 203,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5296981499513145,
      "grad_norm": 12.48902702331543,
      "learning_rate": 0.0001758938698705884,
      "logits/chosen": -0.8725972771644592,
      "logits/rejected": -0.8844687342643738,
      "logps/chosen": -8.939924240112305,
      "logps/rejected": -76.01535034179688,
      "loss": 0.7113180756568909,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39448532462120056,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13535423576831818,
      "rewards/margins": 6.58955192565918,
      "rewards/rejected": -6.454197883605957,
      "step": 204,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5322947095098994,
      "grad_norm": 5.789051055908203,
      "learning_rate": 0.0001756133222502902,
      "logits/chosen": -0.835827648639679,
      "logits/rejected": -0.848646342754364,
      "logps/chosen": -10.195419311523438,
      "logps/rejected": -72.34866333007812,
      "loss": 0.46852654218673706,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31231871247291565,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03889274597167969,
      "rewards/margins": 5.886906147003174,
      "rewards/rejected": -5.925798416137695,
      "step": 205,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5348912690684843,
      "grad_norm": 11.01954174041748,
      "learning_rate": 0.00017533137806020226,
      "logits/chosen": -0.8194984197616577,
      "logits/rejected": -0.8232303261756897,
      "logps/chosen": -8.20671272277832,
      "logps/rejected": -57.72869873046875,
      "loss": 0.5476050972938538,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3084986209869385,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09075944125652313,
      "rewards/margins": 4.59151554107666,
      "rewards/rejected": -4.50075626373291,
      "step": 206,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5374878286270691,
      "grad_norm": 4.141697406768799,
      "learning_rate": 0.0001750480425078029,
      "logits/chosen": -0.7913045287132263,
      "logits/rejected": -0.7959045171737671,
      "logps/chosen": -8.45845890045166,
      "logps/rejected": -47.05601501464844,
      "loss": 0.44736987352371216,
      "memory(GiB)": 46.82,
      "nll_loss": 0.28352388739585876,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23959283530712128,
      "rewards/margins": 3.8303611278533936,
      "rewards/rejected": -3.5907678604125977,
      "step": 207,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.540084388185654,
      "grad_norm": 3.359447717666626,
      "learning_rate": 0.00017476332082626877,
      "logits/chosen": -0.7912741899490356,
      "logits/rejected": -0.796455979347229,
      "logps/chosen": -5.394259452819824,
      "logps/rejected": -45.25196075439453,
      "loss": 0.36434927582740784,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24980391561985016,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2328459620475769,
      "rewards/margins": 3.577873945236206,
      "rewards/rejected": -3.3450279235839844,
      "step": 208,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5426809477442389,
      "grad_norm": 3.0909605026245117,
      "learning_rate": 0.0001744772182743782,
      "logits/chosen": -0.7767283916473389,
      "logits/rejected": -0.7861429452896118,
      "logps/chosen": -6.036921501159668,
      "logps/rejected": -43.4775390625,
      "loss": 0.42594030499458313,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2773384153842926,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3111240863800049,
      "rewards/margins": 3.5763254165649414,
      "rewards/rejected": -3.2652013301849365,
      "step": 209,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5452775073028238,
      "grad_norm": 2.5949363708496094,
      "learning_rate": 0.000174189740136414,
      "logits/chosen": -0.7519343495368958,
      "logits/rejected": -0.758810818195343,
      "logps/chosen": -6.644616603851318,
      "logps/rejected": -38.87832260131836,
      "loss": 0.4237949848175049,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26098260283470154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27363157272338867,
      "rewards/margins": 3.0662317276000977,
      "rewards/rejected": -2.792600393295288,
      "step": 210,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5478740668614086,
      "grad_norm": 1.992567777633667,
      "learning_rate": 0.00017390089172206592,
      "logits/chosen": -0.7961949706077576,
      "logits/rejected": -0.7975110411643982,
      "logps/chosen": -5.077378749847412,
      "logps/rejected": -37.03203201293945,
      "loss": 0.37955719232559204,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23032808303833008,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16159504652023315,
      "rewards/margins": 2.815537929534912,
      "rewards/rejected": -2.653942584991455,
      "step": 211,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5504706264199936,
      "grad_norm": 1.784830093383789,
      "learning_rate": 0.0001736106783663326,
      "logits/chosen": -0.7778050899505615,
      "logits/rejected": -0.7829293012619019,
      "logps/chosen": -6.269719123840332,
      "logps/rejected": -39.09969711303711,
      "loss": 0.6080423593521118,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3602517247200012,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16860854625701904,
      "rewards/margins": 2.596688747406006,
      "rewards/rejected": -2.4280805587768555,
      "step": 212,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5530671859785784,
      "grad_norm": 3.7514870166778564,
      "learning_rate": 0.000173319105429423,
      "logits/chosen": -0.7780061364173889,
      "logits/rejected": -0.7814825773239136,
      "logps/chosen": -9.701554298400879,
      "logps/rejected": -42.14776611328125,
      "loss": 0.5809894800186157,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4458245038986206,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22117674350738525,
      "rewards/margins": 3.311382532119751,
      "rewards/rejected": -3.090205669403076,
      "step": 213,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.5556637455371632,
      "grad_norm": 1.574320912361145,
      "learning_rate": 0.00017302617829665723,
      "logits/chosen": -0.7605854272842407,
      "logits/rejected": -0.7681394219398499,
      "logps/chosen": -7.890401840209961,
      "logps/rejected": -43.07416534423828,
      "loss": 0.46951717138290405,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3557964861392975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2927654981613159,
      "rewards/margins": 3.377017021179199,
      "rewards/rejected": -3.084251880645752,
      "step": 214,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5582603050957481,
      "grad_norm": 3.259098529815674,
      "learning_rate": 0.00017273190237836756,
      "logits/chosen": -0.8040423393249512,
      "logits/rejected": -0.812609076499939,
      "logps/chosen": -8.544995307922363,
      "logps/rejected": -43.30592346191406,
      "loss": 0.39441177248954773,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3016667366027832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24430298805236816,
      "rewards/margins": 3.4823904037475586,
      "rewards/rejected": -3.2380871772766113,
      "step": 215,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 0.560856864654333,
      "grad_norm": 10.052566528320312,
      "learning_rate": 0.0001724362831097979,
      "logits/chosen": -0.8765899538993835,
      "logits/rejected": -0.8773387670516968,
      "logps/chosen": -9.490972518920898,
      "logps/rejected": -41.05526351928711,
      "loss": 0.8065330982208252,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5094266533851624,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13303275406360626,
      "rewards/margins": 3.132627010345459,
      "rewards/rejected": -2.999594211578369,
      "step": 216,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5634534242129179,
      "grad_norm": 2.0193867683410645,
      "learning_rate": 0.00017213932595100384,
      "logits/chosen": -0.839668869972229,
      "logits/rejected": -0.8433064222335815,
      "logps/chosen": -8.96350383758545,
      "logps/rejected": -43.15342330932617,
      "loss": 0.5949581861495972,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4349959194660187,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.156009703874588,
      "rewards/margins": 3.079315662384033,
      "rewards/rejected": -2.9233059883117676,
      "step": 217,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5660499837715027,
      "grad_norm": 7.255380153656006,
      "learning_rate": 0.00017184103638675157,
      "logits/chosen": -0.8508449792861938,
      "logits/rejected": -0.8561905026435852,
      "logps/chosen": -13.53388786315918,
      "logps/rejected": -47.0406608581543,
      "loss": 0.739139974117279,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4106239676475525,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.030060507357120514,
      "rewards/margins": 3.0796077251434326,
      "rewards/rejected": -3.109668254852295,
      "step": 218,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5686465433300877,
      "grad_norm": 2.1521670818328857,
      "learning_rate": 0.0001715414199264168,
      "logits/chosen": -0.8377723693847656,
      "logits/rejected": -0.8416836857795715,
      "logps/chosen": -11.449177742004395,
      "logps/rejected": -51.93082046508789,
      "loss": 0.5678492784500122,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4639051854610443,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.4290061891078949,
      "rewards/margins": 4.352783203125,
      "rewards/rejected": -3.923776865005493,
      "step": 219,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5712431028886725,
      "grad_norm": 5.820508003234863,
      "learning_rate": 0.00017124048210388266,
      "logits/chosen": -0.8189604878425598,
      "logits/rejected": -0.8287642002105713,
      "logps/chosen": -7.280913352966309,
      "logps/rejected": -58.91409683227539,
      "loss": 0.3058544397354126,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19983381032943726,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12054422497749329,
      "rewards/margins": 4.547727108001709,
      "rewards/rejected": -4.427183628082275,
      "step": 220,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 0.5738396624472574,
      "grad_norm": 1.530921220779419,
      "learning_rate": 0.0001709382284774379,
      "logits/chosen": -0.8419314622879028,
      "logits/rejected": -0.8482197523117065,
      "logps/chosen": -10.496171951293945,
      "logps/rejected": -49.721099853515625,
      "loss": 0.4132998585700989,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3439655900001526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18015247583389282,
      "rewards/margins": 3.97615122795105,
      "rewards/rejected": -3.7959985733032227,
      "step": 221,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5764362220058422,
      "grad_norm": 1.9025592803955078,
      "learning_rate": 0.00017063466462967388,
      "logits/chosen": -0.8564705848693848,
      "logits/rejected": -0.8616027235984802,
      "logps/chosen": -7.931392669677734,
      "logps/rejected": -49.22194290161133,
      "loss": 0.610306978225708,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4569806456565857,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22110262513160706,
      "rewards/margins": 4.034682750701904,
      "rewards/rejected": -3.81358003616333,
      "step": 222,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5790327815644272,
      "grad_norm": 3.6240224838256836,
      "learning_rate": 0.00017032979616738169,
      "logits/chosen": -0.8264967203140259,
      "logits/rejected": -0.8325719833374023,
      "logps/chosen": -6.653590202331543,
      "logps/rejected": -59.540992736816406,
      "loss": 0.49167004227638245,
      "memory(GiB)": 46.82,
      "nll_loss": 0.402230829000473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1619190275669098,
      "rewards/margins": 4.9636969566345215,
      "rewards/rejected": -4.8017778396606445,
      "step": 223,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.581629341123012,
      "grad_norm": 26.20123291015625,
      "learning_rate": 0.00017002362872144843,
      "logits/chosen": -0.84416663646698,
      "logits/rejected": -0.8446400761604309,
      "logps/chosen": -7.779309272766113,
      "logps/rejected": -48.635101318359375,
      "loss": 0.4477548599243164,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31716760993003845,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12198957055807114,
      "rewards/margins": 3.744682788848877,
      "rewards/rejected": -3.622692823410034,
      "step": 224,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5842259006815969,
      "grad_norm": 3.45158052444458,
      "learning_rate": 0.0001697161679467534,
      "logits/chosen": -0.8399550914764404,
      "logits/rejected": -0.8448391556739807,
      "logps/chosen": -8.095958709716797,
      "logps/rejected": -57.113128662109375,
      "loss": 0.4480040967464447,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3591548800468445,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.097431480884552,
      "rewards/margins": 4.470404624938965,
      "rewards/rejected": -4.372973442077637,
      "step": 225,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5868224602401818,
      "grad_norm": 9.01218318939209,
      "learning_rate": 0.0001694074195220634,
      "logits/chosen": -0.8154175281524658,
      "logits/rejected": -0.8212026357650757,
      "logps/chosen": -9.016716957092285,
      "logps/rejected": -48.810791015625,
      "loss": 0.6711245179176331,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4295364022254944,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1990734040737152,
      "rewards/margins": 3.8862059116363525,
      "rewards/rejected": -3.6871328353881836,
      "step": 226,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5894190197987667,
      "grad_norm": 5.020142078399658,
      "learning_rate": 0.00016909738914992813,
      "logits/chosen": -0.7923344969749451,
      "logits/rejected": -0.7923219203948975,
      "logps/chosen": -10.089529037475586,
      "logps/rejected": -45.86975860595703,
      "loss": 0.6674864888191223,
      "memory(GiB)": 46.82,
      "nll_loss": 0.47657352685928345,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04541245102882385,
      "rewards/margins": 3.1376214027404785,
      "rewards/rejected": -3.0922091007232666,
      "step": 227,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5920155793573515,
      "grad_norm": 3.975611925125122,
      "learning_rate": 0.00016878608255657457,
      "logits/chosen": -0.7756553292274475,
      "logits/rejected": -0.7805052399635315,
      "logps/chosen": -6.576499938964844,
      "logps/rejected": -48.288204193115234,
      "loss": 0.36480942368507385,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24316585063934326,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21090297400951385,
      "rewards/margins": 3.6886119842529297,
      "rewards/rejected": -3.4777092933654785,
      "step": 228,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 0.5946121389159363,
      "grad_norm": 3.324848175048828,
      "learning_rate": 0.00016847350549180148,
      "logits/chosen": -0.7683195471763611,
      "logits/rejected": -0.7729092836380005,
      "logps/chosen": -5.581432819366455,
      "logps/rejected": -42.25553894042969,
      "loss": 0.5814209580421448,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33762359619140625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08220689743757248,
      "rewards/margins": 3.051085948944092,
      "rewards/rejected": -2.968878984451294,
      "step": 229,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.5972086984745213,
      "grad_norm": 1.8596134185791016,
      "learning_rate": 0.00016815966372887305,
      "logits/chosen": -0.7532698512077332,
      "logits/rejected": -0.7569671273231506,
      "logps/chosen": -6.81831169128418,
      "logps/rejected": -32.88819885253906,
      "loss": 0.4662015438079834,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31704533100128174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3468167185783386,
      "rewards/margins": 2.541092872619629,
      "rewards/rejected": -2.1942760944366455,
      "step": 230,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.5998052580331061,
      "grad_norm": 9.191289901733398,
      "learning_rate": 0.00016784456306441234,
      "logits/chosen": -0.7527615427970886,
      "logits/rejected": -0.7603521347045898,
      "logps/chosen": -8.545462608337402,
      "logps/rejected": -34.113487243652344,
      "loss": 0.6046890020370483,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35081756114959717,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12367656826972961,
      "rewards/margins": 2.2450129985809326,
      "rewards/rejected": -2.1213364601135254,
      "step": 231,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.602401817591691,
      "grad_norm": 2.0885210037231445,
      "learning_rate": 0.0001675282093182941,
      "logits/chosen": -0.7435917854309082,
      "logits/rejected": -0.7469019889831543,
      "logps/chosen": -9.624258041381836,
      "logps/rejected": -38.40724563598633,
      "loss": 0.6471220850944519,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44592684507369995,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2652357518672943,
      "rewards/margins": 2.823615312576294,
      "rewards/rejected": -2.558379650115967,
      "step": 232,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.6049983771502759,
      "grad_norm": 4.651163101196289,
      "learning_rate": 0.0001672106083335374,
      "logits/chosen": -0.7706339955329895,
      "logits/rejected": -0.7752231955528259,
      "logps/chosen": -9.609830856323242,
      "logps/rejected": -36.36707305908203,
      "loss": 0.7082432508468628,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4555765986442566,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22910386323928833,
      "rewards/margins": 2.703925609588623,
      "rewards/rejected": -2.4748215675354004,
      "step": 233,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6075949367088608,
      "grad_norm": 12.39693832397461,
      "learning_rate": 0.00016689176597619774,
      "logits/chosen": -0.7196263074874878,
      "logits/rejected": -0.7302131652832031,
      "logps/chosen": -7.361204624176025,
      "logps/rejected": -40.15000534057617,
      "loss": 0.7524147033691406,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5045353770256042,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14129626750946045,
      "rewards/margins": 2.2885243892669678,
      "rewards/rejected": -2.1472277641296387,
      "step": 234,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6101914962674456,
      "grad_norm": 6.736759185791016,
      "learning_rate": 0.00016657168813525853,
      "logits/chosen": -0.7526311278343201,
      "logits/rejected": -0.752763032913208,
      "logps/chosen": -7.331406116485596,
      "logps/rejected": -32.31086730957031,
      "loss": 0.754887580871582,
      "memory(GiB)": 46.82,
      "nll_loss": 0.48174968361854553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0755733847618103,
      "rewards/margins": 2.1878206729888916,
      "rewards/rejected": -2.1122474670410156,
      "step": 235,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6127880558260305,
      "grad_norm": 2.6155624389648438,
      "learning_rate": 0.0001662503807225225,
      "logits/chosen": -0.7355427742004395,
      "logits/rejected": -0.7384796738624573,
      "logps/chosen": -8.030364036560059,
      "logps/rejected": -31.442228317260742,
      "loss": 0.5243774652481079,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3355557322502136,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19963966310024261,
      "rewards/margins": 2.296480655670166,
      "rewards/rejected": -2.096841335296631,
      "step": 236,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.6153846153846154,
      "grad_norm": 18.65455436706543,
      "learning_rate": 0.0001659278496725024,
      "logits/chosen": -0.7295012474060059,
      "logits/rejected": -0.7436960935592651,
      "logps/chosen": -2.705686569213867,
      "logps/rejected": -41.4896354675293,
      "loss": 0.34673982858657837,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17969390749931335,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24503067135810852,
      "rewards/margins": 3.1294853687286377,
      "rewards/rejected": -2.8844549655914307,
      "step": 237,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.6179811749432003,
      "grad_norm": 7.085238456726074,
      "learning_rate": 0.00016560410094231144,
      "logits/chosen": -0.7452048063278198,
      "logits/rejected": -0.7516958713531494,
      "logps/chosen": -7.530111789703369,
      "logps/rejected": -34.36936569213867,
      "loss": 0.6021293997764587,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3124768137931824,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21694707870483398,
      "rewards/margins": 2.285515785217285,
      "rewards/rejected": -2.068568706512451,
      "step": 238,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6205777345017851,
      "grad_norm": 4.304277420043945,
      "learning_rate": 0.00016527914051155327,
      "logits/chosen": -0.7648165822029114,
      "logits/rejected": -0.7681047320365906,
      "logps/chosen": -10.339492797851562,
      "logps/rejected": -35.120906829833984,
      "loss": 0.6538665890693665,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35871338844299316,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.31186383962631226,
      "rewards/margins": 2.426875591278076,
      "rewards/rejected": -2.11501145362854,
      "step": 239,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6231742940603701,
      "grad_norm": 2.158324718475342,
      "learning_rate": 0.00016495297438221145,
      "logits/chosen": -0.8034281730651855,
      "logits/rejected": -0.807834267616272,
      "logps/chosen": -5.897243976593018,
      "logps/rejected": -28.772375106811523,
      "loss": 0.49097374081611633,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2912865877151489,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.41394859552383423,
      "rewards/margins": 2.25996994972229,
      "rewards/rejected": -1.8460214138031006,
      "step": 240,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6257708536189549,
      "grad_norm": 3.6357457637786865,
      "learning_rate": 0.00016462560857853875,
      "logits/chosen": -0.7996818423271179,
      "logits/rejected": -0.8091530203819275,
      "logps/chosen": -7.473721981048584,
      "logps/rejected": -54.37172317504883,
      "loss": 0.38115888833999634,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21902790665626526,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.033102504909038544,
      "rewards/margins": 3.986940383911133,
      "rewards/rejected": -3.9538376331329346,
      "step": 241,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6283674131775397,
      "grad_norm": 2.7460196018218994,
      "learning_rate": 0.00016429704914694573,
      "logits/chosen": -0.7820178866386414,
      "logits/rejected": -0.7842017412185669,
      "logps/chosen": -5.335976600646973,
      "logps/rejected": -42.033546447753906,
      "loss": 0.3013039231300354,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18570105731487274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1724267154932022,
      "rewards/margins": 3.339083194732666,
      "rewards/rejected": -3.166656494140625,
      "step": 242,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6309639727361246,
      "grad_norm": 5.302358627319336,
      "learning_rate": 0.00016396730215588915,
      "logits/chosen": -0.8216249942779541,
      "logits/rejected": -0.8270251750946045,
      "logps/chosen": -6.513396263122559,
      "logps/rejected": -36.97430419921875,
      "loss": 0.5384330749511719,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2902465760707855,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3697386085987091,
      "rewards/margins": 3.0714187622070312,
      "rewards/rejected": -2.7016794681549072,
      "step": 243,
      "train_speed(iter/s)": 0.005362
    },
    {
      "epoch": 0.6335605322947095,
      "grad_norm": 3.1649816036224365,
      "learning_rate": 0.00016363637369575985,
      "logits/chosen": -0.8752993941307068,
      "logits/rejected": -0.8820919990539551,
      "logps/chosen": -7.153526306152344,
      "logps/rejected": -39.54959487915039,
      "loss": 0.46389204263687134,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27045994997024536,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20367804169654846,
      "rewards/margins": 2.886066436767578,
      "rewards/rejected": -2.6823883056640625,
      "step": 244,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6361570918532944,
      "grad_norm": 1.2264916896820068,
      "learning_rate": 0.00016330426987877029,
      "logits/chosen": -0.8916351199150085,
      "logits/rejected": -0.903096616268158,
      "logps/chosen": -7.63686990737915,
      "logps/rejected": -45.377986907958984,
      "loss": 0.4263359308242798,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33412155508995056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44734618067741394,
      "rewards/margins": 3.656412124633789,
      "rewards/rejected": -3.2090659141540527,
      "step": 245,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6387536514118792,
      "grad_norm": 2.662604570388794,
      "learning_rate": 0.00016297099683884163,
      "logits/chosen": -0.9035799503326416,
      "logits/rejected": -0.9036927223205566,
      "logps/chosen": -7.97056770324707,
      "logps/rejected": -39.277339935302734,
      "loss": 0.4779468774795532,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3122636079788208,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11914081126451492,
      "rewards/margins": 3.0797340869903564,
      "rewards/rejected": -2.9605932235717773,
      "step": 246,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.6413502109704642,
      "grad_norm": 1.7811486721038818,
      "learning_rate": 0.0001626365607314905,
      "logits/chosen": -0.9211519956588745,
      "logits/rejected": -0.9266526699066162,
      "logps/chosen": -12.947280883789062,
      "logps/rejected": -44.871585845947266,
      "loss": 0.4722021222114563,
      "memory(GiB)": 46.82,
      "nll_loss": 0.37982484698295593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2566950023174286,
      "rewards/margins": 3.416555881500244,
      "rewards/rejected": -3.1598610877990723,
      "step": 247,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.643946770529049,
      "grad_norm": 12.365011215209961,
      "learning_rate": 0.00016230096773371514,
      "logits/chosen": -0.9951969385147095,
      "logits/rejected": -0.9942623376846313,
      "logps/chosen": -8.581663131713867,
      "logps/rejected": -43.716217041015625,
      "loss": 0.518119215965271,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38296759128570557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09159248322248459,
      "rewards/margins": 3.335644006729126,
      "rewards/rejected": -3.244051456451416,
      "step": 248,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6465433300876339,
      "grad_norm": 11.83633041381836,
      "learning_rate": 0.00016196422404388157,
      "logits/chosen": -0.9826600551605225,
      "logits/rejected": -0.9850393533706665,
      "logps/chosen": -9.135320663452148,
      "logps/rejected": -46.018798828125,
      "loss": 0.5105503797531128,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39638927578926086,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10775209963321686,
      "rewards/margins": 3.4860193729400635,
      "rewards/rejected": -3.378267526626587,
      "step": 249,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6491398896462187,
      "grad_norm": 4.603750228881836,
      "learning_rate": 0.0001616263358816089,
      "logits/chosen": -1.0238407850265503,
      "logits/rejected": -1.0272432565689087,
      "logps/chosen": -5.940099716186523,
      "logps/rejected": -52.49337387084961,
      "loss": 0.36938077211380005,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2231290638446808,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25854048132896423,
      "rewards/margins": 4.265020847320557,
      "rewards/rejected": -4.0064802169799805,
      "step": 250,
      "train_speed(iter/s)": 0.005361
    },
    {
      "epoch": 0.6517364492048037,
      "grad_norm": 52.063899993896484,
      "learning_rate": 0.00016128730948765447,
      "logits/chosen": -1.0816775560379028,
      "logits/rejected": -1.089053750038147,
      "logps/chosen": -5.90020227432251,
      "logps/rejected": -48.5638542175293,
      "loss": 0.43552514910697937,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3044917583465576,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26977428793907166,
      "rewards/margins": 3.8167455196380615,
      "rewards/rejected": -3.546971321105957,
      "step": 251,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6543330087633885,
      "grad_norm": 4.294113636016846,
      "learning_rate": 0.0001609471511237987,
      "logits/chosen": -1.014509677886963,
      "logits/rejected": -1.0186957120895386,
      "logps/chosen": -7.97694730758667,
      "logps/rejected": -56.545310974121094,
      "loss": 0.36489278078079224,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2762773334980011,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.022740613669157028,
      "rewards/margins": 4.397303581237793,
      "rewards/rejected": -4.4200439453125,
      "step": 252,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6569295683219734,
      "grad_norm": 3.516819477081299,
      "learning_rate": 0.00016060586707272942,
      "logits/chosen": -1.0790157318115234,
      "logits/rejected": -1.0865930318832397,
      "logps/chosen": -8.454242706298828,
      "logps/rejected": -54.464717864990234,
      "loss": 0.3686315417289734,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27849066257476807,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13144642114639282,
      "rewards/margins": 4.120382308959961,
      "rewards/rejected": -3.988936424255371,
      "step": 253,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6595261278805583,
      "grad_norm": 10.735478401184082,
      "learning_rate": 0.00016026346363792567,
      "logits/chosen": -1.0199754238128662,
      "logits/rejected": -1.024275779724121,
      "logps/chosen": -10.477851867675781,
      "logps/rejected": -48.380496978759766,
      "loss": 0.6012676358222961,
      "memory(GiB)": 46.82,
      "nll_loss": 0.42399072647094727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.174387127161026,
      "rewards/margins": 3.8456358909606934,
      "rewards/rejected": -3.671248435974121,
      "step": 254,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6621226874391432,
      "grad_norm": 4.77883243560791,
      "learning_rate": 0.0001599199471435414,
      "logits/chosen": -1.0578190088272095,
      "logits/rejected": -1.0658252239227295,
      "logps/chosen": -9.42254638671875,
      "logps/rejected": -52.36109924316406,
      "loss": 0.4474544823169708,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3092021942138672,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1929168850183487,
      "rewards/margins": 4.269603729248047,
      "rewards/rejected": -4.076685905456543,
      "step": 255,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.664719246997728,
      "grad_norm": 9.897768020629883,
      "learning_rate": 0.00015957532393428872,
      "logits/chosen": -1.0346934795379639,
      "logits/rejected": -1.0372470617294312,
      "logps/chosen": -9.071995735168457,
      "logps/rejected": -49.584144592285156,
      "loss": 0.617793083190918,
      "memory(GiB)": 46.82,
      "nll_loss": 0.49921637773513794,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16571339964866638,
      "rewards/margins": 3.9660375118255615,
      "rewards/rejected": -3.800323963165283,
      "step": 256,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6673158065563128,
      "grad_norm": 1.2311677932739258,
      "learning_rate": 0.00015922960037532055,
      "logits/chosen": -1.108057975769043,
      "logits/rejected": -1.1105315685272217,
      "logps/chosen": -6.960026741027832,
      "logps/rejected": -53.92729568481445,
      "loss": 0.3580763638019562,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32845476269721985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28917384147644043,
      "rewards/margins": 4.447230339050293,
      "rewards/rejected": -4.15805721282959,
      "step": 257,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6699123661148978,
      "grad_norm": 45.17562484741211,
      "learning_rate": 0.0001588827828521133,
      "logits/chosen": -1.050451397895813,
      "logits/rejected": -1.0607093572616577,
      "logps/chosen": -8.958418846130371,
      "logps/rejected": -57.699886322021484,
      "loss": 0.7421323657035828,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4822649359703064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22899343073368073,
      "rewards/margins": 4.515078544616699,
      "rewards/rejected": -4.286085605621338,
      "step": 258,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6725089256734826,
      "grad_norm": 1.422256588935852,
      "learning_rate": 0.00015853487777034862,
      "logits/chosen": -1.0635780096054077,
      "logits/rejected": -1.0673224925994873,
      "logps/chosen": -8.635517120361328,
      "logps/rejected": -58.0097541809082,
      "loss": 0.3220076858997345,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30796006321907043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3498881161212921,
      "rewards/margins": 4.858260154724121,
      "rewards/rejected": -4.5083723068237305,
      "step": 259,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6751054852320675,
      "grad_norm": 2.5571460723876953,
      "learning_rate": 0.0001581858915557953,
      "logits/chosen": -1.0401530265808105,
      "logits/rejected": -1.056994915008545,
      "logps/chosen": -7.485888481140137,
      "logps/rejected": -66.18753814697266,
      "loss": 0.34027281403541565,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24791380763053894,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1610621213912964,
      "rewards/margins": 5.100013732910156,
      "rewards/rejected": -4.938951015472412,
      "step": 260,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6777020447906524,
      "grad_norm": 2.7434134483337402,
      "learning_rate": 0.00015783583065419055,
      "logits/chosen": -1.0726388692855835,
      "logits/rejected": -1.0658763647079468,
      "logps/chosen": -9.934000968933105,
      "logps/rejected": -43.6441764831543,
      "loss": 0.48066970705986023,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3425687551498413,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2836655080318451,
      "rewards/margins": 3.740111827850342,
      "rewards/rejected": -3.456446647644043,
      "step": 261,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6802986043492373,
      "grad_norm": 11.48259162902832,
      "learning_rate": 0.00015748470153112093,
      "logits/chosen": -1.02780020236969,
      "logits/rejected": -1.0415828227996826,
      "logps/chosen": -5.212684631347656,
      "logps/rejected": -61.468292236328125,
      "loss": 0.4467061758041382,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2766522765159607,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25993314385414124,
      "rewards/margins": 5.092804908752441,
      "rewards/rejected": -4.832871437072754,
      "step": 262,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6828951639078221,
      "grad_norm": 8.564838409423828,
      "learning_rate": 0.00015713251067190297,
      "logits/chosen": -1.0391637086868286,
      "logits/rejected": -1.0465219020843506,
      "logps/chosen": -10.817865371704102,
      "logps/rejected": -51.074317932128906,
      "loss": 0.5829335451126099,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32782095670700073,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.04439431428909302,
      "rewards/margins": 3.851675510406494,
      "rewards/rejected": -3.8960695266723633,
      "step": 263,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.685491723466407,
      "grad_norm": 4.39657735824585,
      "learning_rate": 0.00015677926458146325,
      "logits/chosen": -1.0428134202957153,
      "logits/rejected": -1.0442464351654053,
      "logps/chosen": -9.461180686950684,
      "logps/rejected": -50.168216705322266,
      "loss": 0.527190625667572,
      "memory(GiB)": 46.82,
      "nll_loss": 0.39044150710105896,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2586324214935303,
      "rewards/margins": 4.324131488800049,
      "rewards/rejected": -4.065499305725098,
      "step": 264,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6880882830249919,
      "grad_norm": 33.24240493774414,
      "learning_rate": 0.00015642496978421843,
      "logits/chosen": -0.9720567464828491,
      "logits/rejected": -0.9765857458114624,
      "logps/chosen": -6.901163578033447,
      "logps/rejected": -53.93683624267578,
      "loss": 0.42922741174697876,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3468468487262726,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.45161283016204834,
      "rewards/margins": 4.4618635177612305,
      "rewards/rejected": -4.010250568389893,
      "step": 265,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.6906848425835768,
      "grad_norm": 5.137800693511963,
      "learning_rate": 0.00015606963282395467,
      "logits/chosen": -1.0221166610717773,
      "logits/rejected": -1.030777931213379,
      "logps/chosen": -6.099124908447266,
      "logps/rejected": -51.621883392333984,
      "loss": 0.44425708055496216,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34302324056625366,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26895275712013245,
      "rewards/margins": 4.289248943328857,
      "rewards/rejected": -4.020296096801758,
      "step": 266,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6932814021421616,
      "grad_norm": 1.4863890409469604,
      "learning_rate": 0.00015571326026370675,
      "logits/chosen": -1.0046309232711792,
      "logits/rejected": -1.0050278902053833,
      "logps/chosen": -7.830292701721191,
      "logps/rejected": -52.04121017456055,
      "loss": 0.30963078141212463,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23877012729644775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16400472819805145,
      "rewards/margins": 4.016274452209473,
      "rewards/rejected": -3.8522696495056152,
      "step": 267,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6958779617007466,
      "grad_norm": 1.783074140548706,
      "learning_rate": 0.00015535585868563687,
      "logits/chosen": -0.9659662842750549,
      "logits/rejected": -0.9642824530601501,
      "logps/chosen": -7.803462505340576,
      "logps/rejected": -40.019229888916016,
      "loss": 0.45223909616470337,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2870696485042572,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3678834140300751,
      "rewards/margins": 3.325589895248413,
      "rewards/rejected": -2.9577064514160156,
      "step": 268,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.6984745212593314,
      "grad_norm": 2.9299373626708984,
      "learning_rate": 0.00015499743469091305,
      "logits/chosen": -0.9901977181434631,
      "logits/rejected": -0.9931589365005493,
      "logps/chosen": -9.296809196472168,
      "logps/rejected": -42.269630432128906,
      "loss": 0.45571860671043396,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2836253046989441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3169897794723511,
      "rewards/margins": 3.4090187549591064,
      "rewards/rejected": -3.092029094696045,
      "step": 269,
      "train_speed(iter/s)": 0.005357
    },
    {
      "epoch": 0.7010710808179162,
      "grad_norm": 2.768286943435669,
      "learning_rate": 0.00015463799489958728,
      "logits/chosen": -0.9836040139198303,
      "logits/rejected": -0.9821513295173645,
      "logps/chosen": -8.032227516174316,
      "logps/rejected": -46.852630615234375,
      "loss": 0.3594820201396942,
      "memory(GiB)": 46.82,
      "nll_loss": 0.301334023475647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21750850975513458,
      "rewards/margins": 3.9014554023742676,
      "rewards/rejected": -3.6839470863342285,
      "step": 270,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.7036676403765011,
      "grad_norm": 11.645221710205078,
      "learning_rate": 0.0001542775459504732,
      "logits/chosen": -0.977642297744751,
      "logits/rejected": -0.983423113822937,
      "logps/chosen": -7.453811168670654,
      "logps/rejected": -41.04922866821289,
      "loss": 0.7860956788063049,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5319768786430359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1531173586845398,
      "rewards/margins": 3.1887106895446777,
      "rewards/rejected": -3.035593271255493,
      "step": 271,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.706264199935086,
      "grad_norm": 2.451678514480591,
      "learning_rate": 0.00015391609450102344,
      "logits/chosen": -1.023182988166809,
      "logits/rejected": -1.0244897603988647,
      "logps/chosen": -7.354053497314453,
      "logps/rejected": -44.0927619934082,
      "loss": 0.39909517765045166,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3177676796913147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3391517996788025,
      "rewards/margins": 3.611537218093872,
      "rewards/rejected": -3.272385597229004,
      "step": 272,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7088607594936709,
      "grad_norm": 1.9267401695251465,
      "learning_rate": 0.00015355364722720674,
      "logits/chosen": -0.9752072691917419,
      "logits/rejected": -0.9693701863288879,
      "logps/chosen": -11.397211074829102,
      "logps/rejected": -48.029632568359375,
      "loss": 0.39653921127319336,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3201441764831543,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3673996925354004,
      "rewards/margins": 3.925154209136963,
      "rewards/rejected": -3.5577547550201416,
      "step": 273,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7114573190522557,
      "grad_norm": 8.30762767791748,
      "learning_rate": 0.0001531902108233846,
      "logits/chosen": -1.005213737487793,
      "logits/rejected": -1.0123414993286133,
      "logps/chosen": -10.145181655883789,
      "logps/rejected": -43.201988220214844,
      "loss": 0.9267427921295166,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5765597224235535,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0955648422241211,
      "rewards/margins": 3.0650508403778076,
      "rewards/rejected": -2.9694862365722656,
      "step": 274,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7140538786108407,
      "grad_norm": 1.9015525579452515,
      "learning_rate": 0.00015282579200218762,
      "logits/chosen": -0.9652273654937744,
      "logits/rejected": -0.9685376882553101,
      "logps/chosen": -5.937908172607422,
      "logps/rejected": -48.166961669921875,
      "loss": 0.3091265857219696,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19744150340557098,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10177955776453018,
      "rewards/margins": 3.805178642272949,
      "rewards/rejected": -3.7033989429473877,
      "step": 275,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7166504381694255,
      "grad_norm": 6.060371398925781,
      "learning_rate": 0.00015246039749439158,
      "logits/chosen": -0.9658158421516418,
      "logits/rejected": -0.9731608033180237,
      "logps/chosen": -7.255200386047363,
      "logps/rejected": -50.502464294433594,
      "loss": 0.6045569777488708,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3833393156528473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.06260780245065689,
      "rewards/margins": 3.6017262935638428,
      "rewards/rejected": -3.6643340587615967,
      "step": 276,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7192469977280104,
      "grad_norm": 3.960418462753296,
      "learning_rate": 0.00015209403404879303,
      "logits/chosen": -0.8965452909469604,
      "logits/rejected": -0.9077386260032654,
      "logps/chosen": -7.974431991577148,
      "logps/rejected": -51.04905700683594,
      "loss": 0.5292239785194397,
      "memory(GiB)": 46.82,
      "nll_loss": 0.44278621673583984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2756114602088928,
      "rewards/margins": 3.676894187927246,
      "rewards/rejected": -3.401282548904419,
      "step": 277,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7218435572865952,
      "grad_norm": 7.708714008331299,
      "learning_rate": 0.00015172670843208475,
      "logits/chosen": -0.9474626779556274,
      "logits/rejected": -0.9558745622634888,
      "logps/chosen": -6.619680404663086,
      "logps/rejected": -55.76416778564453,
      "loss": 0.33095428347587585,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2111760675907135,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.31980329751968384,
      "rewards/margins": 4.286462783813477,
      "rewards/rejected": -3.9666595458984375,
      "step": 278,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7244401168451802,
      "grad_norm": 3.547145128250122,
      "learning_rate": 0.00015135842742873077,
      "logits/chosen": -0.9616857767105103,
      "logits/rejected": -0.9554501175880432,
      "logps/chosen": -13.64553451538086,
      "logps/rejected": -47.13714599609375,
      "loss": 0.5570240020751953,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34419986605644226,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17980074882507324,
      "rewards/margins": 3.7363922595977783,
      "rewards/rejected": -3.556591510772705,
      "step": 279,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.727036676403765,
      "grad_norm": 2.789520740509033,
      "learning_rate": 0.00015098919784084082,
      "logits/chosen": -0.9526619911193848,
      "logits/rejected": -0.9585291743278503,
      "logps/chosen": -5.527890682220459,
      "logps/rejected": -48.09929656982422,
      "loss": 0.3063603341579437,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19290734827518463,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27611908316612244,
      "rewards/margins": 3.8319613933563232,
      "rewards/rejected": -3.5558416843414307,
      "step": 280,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7296332359623499,
      "grad_norm": 2.256161689758301,
      "learning_rate": 0.00015061902648804502,
      "logits/chosen": -0.913105309009552,
      "logits/rejected": -0.9261531829833984,
      "logps/chosen": -4.366436004638672,
      "logps/rejected": -42.916358947753906,
      "loss": 0.3409702777862549,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2262687385082245,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.35636499524116516,
      "rewards/margins": 3.0527658462524414,
      "rewards/rejected": -2.6964004039764404,
      "step": 281,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7322297955209348,
      "grad_norm": 3.5901756286621094,
      "learning_rate": 0.0001502479202073678,
      "logits/chosen": -0.8968573808670044,
      "logits/rejected": -0.9044336676597595,
      "logps/chosen": -10.585033416748047,
      "logps/rejected": -49.008888244628906,
      "loss": 0.4977288842201233,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3260158896446228,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15746770799160004,
      "rewards/margins": 3.4977617263793945,
      "rewards/rejected": -3.3402938842773438,
      "step": 282,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7348263550795197,
      "grad_norm": 4.433828830718994,
      "learning_rate": 0.00014987588585310154,
      "logits/chosen": -0.9033176898956299,
      "logits/rejected": -0.9071294069290161,
      "logps/chosen": -10.011114120483398,
      "logps/rejected": -41.05360794067383,
      "loss": 0.5514770150184631,
      "memory(GiB)": 46.82,
      "nll_loss": 0.37259769439697266,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07280327379703522,
      "rewards/margins": 3.06974720954895,
      "rewards/rejected": -2.996943950653076,
      "step": 283,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7374229146381045,
      "grad_norm": 4.148340225219727,
      "learning_rate": 0.00014950293029668003,
      "logits/chosen": -0.8784151077270508,
      "logits/rejected": -0.8862117528915405,
      "logps/chosen": -6.820345401763916,
      "logps/rejected": -54.827003479003906,
      "loss": 0.3150921165943146,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24718376994132996,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15254774689674377,
      "rewards/margins": 4.318949222564697,
      "rewards/rejected": -4.1664018630981445,
      "step": 284,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7400194741966893,
      "grad_norm": 1.0749166011810303,
      "learning_rate": 0.00014912906042655164,
      "logits/chosen": -0.8836584091186523,
      "logits/rejected": -0.8885093927383423,
      "logps/chosen": -6.686123847961426,
      "logps/rejected": -51.037071228027344,
      "loss": 0.2929588854312897,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25999218225479126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24088695645332336,
      "rewards/margins": 4.235246658325195,
      "rewards/rejected": -3.9943599700927734,
      "step": 285,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7426160337552743,
      "grad_norm": 3.1975772380828857,
      "learning_rate": 0.00014875428314805195,
      "logits/chosen": -0.8791793584823608,
      "logits/rejected": -0.8838009834289551,
      "logps/chosen": -4.482460975646973,
      "logps/rejected": -42.07259750366211,
      "loss": 0.45191437005996704,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2820110023021698,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.17223408818244934,
      "rewards/margins": 3.3569703102111816,
      "rewards/rejected": -3.1847360134124756,
      "step": 286,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7452125933138591,
      "grad_norm": 4.853325366973877,
      "learning_rate": 0.0001483786053832763,
      "logits/chosen": -0.8666754961013794,
      "logits/rejected": -0.8730036020278931,
      "logps/chosen": -10.173751831054688,
      "logps/rejected": -42.610389709472656,
      "loss": 0.481548547744751,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32319656014442444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1995163857936859,
      "rewards/margins": 3.381746292114258,
      "rewards/rejected": -3.182229518890381,
      "step": 287,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.747809152872444,
      "grad_norm": 5.358548164367676,
      "learning_rate": 0.00014800203407095195,
      "logits/chosen": -0.8995557427406311,
      "logits/rejected": -0.9070794582366943,
      "logps/chosen": -4.7012481689453125,
      "logps/rejected": -49.54352569580078,
      "loss": 0.3500158190727234,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2837740182876587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3831089735031128,
      "rewards/margins": 4.144287586212158,
      "rewards/rejected": -3.761178493499756,
      "step": 288,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.7504057124310289,
      "grad_norm": 2.933692693710327,
      "learning_rate": 0.0001476245761663097,
      "logits/chosen": -0.8843749165534973,
      "logits/rejected": -0.8904112577438354,
      "logps/chosen": -8.129939079284668,
      "logps/rejected": -47.64159393310547,
      "loss": 0.3809664249420166,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30353543162345886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43099674582481384,
      "rewards/margins": 3.9384264945983887,
      "rewards/rejected": -3.507429599761963,
      "step": 289,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.7530022719896138,
      "grad_norm": 5.533115863800049,
      "learning_rate": 0.00014724623864095595,
      "logits/chosen": -0.8974816799163818,
      "logits/rejected": -0.8943885564804077,
      "logps/chosen": -12.756951332092285,
      "logps/rejected": -45.041236877441406,
      "loss": 0.5215253829956055,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4008585512638092,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.28772494196891785,
      "rewards/margins": 3.7097461223602295,
      "rewards/rejected": -3.4220211505889893,
      "step": 290,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7555988315481986,
      "grad_norm": 2.6864752769470215,
      "learning_rate": 0.00014686702848274328,
      "logits/chosen": -0.9088398218154907,
      "logits/rejected": -0.9072700142860413,
      "logps/chosen": -7.4077606201171875,
      "logps/rejected": -46.239906311035156,
      "loss": 0.32609468698501587,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2327779084444046,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20570842921733856,
      "rewards/margins": 3.893001079559326,
      "rewards/rejected": -3.6872925758361816,
      "step": 291,
      "train_speed(iter/s)": 0.00536
    },
    {
      "epoch": 0.7581953911067835,
      "grad_norm": 9.521873474121094,
      "learning_rate": 0.0001464869526956418,
      "logits/chosen": -0.8602750301361084,
      "logits/rejected": -0.8622329235076904,
      "logps/chosen": -8.320989608764648,
      "logps/rejected": -44.78575134277344,
      "loss": 0.5608953237533569,
      "memory(GiB)": 46.82,
      "nll_loss": 0.36838674545288086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0709700807929039,
      "rewards/margins": 3.4943435192108154,
      "rewards/rejected": -3.4233739376068115,
      "step": 292,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7607919506653684,
      "grad_norm": 5.926126480102539,
      "learning_rate": 0.00014610601829960978,
      "logits/chosen": -0.892938494682312,
      "logits/rejected": -0.8982577323913574,
      "logps/chosen": -6.005653381347656,
      "logps/rejected": -44.90076446533203,
      "loss": 0.43931251764297485,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2638612389564514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12249821424484253,
      "rewards/margins": 3.140292167663574,
      "rewards/rejected": -3.017793893814087,
      "step": 293,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7633885102239533,
      "grad_norm": 2.9120235443115234,
      "learning_rate": 0.00014572423233046386,
      "logits/chosen": -0.8966829776763916,
      "logits/rejected": -0.9077416062355042,
      "logps/chosen": -8.57165241241455,
      "logps/rejected": -55.184844970703125,
      "loss": 0.45368537306785583,
      "memory(GiB)": 46.82,
      "nll_loss": 0.36300209164619446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1118740364909172,
      "rewards/margins": 4.270344257354736,
      "rewards/rejected": -4.158470153808594,
      "step": 294,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7659850697825381,
      "grad_norm": 5.669841289520264,
      "learning_rate": 0.00014534160183974907,
      "logits/chosen": -0.8818827271461487,
      "logits/rejected": -0.8899796009063721,
      "logps/chosen": -5.550753593444824,
      "logps/rejected": -44.52480697631836,
      "loss": 0.5001478791236877,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3267979621887207,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22049690783023834,
      "rewards/margins": 3.405883312225342,
      "rewards/rejected": -3.1853861808776855,
      "step": 295,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.7685816293411231,
      "grad_norm": 1.747400164604187,
      "learning_rate": 0.00014495813389460874,
      "logits/chosen": -0.9121143817901611,
      "logits/rejected": -0.9087746739387512,
      "logps/chosen": -9.585158348083496,
      "logps/rejected": -50.017791748046875,
      "loss": 0.4021596610546112,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3450128436088562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37154024839401245,
      "rewards/margins": 4.141083240509033,
      "rewards/rejected": -3.769543170928955,
      "step": 296,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7711781888997079,
      "grad_norm": 11.861947059631348,
      "learning_rate": 0.00014457383557765386,
      "logits/chosen": -0.8657410144805908,
      "logits/rejected": -0.8725622892379761,
      "logps/chosen": -8.42186450958252,
      "logps/rejected": -46.08644104003906,
      "loss": 0.559907078742981,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4467519223690033,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.40526360273361206,
      "rewards/margins": 3.708038568496704,
      "rewards/rejected": -3.3027749061584473,
      "step": 297,
      "train_speed(iter/s)": 0.005358
    },
    {
      "epoch": 0.7737747484582927,
      "grad_norm": 3.2839434146881104,
      "learning_rate": 0.00014418871398683226,
      "logits/chosen": -0.8851770758628845,
      "logits/rejected": -0.8929255604743958,
      "logps/chosen": -6.95133113861084,
      "logps/rejected": -44.50846862792969,
      "loss": 0.3745541274547577,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2655733525753021,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.381335973739624,
      "rewards/margins": 3.8720412254333496,
      "rewards/rejected": -3.4907052516937256,
      "step": 298,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7763713080168776,
      "grad_norm": 2.888361930847168,
      "learning_rate": 0.00014380277623529764,
      "logits/chosen": -0.853575587272644,
      "logits/rejected": -0.8686288595199585,
      "logps/chosen": -3.8131489753723145,
      "logps/rejected": -52.961769104003906,
      "loss": 0.22544604539871216,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15638428926467896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19273503124713898,
      "rewards/margins": 4.502290725708008,
      "rewards/rejected": -4.309555530548096,
      "step": 299,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7789678675754625,
      "grad_norm": 7.9267778396606445,
      "learning_rate": 0.00014341602945127806,
      "logits/chosen": -0.8887860774993896,
      "logits/rejected": -0.8979215621948242,
      "logps/chosen": -5.356024265289307,
      "logps/rejected": -51.7165412902832,
      "loss": 0.3821192979812622,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2632734477519989,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13820704817771912,
      "rewards/margins": 3.850296974182129,
      "rewards/rejected": -3.712090492248535,
      "step": 300,
      "train_speed(iter/s)": 0.005359
    },
    {
      "epoch": 0.7789678675754625,
      "eval_logits/chosen": -0.867992103099823,
      "eval_logits/rejected": -0.874204695224762,
      "eval_logps/chosen": -9.79829216003418,
      "eval_logps/rejected": -51.43035125732422,
      "eval_loss": 0.535804033279419,
      "eval_nll_loss": 0.41711926460266113,
      "eval_rewards/accuracies": 0.9677419066429138,
      "eval_rewards/chosen": 0.2565970718860626,
      "eval_rewards/margins": 4.159191608428955,
      "eval_rewards/rejected": -3.9025943279266357,
      "eval_runtime": 291.9135,
      "eval_samples_per_second": 0.425,
      "eval_steps_per_second": 0.425,
      "step": 300
    },
    {
      "epoch": 0.7815644271340474,
      "grad_norm": 1.242323637008667,
      "learning_rate": 0.00014302848077794427,
      "logits/chosen": -0.8723223209381104,
      "logits/rejected": -0.8770792484283447,
      "logps/chosen": -6.223211288452148,
      "logps/rejected": -52.06251907348633,
      "loss": 0.3036009967327118,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22717300057411194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3022981882095337,
      "rewards/margins": 4.106766700744629,
      "rewards/rejected": -3.8044683933258057,
      "step": 301,
      "train_speed(iter/s)": 0.00533
    },
    {
      "epoch": 0.7841609866926322,
      "grad_norm": 2.6004726886749268,
      "learning_rate": 0.00014264013737327788,
      "logits/chosen": -0.916274905204773,
      "logits/rejected": -0.9221408367156982,
      "logps/chosen": -8.104936599731445,
      "logps/rejected": -52.68116760253906,
      "loss": 0.3741414248943329,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26373815536499023,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1614115685224533,
      "rewards/margins": 4.1823883056640625,
      "rewards/rejected": -4.020977020263672,
      "step": 302,
      "train_speed(iter/s)": 0.00533
    },
    {
      "epoch": 0.7867575462512172,
      "grad_norm": 18.030475616455078,
      "learning_rate": 0.0001422510064099391,
      "logits/chosen": -0.9025695323944092,
      "logits/rejected": -0.9134030342102051,
      "logps/chosen": -5.3788228034973145,
      "logps/rejected": -56.01725769042969,
      "loss": 0.4833014905452728,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35686278343200684,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4239948093891144,
      "rewards/margins": 4.954083442687988,
      "rewards/rejected": -4.530088901519775,
      "step": 303,
      "train_speed(iter/s)": 0.005331
    },
    {
      "epoch": 0.789354105809802,
      "grad_norm": 1.25996732711792,
      "learning_rate": 0.00014186109507513425,
      "logits/chosen": -0.9152663946151733,
      "logits/rejected": -0.9181231260299683,
      "logps/chosen": -8.745201110839844,
      "logps/rejected": -53.348297119140625,
      "loss": 0.3430268168449402,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2737351357936859,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.327117383480072,
      "rewards/margins": 4.604560852050781,
      "rewards/rejected": -4.2774434089660645,
      "step": 304,
      "train_speed(iter/s)": 0.005332
    },
    {
      "epoch": 0.7919506653683869,
      "grad_norm": 2.2379977703094482,
      "learning_rate": 0.00014147041057048303,
      "logits/chosen": -0.9164860248565674,
      "logits/rejected": -0.922378659248352,
      "logps/chosen": -7.221551895141602,
      "logps/rejected": -63.282691955566406,
      "loss": 0.3236457407474518,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2571154534816742,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.33233726024627686,
      "rewards/margins": 5.439297676086426,
      "rewards/rejected": -5.106960296630859,
      "step": 305,
      "train_speed(iter/s)": 0.005332
    },
    {
      "epoch": 0.7945472249269717,
      "grad_norm": 2.401186227798462,
      "learning_rate": 0.00014107896011188547,
      "logits/chosen": -0.9090971946716309,
      "logits/rejected": -0.9093695878982544,
      "logps/chosen": -12.469802856445312,
      "logps/rejected": -52.85637664794922,
      "loss": 0.429090291261673,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34026211500167847,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.262347936630249,
      "rewards/margins": 4.283257007598877,
      "rewards/rejected": -4.020909309387207,
      "step": 306,
      "train_speed(iter/s)": 0.005332
    },
    {
      "epoch": 0.7971437844855567,
      "grad_norm": 11.393203735351562,
      "learning_rate": 0.0001406867509293887,
      "logits/chosen": -0.87870854139328,
      "logits/rejected": -0.890485405921936,
      "logps/chosen": -10.30262279510498,
      "logps/rejected": -65.79576110839844,
      "loss": 0.3643527030944824,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32104456424713135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3608924150466919,
      "rewards/margins": 5.524012088775635,
      "rewards/rejected": -5.163119792938232,
      "step": 307,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.7997403440441415,
      "grad_norm": 2.554598331451416,
      "learning_rate": 0.0001402937902670535,
      "logits/chosen": -0.9264713525772095,
      "logits/rejected": -0.9285149574279785,
      "logps/chosen": -10.006579399108887,
      "logps/rejected": -52.04652404785156,
      "loss": 0.860185444355011,
      "memory(GiB)": 46.82,
      "nll_loss": 0.6048625111579895,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22397533059120178,
      "rewards/margins": 4.25376033782959,
      "rewards/rejected": -4.02978515625,
      "step": 308,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8023369036027264,
      "grad_norm": 1.8770607709884644,
      "learning_rate": 0.00013990008538282027,
      "logits/chosen": -0.8972032070159912,
      "logits/rejected": -0.8980765342712402,
      "logps/chosen": -14.556074142456055,
      "logps/rejected": -53.385929107666016,
      "loss": 0.510574460029602,
      "memory(GiB)": 46.82,
      "nll_loss": 0.45061612129211426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3644930422306061,
      "rewards/margins": 4.575682640075684,
      "rewards/rejected": -4.2111897468566895,
      "step": 309,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8049334631613113,
      "grad_norm": 12.22037124633789,
      "learning_rate": 0.00013950564354837513,
      "logits/chosen": -0.9380626678466797,
      "logits/rejected": -0.9469397664070129,
      "logps/chosen": -10.450864791870117,
      "logps/rejected": -60.86605453491211,
      "loss": 0.47496286034584045,
      "memory(GiB)": 46.82,
      "nll_loss": 0.32565683126449585,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27174729108810425,
      "rewards/margins": 5.215041160583496,
      "rewards/rejected": -4.943294048309326,
      "step": 310,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8075300227198962,
      "grad_norm": 1.5406510829925537,
      "learning_rate": 0.0001391104720490156,
      "logits/chosen": -0.8875188231468201,
      "logits/rejected": -0.889438807964325,
      "logps/chosen": -9.368861198425293,
      "logps/rejected": -60.18291473388672,
      "loss": 0.4254617393016815,
      "memory(GiB)": 46.82,
      "nll_loss": 0.33097419142723083,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2585148215293884,
      "rewards/margins": 5.071025371551514,
      "rewards/rejected": -4.8125104904174805,
      "step": 311,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.810126582278481,
      "grad_norm": 1.0935144424438477,
      "learning_rate": 0.0001387145781835161,
      "logits/chosen": -0.8894232511520386,
      "logits/rejected": -0.8968610763549805,
      "logps/chosen": -6.686229228973389,
      "logps/rejected": -62.9738655090332,
      "loss": 0.32542869448661804,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31048065423965454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4036734998226166,
      "rewards/margins": 5.581910610198975,
      "rewards/rejected": -5.178236961364746,
      "step": 312,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8127231418370658,
      "grad_norm": 3.8420419692993164,
      "learning_rate": 0.00013831796926399293,
      "logits/chosen": -0.8379157185554504,
      "logits/rejected": -0.8429592251777649,
      "logps/chosen": -11.607048034667969,
      "logps/rejected": -59.3102912902832,
      "loss": 0.5591235160827637,
      "memory(GiB)": 46.82,
      "nll_loss": 0.522581934928894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2473653256893158,
      "rewards/margins": 4.927768707275391,
      "rewards/rejected": -4.680403232574463,
      "step": 313,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8153197013956508,
      "grad_norm": 1.6165937185287476,
      "learning_rate": 0.0001379206526157695,
      "logits/chosen": -0.837811291217804,
      "logits/rejected": -0.8444165587425232,
      "logps/chosen": -6.190374851226807,
      "logps/rejected": -51.07488250732422,
      "loss": 0.33069220185279846,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22484685480594635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3744960129261017,
      "rewards/margins": 4.471968173980713,
      "rewards/rejected": -4.097471714019775,
      "step": 314,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8179162609542356,
      "grad_norm": 1.3219656944274902,
      "learning_rate": 0.00013752263557724088,
      "logits/chosen": -0.8476938009262085,
      "logits/rejected": -0.854124903678894,
      "logps/chosen": -4.460864543914795,
      "logps/rejected": -59.64604949951172,
      "loss": 0.24126547574996948,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18051305413246155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3259335160255432,
      "rewards/margins": 4.853468894958496,
      "rewards/rejected": -4.527535915374756,
      "step": 315,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8205128205128205,
      "grad_norm": 1.3838900327682495,
      "learning_rate": 0.00013712392549973813,
      "logits/chosen": -0.8475697636604309,
      "logits/rejected": -0.8508798480033875,
      "logps/chosen": -8.591952323913574,
      "logps/rejected": -54.92388153076172,
      "loss": 0.33699649572372437,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26192301511764526,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.45825159549713135,
      "rewards/margins": 4.5010480880737305,
      "rewards/rejected": -4.042796611785889,
      "step": 316,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8231093800714054,
      "grad_norm": 1.5406709909439087,
      "learning_rate": 0.00013672452974739278,
      "logits/chosen": -0.8334726095199585,
      "logits/rejected": -0.8387458324432373,
      "logps/chosen": -6.5857672691345215,
      "logps/rejected": -57.37644577026367,
      "loss": 0.3329037129878998,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24443216621875763,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26296567916870117,
      "rewards/margins": 4.921113014221191,
      "rewards/rejected": -4.658147811889648,
      "step": 317,
      "train_speed(iter/s)": 0.005333
    },
    {
      "epoch": 0.8257059396299903,
      "grad_norm": 1.1263025999069214,
      "learning_rate": 0.00013632445569700076,
      "logits/chosen": -0.8738721013069153,
      "logits/rejected": -0.8832994699478149,
      "logps/chosen": -4.713293552398682,
      "logps/rejected": -61.70130157470703,
      "loss": 0.24098971486091614,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19781509041786194,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3292510509490967,
      "rewards/margins": 5.506122589111328,
      "rewards/rejected": -5.176871299743652,
      "step": 318,
      "train_speed(iter/s)": 0.005334
    },
    {
      "epoch": 0.8283024991885751,
      "grad_norm": 1.8800857067108154,
      "learning_rate": 0.00013592371073788595,
      "logits/chosen": -0.8317723274230957,
      "logits/rejected": -0.8364999890327454,
      "logps/chosen": -9.805098533630371,
      "logps/rejected": -52.64741516113281,
      "loss": 0.5319538712501526,
      "memory(GiB)": 46.82,
      "nll_loss": 0.422235369682312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20960083603858948,
      "rewards/margins": 4.515375137329102,
      "rewards/rejected": -4.305774211883545,
      "step": 319,
      "train_speed(iter/s)": 0.005334
    },
    {
      "epoch": 0.83089905874716,
      "grad_norm": 1.1608284711837769,
      "learning_rate": 0.0001355223022717639,
      "logits/chosen": -0.8772802948951721,
      "logits/rejected": -0.8848594427108765,
      "logps/chosen": -5.158228397369385,
      "logps/rejected": -62.835357666015625,
      "loss": 0.24610216915607452,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2129330337047577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20435252785682678,
      "rewards/margins": 5.209543228149414,
      "rewards/rejected": -5.005191326141357,
      "step": 320,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8334956183057449,
      "grad_norm": 1.036731243133545,
      "learning_rate": 0.00013512023771260507,
      "logits/chosen": -0.877793550491333,
      "logits/rejected": -0.8856680989265442,
      "logps/chosen": -6.130776882171631,
      "logps/rejected": -65.56432342529297,
      "loss": 0.2860373854637146,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26745346188545227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4021628201007843,
      "rewards/margins": 5.478217124938965,
      "rewards/rejected": -5.076053619384766,
      "step": 321,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8360921778643298,
      "grad_norm": 1.8117263317108154,
      "learning_rate": 0.0001347175244864979,
      "logits/chosen": -0.8334808349609375,
      "logits/rejected": -0.8376457095146179,
      "logps/chosen": -11.004887580871582,
      "logps/rejected": -66.70048522949219,
      "loss": 0.4500907361507416,
      "memory(GiB)": 46.82,
      "nll_loss": 0.42771589756011963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33971378207206726,
      "rewards/margins": 5.777808666229248,
      "rewards/rejected": -5.4380950927734375,
      "step": 322,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8386887374229146,
      "grad_norm": 1.4639251232147217,
      "learning_rate": 0.00013431417003151162,
      "logits/chosen": -0.8392638564109802,
      "logits/rejected": -0.8471822142601013,
      "logps/chosen": -5.59528112411499,
      "logps/rejected": -63.85756301879883,
      "loss": 0.24815644323825836,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22107534110546112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43189477920532227,
      "rewards/margins": 5.589203834533691,
      "rewards/rejected": -5.157309055328369,
      "step": 323,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8412852969814996,
      "grad_norm": 3.701712131500244,
      "learning_rate": 0.00013391018179755886,
      "logits/chosen": -0.862777590751648,
      "logits/rejected": -0.863462507724762,
      "logps/chosen": -9.423402786254883,
      "logps/rejected": -57.09379577636719,
      "loss": 0.5075635313987732,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4723358750343323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17091917991638184,
      "rewards/margins": 4.957292079925537,
      "rewards/rejected": -4.786372661590576,
      "step": 324,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8438818565400844,
      "grad_norm": 3.2106587886810303,
      "learning_rate": 0.0001335055672462581,
      "logits/chosen": -0.8590080738067627,
      "logits/rejected": -0.8611211776733398,
      "logps/chosen": -9.565107345581055,
      "logps/rejected": -56.25175476074219,
      "loss": 0.7309077382087708,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5393710732460022,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.37465783953666687,
      "rewards/margins": 5.114628791809082,
      "rewards/rejected": -4.739970684051514,
      "step": 325,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8464784160986693,
      "grad_norm": 1.718408465385437,
      "learning_rate": 0.00013310033385079587,
      "logits/chosen": -0.8548775911331177,
      "logits/rejected": -0.8569875955581665,
      "logps/chosen": -6.972779273986816,
      "logps/rejected": -53.69463348388672,
      "loss": 0.30815446376800537,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2685162425041199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3408139646053314,
      "rewards/margins": 4.932272911071777,
      "rewards/rejected": -4.591458797454834,
      "step": 326,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8490749756572541,
      "grad_norm": 30.29808235168457,
      "learning_rate": 0.00013269448909578865,
      "logits/chosen": -0.8808773756027222,
      "logits/rejected": -0.8881189227104187,
      "logps/chosen": -7.769625663757324,
      "logps/rejected": -64.69181823730469,
      "loss": 0.4174491763114929,
      "memory(GiB)": 46.82,
      "nll_loss": 0.29413902759552,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1492745727300644,
      "rewards/margins": 5.491776466369629,
      "rewards/rejected": -5.342501640319824,
      "step": 327,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.851671535215839,
      "grad_norm": 1.610142469406128,
      "learning_rate": 0.00013228804047714463,
      "logits/chosen": -0.8723031878471375,
      "logits/rejected": -0.8802396059036255,
      "logps/chosen": -6.0019707679748535,
      "logps/rejected": -67.33113861083984,
      "loss": 0.2763366401195526,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25310149788856506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25009822845458984,
      "rewards/margins": 5.477874755859375,
      "rewards/rejected": -5.227776050567627,
      "step": 328,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8542680947744239,
      "grad_norm": 18.318506240844727,
      "learning_rate": 0.00013188099550192535,
      "logits/chosen": -0.8295834064483643,
      "logits/rejected": -0.838620662689209,
      "logps/chosen": -5.431742191314697,
      "logps/rejected": -62.13144302368164,
      "loss": 0.6254780292510986,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3839109241962433,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09875370562076569,
      "rewards/margins": 5.322996139526367,
      "rewards/rejected": -5.224242210388184,
      "step": 329,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8568646543330087,
      "grad_norm": 1.2672317028045654,
      "learning_rate": 0.000131473361688207,
      "logits/chosen": -0.8911533355712891,
      "logits/rejected": -0.8940534591674805,
      "logps/chosen": -5.854481220245361,
      "logps/rejected": -50.02248001098633,
      "loss": 0.34439149498939514,
      "memory(GiB)": 46.82,
      "nll_loss": 0.268149733543396,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32187986373901367,
      "rewards/margins": 4.458532810211182,
      "rewards/rejected": -4.136653423309326,
      "step": 330,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8594612138915937,
      "grad_norm": 1.345903754234314,
      "learning_rate": 0.00013106514656494148,
      "logits/chosen": -0.8426346182823181,
      "logits/rejected": -0.8419499397277832,
      "logps/chosen": -12.040822982788086,
      "logps/rejected": -51.06175231933594,
      "loss": 0.4810463786125183,
      "memory(GiB)": 46.82,
      "nll_loss": 0.42825689911842346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2871994376182556,
      "rewards/margins": 4.3635358810424805,
      "rewards/rejected": -4.076336860656738,
      "step": 331,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8620577734501785,
      "grad_norm": 7.434391975402832,
      "learning_rate": 0.00013065635767181747,
      "logits/chosen": -0.8796988129615784,
      "logits/rejected": -0.8907098174095154,
      "logps/chosen": -4.122412204742432,
      "logps/rejected": -64.0143051147461,
      "loss": 0.30808770656585693,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21978120505809784,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24143238365650177,
      "rewards/margins": 5.337109088897705,
      "rewards/rejected": -5.095676422119141,
      "step": 332,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8646543330087634,
      "grad_norm": 42.690364837646484,
      "learning_rate": 0.0001302470025591211,
      "logits/chosen": -0.8691973686218262,
      "logits/rejected": -0.8755657076835632,
      "logps/chosen": -6.238379001617432,
      "logps/rejected": -57.67242431640625,
      "loss": 0.5189653635025024,
      "memory(GiB)": 46.82,
      "nll_loss": 0.360885351896286,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26926398277282715,
      "rewards/margins": 4.828179836273193,
      "rewards/rejected": -4.558915615081787,
      "step": 333,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8672508925673482,
      "grad_norm": 8.9744873046875,
      "learning_rate": 0.00012983708878759655,
      "logits/chosen": -0.9184461832046509,
      "logits/rejected": -0.9249346256256104,
      "logps/chosen": -9.006379127502441,
      "logps/rejected": -58.630638122558594,
      "loss": 0.5170425772666931,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4528915584087372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3499585688114166,
      "rewards/margins": 4.760928153991699,
      "rewards/rejected": -4.4109697341918945,
      "step": 334,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8698474521259332,
      "grad_norm": 16.40105628967285,
      "learning_rate": 0.00012942662392830632,
      "logits/chosen": -0.8680149912834167,
      "logits/rejected": -0.8773718476295471,
      "logps/chosen": -4.468932151794434,
      "logps/rejected": -64.07333374023438,
      "loss": 0.29751056432724,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20176543295383453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.29612603783607483,
      "rewards/margins": 5.261467456817627,
      "rewards/rejected": -4.965341567993164,
      "step": 335,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.872444011684518,
      "grad_norm": 13.411872863769531,
      "learning_rate": 0.0001290156155624914,
      "logits/chosen": -0.8806618452072144,
      "logits/rejected": -0.8878685832023621,
      "logps/chosen": -9.750197410583496,
      "logps/rejected": -64.2704849243164,
      "loss": 0.44853177666664124,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30135855078697205,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0591825433075428,
      "rewards/margins": 5.037668228149414,
      "rewards/rejected": -4.978486061096191,
      "step": 336,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8750405712431029,
      "grad_norm": 1.5141712427139282,
      "learning_rate": 0.0001286040712814314,
      "logits/chosen": -0.9008548855781555,
      "logits/rejected": -0.9143273830413818,
      "logps/chosen": -4.234499931335449,
      "logps/rejected": -74.0651626586914,
      "loss": 0.2379544973373413,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2000884711742401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4334224462509155,
      "rewards/margins": 6.484922409057617,
      "rewards/rejected": -6.051499366760254,
      "step": 337,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8776371308016878,
      "grad_norm": 0.8521820306777954,
      "learning_rate": 0.0001281919986863042,
      "logits/chosen": -0.8969717025756836,
      "logits/rejected": -0.9046814441680908,
      "logps/chosen": -7.235101699829102,
      "logps/rejected": -77.14093017578125,
      "loss": 0.273301899433136,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2426755428314209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.320049524307251,
      "rewards/margins": 6.36985445022583,
      "rewards/rejected": -6.049805641174316,
      "step": 338,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8802336903602727,
      "grad_norm": 2.2374491691589355,
      "learning_rate": 0.00012777940538804544,
      "logits/chosen": -0.8919708132743835,
      "logits/rejected": -0.8931483626365662,
      "logps/chosen": -6.93287992477417,
      "logps/rejected": -61.636375427246094,
      "loss": 0.3472403883934021,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2957107722759247,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3734300136566162,
      "rewards/margins": 5.471829891204834,
      "rewards/rejected": -5.098400115966797,
      "step": 339,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8828302499188575,
      "grad_norm": 25.410560607910156,
      "learning_rate": 0.0001273662990072083,
      "logits/chosen": -0.8870843648910522,
      "logits/rejected": -0.8924081325531006,
      "logps/chosen": -7.52494478225708,
      "logps/rejected": -61.48851013183594,
      "loss": 0.7680293321609497,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5508996844291687,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.006981657352298498,
      "rewards/margins": 5.215488433837891,
      "rewards/rejected": -5.222470283508301,
      "step": 340,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8854268094774423,
      "grad_norm": 3.612856864929199,
      "learning_rate": 0.0001269526871738224,
      "logits/chosen": -0.9017181992530823,
      "logits/rejected": -0.906170666217804,
      "logps/chosen": -7.857571125030518,
      "logps/rejected": -63.468265533447266,
      "loss": 0.38538140058517456,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2516745924949646,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.179386705160141,
      "rewards/margins": 5.408325672149658,
      "rewards/rejected": -5.228938579559326,
      "step": 341,
      "train_speed(iter/s)": 0.005335
    },
    {
      "epoch": 0.8880233690360273,
      "grad_norm": 9.315306663513184,
      "learning_rate": 0.00012653857752725303,
      "logits/chosen": -0.88401198387146,
      "logits/rejected": -0.891493022441864,
      "logps/chosen": -7.122734546661377,
      "logps/rejected": -57.48167419433594,
      "loss": 0.5956651568412781,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4049210548400879,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09174284338951111,
      "rewards/margins": 4.561285972595215,
      "rewards/rejected": -4.469542503356934,
      "step": 342,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.8906199285946121,
      "grad_norm": 0.7824282050132751,
      "learning_rate": 0.00012612397771606014,
      "logits/chosen": -0.9107692837715149,
      "logits/rejected": -0.9192830324172974,
      "logps/chosen": -5.18141508102417,
      "logps/rejected": -63.883949279785156,
      "loss": 0.2276170253753662,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2009223848581314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2666219174861908,
      "rewards/margins": 5.457507610321045,
      "rewards/rejected": -5.190885543823242,
      "step": 343,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.893216488153197,
      "grad_norm": 1.105103850364685,
      "learning_rate": 0.0001257088953978568,
      "logits/chosen": -0.8562593460083008,
      "logits/rejected": -0.8662354350090027,
      "logps/chosen": -7.01981258392334,
      "logps/rejected": -60.08414077758789,
      "loss": 0.2912936210632324,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23564513027668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41656795144081116,
      "rewards/margins": 5.034234046936035,
      "rewards/rejected": -4.617666244506836,
      "step": 344,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.8958130477117819,
      "grad_norm": 3.4230287075042725,
      "learning_rate": 0.00012529333823916807,
      "logits/chosen": -0.8783853650093079,
      "logits/rejected": -0.8884921073913574,
      "logps/chosen": -4.188514232635498,
      "logps/rejected": -66.61878967285156,
      "loss": 0.4436146914958954,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2946454584598541,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18403561413288116,
      "rewards/margins": 5.612396240234375,
      "rewards/rejected": -5.428360939025879,
      "step": 345,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.8984096072703668,
      "grad_norm": 2.5396299362182617,
      "learning_rate": 0.00012487731391528918,
      "logits/chosen": -0.9642701745033264,
      "logits/rejected": -0.9674146175384521,
      "logps/chosen": -5.993198871612549,
      "logps/rejected": -60.531856536865234,
      "loss": 0.25893262028694153,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19595590233802795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26401805877685547,
      "rewards/margins": 5.092806339263916,
      "rewards/rejected": -4.828787803649902,
      "step": 346,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9010061668289516,
      "grad_norm": 2.3769822120666504,
      "learning_rate": 0.00012446083011014388,
      "logits/chosen": -0.9396705627441406,
      "logits/rejected": -0.9417209029197693,
      "logps/chosen": -7.586000919342041,
      "logps/rejected": -61.09663391113281,
      "loss": 0.36653196811676025,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26543620228767395,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2201276272535324,
      "rewards/margins": 5.081995010375977,
      "rewards/rejected": -4.8618669509887695,
      "step": 347,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9036027263875365,
      "grad_norm": 3.2100534439086914,
      "learning_rate": 0.00012404389451614253,
      "logits/chosen": -0.9353513121604919,
      "logits/rejected": -0.9401172995567322,
      "logps/chosen": -6.526828765869141,
      "logps/rejected": -56.59849548339844,
      "loss": 0.4251266419887543,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35948801040649414,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3049776554107666,
      "rewards/margins": 4.886538028717041,
      "rewards/rejected": -4.581560134887695,
      "step": 348,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9061992859461214,
      "grad_norm": 22.74829864501953,
      "learning_rate": 0.00012362651483403985,
      "logits/chosen": -0.9125022888183594,
      "logits/rejected": -0.9109125733375549,
      "logps/chosen": -8.987406730651855,
      "logps/rejected": -49.98797607421875,
      "loss": 0.5128880739212036,
      "memory(GiB)": 46.82,
      "nll_loss": 0.43219777941703796,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27204346656799316,
      "rewards/margins": 4.296665191650391,
      "rewards/rejected": -4.024621486663818,
      "step": 349,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9087958455047063,
      "grad_norm": 2.0788931846618652,
      "learning_rate": 0.00012320869877279297,
      "logits/chosen": -0.9203197956085205,
      "logits/rejected": -0.9223061800003052,
      "logps/chosen": -10.433500289916992,
      "logps/rejected": -62.36846160888672,
      "loss": 0.4062730073928833,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35132896900177,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24331118166446686,
      "rewards/margins": 5.01633882522583,
      "rewards/rejected": -4.773027420043945,
      "step": 350,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9113924050632911,
      "grad_norm": 1.2080000638961792,
      "learning_rate": 0.0001227904540494188,
      "logits/chosen": -0.9349120855331421,
      "logits/rejected": -0.9338538646697998,
      "logps/chosen": -8.405929565429688,
      "logps/rejected": -57.93353271484375,
      "loss": 0.366677850484848,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31156450510025024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25386035442352295,
      "rewards/margins": 5.119736671447754,
      "rewards/rejected": -4.865876197814941,
      "step": 351,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.9139889646218761,
      "grad_norm": 5.341267108917236,
      "learning_rate": 0.00012237178838885168,
      "logits/chosen": -0.9456449747085571,
      "logits/rejected": -0.9499503970146179,
      "logps/chosen": -6.785584926605225,
      "logps/rejected": -63.56637954711914,
      "loss": 0.3164372742176056,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21375565230846405,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3775550425052643,
      "rewards/margins": 5.603601455688477,
      "rewards/rejected": -5.226045608520508,
      "step": 352,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.9165855241804609,
      "grad_norm": 20.04754066467285,
      "learning_rate": 0.00012195270952380051,
      "logits/chosen": -0.9753867387771606,
      "logits/rejected": -0.9728374481201172,
      "logps/chosen": -8.288665771484375,
      "logps/rejected": -50.586734771728516,
      "loss": 0.45580485463142395,
      "memory(GiB)": 46.82,
      "nll_loss": 0.38007235527038574,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4403877258300781,
      "rewards/margins": 4.540890216827393,
      "rewards/rejected": -4.100502967834473,
      "step": 353,
      "train_speed(iter/s)": 0.005336
    },
    {
      "epoch": 0.9191820837390458,
      "grad_norm": 1.5026851892471313,
      "learning_rate": 0.0001215332251946061,
      "logits/chosen": -0.9055343866348267,
      "logits/rejected": -0.9097537398338318,
      "logps/chosen": -8.293628692626953,
      "logps/rejected": -65.0361099243164,
      "loss": 0.3375437557697296,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2872999310493469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3178735375404358,
      "rewards/margins": 5.573904991149902,
      "rewards/rejected": -5.2560319900512695,
      "step": 354,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9217786432976306,
      "grad_norm": 9.46897029876709,
      "learning_rate": 0.0001211133431490981,
      "logits/chosen": -0.9408826231956482,
      "logits/rejected": -0.9440443515777588,
      "logps/chosen": -7.679222583770752,
      "logps/rejected": -58.07322311401367,
      "loss": 0.4588177502155304,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3633183538913727,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.35548433661460876,
      "rewards/margins": 5.1678056716918945,
      "rewards/rejected": -4.812321186065674,
      "step": 355,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9243752028562155,
      "grad_norm": 2.6942434310913086,
      "learning_rate": 0.00012069307114245196,
      "logits/chosen": -0.9773699045181274,
      "logits/rejected": -0.9839785099029541,
      "logps/chosen": -6.199317932128906,
      "logps/rejected": -51.811275482177734,
      "loss": 0.33174929022789,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20409320294857025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2277924120426178,
      "rewards/margins": 3.9625682830810547,
      "rewards/rejected": -3.734776020050049,
      "step": 356,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9269717624148004,
      "grad_norm": 1.542070746421814,
      "learning_rate": 0.00012027241693704567,
      "logits/chosen": -0.9564674496650696,
      "logits/rejected": -0.958985447883606,
      "logps/chosen": -4.848186016082764,
      "logps/rejected": -64.11177062988281,
      "loss": 0.2734638750553131,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22212821245193481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2529650330543518,
      "rewards/margins": 5.414011478424072,
      "rewards/rejected": -5.161045074462891,
      "step": 357,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9295683219733852,
      "grad_norm": 1.618640422821045,
      "learning_rate": 0.00011985138830231637,
      "logits/chosen": -0.952854573726654,
      "logits/rejected": -0.9557262063026428,
      "logps/chosen": -7.695744514465332,
      "logps/rejected": -64.7623062133789,
      "loss": 0.27667006850242615,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2487485110759735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18947187066078186,
      "rewards/margins": 5.626814365386963,
      "rewards/rejected": -5.437342166900635,
      "step": 358,
      "train_speed(iter/s)": 0.005337
    },
    {
      "epoch": 0.9321648815319702,
      "grad_norm": 9.904494285583496,
      "learning_rate": 0.00011942999301461693,
      "logits/chosen": -0.9912068247795105,
      "logits/rejected": -1.0118963718414307,
      "logps/chosen": -6.654687881469727,
      "logps/rejected": -68.99776458740234,
      "loss": 0.3964228332042694,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27639806270599365,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22217164933681488,
      "rewards/margins": 5.643857479095459,
      "rewards/rejected": -5.421686172485352,
      "step": 359,
      "train_speed(iter/s)": 0.005338
    },
    {
      "epoch": 0.934761441090555,
      "grad_norm": 2.499595880508423,
      "learning_rate": 0.00011900823885707215,
      "logits/chosen": -0.9873682260513306,
      "logits/rejected": -0.9983680248260498,
      "logps/chosen": -7.571791172027588,
      "logps/rejected": -69.42076110839844,
      "loss": 0.6393591165542603,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4515572786331177,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.013363976031541824,
      "rewards/margins": 5.555707931518555,
      "rewards/rejected": -5.542343616485596,
      "step": 360,
      "train_speed(iter/s)": 0.005338
    },
    {
      "epoch": 0.9373580006491399,
      "grad_norm": 5.06946325302124,
      "learning_rate": 0.00011858613361943518,
      "logits/chosen": -0.9797115325927734,
      "logits/rejected": -0.9833052754402161,
      "logps/chosen": -7.266734600067139,
      "logps/rejected": -66.09844207763672,
      "loss": 0.27793920040130615,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2126695215702057,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08487731218338013,
      "rewards/margins": 5.504128456115723,
      "rewards/rejected": -5.419251441955566,
      "step": 361,
      "train_speed(iter/s)": 0.005338
    },
    {
      "epoch": 0.9399545602077247,
      "grad_norm": 1.6952695846557617,
      "learning_rate": 0.00011816368509794364,
      "logits/chosen": -0.9289021492004395,
      "logits/rejected": -0.933040201663971,
      "logps/chosen": -8.65612506866455,
      "logps/rejected": -65.69309997558594,
      "loss": 0.3674156665802002,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3314497768878937,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4269205629825592,
      "rewards/margins": 5.734676837921143,
      "rewards/rejected": -5.307756423950195,
      "step": 362,
      "train_speed(iter/s)": 0.005338
    },
    {
      "epoch": 0.9425511197663097,
      "grad_norm": 1.0782965421676636,
      "learning_rate": 0.00011774090109517551,
      "logits/chosen": -0.946650505065918,
      "logits/rejected": -0.9464154243469238,
      "logps/chosen": -5.612125396728516,
      "logps/rejected": -56.009986877441406,
      "loss": 0.30003514885902405,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2715224027633667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2884294390678406,
      "rewards/margins": 5.06643009185791,
      "rewards/rejected": -4.778000354766846,
      "step": 363,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 0.9451476793248945,
      "grad_norm": 7.0804009437561035,
      "learning_rate": 0.00011731778941990496,
      "logits/chosen": -0.9121062755584717,
      "logits/rejected": -0.9164551496505737,
      "logps/chosen": -9.16533088684082,
      "logps/rejected": -54.61601257324219,
      "loss": 0.5863555073738098,
      "memory(GiB)": 46.82,
      "nll_loss": 0.43649864196777344,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23763269186019897,
      "rewards/margins": 4.418910026550293,
      "rewards/rejected": -4.181277275085449,
      "step": 364,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.9477442388834794,
      "grad_norm": 1.0685062408447266,
      "learning_rate": 0.00011689435788695844,
      "logits/chosen": -0.9457142353057861,
      "logits/rejected": -0.946734607219696,
      "logps/chosen": -7.535189151763916,
      "logps/rejected": -57.889957427978516,
      "loss": 0.3278339207172394,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3113463819026947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2534429132938385,
      "rewards/margins": 5.122234344482422,
      "rewards/rejected": -4.868791103363037,
      "step": 365,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.9503407984420643,
      "grad_norm": 18.325923919677734,
      "learning_rate": 0.00011647061431707,
      "logits/chosen": -0.9664883017539978,
      "logits/rejected": -0.9706634283065796,
      "logps/chosen": -8.00462818145752,
      "logps/rejected": -58.485496520996094,
      "loss": 0.5252708792686462,
      "memory(GiB)": 46.82,
      "nll_loss": 0.40657514333724976,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16244688630104065,
      "rewards/margins": 4.640269756317139,
      "rewards/rejected": -4.477822303771973,
      "step": 366,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.9529373580006492,
      "grad_norm": 1.7476553916931152,
      "learning_rate": 0.00011604656653673707,
      "logits/chosen": -0.9371016025543213,
      "logits/rejected": -0.9420011639595032,
      "logps/chosen": -8.489301681518555,
      "logps/rejected": -60.067012786865234,
      "loss": 0.2864355146884918,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2481742799282074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3987981975078583,
      "rewards/margins": 5.001021862030029,
      "rewards/rejected": -4.602223873138428,
      "step": 367,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.955533917559234,
      "grad_norm": 0.7240554094314575,
      "learning_rate": 0.0001156222223780757,
      "logits/chosen": -0.9764981865882874,
      "logits/rejected": -0.9823321104049683,
      "logps/chosen": -3.561066150665283,
      "logps/rejected": -64.69168853759766,
      "loss": 0.15193597972393036,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13965152204036713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31303656101226807,
      "rewards/margins": 5.628165245056152,
      "rewards/rejected": -5.315129280090332,
      "step": 368,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.9581304771178188,
      "grad_norm": 0.8999217748641968,
      "learning_rate": 0.00011519758967867606,
      "logits/chosen": -0.9562562704086304,
      "logits/rejected": -0.9621536731719971,
      "logps/chosen": -4.751138210296631,
      "logps/rejected": -66.24119567871094,
      "loss": 0.19877935945987701,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17447549104690552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2585357129573822,
      "rewards/margins": 5.8691558837890625,
      "rewards/rejected": -5.610620498657227,
      "step": 369,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 0.9607270366764038,
      "grad_norm": 7.604972839355469,
      "learning_rate": 0.00011477267628145776,
      "logits/chosen": -0.9861749410629272,
      "logits/rejected": -0.9912722706794739,
      "logps/chosen": -7.531174182891846,
      "logps/rejected": -59.80250549316406,
      "loss": 0.43909671902656555,
      "memory(GiB)": 46.82,
      "nll_loss": 0.34136056900024414,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14466743171215057,
      "rewards/margins": 4.875586986541748,
      "rewards/rejected": -4.730918884277344,
      "step": 370,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 0.9633235962349886,
      "grad_norm": 1.2200195789337158,
      "learning_rate": 0.00011434749003452467,
      "logits/chosen": -1.0024657249450684,
      "logits/rejected": -1.0117108821868896,
      "logps/chosen": -4.926345348358154,
      "logps/rejected": -64.41602325439453,
      "loss": 0.24037624895572662,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19864444434642792,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3361244201660156,
      "rewards/margins": 5.382260322570801,
      "rewards/rejected": -5.046136379241943,
      "step": 371,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 0.9659201557935735,
      "grad_norm": 1.4470072984695435,
      "learning_rate": 0.00011392203879102025,
      "logits/chosen": -0.9764466881752014,
      "logits/rejected": -0.984937846660614,
      "logps/chosen": -4.395660400390625,
      "logps/rejected": -72.33343505859375,
      "loss": 0.21519267559051514,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16430063545703888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4711214303970337,
      "rewards/margins": 6.096909046173096,
      "rewards/rejected": -5.625787734985352,
      "step": 372,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 0.9685167153521584,
      "grad_norm": 3.041203737258911,
      "learning_rate": 0.00011349633040898247,
      "logits/chosen": -1.0063599348068237,
      "logits/rejected": -1.015335202217102,
      "logps/chosen": -5.668739318847656,
      "logps/rejected": -67.33605194091797,
      "loss": 0.2572421133518219,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1989455670118332,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4027881622314453,
      "rewards/margins": 5.818880558013916,
      "rewards/rejected": -5.416092872619629,
      "step": 373,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 0.9711132749107433,
      "grad_norm": 6.189421653747559,
      "learning_rate": 0.00011307037275119853,
      "logits/chosen": -1.072676181793213,
      "logits/rejected": -1.0726059675216675,
      "logps/chosen": -11.18874454498291,
      "logps/rejected": -57.087642669677734,
      "loss": 0.6298056840896606,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4115663468837738,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14192688465118408,
      "rewards/margins": 4.86482572555542,
      "rewards/rejected": -4.722898483276367,
      "step": 374,
      "train_speed(iter/s)": 0.005342
    },
    {
      "epoch": 0.9737098344693281,
      "grad_norm": 7.813536167144775,
      "learning_rate": 0.00011264417368505981,
      "logits/chosen": -0.9870294332504272,
      "logits/rejected": -0.9871564507484436,
      "logps/chosen": -7.496246337890625,
      "logps/rejected": -55.7673454284668,
      "loss": 0.5474833846092224,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4691420793533325,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2661711573600769,
      "rewards/margins": 4.8272881507873535,
      "rewards/rejected": -4.561116695404053,
      "step": 375,
      "train_speed(iter/s)": 0.005342
    },
    {
      "epoch": 0.976306394027913,
      "grad_norm": 1.4118865728378296,
      "learning_rate": 0.00011221774108241645,
      "logits/chosen": -1.0178087949752808,
      "logits/rejected": -1.0260872840881348,
      "logps/chosen": -4.35853910446167,
      "logps/rejected": -63.377872467041016,
      "loss": 0.2507343292236328,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17132747173309326,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22918272018432617,
      "rewards/margins": 5.319854736328125,
      "rewards/rejected": -5.090672016143799,
      "step": 376,
      "train_speed(iter/s)": 0.005342
    },
    {
      "epoch": 0.9789029535864979,
      "grad_norm": 1.3914563655853271,
      "learning_rate": 0.0001117910828194319,
      "logits/chosen": -1.004896640777588,
      "logits/rejected": -1.000983715057373,
      "logps/chosen": -7.156683444976807,
      "logps/rejected": -54.51406478881836,
      "loss": 0.30924680829048157,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25739455223083496,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24743176996707916,
      "rewards/margins": 4.668411731719971,
      "rewards/rejected": -4.4209794998168945,
      "step": 377,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9814995131450828,
      "grad_norm": 2.6997230052948,
      "learning_rate": 0.00011136420677643762,
      "logits/chosen": -1.02091646194458,
      "logits/rejected": -1.0249969959259033,
      "logps/chosen": -5.773136615753174,
      "logps/rejected": -64.33428955078125,
      "loss": 0.4387259781360626,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31376269459724426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2982335686683655,
      "rewards/margins": 5.575780391693115,
      "rewards/rejected": -5.2775468826293945,
      "step": 378,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9840960727036676,
      "grad_norm": 2.8990633487701416,
      "learning_rate": 0.00011093712083778746,
      "logits/chosen": -1.0018008947372437,
      "logits/rejected": -1.0064812898635864,
      "logps/chosen": -6.614377021789551,
      "logps/rejected": -67.86730194091797,
      "loss": 0.309373140335083,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2491704821586609,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.311954140663147,
      "rewards/margins": 5.89847469329834,
      "rewards/rejected": -5.586519718170166,
      "step": 379,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9866926322622526,
      "grad_norm": 1.8598626852035522,
      "learning_rate": 0.00011050983289171195,
      "logits/chosen": -1.0007565021514893,
      "logits/rejected": -1.0053094625473022,
      "logps/chosen": -4.953035354614258,
      "logps/rejected": -60.55445861816406,
      "loss": 0.22382761538028717,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1699100136756897,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2609115540981293,
      "rewards/margins": 5.4445085525512695,
      "rewards/rejected": -5.183596134185791,
      "step": 380,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9892891918208374,
      "grad_norm": 2.610196828842163,
      "learning_rate": 0.00011008235083017271,
      "logits/chosen": -1.0573437213897705,
      "logits/rejected": -1.059203028678894,
      "logps/chosen": -5.614582538604736,
      "logps/rejected": -67.13617706298828,
      "loss": 0.32841193675994873,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21658550202846527,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1962202787399292,
      "rewards/margins": 5.73972225189209,
      "rewards/rejected": -5.543501853942871,
      "step": 381,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9918857513794223,
      "grad_norm": 2.123706102371216,
      "learning_rate": 0.00010965468254871669,
      "logits/chosen": -1.0279793739318848,
      "logits/rejected": -1.027879238128662,
      "logps/chosen": -7.853454113006592,
      "logps/rejected": -57.6713752746582,
      "loss": 0.3230496346950531,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2656661570072174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1339038908481598,
      "rewards/margins": 4.969094276428223,
      "rewards/rejected": -4.835190296173096,
      "step": 382,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9944823109380071,
      "grad_norm": 1.48650062084198,
      "learning_rate": 0.00010922683594633021,
      "logits/chosen": -1.0191617012023926,
      "logits/rejected": -1.0311510562896729,
      "logps/chosen": -5.478486061096191,
      "logps/rejected": -63.874629974365234,
      "loss": 0.2811015844345093,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2152327299118042,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2903039753437042,
      "rewards/margins": 5.215958118438721,
      "rewards/rejected": -4.92565393447876,
      "step": 383,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.997078870496592,
      "grad_norm": 6.09173059463501,
      "learning_rate": 0.00010879881892529324,
      "logits/chosen": -1.013049602508545,
      "logits/rejected": -1.0167346000671387,
      "logps/chosen": -6.224278450012207,
      "logps/rejected": -59.02324295043945,
      "loss": 0.29393669962882996,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21503983438014984,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43577679991722107,
      "rewards/margins": 4.863895416259766,
      "rewards/rejected": -4.428118705749512,
      "step": 384,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 0.9996754300551769,
      "grad_norm": 3.398437976837158,
      "learning_rate": 0.00010837063939103332,
      "logits/chosen": -1.006939172744751,
      "logits/rejected": -1.0076408386230469,
      "logps/chosen": -7.99960470199585,
      "logps/rejected": -55.3328971862793,
      "loss": 0.586876392364502,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4625084102153778,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.061562374234199524,
      "rewards/margins": 4.382445812225342,
      "rewards/rejected": -4.320883750915527,
      "step": 385,
      "train_speed(iter/s)": 0.005343
    },
    {
      "epoch": 1.0,
      "grad_norm": 3.398437976837158,
      "learning_rate": 0.0001079423052519796,
      "logits/chosen": -1.0445483922958374,
      "logits/rejected": -1.067384958267212,
      "logps/chosen": -0.5868608355522156,
      "logps/rejected": -66.0391616821289,
      "loss": 0.007936220616102219,
      "memory(GiB)": 46.82,
      "nll_loss": 0.04896550625562668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21240469813346863,
      "rewards/margins": 5.356934070587158,
      "rewards/rejected": -5.144529819488525,
      "step": 386,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0025965595585848,
      "grad_norm": 19.228408813476562,
      "learning_rate": 0.00010751382441941678,
      "logits/chosen": -0.9817072749137878,
      "logits/rejected": -0.9876554012298584,
      "logps/chosen": -10.516595840454102,
      "logps/rejected": -58.05045700073242,
      "loss": 0.6290792226791382,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4879079759120941,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6398650407791138,
      "rewards/margins": 4.8145246505737305,
      "rewards/rejected": -4.174659729003906,
      "step": 387,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0051931191171697,
      "grad_norm": 20.29606819152832,
      "learning_rate": 0.00010708520480733894,
      "logits/chosen": -0.9528878331184387,
      "logits/rejected": -0.9571166038513184,
      "logps/chosen": -5.756232738494873,
      "logps/rejected": -55.34486389160156,
      "loss": 0.36920851469039917,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3217727243900299,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43254968523979187,
      "rewards/margins": 4.792313575744629,
      "rewards/rejected": -4.3597636222839355,
      "step": 388,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0077896786757545,
      "grad_norm": 1.2253527641296387,
      "learning_rate": 0.00010665645433230344,
      "logits/chosen": -0.963991105556488,
      "logits/rejected": -0.9693760275840759,
      "logps/chosen": -4.665340900421143,
      "logps/rejected": -49.67149353027344,
      "loss": 0.2663205564022064,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2073584645986557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5366206765174866,
      "rewards/margins": 4.278485298156738,
      "rewards/rejected": -3.7418649196624756,
      "step": 389,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 1.0103862382343396,
      "grad_norm": 1.7141112089157104,
      "learning_rate": 0.00010622758091328469,
      "logits/chosen": -0.9876523613929749,
      "logits/rejected": -0.9932205677032471,
      "logps/chosen": -5.421288967132568,
      "logps/rejected": -53.175601959228516,
      "loss": 0.2864997982978821,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22578980028629303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.650011420249939,
      "rewards/margins": 4.690655708312988,
      "rewards/rejected": -4.04064416885376,
      "step": 390,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0129827977929244,
      "grad_norm": 0.9784797430038452,
      "learning_rate": 0.00010579859247152772,
      "logits/chosen": -0.9702428579330444,
      "logits/rejected": -0.9770087599754333,
      "logps/chosen": -3.717294692993164,
      "logps/rejected": -54.88421630859375,
      "loss": 0.18811215460300446,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15398232638835907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5122168660163879,
      "rewards/margins": 4.824542999267578,
      "rewards/rejected": -4.312326431274414,
      "step": 391,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 1.0155793573515093,
      "grad_norm": 1.4829105138778687,
      "learning_rate": 0.00010536949693040224,
      "logits/chosen": -0.9901302456855774,
      "logits/rejected": -0.9919001460075378,
      "logps/chosen": -4.703402519226074,
      "logps/rejected": -46.348514556884766,
      "loss": 0.23413625359535217,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1849011927843094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5978916883468628,
      "rewards/margins": 4.413217544555664,
      "rewards/rejected": -3.8153257369995117,
      "step": 392,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 1.0181759169100941,
      "grad_norm": 0.8022202253341675,
      "learning_rate": 0.00010494030221525581,
      "logits/chosen": -1.0340824127197266,
      "logits/rejected": -1.040837049484253,
      "logps/chosen": -6.395625114440918,
      "logps/rejected": -58.842002868652344,
      "loss": 0.23915542662143707,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2224937230348587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25003036856651306,
      "rewards/margins": 4.86290168762207,
      "rewards/rejected": -4.612870693206787,
      "step": 393,
      "train_speed(iter/s)": 0.005356
    },
    {
      "epoch": 1.020772476468679,
      "grad_norm": 2.875991106033325,
      "learning_rate": 0.00010451101625326798,
      "logits/chosen": -1.0399117469787598,
      "logits/rejected": -1.0477368831634521,
      "logps/chosen": -5.4347615242004395,
      "logps/rejected": -62.94850158691406,
      "loss": 0.2218306064605713,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18997295200824738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5420814752578735,
      "rewards/margins": 5.414422988891602,
      "rewards/rejected": -4.872342109680176,
      "step": 394,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0233690360272638,
      "grad_norm": 2.076349973678589,
      "learning_rate": 0.00010408164697330347,
      "logits/chosen": -1.0666667222976685,
      "logits/rejected": -1.072896122932434,
      "logps/chosen": -4.46807336807251,
      "logps/rejected": -63.26573944091797,
      "loss": 0.25682011246681213,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2303263247013092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3944869935512543,
      "rewards/margins": 5.512956619262695,
      "rewards/rejected": -5.118469715118408,
      "step": 395,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0259655955858487,
      "grad_norm": 0.7809150815010071,
      "learning_rate": 0.0001036522023057659,
      "logits/chosen": -1.0214073657989502,
      "logits/rejected": -1.0335733890533447,
      "logps/chosen": -6.993215560913086,
      "logps/rejected": -76.31700134277344,
      "loss": 0.2290063351392746,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21447491645812988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6252080202102661,
      "rewards/margins": 6.392251014709473,
      "rewards/rejected": -5.767043113708496,
      "step": 396,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0285621551444337,
      "grad_norm": 0.9348388910293579,
      "learning_rate": 0.00010322269018245128,
      "logits/chosen": -1.0821815729141235,
      "logits/rejected": -1.09281587600708,
      "logps/chosen": -5.995283126831055,
      "logps/rejected": -68.71944427490234,
      "loss": 0.2534785866737366,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23875081539154053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42260050773620605,
      "rewards/margins": 6.026902198791504,
      "rewards/rejected": -5.604302406311035,
      "step": 397,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0311587147030186,
      "grad_norm": 2.3803582191467285,
      "learning_rate": 0.00010279311853640156,
      "logits/chosen": -1.0678625106811523,
      "logits/rejected": -1.0740766525268555,
      "logps/chosen": -4.724799633026123,
      "logps/rejected": -59.92235565185547,
      "loss": 0.26597142219543457,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21491315960884094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3383842706680298,
      "rewards/margins": 5.239107131958008,
      "rewards/rejected": -4.900722503662109,
      "step": 398,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0337552742616034,
      "grad_norm": 0.9540591835975647,
      "learning_rate": 0.00010236349530175806,
      "logits/chosen": -1.101372480392456,
      "logits/rejected": -1.1012152433395386,
      "logps/chosen": -8.496014595031738,
      "logps/rejected": -63.654239654541016,
      "loss": 0.2421647608280182,
      "memory(GiB)": 46.82,
      "nll_loss": 0.229741632938385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5306934118270874,
      "rewards/margins": 5.865250587463379,
      "rewards/rejected": -5.33455753326416,
      "step": 399,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.0363518338201883,
      "grad_norm": 38.899085998535156,
      "learning_rate": 0.0001019338284136149,
      "logits/chosen": -1.141862154006958,
      "logits/rejected": -1.1475955247879028,
      "logps/chosen": -7.130688667297363,
      "logps/rejected": -71.88350677490234,
      "loss": 0.2930022180080414,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2670814096927643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2902993857860565,
      "rewards/margins": 5.992380142211914,
      "rewards/rejected": -5.702080726623535,
      "step": 400,
      "train_speed(iter/s)": 0.005355
    },
    {
      "epoch": 1.038948393378773,
      "grad_norm": 2.7088816165924072,
      "learning_rate": 0.0001015041258078725,
      "logits/chosen": -1.1285897493362427,
      "logits/rejected": -1.1310758590698242,
      "logps/chosen": -5.603566646575928,
      "logps/rejected": -61.33863067626953,
      "loss": 0.3561743497848511,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2987910509109497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5551941990852356,
      "rewards/margins": 5.580060005187988,
      "rewards/rejected": -5.024865627288818,
      "step": 401,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.041544952937358,
      "grad_norm": 0.8148896098136902,
      "learning_rate": 0.00010107439542109096,
      "logits/chosen": -1.121638298034668,
      "logits/rejected": -1.1334351301193237,
      "logps/chosen": -3.9921107292175293,
      "logps/rejected": -73.31179809570312,
      "loss": 0.18677955865859985,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1822265237569809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5457329750061035,
      "rewards/margins": 6.892066955566406,
      "rewards/rejected": -6.346334457397461,
      "step": 402,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.0441415124959428,
      "grad_norm": 1.029624104499817,
      "learning_rate": 0.00010064464519034358,
      "logits/chosen": -1.1337131261825562,
      "logits/rejected": -1.1473417282104492,
      "logps/chosen": -4.090713024139404,
      "logps/rejected": -77.2343978881836,
      "loss": 0.21103273332118988,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18134111166000366,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30427786707878113,
      "rewards/margins": 6.670842170715332,
      "rewards/rejected": -6.366564750671387,
      "step": 403,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.0467380720545278,
      "grad_norm": 1.2710298299789429,
      "learning_rate": 0.00010021488305307002,
      "logits/chosen": -1.1442415714263916,
      "logits/rejected": -1.1480284929275513,
      "logps/chosen": -4.857927322387695,
      "logps/rejected": -64.70890045166016,
      "loss": 0.24422866106033325,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21266308426856995,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46621280908584595,
      "rewards/margins": 5.662542343139648,
      "rewards/rejected": -5.1963300704956055,
      "step": 404,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0493346316131127,
      "grad_norm": 0.7117511034011841,
      "learning_rate": 9.978511694692999e-05,
      "logits/chosen": -1.1403015851974487,
      "logits/rejected": -1.1483004093170166,
      "logps/chosen": -6.501082897186279,
      "logps/rejected": -68.161865234375,
      "loss": 0.2804066836833954,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22950297594070435,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26065731048583984,
      "rewards/margins": 5.946385383605957,
      "rewards/rejected": -5.685728073120117,
      "step": 405,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0519311911716975,
      "grad_norm": 1.8634769916534424,
      "learning_rate": 9.935535480965648e-05,
      "logits/chosen": -1.174145221710205,
      "logits/rejected": -1.1830925941467285,
      "logps/chosen": -5.915862083435059,
      "logps/rejected": -81.09744262695312,
      "loss": 0.18793915212154388,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18054337799549103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49257692694664,
      "rewards/margins": 6.884129047393799,
      "rewards/rejected": -6.391551971435547,
      "step": 406,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0545277507302824,
      "grad_norm": 2.999732732772827,
      "learning_rate": 9.892560457890906e-05,
      "logits/chosen": -1.1771910190582275,
      "logits/rejected": -1.182723045349121,
      "logps/chosen": -4.092665672302246,
      "logps/rejected": -75.03519439697266,
      "loss": 0.2703268527984619,
      "memory(GiB)": 46.82,
      "nll_loss": 0.217498779296875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5273224115371704,
      "rewards/margins": 6.906276226043701,
      "rewards/rejected": -6.378953456878662,
      "step": 407,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0571243102888672,
      "grad_norm": 1.1191706657409668,
      "learning_rate": 9.849587419212751e-05,
      "logits/chosen": -1.1794843673706055,
      "logits/rejected": -1.1921828985214233,
      "logps/chosen": -2.9631309509277344,
      "logps/rejected": -65.27130126953125,
      "loss": 0.15160685777664185,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12793675065040588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5282881855964661,
      "rewards/margins": 6.006821155548096,
      "rewards/rejected": -5.478532791137695,
      "step": 408,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.059720869847452,
      "grad_norm": 1.0475082397460938,
      "learning_rate": 9.806617158638515e-05,
      "logits/chosen": -1.2007383108139038,
      "logits/rejected": -1.2024704217910767,
      "logps/chosen": -7.0153350830078125,
      "logps/rejected": -73.69622039794922,
      "loss": 0.22654809057712555,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21019339561462402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3381541073322296,
      "rewards/margins": 6.361845970153809,
      "rewards/rejected": -6.023691654205322,
      "step": 409,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.062317429406037,
      "grad_norm": 0.9155303835868835,
      "learning_rate": 9.763650469824196e-05,
      "logits/chosen": -1.1522904634475708,
      "logits/rejected": -1.1580644845962524,
      "logps/chosen": -7.180323600769043,
      "logps/rejected": -70.96688842773438,
      "loss": 0.23980437219142914,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23318149149417877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4015149474143982,
      "rewards/margins": 6.299011707305908,
      "rewards/rejected": -5.897497177124023,
      "step": 410,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.064913988964622,
      "grad_norm": 10.754075050354004,
      "learning_rate": 9.720688146359843e-05,
      "logits/chosen": -1.189253568649292,
      "logits/rejected": -1.1925952434539795,
      "logps/chosen": -9.017038345336914,
      "logps/rejected": -68.37117767333984,
      "loss": 0.346275269985199,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2631103992462158,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2936975359916687,
      "rewards/margins": 6.014129161834717,
      "rewards/rejected": -5.720431327819824,
      "step": 411,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0675105485232068,
      "grad_norm": 5.042325019836426,
      "learning_rate": 9.677730981754876e-05,
      "logits/chosen": -1.2106777429580688,
      "logits/rejected": -1.2027477025985718,
      "logps/chosen": -10.420255661010742,
      "logps/rejected": -61.35327911376953,
      "loss": 0.35904085636138916,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3351891040802002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5399136543273926,
      "rewards/margins": 5.419702529907227,
      "rewards/rejected": -4.879788875579834,
      "step": 412,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0701071080817917,
      "grad_norm": 1.102660894393921,
      "learning_rate": 9.63477976942341e-05,
      "logits/chosen": -1.2155516147613525,
      "logits/rejected": -1.2208374738693237,
      "logps/chosen": -4.927253723144531,
      "logps/rejected": -68.66384887695312,
      "loss": 0.2387477606534958,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20795047283172607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48150452971458435,
      "rewards/margins": 6.018568992614746,
      "rewards/rejected": -5.537064552307129,
      "step": 413,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0727036676403765,
      "grad_norm": 2.5885398387908936,
      "learning_rate": 9.591835302669657e-05,
      "logits/chosen": -1.2256488800048828,
      "logits/rejected": -1.2182636260986328,
      "logps/chosen": -7.875158309936523,
      "logps/rejected": -61.765113830566406,
      "loss": 0.28974008560180664,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25022780895233154,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4521024823188782,
      "rewards/margins": 5.778402328491211,
      "rewards/rejected": -5.326300144195557,
      "step": 414,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0753002271989613,
      "grad_norm": 2.834430456161499,
      "learning_rate": 9.548898374673204e-05,
      "logits/chosen": -1.2157726287841797,
      "logits/rejected": -1.220966100692749,
      "logps/chosen": -3.8334553241729736,
      "logps/rejected": -61.09181213378906,
      "loss": 0.41990533471107483,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3681584596633911,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19060057401657104,
      "rewards/margins": 5.299651622772217,
      "rewards/rejected": -5.109050750732422,
      "step": 415,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0778967867575462,
      "grad_norm": 1.1838082075119019,
      "learning_rate": 9.505969778474417e-05,
      "logits/chosen": -1.2442048788070679,
      "logits/rejected": -1.2560878992080688,
      "logps/chosen": -6.319258689880371,
      "logps/rejected": -69.6029052734375,
      "loss": 0.3231492340564728,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30189916491508484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43841037154197693,
      "rewards/margins": 6.017553329467773,
      "rewards/rejected": -5.5791425704956055,
      "step": 416,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.080493346316131,
      "grad_norm": 1.357805609703064,
      "learning_rate": 9.463050306959781e-05,
      "logits/chosen": -1.2448838949203491,
      "logits/rejected": -1.2514715194702148,
      "logps/chosen": -4.3993000984191895,
      "logps/rejected": -73.56219482421875,
      "loss": 0.1892472356557846,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16478529572486877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26193705201148987,
      "rewards/margins": 6.293513298034668,
      "rewards/rejected": -6.031576156616211,
      "step": 417,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.083089905874716,
      "grad_norm": 0.948700487613678,
      "learning_rate": 9.42014075284723e-05,
      "logits/chosen": -1.2140638828277588,
      "logits/rejected": -1.2203437089920044,
      "logps/chosen": -5.3918375968933105,
      "logps/rejected": -79.55841064453125,
      "loss": 0.1978967785835266,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18335530161857605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5024500489234924,
      "rewards/margins": 6.943933963775635,
      "rewards/rejected": -6.441484451293945,
      "step": 418,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.085686465433301,
      "grad_norm": 0.8673838376998901,
      "learning_rate": 9.377241908671532e-05,
      "logits/chosen": -1.2066748142242432,
      "logits/rejected": -1.2199989557266235,
      "logps/chosen": -2.639554738998413,
      "logps/rejected": -72.2052001953125,
      "loss": 0.1263066977262497,
      "memory(GiB)": 46.82,
      "nll_loss": 0.09473444521427155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.511016845703125,
      "rewards/margins": 6.307302474975586,
      "rewards/rejected": -5.796286106109619,
      "step": 419,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.0882830249918858,
      "grad_norm": 6.662075996398926,
      "learning_rate": 9.334354566769658e-05,
      "logits/chosen": -1.2155200242996216,
      "logits/rejected": -1.2136273384094238,
      "logps/chosen": -5.998470306396484,
      "logps/rejected": -68.34226989746094,
      "loss": 0.24206891655921936,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17550605535507202,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3885684013366699,
      "rewards/margins": 6.167290210723877,
      "rewards/rejected": -5.778721809387207,
      "step": 420,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.0908795845504706,
      "grad_norm": 3.4863972663879395,
      "learning_rate": 9.291479519266108e-05,
      "logits/chosen": -1.1982496976852417,
      "logits/rejected": -1.2094392776489258,
      "logps/chosen": -4.311284065246582,
      "logps/rejected": -67.43600463867188,
      "loss": 0.24855858087539673,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19574138522148132,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.373134583234787,
      "rewards/margins": 5.80324125289917,
      "rewards/rejected": -5.430107116699219,
      "step": 421,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0934761441090555,
      "grad_norm": 3.9711451530456543,
      "learning_rate": 9.248617558058327e-05,
      "logits/chosen": -1.1784768104553223,
      "logits/rejected": -1.190112590789795,
      "logps/chosen": -5.879142761230469,
      "logps/rejected": -73.49214935302734,
      "loss": 0.22467440366744995,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2147173136472702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48001062870025635,
      "rewards/margins": 6.742798805236816,
      "rewards/rejected": -6.26278829574585,
      "step": 422,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.0960727036676403,
      "grad_norm": 1.5909618139266968,
      "learning_rate": 9.205769474802044e-05,
      "logits/chosen": -1.225572943687439,
      "logits/rejected": -1.237328052520752,
      "logps/chosen": -3.7763729095458984,
      "logps/rejected": -71.20807647705078,
      "loss": 0.17804725468158722,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1526719480752945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4742431342601776,
      "rewards/margins": 6.401832103729248,
      "rewards/rejected": -5.92758846282959,
      "step": 423,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.0986692632262252,
      "grad_norm": 0.8102360963821411,
      "learning_rate": 9.16293606089667e-05,
      "logits/chosen": -1.240356683731079,
      "logits/rejected": -1.2470406293869019,
      "logps/chosen": -4.268908500671387,
      "logps/rejected": -65.2350845336914,
      "loss": 0.19071687757968903,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17185813188552856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41862109303474426,
      "rewards/margins": 5.682591438293457,
      "rewards/rejected": -5.263970375061035,
      "step": 424,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1012658227848102,
      "grad_norm": 0.9778416156768799,
      "learning_rate": 9.120118107470679e-05,
      "logits/chosen": -1.2163119316101074,
      "logits/rejected": -1.2221122980117798,
      "logps/chosen": -4.372983455657959,
      "logps/rejected": -78.68803405761719,
      "loss": 0.20959261059761047,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19133754074573517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.60094153881073,
      "rewards/margins": 7.121860504150391,
      "rewards/rejected": -6.520918846130371,
      "step": 425,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.103862382343395,
      "grad_norm": 0.983422040939331,
      "learning_rate": 9.077316405366981e-05,
      "logits/chosen": -1.2027230262756348,
      "logits/rejected": -1.2142469882965088,
      "logps/chosen": -5.299243927001953,
      "logps/rejected": -76.34408569335938,
      "loss": 0.22110909223556519,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20182488858699799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34696075320243835,
      "rewards/margins": 6.32121467590332,
      "rewards/rejected": -5.9742536544799805,
      "step": 426,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.10645894190198,
      "grad_norm": 2.735224723815918,
      "learning_rate": 9.034531745128334e-05,
      "logits/chosen": -1.226122498512268,
      "logits/rejected": -1.2391126155853271,
      "logps/chosen": -3.4195706844329834,
      "logps/rejected": -81.26631927490234,
      "loss": 0.18169909715652466,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14818009734153748,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.37102988362312317,
      "rewards/margins": 7.223745346069336,
      "rewards/rejected": -6.852715492248535,
      "step": 427,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1090555014605648,
      "grad_norm": 0.81003338098526,
      "learning_rate": 8.991764916982731e-05,
      "logits/chosen": -1.2129402160644531,
      "logits/rejected": -1.2193183898925781,
      "logps/chosen": -6.524440765380859,
      "logps/rejected": -79.71651458740234,
      "loss": 0.18506552278995514,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17972829937934875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5361291170120239,
      "rewards/margins": 7.205081939697266,
      "rewards/rejected": -6.668951988220215,
      "step": 428,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1116520610191496,
      "grad_norm": 1.204606056213379,
      "learning_rate": 8.949016710828808e-05,
      "logits/chosen": -1.2344969511032104,
      "logits/rejected": -1.2368295192718506,
      "logps/chosen": -6.601044178009033,
      "logps/rejected": -65.16004943847656,
      "loss": 0.29616430401802063,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26787662506103516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29541534185409546,
      "rewards/margins": 5.769859313964844,
      "rewards/rejected": -5.4744439125061035,
      "step": 429,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1142486205777344,
      "grad_norm": 1.5188134908676147,
      "learning_rate": 8.906287916221259e-05,
      "logits/chosen": -1.2113444805145264,
      "logits/rejected": -1.2227927446365356,
      "logps/chosen": -6.77754545211792,
      "logps/rejected": -77.72196960449219,
      "loss": 0.2043219655752182,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19175611436367035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29821890592575073,
      "rewards/margins": 6.7826690673828125,
      "rewards/rejected": -6.484450340270996,
      "step": 430,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1168451801363193,
      "grad_norm": 56.287654876708984,
      "learning_rate": 8.863579322356242e-05,
      "logits/chosen": -1.2380510568618774,
      "logits/rejected": -1.2490745782852173,
      "logps/chosen": -4.15383768081665,
      "logps/rejected": -79.29361724853516,
      "loss": 0.2723255157470703,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2262512445449829,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.42212826013565063,
      "rewards/margins": 6.8273210525512695,
      "rewards/rejected": -6.405192852020264,
      "step": 431,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1194417396949043,
      "grad_norm": 0.6230031251907349,
      "learning_rate": 8.820891718056814e-05,
      "logits/chosen": -1.1983201503753662,
      "logits/rejected": -1.2093560695648193,
      "logps/chosen": -2.4044957160949707,
      "logps/rejected": -79.75895690917969,
      "loss": 0.13464465737342834,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11272096633911133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4173327684402466,
      "rewards/margins": 7.142539978027344,
      "rewards/rejected": -6.7252068519592285,
      "step": 432,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1220382992534892,
      "grad_norm": 4.727012634277344,
      "learning_rate": 8.77822589175836e-05,
      "logits/chosen": -1.200883388519287,
      "logits/rejected": -1.2039451599121094,
      "logps/chosen": -6.484433174133301,
      "logps/rejected": -62.913082122802734,
      "loss": 0.3451792001724243,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25801709294319153,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5044211149215698,
      "rewards/margins": 5.68193244934082,
      "rewards/rejected": -5.1775102615356445,
      "step": 433,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.124634858812074,
      "grad_norm": 0.9477587342262268,
      "learning_rate": 8.73558263149402e-05,
      "logits/chosen": -1.2326602935791016,
      "logits/rejected": -1.2371294498443604,
      "logps/chosen": -3.9643776416778564,
      "logps/rejected": -64.6446304321289,
      "loss": 0.17973466217517853,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16231758892536163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5142005681991577,
      "rewards/margins": 5.624721050262451,
      "rewards/rejected": -5.110519886016846,
      "step": 434,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1272314183706589,
      "grad_norm": 1.0718938112258911,
      "learning_rate": 8.692962724880148e-05,
      "logits/chosen": -1.1947612762451172,
      "logits/rejected": -1.1951274871826172,
      "logps/chosen": -11.542988777160645,
      "logps/rejected": -57.942604064941406,
      "loss": 0.4030612111091614,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3767915964126587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47994565963745117,
      "rewards/margins": 5.252399444580078,
      "rewards/rejected": -4.772454261779785,
      "step": 435,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1298279779292437,
      "grad_norm": 1.3139375448226929,
      "learning_rate": 8.650366959101757e-05,
      "logits/chosen": -1.2127829790115356,
      "logits/rejected": -1.2069275379180908,
      "logps/chosen": -7.090031623840332,
      "logps/rejected": -60.13610076904297,
      "loss": 0.2780110538005829,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25617653131484985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38807040452957153,
      "rewards/margins": 5.502678871154785,
      "rewards/rejected": -5.1146087646484375,
      "step": 436,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1324245374878286,
      "grad_norm": 2.8402469158172607,
      "learning_rate": 8.607796120897977e-05,
      "logits/chosen": -1.1895822286605835,
      "logits/rejected": -1.2009150981903076,
      "logps/chosen": -5.18463134765625,
      "logps/rejected": -76.09211730957031,
      "loss": 0.2019975334405899,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1788557916879654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44426459074020386,
      "rewards/margins": 6.382430553436279,
      "rewards/rejected": -5.938166618347168,
      "step": 437,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1350210970464134,
      "grad_norm": 0.8108645677566528,
      "learning_rate": 8.565250996547538e-05,
      "logits/chosen": -1.1975806951522827,
      "logits/rejected": -1.2050762176513672,
      "logps/chosen": -5.465963363647461,
      "logps/rejected": -73.16693878173828,
      "loss": 0.21301648020744324,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1857440173625946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45735448598861694,
      "rewards/margins": 6.357089042663574,
      "rewards/rejected": -5.8997344970703125,
      "step": 438,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1376176566049985,
      "grad_norm": 1.0116618871688843,
      "learning_rate": 8.522732371854228e-05,
      "logits/chosen": -1.2085466384887695,
      "logits/rejected": -1.211836814880371,
      "logps/chosen": -5.572229385375977,
      "logps/rejected": -65.51385498046875,
      "loss": 0.21719352900981903,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18132297694683075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5146795511245728,
      "rewards/margins": 5.9277167320251465,
      "rewards/rejected": -5.413036346435547,
      "step": 439,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1402142161635833,
      "grad_norm": 0.6072871088981628,
      "learning_rate": 8.480241032132393e-05,
      "logits/chosen": -1.2081148624420166,
      "logits/rejected": -1.2092756032943726,
      "logps/chosen": -4.102461814880371,
      "logps/rejected": -69.6110610961914,
      "loss": 0.17899087071418762,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1723383665084839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5367128849029541,
      "rewards/margins": 6.382415294647217,
      "rewards/rejected": -5.845702648162842,
      "step": 440,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1428107757221682,
      "grad_norm": 1.1503797769546509,
      "learning_rate": 8.437777762192434e-05,
      "logits/chosen": -1.1785545349121094,
      "logits/rejected": -1.1898421049118042,
      "logps/chosen": -5.658975601196289,
      "logps/rejected": -66.4476089477539,
      "loss": 0.2457386553287506,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22795403003692627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5527963638305664,
      "rewards/margins": 6.060329437255859,
      "rewards/rejected": -5.507533073425293,
      "step": 441,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.145407335280753,
      "grad_norm": 1.063011884689331,
      "learning_rate": 8.395343346326295e-05,
      "logits/chosen": -1.2125415802001953,
      "logits/rejected": -1.2166345119476318,
      "logps/chosen": -11.403976440429688,
      "logps/rejected": -65.69091033935547,
      "loss": 0.3279325067996979,
      "memory(GiB)": 46.82,
      "nll_loss": 0.31457963585853577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48648983240127563,
      "rewards/margins": 5.955573558807373,
      "rewards/rejected": -5.469082832336426,
      "step": 442,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1480038948393378,
      "grad_norm": 15.399893760681152,
      "learning_rate": 8.352938568292999e-05,
      "logits/chosen": -1.212915062904358,
      "logits/rejected": -1.2257941961288452,
      "logps/chosen": -5.4974541664123535,
      "logps/rejected": -74.6421890258789,
      "loss": 0.43098506331443787,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3471585512161255,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2913598418235779,
      "rewards/margins": 6.395876407623291,
      "rewards/rejected": -6.10451602935791,
      "step": 443,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1506004543979227,
      "grad_norm": 0.8971589803695679,
      "learning_rate": 8.310564211304158e-05,
      "logits/chosen": -1.2243199348449707,
      "logits/rejected": -1.233266830444336,
      "logps/chosen": -4.356855869293213,
      "logps/rejected": -82.03324127197266,
      "loss": 0.16546428203582764,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15822210907936096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4632646441459656,
      "rewards/margins": 7.142735481262207,
      "rewards/rejected": -6.679471015930176,
      "step": 444,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1531970139565075,
      "grad_norm": 0.9893987774848938,
      "learning_rate": 8.268221058009506e-05,
      "logits/chosen": -1.1960678100585938,
      "logits/rejected": -1.1931755542755127,
      "logps/chosen": -5.763791084289551,
      "logps/rejected": -67.48243713378906,
      "loss": 0.22159884870052338,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20868626236915588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44276654720306396,
      "rewards/margins": 6.342441558837891,
      "rewards/rejected": -5.899675369262695,
      "step": 445,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1557935735150926,
      "grad_norm": 0.9730236530303955,
      "learning_rate": 8.225909890482456e-05,
      "logits/chosen": -1.2410675287246704,
      "logits/rejected": -1.2531105279922485,
      "logps/chosen": -3.777689218521118,
      "logps/rejected": -73.96142578125,
      "loss": 0.17882992327213287,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15262357890605927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30238115787506104,
      "rewards/margins": 6.5273051261901855,
      "rewards/rejected": -6.224924087524414,
      "step": 446,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.1583901330736774,
      "grad_norm": 0.9676756262779236,
      "learning_rate": 8.183631490205637e-05,
      "logits/chosen": -1.2334588766098022,
      "logits/rejected": -1.23115074634552,
      "logps/chosen": -5.909482002258301,
      "logps/rejected": -71.83121490478516,
      "loss": 0.21972762048244476,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20232652127742767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3557296097278595,
      "rewards/margins": 6.261346340179443,
      "rewards/rejected": -5.905617713928223,
      "step": 447,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.1609866926322623,
      "grad_norm": 3.615070104598999,
      "learning_rate": 8.141386638056481e-05,
      "logits/chosen": -1.1984878778457642,
      "logits/rejected": -1.2202084064483643,
      "logps/chosen": -7.201460838317871,
      "logps/rejected": -83.67745208740234,
      "loss": 0.3116104006767273,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25579264760017395,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5771140456199646,
      "rewards/margins": 7.238218784332275,
      "rewards/rejected": -6.661106109619141,
      "step": 448,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.1635832521908471,
      "grad_norm": 5.897024631500244,
      "learning_rate": 8.099176114292788e-05,
      "logits/chosen": -1.2283486127853394,
      "logits/rejected": -1.2268238067626953,
      "logps/chosen": -4.299000263214111,
      "logps/rejected": -63.7210578918457,
      "loss": 0.3152792155742645,
      "memory(GiB)": 46.82,
      "nll_loss": 0.25136417150497437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5095170140266418,
      "rewards/margins": 6.158134937286377,
      "rewards/rejected": -5.648618221282959,
      "step": 449,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.166179811749432,
      "grad_norm": 0.8267269730567932,
      "learning_rate": 8.05700069853831e-05,
      "logits/chosen": -1.2181183099746704,
      "logits/rejected": -1.232823371887207,
      "logps/chosen": -2.3422133922576904,
      "logps/rejected": -81.08943176269531,
      "loss": 0.1261310577392578,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11953935027122498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5450170040130615,
      "rewards/margins": 7.200981140136719,
      "rewards/rejected": -6.655963897705078,
      "step": 450,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.1687763713080168,
      "grad_norm": 0.7152145504951477,
      "learning_rate": 8.014861169768363e-05,
      "logits/chosen": -1.2072569131851196,
      "logits/rejected": -1.2154910564422607,
      "logps/chosen": -5.663619041442871,
      "logps/rejected": -83.33345031738281,
      "loss": 0.19898222386837006,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18870678544044495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6240676045417786,
      "rewards/margins": 7.413023948669434,
      "rewards/rejected": -6.788956165313721,
      "step": 451,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1713729308666017,
      "grad_norm": 1.1054271459579468,
      "learning_rate": 7.972758306295435e-05,
      "logits/chosen": -1.2423603534698486,
      "logits/rejected": -1.238939642906189,
      "logps/chosen": -9.10930347442627,
      "logps/rejected": -68.92269897460938,
      "loss": 0.2644210159778595,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23959574103355408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6885038018226624,
      "rewards/margins": 6.226180076599121,
      "rewards/rejected": -5.537675857543945,
      "step": 452,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1739694904251867,
      "grad_norm": 0.8297804594039917,
      "learning_rate": 7.930692885754806e-05,
      "logits/chosen": -1.2389461994171143,
      "logits/rejected": -1.2466444969177246,
      "logps/chosen": -4.285374641418457,
      "logps/rejected": -81.12661743164062,
      "loss": 0.20591987669467926,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18140685558319092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5226142406463623,
      "rewards/margins": 7.35651159286499,
      "rewards/rejected": -6.833897590637207,
      "step": 453,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1765660499837716,
      "grad_norm": 1.7889838218688965,
      "learning_rate": 7.888665685090193e-05,
      "logits/chosen": -1.2282047271728516,
      "logits/rejected": -1.240710973739624,
      "logps/chosen": -3.9668195247650146,
      "logps/rejected": -83.85064697265625,
      "loss": 0.3006521463394165,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2842242121696472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5261533856391907,
      "rewards/margins": 7.802785873413086,
      "rewards/rejected": -7.276631832122803,
      "step": 454,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1791626095423564,
      "grad_norm": 0.6768118143081665,
      "learning_rate": 7.846677480539392e-05,
      "logits/chosen": -1.2407575845718384,
      "logits/rejected": -1.2590632438659668,
      "logps/chosen": -3.9896676540374756,
      "logps/rejected": -88.54475402832031,
      "loss": 0.14948198199272156,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14015856385231018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5012010335922241,
      "rewards/margins": 7.759642601013184,
      "rewards/rejected": -7.25844144821167,
      "step": 455,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1817591691009413,
      "grad_norm": 1.0748811960220337,
      "learning_rate": 7.804729047619948e-05,
      "logits/chosen": -1.2838971614837646,
      "logits/rejected": -1.2972068786621094,
      "logps/chosen": -4.9451680183410645,
      "logps/rejected": -75.96924591064453,
      "loss": 0.19334405660629272,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17472317814826965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.43368232250213623,
      "rewards/margins": 6.9603376388549805,
      "rewards/rejected": -6.526655197143555,
      "step": 456,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.184355728659526,
      "grad_norm": 27.02585792541504,
      "learning_rate": 7.762821161114833e-05,
      "logits/chosen": -1.2738770246505737,
      "logits/rejected": -1.2871417999267578,
      "logps/chosen": -6.898077964782715,
      "logps/rejected": -70.33586883544922,
      "loss": 0.3962981700897217,
      "memory(GiB)": 46.82,
      "nll_loss": 0.30386969447135925,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.49996471405029297,
      "rewards/margins": 6.2457451820373535,
      "rewards/rejected": -5.745780944824219,
      "step": 457,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.186952288218111,
      "grad_norm": 0.9774232506752014,
      "learning_rate": 7.720954595058118e-05,
      "logits/chosen": -1.270230770111084,
      "logits/rejected": -1.283980131149292,
      "logps/chosen": -3.3140833377838135,
      "logps/rejected": -72.56609344482422,
      "loss": 0.1822766661643982,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13324937224388123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31882578134536743,
      "rewards/margins": 6.227068901062012,
      "rewards/rejected": -5.908243179321289,
      "step": 458,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1895488477766958,
      "grad_norm": 0.9652191400527954,
      "learning_rate": 7.679130122720704e-05,
      "logits/chosen": -1.3034790754318237,
      "logits/rejected": -1.314261555671692,
      "logps/chosen": -7.06810998916626,
      "logps/rejected": -81.12928771972656,
      "loss": 0.2216591238975525,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20493057370185852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4113575220108032,
      "rewards/margins": 6.991024971008301,
      "rewards/rejected": -6.579666614532471,
      "step": 459,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1921454073352808,
      "grad_norm": 4.3938422203063965,
      "learning_rate": 7.637348516596016e-05,
      "logits/chosen": -1.288078784942627,
      "logits/rejected": -1.289923071861267,
      "logps/chosen": -6.353469371795654,
      "logps/rejected": -73.09895324707031,
      "loss": 0.275894433259964,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2141137421131134,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43534383177757263,
      "rewards/margins": 6.68284797668457,
      "rewards/rejected": -6.247504234313965,
      "step": 460,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1947419668938657,
      "grad_norm": 0.9771380424499512,
      "learning_rate": 7.59561054838575e-05,
      "logits/chosen": -1.3042765855789185,
      "logits/rejected": -1.317743182182312,
      "logps/chosen": -3.9304447174072266,
      "logps/rejected": -83.80009460449219,
      "loss": 0.1489650011062622,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1390589475631714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3365148901939392,
      "rewards/margins": 7.487484931945801,
      "rewards/rejected": -7.150969505310059,
      "step": 461,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1973385264524505,
      "grad_norm": 0.7796929478645325,
      "learning_rate": 7.553916988985613e-05,
      "logits/chosen": -1.3301423788070679,
      "logits/rejected": -1.3388516902923584,
      "logps/chosen": -3.854522943496704,
      "logps/rejected": -77.66817474365234,
      "loss": 0.16134774684906006,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1557087004184723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46156132221221924,
      "rewards/margins": 7.119222164154053,
      "rewards/rejected": -6.657661437988281,
      "step": 462,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.1999350860110354,
      "grad_norm": 0.8544324636459351,
      "learning_rate": 7.512268608471083e-05,
      "logits/chosen": -1.3370078802108765,
      "logits/rejected": -1.3487505912780762,
      "logps/chosen": -4.693976879119873,
      "logps/rejected": -82.53611755371094,
      "loss": 0.1619998812675476,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15308500826358795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5657608509063721,
      "rewards/margins": 7.620563507080078,
      "rewards/rejected": -7.054802417755127,
      "step": 463,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.2025316455696202,
      "grad_norm": 1.2491400241851807,
      "learning_rate": 7.470666176083192e-05,
      "logits/chosen": -1.3235511779785156,
      "logits/rejected": -1.3217864036560059,
      "logps/chosen": -5.475301265716553,
      "logps/rejected": -66.96241760253906,
      "loss": 0.22019821405410767,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2117622047662735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5525124669075012,
      "rewards/margins": 6.302184104919434,
      "rewards/rejected": -5.749671936035156,
      "step": 464,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.205128205128205,
      "grad_norm": 4.391453266143799,
      "learning_rate": 7.429110460214319e-05,
      "logits/chosen": -1.324596881866455,
      "logits/rejected": -1.3249270915985107,
      "logps/chosen": -5.006054401397705,
      "logps/rejected": -79.24930572509766,
      "loss": 0.19958798587322235,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17760564386844635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.529591977596283,
      "rewards/margins": 7.06871223449707,
      "rewards/rejected": -6.539119243621826,
      "step": 465,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.20772476468679,
      "grad_norm": 1.2282588481903076,
      "learning_rate": 7.387602228393987e-05,
      "logits/chosen": -1.3140517473220825,
      "logits/rejected": -1.3105956315994263,
      "logps/chosen": -7.42270565032959,
      "logps/rejected": -66.26721954345703,
      "loss": 0.294545441865921,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2685684859752655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6955150961875916,
      "rewards/margins": 6.379220008850098,
      "rewards/rejected": -5.683704853057861,
      "step": 466,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.210321324245375,
      "grad_norm": 0.9563242197036743,
      "learning_rate": 7.346142247274694e-05,
      "logits/chosen": -1.3215587139129639,
      "logits/rejected": -1.3361427783966064,
      "logps/chosen": -3.699913263320923,
      "logps/rejected": -88.9631118774414,
      "loss": 0.21111004054546356,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18570095300674438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40791285037994385,
      "rewards/margins": 8.11806583404541,
      "rewards/rejected": -7.710152626037598,
      "step": 467,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.2129178838039598,
      "grad_norm": 1.081046462059021,
      "learning_rate": 7.304731282617762e-05,
      "logits/chosen": -1.3312351703643799,
      "logits/rejected": -1.3387830257415771,
      "logps/chosen": -5.243213653564453,
      "logps/rejected": -84.92706298828125,
      "loss": 0.2157626897096634,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21122369170188904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4976283013820648,
      "rewards/margins": 8.005891799926758,
      "rewards/rejected": -7.508264064788818,
      "step": 468,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.2155144433625447,
      "grad_norm": 1.0331863164901733,
      "learning_rate": 7.263370099279172e-05,
      "logits/chosen": -1.3447762727737427,
      "logits/rejected": -1.3390498161315918,
      "logps/chosen": -6.393280982971191,
      "logps/rejected": -68.75172424316406,
      "loss": 0.2636359632015228,
      "memory(GiB)": 46.82,
      "nll_loss": 0.246793732047081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6458650827407837,
      "rewards/margins": 6.282301425933838,
      "rewards/rejected": -5.636436939239502,
      "step": 469,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.2181110029211295,
      "grad_norm": 0.8750536441802979,
      "learning_rate": 7.22205946119546e-05,
      "logits/chosen": -1.3133454322814941,
      "logits/rejected": -1.3110582828521729,
      "logps/chosen": -5.820354461669922,
      "logps/rejected": -71.29835510253906,
      "loss": 0.20814698934555054,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1895810067653656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5639092922210693,
      "rewards/margins": 6.466952323913574,
      "rewards/rejected": -5.903042793273926,
      "step": 470,
      "train_speed(iter/s)": 0.00535
    },
    {
      "epoch": 1.2207075624797143,
      "grad_norm": 1.3065314292907715,
      "learning_rate": 7.180800131369584e-05,
      "logits/chosen": -1.3447749614715576,
      "logits/rejected": -1.3477418422698975,
      "logps/chosen": -6.139316558837891,
      "logps/rejected": -62.90933609008789,
      "loss": 0.26562365889549255,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21039319038391113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.537449836730957,
      "rewards/margins": 5.653042793273926,
      "rewards/rejected": -5.115592956542969,
      "step": 471,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2233041220382992,
      "grad_norm": 1.267687439918518,
      "learning_rate": 7.13959287185686e-05,
      "logits/chosen": -1.3199489116668701,
      "logits/rejected": -1.3398852348327637,
      "logps/chosen": -2.706737995147705,
      "logps/rejected": -79.95001983642578,
      "loss": 0.12426899373531342,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11003828048706055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47334274649620056,
      "rewards/margins": 7.157402992248535,
      "rewards/rejected": -6.684060096740723,
      "step": 472,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.225900681596884,
      "grad_norm": 7.132215976715088,
      "learning_rate": 7.098438443750865e-05,
      "logits/chosen": -1.3689088821411133,
      "logits/rejected": -1.3883635997772217,
      "logps/chosen": -8.076455116271973,
      "logps/rejected": -82.17918395996094,
      "loss": 0.42827925086021423,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3502614498138428,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4545542001724243,
      "rewards/margins": 7.125252723693848,
      "rewards/rejected": -6.6706976890563965,
      "step": 473,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.228497241155469,
      "grad_norm": 0.8221337795257568,
      "learning_rate": 7.057337607169371e-05,
      "logits/chosen": -1.3182464838027954,
      "logits/rejected": -1.3349087238311768,
      "logps/chosen": -4.247696876525879,
      "logps/rejected": -82.20696258544922,
      "loss": 0.16310645639896393,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14582528173923492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3645007908344269,
      "rewards/margins": 7.333131313323975,
      "rewards/rejected": -6.968629837036133,
      "step": 474,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.231093800714054,
      "grad_norm": 0.9309591054916382,
      "learning_rate": 7.016291121240346e-05,
      "logits/chosen": -1.3715977668762207,
      "logits/rejected": -1.3699740171432495,
      "logps/chosen": -5.316465377807617,
      "logps/rejected": -68.14669799804688,
      "loss": 0.2627433240413666,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21229830384254456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5413371920585632,
      "rewards/margins": 6.536716461181641,
      "rewards/rejected": -5.995378494262695,
      "step": 475,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2336903602726388,
      "grad_norm": 8.09181022644043,
      "learning_rate": 6.975299744087891e-05,
      "logits/chosen": -1.3668107986450195,
      "logits/rejected": -1.371617078781128,
      "logps/chosen": -9.833568572998047,
      "logps/rejected": -82.02591705322266,
      "loss": 0.8664286136627197,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5704513192176819,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.256930947303772,
      "rewards/margins": 7.255249977111816,
      "rewards/rejected": -6.998319149017334,
      "step": 476,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2362869198312236,
      "grad_norm": 2.064409017562866,
      "learning_rate": 6.934364232818254e-05,
      "logits/chosen": -1.3650354146957397,
      "logits/rejected": -1.3771873712539673,
      "logps/chosen": -4.302821636199951,
      "logps/rejected": -75.79000854492188,
      "loss": 0.16696852445602417,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1503523886203766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4869020879268646,
      "rewards/margins": 6.73642110824585,
      "rewards/rejected": -6.249518871307373,
      "step": 477,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2388834793898085,
      "grad_norm": 0.798882782459259,
      "learning_rate": 6.893485343505857e-05,
      "logits/chosen": -1.3637698888778687,
      "logits/rejected": -1.3578245639801025,
      "logps/chosen": -5.138448715209961,
      "logps/rejected": -66.73672485351562,
      "loss": 0.21367499232292175,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1953822672367096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3032958507537842,
      "rewards/margins": 6.190895080566406,
      "rewards/rejected": -5.887599945068359,
      "step": 478,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2414800389483933,
      "grad_norm": 1.1406197547912598,
      "learning_rate": 6.852663831179302e-05,
      "logits/chosen": -1.352840781211853,
      "logits/rejected": -1.35397207736969,
      "logps/chosen": -4.986490249633789,
      "logps/rejected": -73.43693542480469,
      "loss": 0.205072820186615,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19722062349319458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5596895217895508,
      "rewards/margins": 6.6583757400512695,
      "rewards/rejected": -6.0986857414245605,
      "step": 479,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2440765985069782,
      "grad_norm": 0.766574501991272,
      "learning_rate": 6.811900449807465e-05,
      "logits/chosen": -1.3501691818237305,
      "logits/rejected": -1.3594365119934082,
      "logps/chosen": -6.533414363861084,
      "logps/rejected": -66.47720336914062,
      "loss": 0.23299582302570343,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22283174097537994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7748389840126038,
      "rewards/margins": 6.291479587554932,
      "rewards/rejected": -5.516641139984131,
      "step": 480,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2466731580655632,
      "grad_norm": 0.9920905828475952,
      "learning_rate": 6.77119595228554e-05,
      "logits/chosen": -1.3646950721740723,
      "logits/rejected": -1.3720245361328125,
      "logps/chosen": -6.916625022888184,
      "logps/rejected": -73.02135467529297,
      "loss": 0.23078814148902893,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21652816236019135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4459760785102844,
      "rewards/margins": 6.14084005355835,
      "rewards/rejected": -5.694863796234131,
      "step": 481,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.249269717624148,
      "grad_norm": 1.0885769128799438,
      "learning_rate": 6.730551090421137e-05,
      "logits/chosen": -1.3644369840621948,
      "logits/rejected": -1.3590939044952393,
      "logps/chosen": -7.648492813110352,
      "logps/rejected": -63.562782287597656,
      "loss": 0.24852637946605682,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21991106867790222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6735080480575562,
      "rewards/margins": 5.918203353881836,
      "rewards/rejected": -5.244695663452148,
      "step": 482,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.251866277182733,
      "grad_norm": 0.7712500691413879,
      "learning_rate": 6.689966614920413e-05,
      "logits/chosen": -1.3166533708572388,
      "logits/rejected": -1.3218377828598022,
      "logps/chosen": -4.022350788116455,
      "logps/rejected": -71.528076171875,
      "loss": 0.13269458711147308,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11788547039031982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5740154981613159,
      "rewards/margins": 6.363125324249268,
      "rewards/rejected": -5.789109230041504,
      "step": 483,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2544628367413178,
      "grad_norm": 0.8549020886421204,
      "learning_rate": 6.649443275374193e-05,
      "logits/chosen": -1.3651435375213623,
      "logits/rejected": -1.3780406713485718,
      "logps/chosen": -4.872490406036377,
      "logps/rejected": -68.51624298095703,
      "loss": 0.1938597708940506,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18332600593566895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5763934254646301,
      "rewards/margins": 5.976587295532227,
      "rewards/rejected": -5.400193691253662,
      "step": 484,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2570593962999026,
      "grad_norm": 0.9369620084762573,
      "learning_rate": 6.608981820244115e-05,
      "logits/chosen": -1.3296164274215698,
      "logits/rejected": -1.3332488536834717,
      "logps/chosen": -4.46505069732666,
      "logps/rejected": -64.79832458496094,
      "loss": 0.17813126742839813,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1691313236951828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5418381690979004,
      "rewards/margins": 5.926529407501221,
      "rewards/rejected": -5.38469123840332,
      "step": 485,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2596559558584874,
      "grad_norm": 0.844835102558136,
      "learning_rate": 6.568582996848843e-05,
      "logits/chosen": -1.3325531482696533,
      "logits/rejected": -1.3392637968063354,
      "logps/chosen": -4.008181571960449,
      "logps/rejected": -71.48811340332031,
      "loss": 0.17114686965942383,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15825404226779938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5624055862426758,
      "rewards/margins": 6.441608905792236,
      "rewards/rejected": -5.8792033195495605,
      "step": 486,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2622525154170723,
      "grad_norm": 1.130921721458435,
      "learning_rate": 6.528247551350213e-05,
      "logits/chosen": -1.3767224550247192,
      "logits/rejected": -1.3819544315338135,
      "logps/chosen": -3.62576961517334,
      "logps/rejected": -68.78441619873047,
      "loss": 0.21932178735733032,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19097000360488892,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4108361601829529,
      "rewards/margins": 6.289252758026123,
      "rewards/rejected": -5.878417015075684,
      "step": 487,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2648490749756571,
      "grad_norm": 1.452879786491394,
      "learning_rate": 6.487976228739494e-05,
      "logits/chosen": -1.3391304016113281,
      "logits/rejected": -1.356032371520996,
      "logps/chosen": -4.974707126617432,
      "logps/rejected": -70.37326049804688,
      "loss": 0.21740569174289703,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20510534942150116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5399238467216492,
      "rewards/margins": 6.117460250854492,
      "rewards/rejected": -5.577536106109619,
      "step": 488,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2674456345342422,
      "grad_norm": 0.7685065865516663,
      "learning_rate": 6.447769772823614e-05,
      "logits/chosen": -1.319055199623108,
      "logits/rejected": -1.3363419771194458,
      "logps/chosen": -6.615801811218262,
      "logps/rejected": -83.4671630859375,
      "loss": 0.21475572884082794,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18952563405036926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4121897220611572,
      "rewards/margins": 7.273510932922363,
      "rewards/rejected": -6.861320972442627,
      "step": 489,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.270042194092827,
      "grad_norm": 1.0484442710876465,
      "learning_rate": 6.407628926211409e-05,
      "logits/chosen": -1.33793044090271,
      "logits/rejected": -1.3429871797561646,
      "logps/chosen": -7.6060333251953125,
      "logps/rejected": -65.28913879394531,
      "loss": 0.2497866153717041,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24507665634155273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8890218734741211,
      "rewards/margins": 6.331787109375,
      "rewards/rejected": -5.442765235900879,
      "step": 490,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2726387536514119,
      "grad_norm": 0.9693198204040527,
      "learning_rate": 6.367554430299924e-05,
      "logits/chosen": -1.3228676319122314,
      "logits/rejected": -1.3340733051300049,
      "logps/chosen": -4.008767604827881,
      "logps/rejected": -71.55172729492188,
      "loss": 0.18880397081375122,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1601686179637909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4935658276081085,
      "rewards/margins": 6.400652885437012,
      "rewards/rejected": -5.907087326049805,
      "step": 491,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2752353132099967,
      "grad_norm": 1.0116583108901978,
      "learning_rate": 6.327547025260723e-05,
      "logits/chosen": -1.3240255117416382,
      "logits/rejected": -1.3320386409759521,
      "logps/chosen": -4.213266849517822,
      "logps/rejected": -72.59737396240234,
      "loss": 0.1900216042995453,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1726846843957901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48582398891448975,
      "rewards/margins": 6.556468963623047,
      "rewards/rejected": -6.070645332336426,
      "step": 492,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.2778318727685816,
      "grad_norm": 1.002138614654541,
      "learning_rate": 6.287607450026189e-05,
      "logits/chosen": -1.3404346704483032,
      "logits/rejected": -1.34886634349823,
      "logps/chosen": -4.731965065002441,
      "logps/rejected": -74.24478149414062,
      "loss": 0.1853921115398407,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1709199845790863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44357895851135254,
      "rewards/margins": 6.4645490646362305,
      "rewards/rejected": -6.020970344543457,
      "step": 493,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2804284323271666,
      "grad_norm": 0.8741234540939331,
      "learning_rate": 6.247736442275917e-05,
      "logits/chosen": -1.287043571472168,
      "logits/rejected": -1.2934644222259521,
      "logps/chosen": -4.63891077041626,
      "logps/rejected": -74.50043487548828,
      "loss": 0.19321085512638092,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1751416027545929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.582757830619812,
      "rewards/margins": 6.900094509124756,
      "rewards/rejected": -6.3173370361328125,
      "step": 494,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2830249918857515,
      "grad_norm": 0.6828720569610596,
      "learning_rate": 6.20793473842305e-05,
      "logits/chosen": -1.2890350818634033,
      "logits/rejected": -1.2999300956726074,
      "logps/chosen": -4.856447219848633,
      "logps/rejected": -72.6617431640625,
      "loss": 0.18545033037662506,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16962683200836182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5169156193733215,
      "rewards/margins": 6.635002136230469,
      "rewards/rejected": -6.118086338043213,
      "step": 495,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2856215514443363,
      "grad_norm": 1.0596202611923218,
      "learning_rate": 6.168203073600705e-05,
      "logits/chosen": -1.303747534751892,
      "logits/rejected": -1.3145846128463745,
      "logps/chosen": -5.57688045501709,
      "logps/rejected": -77.08624267578125,
      "loss": 0.19950714707374573,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1948622763156891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5272884368896484,
      "rewards/margins": 6.925128936767578,
      "rewards/rejected": -6.3978400230407715,
      "step": 496,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2882181110029212,
      "grad_norm": 1.029574990272522,
      "learning_rate": 6.128542181648394e-05,
      "logits/chosen": -1.3281769752502441,
      "logits/rejected": -1.3378801345825195,
      "logps/chosen": -5.10857629776001,
      "logps/rejected": -78.54777526855469,
      "loss": 0.2200167030096054,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21050511300563812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39650988578796387,
      "rewards/margins": 7.069237232208252,
      "rewards/rejected": -6.672727108001709,
      "step": 497,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.290814670561506,
      "grad_norm": 2.646754741668701,
      "learning_rate": 6.0889527950984416e-05,
      "logits/chosen": -1.303905963897705,
      "logits/rejected": -1.295387625694275,
      "logps/chosen": -6.794533729553223,
      "logps/rejected": -66.7017822265625,
      "loss": 0.412862628698349,
      "memory(GiB)": 46.82,
      "nll_loss": 0.35357925295829773,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3687395453453064,
      "rewards/margins": 6.016590118408203,
      "rewards/rejected": -5.647850036621094,
      "step": 498,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2934112301200908,
      "grad_norm": 0.8122525811195374,
      "learning_rate": 6.049435645162487e-05,
      "logits/chosen": -1.3104355335235596,
      "logits/rejected": -1.3152062892913818,
      "logps/chosen": -5.562468528747559,
      "logps/rejected": -74.46591186523438,
      "loss": 0.21052336692810059,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20442822575569153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6305204033851624,
      "rewards/margins": 7.020031452178955,
      "rewards/rejected": -6.3895111083984375,
      "step": 499,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2960077896786757,
      "grad_norm": 0.7511181831359863,
      "learning_rate": 6.0099914617179764e-05,
      "logits/chosen": -1.2803047895431519,
      "logits/rejected": -1.2894065380096436,
      "logps/chosen": -4.588591575622559,
      "logps/rejected": -85.19679260253906,
      "loss": 0.17056904733181,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1638702154159546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4681891202926636,
      "rewards/margins": 7.461175918579102,
      "rewards/rejected": -6.992986679077148,
      "step": 500,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.2986043492372605,
      "grad_norm": 0.708882749080658,
      "learning_rate": 5.970620973294649e-05,
      "logits/chosen": -1.2864353656768799,
      "logits/rejected": -1.2991129159927368,
      "logps/chosen": -2.7639682292938232,
      "logps/rejected": -77.82109832763672,
      "loss": 0.13476227223873138,
      "memory(GiB)": 46.82,
      "nll_loss": 0.129118874669075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4478917717933655,
      "rewards/margins": 6.971372604370117,
      "rewards/rejected": -6.5234808921813965,
      "step": 501,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3012009087958454,
      "grad_norm": 2.3029229640960693,
      "learning_rate": 5.931324907061131e-05,
      "logits/chosen": -1.2641024589538574,
      "logits/rejected": -1.2685556411743164,
      "logps/chosen": -5.792514324188232,
      "logps/rejected": -76.38786315917969,
      "loss": 0.4058936536312103,
      "memory(GiB)": 46.82,
      "nll_loss": 0.36178189516067505,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32265374064445496,
      "rewards/margins": 6.516972064971924,
      "rewards/rejected": -6.1943182945251465,
      "step": 502,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3037974683544304,
      "grad_norm": 0.9166151285171509,
      "learning_rate": 5.892103988811457e-05,
      "logits/chosen": -1.2319368124008179,
      "logits/rejected": -1.2350472211837769,
      "logps/chosen": -6.415447235107422,
      "logps/rejected": -75.25454711914062,
      "loss": 0.2667836844921112,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23221434652805328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6779787540435791,
      "rewards/margins": 6.834658145904541,
      "rewards/rejected": -6.156679153442383,
      "step": 503,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3063940279130153,
      "grad_norm": 6.748708248138428,
      "learning_rate": 5.8529589429517003e-05,
      "logits/chosen": -1.2778469324111938,
      "logits/rejected": -1.2807527780532837,
      "logps/chosen": -6.256143569946289,
      "logps/rejected": -72.61140441894531,
      "loss": 0.24257849156856537,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2350560873746872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6175004243850708,
      "rewards/margins": 6.710084438323975,
      "rewards/rejected": -6.092584609985352,
      "step": 504,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3089905874716001,
      "grad_norm": 0.8000685572624207,
      "learning_rate": 5.8138904924865766e-05,
      "logits/chosen": -1.2935757637023926,
      "logits/rejected": -1.3018581867218018,
      "logps/chosen": -4.087766647338867,
      "logps/rejected": -79.02437591552734,
      "loss": 0.2187802791595459,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19569718837738037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4645954966545105,
      "rewards/margins": 7.049061298370361,
      "rewards/rejected": -6.584465980529785,
      "step": 505,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.311587147030185,
      "grad_norm": 1.0485024452209473,
      "learning_rate": 5.774899359006092e-05,
      "logits/chosen": -1.2387014627456665,
      "logits/rejected": -1.2328811883926392,
      "logps/chosen": -5.889930248260498,
      "logps/rejected": -71.13211059570312,
      "loss": 0.20536896586418152,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18820105493068695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4390983283519745,
      "rewards/margins": 6.4834065437316895,
      "rewards/rejected": -6.044308185577393,
      "step": 506,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3141837065887698,
      "grad_norm": 1.3112807273864746,
      "learning_rate": 5.7359862626722106e-05,
      "logits/chosen": -1.221153736114502,
      "logits/rejected": -1.2322664260864258,
      "logps/chosen": -8.636886596679688,
      "logps/rejected": -80.09297180175781,
      "loss": 0.276731938123703,
      "memory(GiB)": 46.82,
      "nll_loss": 0.26180499792099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34714198112487793,
      "rewards/margins": 6.943133354187012,
      "rewards/rejected": -6.595992088317871,
      "step": 507,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3167802661473549,
      "grad_norm": 4.199530601501465,
      "learning_rate": 5.6971519222055745e-05,
      "logits/chosen": -1.2388005256652832,
      "logits/rejected": -1.2482101917266846,
      "logps/chosen": -3.7984771728515625,
      "logps/rejected": -67.59223937988281,
      "loss": 0.2313919961452484,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15446686744689941,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.46181103587150574,
      "rewards/margins": 6.2272748947143555,
      "rewards/rejected": -5.765463829040527,
      "step": 508,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3193768257059397,
      "grad_norm": 0.5734891891479492,
      "learning_rate": 5.658397054872197e-05,
      "logits/chosen": -1.2510944604873657,
      "logits/rejected": -1.2634743452072144,
      "logps/chosen": -2.7403945922851562,
      "logps/rejected": -83.09203338623047,
      "loss": 0.10001116245985031,
      "memory(GiB)": 46.82,
      "nll_loss": 0.09578342735767365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4184391498565674,
      "rewards/margins": 7.603847980499268,
      "rewards/rejected": -7.185409069061279,
      "step": 509,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3219733852645246,
      "grad_norm": 2.557685136795044,
      "learning_rate": 5.6197223764702376e-05,
      "logits/chosen": -1.2410025596618652,
      "logits/rejected": -1.2566806077957153,
      "logps/chosen": -5.59817361831665,
      "logps/rejected": -81.26180267333984,
      "loss": 0.2501680254936218,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22971931099891663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1663077175617218,
      "rewards/margins": 6.9138503074646,
      "rewards/rejected": -6.747541904449463,
      "step": 510,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3245699448231094,
      "grad_norm": 0.7034062743186951,
      "learning_rate": 5.581128601316774e-05,
      "logits/chosen": -1.271955132484436,
      "logits/rejected": -1.2791032791137695,
      "logps/chosen": -5.076821804046631,
      "logps/rejected": -72.70384216308594,
      "loss": 0.21184766292572021,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18928952515125275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5331236124038696,
      "rewards/margins": 6.811183929443359,
      "rewards/rejected": -6.278060436248779,
      "step": 511,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.3271665043816943,
      "grad_norm": 0.8642941117286682,
      "learning_rate": 5.542616442234618e-05,
      "logits/chosen": -1.1959160566329956,
      "logits/rejected": -1.1944928169250488,
      "logps/chosen": -4.370607376098633,
      "logps/rejected": -59.13761520385742,
      "loss": 0.210297629237175,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1646156907081604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49285203218460083,
      "rewards/margins": 5.4679365158081055,
      "rewards/rejected": -4.9750847816467285,
      "step": 512,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.329763063940279,
      "grad_norm": 0.9332461953163147,
      "learning_rate": 5.50418661053913e-05,
      "logits/chosen": -1.2382476329803467,
      "logits/rejected": -1.242477536201477,
      "logps/chosen": -6.872227191925049,
      "logps/rejected": -74.04458618164062,
      "loss": 0.2718394994735718,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2601723074913025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39546260237693787,
      "rewards/margins": 6.640940189361572,
      "rewards/rejected": -6.245477199554443,
      "step": 513,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.332359623498864,
      "grad_norm": 23.864660263061523,
      "learning_rate": 5.4658398160250935e-05,
      "logits/chosen": -1.2790114879608154,
      "logits/rejected": -1.2800090312957764,
      "logps/chosen": -5.093778610229492,
      "logps/rejected": -66.47357177734375,
      "loss": 0.3131547272205353,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24423646926879883,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4665161073207855,
      "rewards/margins": 5.967519283294678,
      "rewards/rejected": -5.501004219055176,
      "step": 514,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3349561830574488,
      "grad_norm": 1.120458960533142,
      "learning_rate": 5.4275767669536146e-05,
      "logits/chosen": -1.2243143320083618,
      "logits/rejected": -1.23822021484375,
      "logps/chosen": -5.449795722961426,
      "logps/rejected": -68.27152252197266,
      "loss": 0.2214929461479187,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20409227907657623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.484249085187912,
      "rewards/margins": 6.0561323165893555,
      "rewards/rejected": -5.571883201599121,
      "step": 515,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3375527426160336,
      "grad_norm": 0.6455298066139221,
      "learning_rate": 5.3893981700390217e-05,
      "logits/chosen": -1.2467719316482544,
      "logits/rejected": -1.248091459274292,
      "logps/chosen": -5.1662678718566895,
      "logps/rejected": -69.29208374023438,
      "loss": 0.16640540957450867,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1609758585691452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7331331968307495,
      "rewards/margins": 6.481633186340332,
      "rewards/rejected": -5.748499870300293,
      "step": 516,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3401493021746187,
      "grad_norm": 1.8854632377624512,
      "learning_rate": 5.35130473043582e-05,
      "logits/chosen": -1.2681548595428467,
      "logits/rejected": -1.2783433198928833,
      "logps/chosen": -4.961431503295898,
      "logps/rejected": -66.57093811035156,
      "loss": 0.21522153913974762,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17638429999351501,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.37839043140411377,
      "rewards/margins": 5.8021626472473145,
      "rewards/rejected": -5.423771858215332,
      "step": 517,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3427458617332035,
      "grad_norm": 2.53611159324646,
      "learning_rate": 5.313297151725679e-05,
      "logits/chosen": -1.2609970569610596,
      "logits/rejected": -1.2676525115966797,
      "logps/chosen": -3.5458755493164062,
      "logps/rejected": -64.48684692382812,
      "loss": 0.17321892082691193,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1277797669172287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5875951051712036,
      "rewards/margins": 5.845522403717041,
      "rewards/rejected": -5.2579264640808105,
      "step": 518,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3453424212917884,
      "grad_norm": 0.7559269666671753,
      "learning_rate": 5.275376135904408e-05,
      "logits/chosen": -1.2371904850006104,
      "logits/rejected": -1.251573085784912,
      "logps/chosen": -3.288680076599121,
      "logps/rejected": -72.45018005371094,
      "loss": 0.14326097071170807,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12870629131793976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5437870025634766,
      "rewards/margins": 6.532266616821289,
      "rewards/rejected": -5.988478660583496,
      "step": 519,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3479389808503732,
      "grad_norm": 0.9264482855796814,
      "learning_rate": 5.237542383369025e-05,
      "logits/chosen": -1.2770700454711914,
      "logits/rejected": -1.2863562107086182,
      "logps/chosen": -6.186993598937988,
      "logps/rejected": -72.9734115600586,
      "loss": 0.21653519570827484,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2064797580242157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5985318422317505,
      "rewards/margins": 6.420475482940674,
      "rewards/rejected": -5.821944236755371,
      "step": 520,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.350535540408958,
      "grad_norm": 0.927513062953949,
      "learning_rate": 5.199796592904812e-05,
      "logits/chosen": -1.3056390285491943,
      "logits/rejected": -1.3078558444976807,
      "logps/chosen": -6.011695861816406,
      "logps/rejected": -61.83517837524414,
      "loss": 0.2499130517244339,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22844329476356506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6435909271240234,
      "rewards/margins": 5.511775016784668,
      "rewards/rejected": -4.868184566497803,
      "step": 521,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3531320999675431,
      "grad_norm": 0.8462246060371399,
      "learning_rate": 5.16213946167237e-05,
      "logits/chosen": -1.2712773084640503,
      "logits/rejected": -1.2814041376113892,
      "logps/chosen": -4.3128533363342285,
      "logps/rejected": -75.32404327392578,
      "loss": 0.18354232609272003,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14595916867256165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44975709915161133,
      "rewards/margins": 6.685505390167236,
      "rewards/rejected": -6.235748291015625,
      "step": 522,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.355728659526128,
      "grad_norm": 0.7453184127807617,
      "learning_rate": 5.1245716851948036e-05,
      "logits/chosen": -1.282777190208435,
      "logits/rejected": -1.2888199090957642,
      "logps/chosen": -4.018249988555908,
      "logps/rejected": -71.82275390625,
      "loss": 0.17397840321063995,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17042097449302673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5543403625488281,
      "rewards/margins": 6.740978717803955,
      "rewards/rejected": -6.186638355255127,
      "step": 523,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3583252190847128,
      "grad_norm": 0.7847375273704529,
      "learning_rate": 5.087093957344841e-05,
      "logits/chosen": -1.263948678970337,
      "logits/rejected": -1.285239577293396,
      "logps/chosen": -2.6982815265655518,
      "logps/rejected": -83.65795135498047,
      "loss": 0.13130351901054382,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11251454055309296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41817164421081543,
      "rewards/margins": 7.289690017700195,
      "rewards/rejected": -6.871518135070801,
      "step": 524,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.3609217786432977,
      "grad_norm": 7.355116844177246,
      "learning_rate": 5.049706970332e-05,
      "logits/chosen": -1.3026247024536133,
      "logits/rejected": -1.3189667463302612,
      "logps/chosen": -4.715170383453369,
      "logps/rejected": -71.38179016113281,
      "loss": 0.1960161328315735,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16761450469493866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4897134304046631,
      "rewards/margins": 6.54774284362793,
      "rewards/rejected": -6.058030128479004,
      "step": 525,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3635183382018825,
      "grad_norm": 0.5812721848487854,
      "learning_rate": 5.012411414689851e-05,
      "logits/chosen": -1.2992465496063232,
      "logits/rejected": -1.32021164894104,
      "logps/chosen": -2.617116928100586,
      "logps/rejected": -77.7351303100586,
      "loss": 0.13271179795265198,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11518695950508118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45856040716171265,
      "rewards/margins": 6.7828898429870605,
      "rewards/rejected": -6.324328899383545,
      "step": 526,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3661148977604674,
      "grad_norm": 1.8369522094726562,
      "learning_rate": 4.9752079792632246e-05,
      "logits/chosen": -1.318105936050415,
      "logits/rejected": -1.327620029449463,
      "logps/chosen": -4.5236616134643555,
      "logps/rejected": -76.44921112060547,
      "loss": 0.18702739477157593,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15841788053512573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5277929306030273,
      "rewards/margins": 6.9209723472595215,
      "rewards/rejected": -6.393178939819336,
      "step": 527,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3687114573190522,
      "grad_norm": 0.8088624477386475,
      "learning_rate": 4.938097351195499e-05,
      "logits/chosen": -1.3427401781082153,
      "logits/rejected": -1.3583595752716064,
      "logps/chosen": -4.807055473327637,
      "logps/rejected": -86.13604736328125,
      "loss": 0.16440600156784058,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14281649887561798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42351657152175903,
      "rewards/margins": 7.823343753814697,
      "rewards/rejected": -7.399827003479004,
      "step": 528,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.371308016877637,
      "grad_norm": 0.6622845530509949,
      "learning_rate": 4.901080215915922e-05,
      "logits/chosen": -1.2923277616500854,
      "logits/rejected": -1.307023525238037,
      "logps/chosen": -5.0182881355285645,
      "logps/rejected": -82.5390853881836,
      "loss": 0.21107839047908783,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20138172805309296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5142525434494019,
      "rewards/margins": 7.308236122131348,
      "rewards/rejected": -6.7939839363098145,
      "step": 529,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3739045764362219,
      "grad_norm": 1.2780404090881348,
      "learning_rate": 4.864157257126928e-05,
      "logits/chosen": -1.3358229398727417,
      "logits/rejected": -1.3411754369735718,
      "logps/chosen": -6.996187210083008,
      "logps/rejected": -75.58708953857422,
      "loss": 0.2538679838180542,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24866743385791779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5523662567138672,
      "rewards/margins": 6.849069595336914,
      "rewards/rejected": -6.296703815460205,
      "step": 530,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.376501135994807,
      "grad_norm": 1.143084168434143,
      "learning_rate": 4.827329156791522e-05,
      "logits/chosen": -1.3413739204406738,
      "logits/rejected": -1.3325977325439453,
      "logps/chosen": -6.320285797119141,
      "logps/rejected": -55.596866607666016,
      "loss": 0.2689521014690399,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2311260998249054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7088637948036194,
      "rewards/margins": 5.551700592041016,
      "rewards/rejected": -4.842836856842041,
      "step": 531,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3790976955533918,
      "grad_norm": 0.9891291856765747,
      "learning_rate": 4.790596595120699e-05,
      "logits/chosen": -1.353549838066101,
      "logits/rejected": -1.3516517877578735,
      "logps/chosen": -5.421792984008789,
      "logps/rejected": -74.89588165283203,
      "loss": 0.19859538972377777,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19306476414203644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5956583619117737,
      "rewards/margins": 6.918511390686035,
      "rewards/rejected": -6.322854042053223,
      "step": 532,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3816942551119766,
      "grad_norm": 0.7545592784881592,
      "learning_rate": 4.753960250560843e-05,
      "logits/chosen": -1.3177759647369385,
      "logits/rejected": -1.3262300491333008,
      "logps/chosen": -4.168139934539795,
      "logps/rejected": -80.60372924804688,
      "loss": 0.15544827282428741,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13690081238746643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5192233324050903,
      "rewards/margins": 7.489371299743652,
      "rewards/rejected": -6.970148086547852,
      "step": 533,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3842908146705615,
      "grad_norm": 0.9258074164390564,
      "learning_rate": 4.7174207997812434e-05,
      "logits/chosen": -1.3286628723144531,
      "logits/rejected": -1.3313071727752686,
      "logps/chosen": -5.146695613861084,
      "logps/rejected": -74.13917541503906,
      "loss": 0.18875287473201752,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18020354211330414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6697993874549866,
      "rewards/margins": 7.111582279205322,
      "rewards/rejected": -6.441783428192139,
      "step": 534,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3868873742291463,
      "grad_norm": 0.7437857389450073,
      "learning_rate": 4.680978917661544e-05,
      "logits/chosen": -1.3763070106506348,
      "logits/rejected": -1.3904519081115723,
      "logps/chosen": -3.5649471282958984,
      "logps/rejected": -98.40135955810547,
      "loss": 0.15277497470378876,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14997752010822296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4047650396823883,
      "rewards/margins": 8.529120445251465,
      "rewards/rejected": -8.12435531616211,
      "step": 535,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3894839337877314,
      "grad_norm": 0.7930772304534912,
      "learning_rate": 4.644635277279326e-05,
      "logits/chosen": -1.2950698137283325,
      "logits/rejected": -1.301226019859314,
      "logps/chosen": -3.2517616748809814,
      "logps/rejected": -73.68115997314453,
      "loss": 0.13988333940505981,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1251326948404312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5784369707107544,
      "rewards/margins": 6.892407417297363,
      "rewards/rejected": -6.313970565795898,
      "step": 536,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3920804933463162,
      "grad_norm": 3.1857376098632812,
      "learning_rate": 4.608390549897661e-05,
      "logits/chosen": -1.312851071357727,
      "logits/rejected": -1.31553316116333,
      "logps/chosen": -4.813056945800781,
      "logps/rejected": -72.02601623535156,
      "loss": 0.17908523976802826,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1712900847196579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5639367699623108,
      "rewards/margins": 6.651814937591553,
      "rewards/rejected": -6.087879180908203,
      "step": 537,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.394677052904901,
      "grad_norm": 0.6969592571258545,
      "learning_rate": 4.572245404952682e-05,
      "logits/chosen": -1.3196005821228027,
      "logits/rejected": -1.3400623798370361,
      "logps/chosen": -2.851443290710449,
      "logps/rejected": -84.36784362792969,
      "loss": 0.14703302085399628,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13512712717056274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4452558755874634,
      "rewards/margins": 7.475783824920654,
      "rewards/rejected": -7.030528545379639,
      "step": 538,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.397273612463486,
      "grad_norm": 1.1035842895507812,
      "learning_rate": 4.536200510041271e-05,
      "logits/chosen": -1.3497779369354248,
      "logits/rejected": -1.3533220291137695,
      "logps/chosen": -6.113505840301514,
      "logps/rejected": -82.510986328125,
      "loss": 0.1953519731760025,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17564868927001953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2876747250556946,
      "rewards/margins": 7.36361837387085,
      "rewards/rejected": -7.075943470001221,
      "step": 539,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.3998701720220708,
      "grad_norm": 1.0004595518112183,
      "learning_rate": 4.5002565309087e-05,
      "logits/chosen": -1.320373296737671,
      "logits/rejected": -1.3390169143676758,
      "logps/chosen": -3.581512689590454,
      "logps/rejected": -83.01679992675781,
      "loss": 0.1440872699022293,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13884009420871735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5866944789886475,
      "rewards/margins": 7.652634143829346,
      "rewards/rejected": -7.065939903259277,
      "step": 540,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4024667315806556,
      "grad_norm": 0.9309626221656799,
      "learning_rate": 4.464414131436317e-05,
      "logits/chosen": -1.3228111267089844,
      "logits/rejected": -1.3259844779968262,
      "logps/chosen": -5.710712432861328,
      "logps/rejected": -86.91190338134766,
      "loss": 0.18705937266349792,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17348426580429077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5686015486717224,
      "rewards/margins": 8.052790641784668,
      "rewards/rejected": -7.484189033508301,
      "step": 541,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4050632911392404,
      "grad_norm": 2.3234517574310303,
      "learning_rate": 4.428673973629328e-05,
      "logits/chosen": -1.2998096942901611,
      "logits/rejected": -1.30985689163208,
      "logps/chosen": -7.139487266540527,
      "logps/rejected": -82.738037109375,
      "loss": 0.25522780418395996,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22490915656089783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5230386257171631,
      "rewards/margins": 7.560036659240723,
      "rewards/rejected": -7.036998748779297,
      "step": 542,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4076598506978253,
      "grad_norm": 0.7589312195777893,
      "learning_rate": 4.393036717604536e-05,
      "logits/chosen": -1.3219821453094482,
      "logits/rejected": -1.3300949335098267,
      "logps/chosen": -3.9188337326049805,
      "logps/rejected": -79.20855712890625,
      "loss": 0.15153641998767853,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14401042461395264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4560588300228119,
      "rewards/margins": 7.192531108856201,
      "rewards/rejected": -6.736471652984619,
      "step": 543,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4102564102564101,
      "grad_norm": 0.7089377045631409,
      "learning_rate": 4.357503021578157e-05,
      "logits/chosen": -1.3246771097183228,
      "logits/rejected": -1.331357479095459,
      "logps/chosen": -5.504782199859619,
      "logps/rejected": -79.88471221923828,
      "loss": 0.5075070858001709,
      "memory(GiB)": 46.82,
      "nll_loss": 0.371629536151886,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43268999457359314,
      "rewards/margins": 7.278794288635254,
      "rewards/rejected": -6.846104145050049,
      "step": 544,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4128529698149952,
      "grad_norm": 6.009806156158447,
      "learning_rate": 4.3220735418536774e-05,
      "logits/chosen": -1.3304033279418945,
      "logits/rejected": -1.3362674713134766,
      "logps/chosen": -4.7876715660095215,
      "logps/rejected": -77.58482360839844,
      "loss": 0.21589714288711548,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17250403761863708,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.43926605582237244,
      "rewards/margins": 7.207821369171143,
      "rewards/rejected": -6.7685546875,
      "step": 545,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.41544952937358,
      "grad_norm": 0.8883028626441956,
      "learning_rate": 4.2867489328097066e-05,
      "logits/chosen": -1.286049723625183,
      "logits/rejected": -1.3016682863235474,
      "logps/chosen": -4.48312520980835,
      "logps/rejected": -80.04753112792969,
      "loss": 0.18265032768249512,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16458269953727722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3826461732387543,
      "rewards/margins": 7.131008148193359,
      "rewards/rejected": -6.748361587524414,
      "step": 546,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4180460889321649,
      "grad_norm": 0.8117271661758423,
      "learning_rate": 4.251529846887906e-05,
      "logits/chosen": -1.3500478267669678,
      "logits/rejected": -1.3516051769256592,
      "logps/chosen": -5.849126815795898,
      "logps/rejected": -75.34406280517578,
      "loss": 0.21577468514442444,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21225160360336304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.787651538848877,
      "rewards/margins": 7.277240753173828,
      "rewards/rejected": -6.489588737487793,
      "step": 547,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4206426484907497,
      "grad_norm": 0.7191773056983948,
      "learning_rate": 4.216416934580947e-05,
      "logits/chosen": -1.3170534372329712,
      "logits/rejected": -1.3186712265014648,
      "logps/chosen": -5.588179111480713,
      "logps/rejected": -73.8975830078125,
      "loss": 0.19539664685726166,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19075627624988556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46979498863220215,
      "rewards/margins": 6.78049373626709,
      "rewards/rejected": -6.310698986053467,
      "step": 548,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4232392080493346,
      "grad_norm": 0.6664804816246033,
      "learning_rate": 4.181410844420474e-05,
      "logits/chosen": -1.304147481918335,
      "logits/rejected": -1.3245407342910767,
      "logps/chosen": -3.386530876159668,
      "logps/rejected": -84.88090515136719,
      "loss": 0.14330296218395233,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13968124985694885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6671038269996643,
      "rewards/margins": 8.031911849975586,
      "rewards/rejected": -7.364808082580566,
      "step": 549,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4258357676079196,
      "grad_norm": 0.7066012024879456,
      "learning_rate": 4.146512222965143e-05,
      "logits/chosen": -1.3411874771118164,
      "logits/rejected": -1.3574979305267334,
      "logps/chosen": -2.9142534732818604,
      "logps/rejected": -88.72208404541016,
      "loss": 0.1130063459277153,
      "memory(GiB)": 46.82,
      "nll_loss": 0.10818038135766983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4542260766029358,
      "rewards/margins": 8.142085075378418,
      "rewards/rejected": -7.687858581542969,
      "step": 550,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.4284323271665045,
      "grad_norm": 0.7845104932785034,
      "learning_rate": 4.111721714788671e-05,
      "logits/chosen": -1.3469820022583008,
      "logits/rejected": -1.3495479822158813,
      "logps/chosen": -4.226562976837158,
      "logps/rejected": -70.08132934570312,
      "loss": 0.18562032282352448,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16532127559185028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6004529595375061,
      "rewards/margins": 6.635247230529785,
      "rewards/rejected": -6.034794330596924,
      "step": 551,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4310288867250893,
      "grad_norm": 0.9597321152687073,
      "learning_rate": 4.0770399624679456e-05,
      "logits/chosen": -1.3411657810211182,
      "logits/rejected": -1.353317379951477,
      "logps/chosen": -5.72759485244751,
      "logps/rejected": -84.97734069824219,
      "loss": 0.22993287444114685,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21702148020267487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5299229025840759,
      "rewards/margins": 7.509583473205566,
      "rewards/rejected": -6.979660511016846,
      "step": 552,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4336254462836742,
      "grad_norm": 2.079641580581665,
      "learning_rate": 4.042467606571134e-05,
      "logits/chosen": -1.3426014184951782,
      "logits/rejected": -1.3533074855804443,
      "logps/chosen": -7.159404754638672,
      "logps/rejected": -73.98983764648438,
      "loss": 0.5649017691612244,
      "memory(GiB)": 46.82,
      "nll_loss": 0.4176748991012573,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.36085012555122375,
      "rewards/margins": 6.391058444976807,
      "rewards/rejected": -6.030208110809326,
      "step": 553,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.436222005842259,
      "grad_norm": 0.7903665900230408,
      "learning_rate": 4.0080052856458626e-05,
      "logits/chosen": -1.3065292835235596,
      "logits/rejected": -1.326464056968689,
      "logps/chosen": -3.64105224609375,
      "logps/rejected": -86.52337646484375,
      "loss": 0.13767823576927185,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1341703087091446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3208577036857605,
      "rewards/margins": 7.719215393066406,
      "rewards/rejected": -7.39835786819458,
      "step": 554,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4388185654008439,
      "grad_norm": 5.593478679656982,
      "learning_rate": 3.973653636207437e-05,
      "logits/chosen": -1.3405157327651978,
      "logits/rejected": -1.3490185737609863,
      "logps/chosen": -5.513555526733398,
      "logps/rejected": -79.61082458496094,
      "loss": 0.4690606892108917,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2211458384990692,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24729761481285095,
      "rewards/margins": 7.00010871887207,
      "rewards/rejected": -6.752811431884766,
      "step": 555,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4414151249594287,
      "grad_norm": 0.7707073092460632,
      "learning_rate": 3.9394132927270613e-05,
      "logits/chosen": -1.3235504627227783,
      "logits/rejected": -1.3401741981506348,
      "logps/chosen": -4.119601249694824,
      "logps/rejected": -86.29273986816406,
      "loss": 0.16342757642269135,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1537020206451416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46904829144477844,
      "rewards/margins": 7.555969715118408,
      "rewards/rejected": -7.086921215057373,
      "step": 556,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4440116845180135,
      "grad_norm": 1.0258326530456543,
      "learning_rate": 3.9052848876201284e-05,
      "logits/chosen": -1.2859511375427246,
      "logits/rejected": -1.2992178201675415,
      "logps/chosen": -4.974706172943115,
      "logps/rejected": -72.67876434326172,
      "loss": 0.20728819072246552,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19392336905002594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6317263841629028,
      "rewards/margins": 6.647282123565674,
      "rewards/rejected": -6.0155558586120605,
      "step": 557,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4466082440765984,
      "grad_norm": 0.8165487051010132,
      "learning_rate": 3.871269051234555e-05,
      "logits/chosen": -1.3127992153167725,
      "logits/rejected": -1.329878330230713,
      "logps/chosen": -3.2941739559173584,
      "logps/rejected": -87.43807220458984,
      "loss": 0.17191743850708008,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16360032558441162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5402951240539551,
      "rewards/margins": 7.842915058135986,
      "rewards/rejected": -7.3026204109191895,
      "step": 558,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4492048036351834,
      "grad_norm": 0.7644895315170288,
      "learning_rate": 3.837366411839114e-05,
      "logits/chosen": -1.322697639465332,
      "logits/rejected": -1.3291290998458862,
      "logps/chosen": -3.767364025115967,
      "logps/rejected": -79.43592071533203,
      "loss": 0.13489316403865814,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12984776496887207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47781094908714294,
      "rewards/margins": 7.167353630065918,
      "rewards/rejected": -6.6895432472229,
      "step": 559,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4518013631937683,
      "grad_norm": 0.6488920450210571,
      "learning_rate": 3.8035775956118415e-05,
      "logits/chosen": -1.3089568614959717,
      "logits/rejected": -1.333059549331665,
      "logps/chosen": -3.604278087615967,
      "logps/rejected": -75.26642608642578,
      "loss": 0.12487327307462692,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11769910156726837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5058732032775879,
      "rewards/margins": 6.758044242858887,
      "rewards/rejected": -6.252170562744141,
      "step": 560,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.4543979227523531,
      "grad_norm": 0.7780476212501526,
      "learning_rate": 3.7699032266284865e-05,
      "logits/chosen": -1.3105119466781616,
      "logits/rejected": -1.3256127834320068,
      "logps/chosen": -4.206973075866699,
      "logps/rejected": -81.01417541503906,
      "loss": 0.1642312854528427,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1462475061416626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.469632625579834,
      "rewards/margins": 7.197355270385742,
      "rewards/rejected": -6.72772216796875,
      "step": 561,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.456994482310938,
      "grad_norm": 1.534871220588684,
      "learning_rate": 3.7363439268509535e-05,
      "logits/chosen": -1.324033498764038,
      "logits/rejected": -1.3254599571228027,
      "logps/chosen": -6.707502365112305,
      "logps/rejected": -66.20709991455078,
      "loss": 0.34620898962020874,
      "memory(GiB)": 46.82,
      "nll_loss": 0.3185166120529175,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.49867385625839233,
      "rewards/margins": 6.206461429595947,
      "rewards/rejected": -5.707787036895752,
      "step": 562,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4595910418695228,
      "grad_norm": 0.9614840745925903,
      "learning_rate": 3.702900316115836e-05,
      "logits/chosen": -1.312652587890625,
      "logits/rejected": -1.3183560371398926,
      "logps/chosen": -6.137225151062012,
      "logps/rejected": -76.21929931640625,
      "loss": 0.24062488973140717,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20657041668891907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38741621375083923,
      "rewards/margins": 6.601250648498535,
      "rewards/rejected": -6.213834285736084,
      "step": 563,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4621876014281079,
      "grad_norm": 1.2924113273620605,
      "learning_rate": 3.669573012122973e-05,
      "logits/chosen": -1.2920567989349365,
      "logits/rejected": -1.29514741897583,
      "logps/chosen": -6.05202054977417,
      "logps/rejected": -62.32487106323242,
      "loss": 0.3004983067512512,
      "memory(GiB)": 46.82,
      "nll_loss": 0.27208033204078674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6880547404289246,
      "rewards/margins": 5.9696760177612305,
      "rewards/rejected": -5.281620979309082,
      "step": 564,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4647841609866927,
      "grad_norm": 1.697675347328186,
      "learning_rate": 3.6363626304240184e-05,
      "logits/chosen": -1.3259660005569458,
      "logits/rejected": -1.3245418071746826,
      "logps/chosen": -4.952503681182861,
      "logps/rejected": -66.86807250976562,
      "loss": 0.20483994483947754,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17799176275730133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.64216148853302,
      "rewards/margins": 6.409905433654785,
      "rewards/rejected": -5.767744064331055,
      "step": 565,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4673807205452776,
      "grad_norm": 0.9418896436691284,
      "learning_rate": 3.60326978441109e-05,
      "logits/chosen": -1.2750047445297241,
      "logits/rejected": -1.2927812337875366,
      "logps/chosen": -5.463886260986328,
      "logps/rejected": -86.8280258178711,
      "loss": 0.19925126433372498,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18372556567192078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38306546211242676,
      "rewards/margins": 7.263001441955566,
      "rewards/rejected": -6.879936218261719,
      "step": 566,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4699772801038624,
      "grad_norm": 1.4005364179611206,
      "learning_rate": 3.5702950853054284e-05,
      "logits/chosen": -1.2621244192123413,
      "logits/rejected": -1.2756032943725586,
      "logps/chosen": -4.0591864585876465,
      "logps/rejected": -76.57896423339844,
      "loss": 0.16474060714244843,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15141594409942627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3780522644519806,
      "rewards/margins": 6.907110691070557,
      "rewards/rejected": -6.529058933258057,
      "step": 567,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4725738396624473,
      "grad_norm": 0.6262617111206055,
      "learning_rate": 3.5374391421461274e-05,
      "logits/chosen": -1.3366174697875977,
      "logits/rejected": -1.3425589799880981,
      "logps/chosen": -5.0637078285217285,
      "logps/rejected": -75.93576049804688,
      "loss": 0.14967593550682068,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13986575603485107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45764419436454773,
      "rewards/margins": 6.864101409912109,
      "rewards/rejected": -6.406457901000977,
      "step": 568,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.475170399221032,
      "grad_norm": 1.0724283456802368,
      "learning_rate": 3.5047025617788576e-05,
      "logits/chosen": -1.318103551864624,
      "logits/rejected": -1.3257315158843994,
      "logps/chosen": -4.4997782707214355,
      "logps/rejected": -69.55330657958984,
      "loss": 0.21760107576847076,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20659759640693665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36253267526626587,
      "rewards/margins": 6.121118545532227,
      "rewards/rejected": -5.758585453033447,
      "step": 569,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.477766958779617,
      "grad_norm": 0.8618238568305969,
      "learning_rate": 3.472085948844674e-05,
      "logits/chosen": -1.3077456951141357,
      "logits/rejected": -1.3146158456802368,
      "logps/chosen": -4.7398481369018555,
      "logps/rejected": -75.23126220703125,
      "loss": 0.2030331939458847,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19906149804592133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6159305572509766,
      "rewards/margins": 6.953237533569336,
      "rewards/rejected": -6.337306499481201,
      "step": 570,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4803635183382018,
      "grad_norm": 2.220825433731079,
      "learning_rate": 3.439589905768857e-05,
      "logits/chosen": -1.3038007020950317,
      "logits/rejected": -1.314782977104187,
      "logps/chosen": -4.8768205642700195,
      "logps/rejected": -78.61490631103516,
      "loss": 0.2124282568693161,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20312494039535522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42080211639404297,
      "rewards/margins": 6.928518295288086,
      "rewards/rejected": -6.507716178894043,
      "step": 571,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4829600778967866,
      "grad_norm": 0.9802061915397644,
      "learning_rate": 3.407215032749763e-05,
      "logits/chosen": -1.2862904071807861,
      "logits/rejected": -1.2907946109771729,
      "logps/chosen": -6.141645431518555,
      "logps/rejected": -61.32841110229492,
      "loss": 0.2664101719856262,
      "memory(GiB)": 46.82,
      "nll_loss": 0.23925836384296417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6145697832107544,
      "rewards/margins": 5.361448764801025,
      "rewards/rejected": -4.7468791007995605,
      "step": 572,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4855566374553717,
      "grad_norm": 0.7380911111831665,
      "learning_rate": 3.374961927747751e-05,
      "logits/chosen": -1.2636692523956299,
      "logits/rejected": -1.259122610092163,
      "logps/chosen": -5.851538181304932,
      "logps/rejected": -62.0422477722168,
      "loss": 0.2571408450603485,
      "memory(GiB)": 46.82,
      "nll_loss": 0.24986571073532104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6978765726089478,
      "rewards/margins": 5.906915664672852,
      "rewards/rejected": -5.209038734436035,
      "step": 573,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4881531970139565,
      "grad_norm": 1.2015725374221802,
      "learning_rate": 3.342831186474149e-05,
      "logits/chosen": -1.2451616525650024,
      "logits/rejected": -1.2526040077209473,
      "logps/chosen": -6.031259536743164,
      "logps/rejected": -66.83104705810547,
      "loss": 0.23752854764461517,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20862002670764923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3990924656391144,
      "rewards/margins": 5.784240245819092,
      "rewards/rejected": -5.385148525238037,
      "step": 574,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4907497565725414,
      "grad_norm": 26.948949813842773,
      "learning_rate": 3.31082340238023e-05,
      "logits/chosen": -1.2702953815460205,
      "logits/rejected": -1.2942155599594116,
      "logps/chosen": -5.758556365966797,
      "logps/rejected": -79.33439636230469,
      "loss": 0.7174453735351562,
      "memory(GiB)": 46.82,
      "nll_loss": 0.5235574245452881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16809692978858948,
      "rewards/margins": 6.522285461425781,
      "rewards/rejected": -6.354187965393066,
      "step": 575,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4933463161311262,
      "grad_norm": 0.7220916748046875,
      "learning_rate": 3.2789391666462596e-05,
      "logits/chosen": -1.2505327463150024,
      "logits/rejected": -1.2620594501495361,
      "logps/chosen": -3.8762307167053223,
      "logps/rejected": -84.97810363769531,
      "loss": 0.15173956751823425,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1497780978679657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4926835298538208,
      "rewards/margins": 7.460906028747559,
      "rewards/rejected": -6.968222618103027,
      "step": 576,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.495942875689711,
      "grad_norm": 1.8231993913650513,
      "learning_rate": 3.247179068170593e-05,
      "logits/chosen": -1.263367772102356,
      "logits/rejected": -1.272806167602539,
      "logps/chosen": -3.779219388961792,
      "logps/rejected": -70.675048828125,
      "loss": 0.18493540585041046,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15074032545089722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5790086984634399,
      "rewards/margins": 6.532997131347656,
      "rewards/rejected": -5.953989028930664,
      "step": 577,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.4985394352482961,
      "grad_norm": 1.029189109802246,
      "learning_rate": 3.215543693558769e-05,
      "logits/chosen": -1.2609919309616089,
      "logits/rejected": -1.2647769451141357,
      "logps/chosen": -5.3333539962768555,
      "logps/rejected": -65.2447509765625,
      "loss": 0.23766495287418365,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19680431485176086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5142011642456055,
      "rewards/margins": 6.050221920013428,
      "rewards/rejected": -5.536020755767822,
      "step": 578,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.501135994806881,
      "grad_norm": 1.0110372304916382,
      "learning_rate": 3.184033627112694e-05,
      "logits/chosen": -1.233156681060791,
      "logits/rejected": -1.2387574911117554,
      "logps/chosen": -7.914291858673096,
      "logps/rejected": -66.5923080444336,
      "loss": 0.26373106241226196,
      "memory(GiB)": 46.82,
      "nll_loss": 0.233109250664711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7227498888969421,
      "rewards/margins": 6.343271255493164,
      "rewards/rejected": -5.620521545410156,
      "step": 579,
      "train_speed(iter/s)": 0.005351
    },
    {
      "epoch": 1.5037325543654658,
      "grad_norm": 0.8185076117515564,
      "learning_rate": 3.1526494508198525e-05,
      "logits/chosen": -1.2909034490585327,
      "logits/rejected": -1.3032139539718628,
      "logps/chosen": -5.271670818328857,
      "logps/rejected": -70.27622985839844,
      "loss": 0.22915054857730865,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2062722146511078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4416859745979309,
      "rewards/margins": 6.420576095581055,
      "rewards/rejected": -5.978890419006348,
      "step": 580,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5063291139240507,
      "grad_norm": 0.7780730128288269,
      "learning_rate": 3.121391744342546e-05,
      "logits/chosen": -1.27274489402771,
      "logits/rejected": -1.2946935892105103,
      "logps/chosen": -2.167703628540039,
      "logps/rejected": -82.38484191894531,
      "loss": 0.1313847005367279,
      "memory(GiB)": 46.82,
      "nll_loss": 0.124984011054039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4470738470554352,
      "rewards/margins": 7.279787063598633,
      "rewards/rejected": -6.832714557647705,
      "step": 581,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5089256734826355,
      "grad_norm": 0.7834947109222412,
      "learning_rate": 3.0902610850071924e-05,
      "logits/chosen": -1.2649881839752197,
      "logits/rejected": -1.270396113395691,
      "logps/chosen": -4.809529781341553,
      "logps/rejected": -75.95304107666016,
      "loss": 0.21607160568237305,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19319041073322296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6121608018875122,
      "rewards/margins": 7.184690475463867,
      "rewards/rejected": -6.572530269622803,
      "step": 582,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5115222330412204,
      "grad_norm": 1.0974911451339722,
      "learning_rate": 3.059258047793661e-05,
      "logits/chosen": -1.2686303853988647,
      "logits/rejected": -1.266996145248413,
      "logps/chosen": -6.2699503898620605,
      "logps/rejected": -75.85345458984375,
      "loss": 0.20134985446929932,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19503507018089294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5324012041091919,
      "rewards/margins": 6.867381572723389,
      "rewards/rejected": -6.334980487823486,
      "step": 583,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5141187925998052,
      "grad_norm": 0.8552331924438477,
      "learning_rate": 3.028383205324664e-05,
      "logits/chosen": -1.2964813709259033,
      "logits/rejected": -1.2977771759033203,
      "logps/chosen": -4.300373077392578,
      "logps/rejected": -71.34073638916016,
      "loss": 0.1761379987001419,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16346776485443115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5423522591590881,
      "rewards/margins": 6.4881744384765625,
      "rewards/rejected": -5.945821762084961,
      "step": 584,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.51671535215839,
      "grad_norm": 2.7961208820343018,
      "learning_rate": 2.9976371278551596e-05,
      "logits/chosen": -1.26725172996521,
      "logits/rejected": -1.2702522277832031,
      "logps/chosen": -6.598125457763672,
      "logps/rejected": -75.9102783203125,
      "loss": 0.21579381823539734,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2034890353679657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5635837912559509,
      "rewards/margins": 6.742549419403076,
      "rewards/rejected": -6.178965091705322,
      "step": 585,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5193119117169749,
      "grad_norm": 0.7595794796943665,
      "learning_rate": 2.9670203832618338e-05,
      "logits/chosen": -1.3211987018585205,
      "logits/rejected": -1.3264515399932861,
      "logps/chosen": -5.026176452636719,
      "logps/rejected": -73.40199279785156,
      "loss": 0.2131197303533554,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20605915784835815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5510646104812622,
      "rewards/margins": 6.663293361663818,
      "rewards/rejected": -6.112229347229004,
      "step": 586,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5219084712755597,
      "grad_norm": 0.5875643491744995,
      "learning_rate": 2.9365335370326142e-05,
      "logits/chosen": -1.2664649486541748,
      "logits/rejected": -1.2790292501449585,
      "logps/chosen": -4.272137641906738,
      "logps/rejected": -78.49589538574219,
      "loss": 0.16073563694953918,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14196071028709412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5238849520683289,
      "rewards/margins": 7.083423614501953,
      "rewards/rejected": -6.559538841247559,
      "step": 587,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5245050308341448,
      "grad_norm": 0.5453342795372009,
      "learning_rate": 2.906177152256214e-05,
      "logits/chosen": -1.2802778482437134,
      "logits/rejected": -1.287778615951538,
      "logps/chosen": -3.453296661376953,
      "logps/rejected": -80.28994750976562,
      "loss": 0.11858940124511719,
      "memory(GiB)": 46.82,
      "nll_loss": 0.114979088306427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3646010160446167,
      "rewards/margins": 7.368077278137207,
      "rewards/rejected": -7.003476619720459,
      "step": 588,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5271015903927296,
      "grad_norm": 0.8347062468528748,
      "learning_rate": 2.875951789611734e-05,
      "logits/chosen": -1.2904810905456543,
      "logits/rejected": -1.2961167097091675,
      "logps/chosen": -7.039205551147461,
      "logps/rejected": -79.34686279296875,
      "loss": 0.20924313366413116,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18833908438682556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5033472180366516,
      "rewards/margins": 7.3004374504089355,
      "rewards/rejected": -6.797090530395508,
      "step": 589,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5296981499513145,
      "grad_norm": 0.6540753841400146,
      "learning_rate": 2.8458580073583264e-05,
      "logits/chosen": -1.266171932220459,
      "logits/rejected": -1.2721834182739258,
      "logps/chosen": -4.1306471824646,
      "logps/rejected": -74.91023254394531,
      "loss": 0.1655585914850235,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1580181121826172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32985809445381165,
      "rewards/margins": 6.546298980712891,
      "rewards/rejected": -6.216440677642822,
      "step": 590,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5322947095098995,
      "grad_norm": 0.6838015913963318,
      "learning_rate": 2.8158963613248433e-05,
      "logits/chosen": -1.303230881690979,
      "logits/rejected": -1.3014085292816162,
      "logps/chosen": -5.523627281188965,
      "logps/rejected": -60.67312240600586,
      "loss": 0.19359934329986572,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15228185057640076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5309136509895325,
      "rewards/margins": 5.451134204864502,
      "rewards/rejected": -4.920220851898193,
      "step": 591,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5348912690684844,
      "grad_norm": 1.0530349016189575,
      "learning_rate": 2.786067404899617e-05,
      "logits/chosen": -1.2414886951446533,
      "logits/rejected": -1.2445082664489746,
      "logps/chosen": -5.888449192047119,
      "logps/rejected": -71.38929748535156,
      "loss": 0.22650176286697388,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20438861846923828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49599599838256836,
      "rewards/margins": 6.550296306610107,
      "rewards/rejected": -6.054300785064697,
      "step": 592,
      "train_speed(iter/s)": 0.005352
    },
    {
      "epoch": 1.5374878286270692,
      "grad_norm": 0.9590216875076294,
      "learning_rate": 2.756371689020214e-05,
      "logits/chosen": -1.284089207649231,
      "logits/rejected": -1.291182041168213,
      "logps/chosen": -5.660233497619629,
      "logps/rejected": -78.09732055664062,
      "loss": 0.21897563338279724,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20564842224121094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6212056279182434,
      "rewards/margins": 7.130356311798096,
      "rewards/rejected": -6.509150981903076,
      "step": 593,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.540084388185654,
      "grad_norm": 0.8171280026435852,
      "learning_rate": 2.726809762163247e-05,
      "logits/chosen": -1.2626338005065918,
      "logits/rejected": -1.2816903591156006,
      "logps/chosen": -3.3798415660858154,
      "logps/rejected": -79.01041412353516,
      "loss": 0.1683107167482376,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1568489968776703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6359814405441284,
      "rewards/margins": 7.243876934051514,
      "rewards/rejected": -6.607895374298096,
      "step": 594,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.542680947744239,
      "grad_norm": 0.7614494562149048,
      "learning_rate": 2.697382170334275e-05,
      "logits/chosen": -1.2870750427246094,
      "logits/rejected": -1.2988437414169312,
      "logps/chosen": -3.8276844024658203,
      "logps/rejected": -80.0929946899414,
      "loss": 0.15257692337036133,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14827337861061096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44497260451316833,
      "rewards/margins": 7.25712776184082,
      "rewards/rejected": -6.812155246734619,
      "step": 595,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.5452775073028238,
      "grad_norm": 0.7709101438522339,
      "learning_rate": 2.668089457057704e-05,
      "logits/chosen": -1.3179664611816406,
      "logits/rejected": -1.3277684450149536,
      "logps/chosen": -3.917905807495117,
      "logps/rejected": -73.78385162353516,
      "loss": 0.20244568586349487,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1898242086172104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.408854603767395,
      "rewards/margins": 6.5159125328063965,
      "rewards/rejected": -6.107057571411133,
      "step": 596,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.5478740668614086,
      "grad_norm": 1.1506835222244263,
      "learning_rate": 2.638932163366742e-05,
      "logits/chosen": -1.2968817949295044,
      "logits/rejected": -1.307157278060913,
      "logps/chosen": -3.7310962677001953,
      "logps/rejected": -79.50012969970703,
      "loss": 0.1415027379989624,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12706293165683746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38727837800979614,
      "rewards/margins": 7.095935344696045,
      "rewards/rejected": -6.708656311035156,
      "step": 597,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 1.5504706264199934,
      "grad_norm": 1.2147072553634644,
      "learning_rate": 2.6099108277934103e-05,
      "logits/chosen": -1.3259124755859375,
      "logits/rejected": -1.3342318534851074,
      "logps/chosen": -5.113812446594238,
      "logps/rejected": -72.30021667480469,
      "loss": 0.23814579844474792,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1989901065826416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3370703458786011,
      "rewards/margins": 6.340109825134277,
      "rewards/rejected": -6.003039836883545,
      "step": 598,
      "train_speed(iter/s)": 0.005353
    },
    {
      "epoch": 1.5530671859785783,
      "grad_norm": 0.6469516754150391,
      "learning_rate": 2.5810259863586018e-05,
      "logits/chosen": -1.3008332252502441,
      "logits/rejected": -1.3114441633224487,
      "logps/chosen": -3.1887383460998535,
      "logps/rejected": -89.48507690429688,
      "loss": 0.10776513069868088,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1028357744216919,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.410270631313324,
      "rewards/margins": 7.939424514770508,
      "rewards/rejected": -7.529153823852539,
      "step": 599,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 1.5556637455371631,
      "grad_norm": 0.5676211714744568,
      "learning_rate": 2.5522781725621813e-05,
      "logits/chosen": -1.3371210098266602,
      "logits/rejected": -1.3387823104858398,
      "logps/chosen": -3.7307708263397217,
      "logps/rejected": -80.85253143310547,
      "loss": 0.13133268058300018,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12862281501293182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5126275420188904,
      "rewards/margins": 7.358179092407227,
      "rewards/rejected": -6.845551013946533,
      "step": 600,
      "train_speed(iter/s)": 0.005354
    },
    {
      "epoch": 1.5556637455371631,
      "eval_logits/chosen": -1.304226040840149,
      "eval_logits/rejected": -1.3143255710601807,
      "eval_logps/chosen": -7.2022528648376465,
      "eval_logps/rejected": -78.7955551147461,
      "eval_loss": 0.2647717297077179,
      "eval_nll_loss": 0.2476370930671692,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": 0.5162011384963989,
      "eval_rewards/margins": 7.155314922332764,
      "eval_rewards/rejected": -6.639112949371338,
      "eval_runtime": 291.2848,
      "eval_samples_per_second": 0.426,
      "eval_steps_per_second": 0.426,
      "step": 600
    },
    {
      "epoch": 1.558260305095748,
      "grad_norm": 0.5284009575843811,
      "learning_rate": 2.523667917373125e-05,
      "logits/chosen": -1.2985765933990479,
      "logits/rejected": -1.3168220520019531,
      "logps/chosen": -2.943354368209839,
      "logps/rejected": -91.36310577392578,
      "loss": 0.09971652179956436,
      "memory(GiB)": 46.82,
      "nll_loss": 0.0945102795958519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5126221776008606,
      "rewards/margins": 8.36147403717041,
      "rewards/rejected": -7.848851203918457,
      "step": 601,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.560856864654333,
      "grad_norm": 0.5907682776451111,
      "learning_rate": 2.4951957492197097e-05,
      "logits/chosen": -1.2848440408706665,
      "logits/rejected": -1.2836586236953735,
      "logps/chosen": -3.989741563796997,
      "logps/rejected": -70.67216491699219,
      "loss": 0.14021284878253937,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13644501566886902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7150586843490601,
      "rewards/margins": 6.884438514709473,
      "rewards/rejected": -6.169380187988281,
      "step": 602,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5634534242129179,
      "grad_norm": 0.6943731904029846,
      "learning_rate": 2.4668621939797743e-05,
      "logits/chosen": -1.3670181035995483,
      "logits/rejected": -1.380336880683899,
      "logps/chosen": -4.377835273742676,
      "logps/rejected": -79.83389282226562,
      "loss": 0.16850493848323822,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1525357961654663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3955461382865906,
      "rewards/margins": 7.191918849945068,
      "rewards/rejected": -6.79637336730957,
      "step": 603,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5660499837715027,
      "grad_norm": 0.7594765424728394,
      "learning_rate": 2.438667774970981e-05,
      "logits/chosen": -1.3331427574157715,
      "logits/rejected": -1.3355261087417603,
      "logps/chosen": -3.4951040744781494,
      "logps/rejected": -73.5210189819336,
      "loss": 0.15708421170711517,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14821352064609528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6260663270950317,
      "rewards/margins": 6.858430862426758,
      "rewards/rejected": -6.232364654541016,
      "step": 604,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5686465433300878,
      "grad_norm": 0.7032631039619446,
      "learning_rate": 2.410613012941161e-05,
      "logits/chosen": -1.3653652667999268,
      "logits/rejected": -1.3712530136108398,
      "logps/chosen": -4.2118754386901855,
      "logps/rejected": -76.30243682861328,
      "loss": 0.1958620846271515,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1903282105922699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6078907251358032,
      "rewards/margins": 7.107419490814209,
      "rewards/rejected": -6.499528884887695,
      "step": 605,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5712431028886726,
      "grad_norm": 0.849923849105835,
      "learning_rate": 2.382698426058708e-05,
      "logits/chosen": -1.335965871810913,
      "logits/rejected": -1.3330975770950317,
      "logps/chosen": -6.150424003601074,
      "logps/rejected": -76.69558715820312,
      "loss": 0.20354051887989044,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19987499713897705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6220974922180176,
      "rewards/margins": 7.106598377227783,
      "rewards/rejected": -6.484500885009766,
      "step": 606,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5738396624472575,
      "grad_norm": 0.6774344444274902,
      "learning_rate": 2.3549245299029777e-05,
      "logits/chosen": -1.349198341369629,
      "logits/rejected": -1.3615375757217407,
      "logps/chosen": -3.236879348754883,
      "logps/rejected": -86.70790100097656,
      "loss": 0.104849673807621,
      "memory(GiB)": 46.82,
      "nll_loss": 0.10025446116924286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6544709205627441,
      "rewards/margins": 7.710179805755615,
      "rewards/rejected": -7.055708885192871,
      "step": 607,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5764362220058423,
      "grad_norm": 0.9244331121444702,
      "learning_rate": 2.3272918374547993e-05,
      "logits/chosen": -1.353797197341919,
      "logits/rejected": -1.3594809770584106,
      "logps/chosen": -5.656945705413818,
      "logps/rejected": -75.61239624023438,
      "loss": 0.1939372569322586,
      "memory(GiB)": 46.82,
      "nll_loss": 0.172727569937706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4641644358634949,
      "rewards/margins": 6.93808650970459,
      "rewards/rejected": -6.473921775817871,
      "step": 608,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.5790327815644272,
      "grad_norm": 0.7308290600776672,
      "learning_rate": 2.2998008590869836e-05,
      "logits/chosen": -1.3354146480560303,
      "logits/rejected": -1.3375046253204346,
      "logps/chosen": -5.986130714416504,
      "logps/rejected": -75.91438293457031,
      "loss": 0.19244912266731262,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1849290430545807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5376923680305481,
      "rewards/margins": 6.803253173828125,
      "rewards/rejected": -6.265561103820801,
      "step": 609,
      "train_speed(iter/s)": 0.005339
    },
    {
      "epoch": 1.581629341123012,
      "grad_norm": 0.8735436797142029,
      "learning_rate": 2.2724521025548828e-05,
      "logits/chosen": -1.3188555240631104,
      "logits/rejected": -1.3274950981140137,
      "logps/chosen": -4.066837310791016,
      "logps/rejected": -85.56442260742188,
      "loss": 0.17418046295642853,
      "memory(GiB)": 46.82,
      "nll_loss": 0.16407504677772522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4197569489479065,
      "rewards/margins": 7.919032096862793,
      "rewards/rejected": -7.499274253845215,
      "step": 610,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 1.5842259006815969,
      "grad_norm": 2.2200708389282227,
      "learning_rate": 2.245246072987045e-05,
      "logits/chosen": -1.2984387874603271,
      "logits/rejected": -1.3079633712768555,
      "logps/chosen": -6.325509071350098,
      "logps/rejected": -74.75194549560547,
      "loss": 0.26426559686660767,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22844800353050232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2353907823562622,
      "rewards/margins": 6.702395439147949,
      "rewards/rejected": -6.467005252838135,
      "step": 611,
      "train_speed(iter/s)": 0.00534
    },
    {
      "epoch": 1.5868224602401817,
      "grad_norm": 0.7086100578308105,
      "learning_rate": 2.2181832728758632e-05,
      "logits/chosen": -1.3292683362960815,
      "logits/rejected": -1.342342495918274,
      "logps/chosen": -7.971512794494629,
      "logps/rejected": -74.80581665039062,
      "loss": 0.2390933483839035,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2257128208875656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5516778826713562,
      "rewards/margins": 6.869455337524414,
      "rewards/rejected": -6.317777633666992,
      "step": 612,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.5894190197987665,
      "grad_norm": 6.074515342712402,
      "learning_rate": 2.191264202068286e-05,
      "logits/chosen": -1.330883264541626,
      "logits/rejected": -1.3335540294647217,
      "logps/chosen": -4.599937438964844,
      "logps/rejected": -75.81354522705078,
      "loss": 0.20728431642055511,
      "memory(GiB)": 46.82,
      "nll_loss": 0.19485875964164734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49222689867019653,
      "rewards/margins": 7.163850784301758,
      "rewards/rejected": -6.671623706817627,
      "step": 613,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.5920155793573514,
      "grad_norm": 0.7445679306983948,
      "learning_rate": 2.1644893577566116e-05,
      "logits/chosen": -1.3524935245513916,
      "logits/rejected": -1.3670296669006348,
      "logps/chosen": -4.401839733123779,
      "logps/rejected": -75.92784881591797,
      "loss": 0.17038778960704803,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1660994589328766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6142419576644897,
      "rewards/margins": 6.9780192375183105,
      "rewards/rejected": -6.363776683807373,
      "step": 614,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.5946121389159362,
      "grad_norm": 0.6692366600036621,
      "learning_rate": 2.1378592344692862e-05,
      "logits/chosen": -1.3450599908828735,
      "logits/rejected": -1.3609131574630737,
      "logps/chosen": -2.963831901550293,
      "logps/rejected": -85.60203552246094,
      "loss": 0.11893543601036072,
      "memory(GiB)": 46.82,
      "nll_loss": 0.10431487113237381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.49938374757766724,
      "rewards/margins": 7.664846897125244,
      "rewards/rejected": -7.165462017059326,
      "step": 615,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.5972086984745213,
      "grad_norm": 1.2328548431396484,
      "learning_rate": 2.1113743240617668e-05,
      "logits/chosen": -1.343220829963684,
      "logits/rejected": -1.3497005701065063,
      "logps/chosen": -4.603923797607422,
      "logps/rejected": -75.76362609863281,
      "loss": 0.18495653569698334,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15200993418693542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6504652500152588,
      "rewards/margins": 6.986648082733154,
      "rewards/rejected": -6.336182594299316,
      "step": 616,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.5998052580331061,
      "grad_norm": 0.7638961672782898,
      "learning_rate": 2.0850351157074598e-05,
      "logits/chosen": -1.3338675498962402,
      "logits/rejected": -1.3393734693527222,
      "logps/chosen": -5.415805339813232,
      "logps/rejected": -76.63778686523438,
      "loss": 0.19142858684062958,
      "memory(GiB)": 46.82,
      "nll_loss": 0.17946861684322357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5905900597572327,
      "rewards/margins": 7.105247974395752,
      "rewards/rejected": -6.514657974243164,
      "step": 617,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.602401817591691,
      "grad_norm": 0.7070993185043335,
      "learning_rate": 2.0588420958886578e-05,
      "logits/chosen": -1.3037599325180054,
      "logits/rejected": -1.3092586994171143,
      "logps/chosen": -3.857360363006592,
      "logps/rejected": -74.8001708984375,
      "loss": 0.16437619924545288,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13853932917118073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31387850642204285,
      "rewards/margins": 6.748230934143066,
      "rewards/rejected": -6.434351921081543,
      "step": 618,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.604998377150276,
      "grad_norm": 0.942775309085846,
      "learning_rate": 2.0327957483875694e-05,
      "logits/chosen": -1.3658478260040283,
      "logits/rejected": -1.3638713359832764,
      "logps/chosen": -5.2065839767456055,
      "logps/rejected": -75.38078308105469,
      "loss": 0.19663846492767334,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15267634391784668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4773911237716675,
      "rewards/margins": 6.9232282638549805,
      "rewards/rejected": -6.445837497711182,
      "step": 619,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6075949367088609,
      "grad_norm": 0.7806491255760193,
      "learning_rate": 2.0068965542773876e-05,
      "logits/chosen": -1.324061632156372,
      "logits/rejected": -1.3434507846832275,
      "logps/chosen": -2.706183671951294,
      "logps/rejected": -81.15982055664062,
      "loss": 0.14189621806144714,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12525126338005066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5728058815002441,
      "rewards/margins": 7.6588029861450195,
      "rewards/rejected": -7.085996150970459,
      "step": 620,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6101914962674457,
      "grad_norm": 0.7165568470954895,
      "learning_rate": 1.981144991913392e-05,
      "logits/chosen": -1.347527027130127,
      "logits/rejected": -1.3617969751358032,
      "logps/chosen": -3.1274702548980713,
      "logps/rejected": -89.35918426513672,
      "loss": 0.13759316504001617,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12061534821987152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5373324751853943,
      "rewards/margins": 8.121000289916992,
      "rewards/rejected": -7.583667755126953,
      "step": 621,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6127880558260306,
      "grad_norm": 0.7025841474533081,
      "learning_rate": 1.9555415369241225e-05,
      "logits/chosen": -1.3251571655273438,
      "logits/rejected": -1.3403229713439941,
      "logps/chosen": -2.860119104385376,
      "logps/rejected": -89.72604370117188,
      "loss": 0.12598752975463867,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12241947650909424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48666584491729736,
      "rewards/margins": 8.185951232910156,
      "rewards/rejected": -7.69928503036499,
      "step": 622,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6153846153846154,
      "grad_norm": 0.7084252834320068,
      "learning_rate": 1.9300866622025893e-05,
      "logits/chosen": -1.371070384979248,
      "logits/rejected": -1.383573293685913,
      "logps/chosen": -5.293520927429199,
      "logps/rejected": -87.0187759399414,
      "loss": 0.15480637550354004,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1465662270784378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3128480911254883,
      "rewards/margins": 7.824625492095947,
      "rewards/rejected": -7.511777877807617,
      "step": 623,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6179811749432003,
      "grad_norm": 0.7480162978172302,
      "learning_rate": 1.9047808378975486e-05,
      "logits/chosen": -1.3239214420318604,
      "logits/rejected": -1.3253374099731445,
      "logps/chosen": -6.071501731872559,
      "logps/rejected": -89.98245239257812,
      "loss": 0.18201348185539246,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1786748617887497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5310088396072388,
      "rewards/margins": 8.276474952697754,
      "rewards/rejected": -7.745467185974121,
      "step": 624,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.620577734501785,
      "grad_norm": 0.7341568470001221,
      "learning_rate": 1.8796245314048044e-05,
      "logits/chosen": -1.350547432899475,
      "logits/rejected": -1.3660997152328491,
      "logps/chosen": -2.5804781913757324,
      "logps/rejected": -85.6153793334961,
      "loss": 0.12906776368618011,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1090274304151535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5288074612617493,
      "rewards/margins": 7.744842052459717,
      "rewards/rejected": -7.216034412384033,
      "step": 625,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.62317429406037,
      "grad_norm": 0.6811425089836121,
      "learning_rate": 1.8546182073585828e-05,
      "logits/chosen": -1.3307138681411743,
      "logits/rejected": -1.3467280864715576,
      "logps/chosen": -4.399513244628906,
      "logps/rejected": -95.36567687988281,
      "loss": 0.17138627171516418,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1693284511566162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36113083362579346,
      "rewards/margins": 8.433231353759766,
      "rewards/rejected": -8.072100639343262,
      "step": 626,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6257708536189548,
      "grad_norm": 0.8490860462188721,
      "learning_rate": 1.829762327622958e-05,
      "logits/chosen": -1.328543782234192,
      "logits/rejected": -1.346970558166504,
      "logps/chosen": -4.569067001342773,
      "logps/rejected": -94.38148498535156,
      "loss": 0.15949155390262604,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15811914205551147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4599432945251465,
      "rewards/margins": 8.337040901184082,
      "rewards/rejected": -7.8770976066589355,
      "step": 627,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6283674131775396,
      "grad_norm": 0.8323735594749451,
      "learning_rate": 1.8050573512833068e-05,
      "logits/chosen": -1.3728554248809814,
      "logits/rejected": -1.388077735900879,
      "logps/chosen": -3.635087490081787,
      "logps/rejected": -84.30162811279297,
      "loss": 0.16639482975006104,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1421230286359787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4008945822715759,
      "rewards/margins": 7.5037126541137695,
      "rewards/rejected": -7.102817535400391,
      "step": 628,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6309639727361245,
      "grad_norm": 0.7016049027442932,
      "learning_rate": 1.7805037346378384e-05,
      "logits/chosen": -1.3624804019927979,
      "logits/rejected": -1.3788541555404663,
      "logps/chosen": -4.170276165008545,
      "logps/rejected": -90.93599700927734,
      "loss": 0.15539219975471497,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14993149042129517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5223506689071655,
      "rewards/margins": 8.19238567352295,
      "rewards/rejected": -7.670035362243652,
      "step": 629,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6335605322947095,
      "grad_norm": 5.010850429534912,
      "learning_rate": 1.7561019311891692e-05,
      "logits/chosen": -1.3748408555984497,
      "logits/rejected": -1.3848907947540283,
      "logps/chosen": -5.912624835968018,
      "logps/rejected": -90.88970184326172,
      "loss": 0.2722803056240082,
      "memory(GiB)": 46.82,
      "nll_loss": 0.22955626249313354,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.46179163455963135,
      "rewards/margins": 8.070945739746094,
      "rewards/rejected": -7.60915470123291,
      "step": 630,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6361570918532944,
      "grad_norm": 0.763546347618103,
      "learning_rate": 1.7318523916359375e-05,
      "logits/chosen": -1.324263572692871,
      "logits/rejected": -1.3297150135040283,
      "logps/chosen": -3.0377016067504883,
      "logps/rejected": -80.36509704589844,
      "loss": 0.13964565098285675,
      "memory(GiB)": 46.82,
      "nll_loss": 0.11970542371273041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6791478395462036,
      "rewards/margins": 7.430585861206055,
      "rewards/rejected": -6.751438140869141,
      "step": 631,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6387536514118792,
      "grad_norm": 0.7758535742759705,
      "learning_rate": 1.707755563864484e-05,
      "logits/chosen": -1.3663057088851929,
      "logits/rejected": -1.3660892248153687,
      "logps/chosen": -7.182569980621338,
      "logps/rejected": -77.53275299072266,
      "loss": 0.20710787177085876,
      "memory(GiB)": 46.82,
      "nll_loss": 0.18503035604953766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5746399164199829,
      "rewards/margins": 7.321009635925293,
      "rewards/rejected": -6.746369361877441,
      "step": 632,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6413502109704643,
      "grad_norm": 18.8266544342041,
      "learning_rate": 1.6838118929405854e-05,
      "logits/chosen": -1.3572516441345215,
      "logits/rejected": -1.3695107698440552,
      "logps/chosen": -4.2153215408325195,
      "logps/rejected": -70.64273834228516,
      "loss": 0.2971822917461395,
      "memory(GiB)": 46.82,
      "nll_loss": 0.28246480226516724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.355130672454834,
      "rewards/margins": 6.3078107833862305,
      "rewards/rejected": -5.952679634094238,
      "step": 633,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6439467705290491,
      "grad_norm": 0.8900645971298218,
      "learning_rate": 1.660021821101222e-05,
      "logits/chosen": -1.3750463724136353,
      "logits/rejected": -1.3676670789718628,
      "logps/chosen": -8.479406356811523,
      "logps/rejected": -74.21772003173828,
      "loss": 0.2598121166229248,
      "memory(GiB)": 46.82,
      "nll_loss": 0.2488086223602295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7162144184112549,
      "rewards/margins": 7.078793048858643,
      "rewards/rejected": -6.36257791519165,
      "step": 634,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.646543330087634,
      "grad_norm": 1.081635594367981,
      "learning_rate": 1.636385787746416e-05,
      "logits/chosen": -1.3343584537506104,
      "logits/rejected": -1.3506702184677124,
      "logps/chosen": -4.287896633148193,
      "logps/rejected": -82.9725112915039,
      "loss": 0.17199589312076569,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14536339044570923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6349250078201294,
      "rewards/margins": 7.4133148193359375,
      "rewards/rejected": -6.778390407562256,
      "step": 635,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6491398896462188,
      "grad_norm": 5.774163722991943,
      "learning_rate": 1.6129042294311226e-05,
      "logits/chosen": -1.343154788017273,
      "logits/rejected": -1.3550117015838623,
      "logps/chosen": -4.13479471206665,
      "logps/rejected": -68.96117401123047,
      "loss": 0.5674282908439636,
      "memory(GiB)": 46.82,
      "nll_loss": 0.40918517112731934,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3416549861431122,
      "rewards/margins": 6.578398704528809,
      "rewards/rejected": -6.236744403839111,
      "step": 636,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6517364492048037,
      "grad_norm": 0.8650281429290771,
      "learning_rate": 1.5895775798571522e-05,
      "logits/chosen": -1.3902291059494019,
      "logits/rejected": -1.3937687873840332,
      "logps/chosen": -5.421413421630859,
      "logps/rejected": -78.23155212402344,
      "loss": 0.1860850304365158,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1803363710641861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4473956525325775,
      "rewards/margins": 7.034005641937256,
      "rewards/rejected": -6.586610317230225,
      "step": 637,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6543330087633885,
      "grad_norm": 0.75139319896698,
      "learning_rate": 1.5664062698651705e-05,
      "logits/chosen": -1.3419991731643677,
      "logits/rejected": -1.3535465002059937,
      "logps/chosen": -3.7635464668273926,
      "logps/rejected": -94.62512969970703,
      "loss": 0.1265435814857483,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12444484233856201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40557020902633667,
      "rewards/margins": 8.491599082946777,
      "rewards/rejected": -8.086028099060059,
      "step": 638,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6569295683219734,
      "grad_norm": 1.0433430671691895,
      "learning_rate": 1.5433907274267355e-05,
      "logits/chosen": -1.328079104423523,
      "logits/rejected": -1.3367297649383545,
      "logps/chosen": -5.806182861328125,
      "logps/rejected": -84.19864654541016,
      "loss": 0.2376406192779541,
      "memory(GiB)": 46.82,
      "nll_loss": 0.235676571726799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.45906656980514526,
      "rewards/margins": 7.460108757019043,
      "rewards/rejected": -7.001041889190674,
      "step": 639,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6595261278805582,
      "grad_norm": 0.6470867991447449,
      "learning_rate": 1.5205313776364027e-05,
      "logits/chosen": -1.3525469303131104,
      "logits/rejected": -1.3603522777557373,
      "logps/chosen": -3.8683226108551025,
      "logps/rejected": -82.79778289794922,
      "loss": 0.15905708074569702,
      "memory(GiB)": 46.82,
      "nll_loss": 0.15342724323272705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5418492555618286,
      "rewards/margins": 7.654129505157471,
      "rewards/rejected": -7.11228084564209,
      "step": 640,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.662122687439143,
      "grad_norm": 0.8814226388931274,
      "learning_rate": 1.4978286427038601e-05,
      "logits/chosen": -1.2953623533248901,
      "logits/rejected": -1.3175257444381714,
      "logps/chosen": -3.588663101196289,
      "logps/rejected": -83.4087905883789,
      "loss": 0.17859791219234467,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14969876408576965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3954733610153198,
      "rewards/margins": 7.318519115447998,
      "rewards/rejected": -6.923046112060547,
      "step": 641,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6647192469977279,
      "grad_norm": 0.9600425362586975,
      "learning_rate": 1.4752829419461356e-05,
      "logits/chosen": -1.3675446510314941,
      "logits/rejected": -1.3715183734893799,
      "logps/chosen": -5.777808666229248,
      "logps/rejected": -72.55799102783203,
      "loss": 0.21355006098747253,
      "memory(GiB)": 46.82,
      "nll_loss": 0.20122846961021423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6262421607971191,
      "rewards/margins": 6.9362077713012695,
      "rewards/rejected": -6.309965133666992,
      "step": 642,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6673158065563127,
      "grad_norm": 0.7960318326950073,
      "learning_rate": 1.4528946917798603e-05,
      "logits/chosen": -1.3032656908035278,
      "logits/rejected": -1.308361291885376,
      "logps/chosen": -4.921981334686279,
      "logps/rejected": -72.86154174804688,
      "loss": 0.18860392272472382,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1692950427532196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5242667198181152,
      "rewards/margins": 6.5315327644348145,
      "rewards/rejected": -6.007266044616699,
      "step": 643,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6699123661148978,
      "grad_norm": 0.7154169678688049,
      "learning_rate": 1.4306643057135637e-05,
      "logits/chosen": -1.3495804071426392,
      "logits/rejected": -1.3487272262573242,
      "logps/chosen": -4.284914016723633,
      "logps/rejected": -82.88322448730469,
      "loss": 0.1530197262763977,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13098730146884918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6068970561027527,
      "rewards/margins": 7.615091323852539,
      "rewards/rejected": -7.008193492889404,
      "step": 644,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6725089256734826,
      "grad_norm": 0.9531897306442261,
      "learning_rate": 1.4085921943400415e-05,
      "logits/chosen": -1.3348288536071777,
      "logits/rejected": -1.3515242338180542,
      "logps/chosen": -5.095047950744629,
      "logps/rejected": -75.7342758178711,
      "loss": 0.24714748561382294,
      "memory(GiB)": 46.82,
      "nll_loss": 0.21035797894001007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38048332929611206,
      "rewards/margins": 6.424774169921875,
      "rewards/rejected": -6.044290542602539,
      "step": 645,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6751054852320675,
      "grad_norm": 16.722322463989258,
      "learning_rate": 1.3866787653287805e-05,
      "logits/chosen": -1.3415000438690186,
      "logits/rejected": -1.359058141708374,
      "logps/chosen": -3.7091660499572754,
      "logps/rejected": -87.80061340332031,
      "loss": 0.1800370067358017,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14961153268814087,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5320316553115845,
      "rewards/margins": 7.5755228996276855,
      "rewards/rejected": -7.043491363525391,
      "step": 646,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6777020447906525,
      "grad_norm": 0.6856510043144226,
      "learning_rate": 1.3649244234184156e-05,
      "logits/chosen": -1.3474607467651367,
      "logits/rejected": -1.353614091873169,
      "logps/chosen": -3.7221789360046387,
      "logps/rejected": -81.68478393554688,
      "loss": 0.1335882842540741,
      "memory(GiB)": 46.82,
      "nll_loss": 0.13118202984333038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6159143447875977,
      "rewards/margins": 7.432640075683594,
      "rewards/rejected": -6.816725730895996,
      "step": 647,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6802986043492374,
      "grad_norm": 0.6763714551925659,
      "learning_rate": 1.3433295704092585e-05,
      "logits/chosen": -1.3173692226409912,
      "logits/rejected": -1.3308582305908203,
      "logps/chosen": -4.244126796722412,
      "logps/rejected": -91.7069320678711,
      "loss": 0.15246841311454773,
      "memory(GiB)": 46.82,
      "nll_loss": 0.14819884300231934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5995199084281921,
      "rewards/margins": 8.185004234313965,
      "rewards/rejected": -7.585484027862549,
      "step": 648,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.6828951639078222,
      "grad_norm": 0.7699263691902161,
      "learning_rate": 1.3218946051558867e-05,
      "logits/chosen": -1.355929970741272,
      "logits/rejected": -1.3632704019546509,
      "logps/chosen": -3.4825494289398193,
      "logps/rejected": -76.4013442993164,
      "loss": 0.15149284899234772,
      "memory(GiB)": 46.82,
      "nll_loss": 0.12583358585834503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6053353548049927,
      "rewards/margins": 7.040757179260254,
      "rewards/rejected": -6.435421943664551,
      "step": 649,
      "train_speed(iter/s)": 0.005341
    },
    {
      "epoch": 1.685491723466407,
      "grad_norm": 1.0006968975067139,
      "learning_rate": 1.3006199235597627e-05,
      "logits/chosen": -1.3747293949127197,
      "logits/rejected": -1.378046989440918,
      "logps/chosen": -4.248571395874023,
      "logps/rejected": -78.72927856445312,
      "loss": 0.21873235702514648,
      "memory(GiB)": 46.82,
      "nll_loss": 0.1984972208738327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36940085887908936,
      "rewards/margins": 7.074660778045654,
      "rewards/rejected": -6.705259799957275,
      "step": 650,
      "train_speed(iter/s)": 0.005341
    }
  ],
  "logging_steps": 1,
  "max_steps": 770,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 4.628166896246587e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}