diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,7251 @@
+{
+  "best_global_step": 300,
+  "best_metric": 0.41139093,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_image_gt10_gt2_rewrite_False/v1-20250614-001049/checkpoint-300",
+  "epoch": 0.8871023632961397,
+  "eval_steps": 300,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0022177559082403493,
+      "grad_norm": 7.276436805725098,
+      "learning_rate": 4.444444444444445e-06,
+      "logits/chosen": -0.7332726716995239,
+      "logits/rejected": -0.7375782132148743,
+      "logps/chosen": -10.463276863098145,
+      "logps/rejected": -16.490936279296875,
+      "loss": 1.3974095582962036,
+      "memory(GiB)": 44.63,
+      "nll_loss": 0.7042622566223145,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.007748
+    },
+    {
+      "epoch": 0.004435511816480699,
+      "grad_norm": 10.07150936126709,
+      "learning_rate": 8.88888888888889e-06,
+      "logits/chosen": -0.7215312719345093,
+      "logits/rejected": -0.7478011250495911,
+      "logps/chosen": -8.118193626403809,
+      "logps/rejected": -19.331707000732422,
+      "loss": 1.5197914838790894,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.8266440629959106,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.007824
+    },
+    {
+      "epoch": 0.006653267724721048,
+      "grad_norm": 10.19728946685791,
+      "learning_rate": 1.3333333333333333e-05,
+      "logits/chosen": -0.685354471206665,
+      "logits/rejected": -0.7223411202430725,
+      "logps/chosen": -8.368087768554688,
+      "logps/rejected": -23.5802001953125,
+      "loss": 1.4096384048461914,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7168383002281189,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -3.958120942115784e-05,
+      "rewards/margins": 0.0009151366539299488,
+      "rewards/rejected": -0.0009547180961817503,
+      "step": 3,
+      "train_speed(iter/s)": 0.007859
+    },
+    {
+      "epoch": 0.008871023632961397,
+      "grad_norm": 8.582710266113281,
+      "learning_rate": 1.777777777777778e-05,
+      "logits/chosen": -0.6947981119155884,
+      "logits/rejected": -0.7241417765617371,
+      "logps/chosen": -13.892762184143066,
+      "logps/rejected": -24.124719619750977,
+      "loss": 1.5422468185424805,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.8507797122001648,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0054289610125124454,
+      "rewards/margins": 0.0035484125837683678,
+      "rewards/rejected": 0.0018805486615747213,
+      "step": 4,
+      "train_speed(iter/s)": 0.007846
+    },
+    {
+      "epoch": 0.011088779541201747,
+      "grad_norm": 8.61772632598877,
+      "learning_rate": 2.2222222222222223e-05,
+      "logits/chosen": -0.7581620216369629,
+      "logits/rejected": -0.7803558111190796,
+      "logps/chosen": -11.490116119384766,
+      "logps/rejected": -17.942598342895508,
+      "loss": 1.456505298614502,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7721378803253174,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01581314392387867,
+      "rewards/margins": 0.01840357482433319,
+      "rewards/rejected": -0.0025904285721480846,
+      "step": 5,
+      "train_speed(iter/s)": 0.007817
+    },
+    {
+      "epoch": 0.013306535449442096,
+      "grad_norm": 11.505023956298828,
+      "learning_rate": 2.6666666666666667e-05,
+      "logits/chosen": -0.7362898588180542,
+      "logits/rejected": -0.7709215879440308,
+      "logps/chosen": -8.461962699890137,
+      "logps/rejected": -27.864641189575195,
+      "loss": 1.3553998470306396,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6918811798095703,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06961186230182648,
+      "rewards/margins": 0.06262461841106415,
+      "rewards/rejected": 0.00698724202811718,
+      "step": 6,
+      "train_speed(iter/s)": 0.007858
+    },
+    {
+      "epoch": 0.015524291357682445,
+      "grad_norm": 7.58583402633667,
+      "learning_rate": 3.111111111111111e-05,
+      "logits/chosen": -0.7080317139625549,
+      "logits/rejected": -0.7482547760009766,
+      "logps/chosen": -6.92811393737793,
+      "logps/rejected": -22.422035217285156,
+      "loss": 1.1541662216186523,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5072559118270874,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.10978628695011139,
+      "rewards/margins": 0.10527929663658142,
+      "rewards/rejected": 0.004506995901465416,
+      "step": 7,
+      "train_speed(iter/s)": 0.007928
+    },
+    {
+      "epoch": 0.017742047265922795,
+      "grad_norm": 7.752142429351807,
+      "learning_rate": 3.555555555555556e-05,
+      "logits/chosen": -0.68747478723526,
+      "logits/rejected": -0.7212347388267517,
+      "logps/chosen": -9.500894546508789,
+      "logps/rejected": -23.92633056640625,
+      "loss": 1.3329849243164062,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7127149701118469,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.030532313510775566,
+      "rewards/margins": 0.18240433931350708,
+      "rewards/rejected": -0.15187203884124756,
+      "step": 8,
+      "train_speed(iter/s)": 0.007913
+    },
+    {
+      "epoch": 0.019959803174163144,
+      "grad_norm": 3.5858492851257324,
+      "learning_rate": 4e-05,
+      "logits/chosen": -0.73879075050354,
+      "logits/rejected": -0.7796315550804138,
+      "logps/chosen": -5.0881805419921875,
+      "logps/rejected": -24.7738037109375,
+      "loss": 0.8303911685943604,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2771793305873871,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.11474708467721939,
+      "rewards/margins": 0.37330126762390137,
+      "rewards/rejected": -0.2585541307926178,
+      "step": 9,
+      "train_speed(iter/s)": 0.007944
+    },
+    {
+      "epoch": 0.022177559082403493,
+      "grad_norm": 7.612152099609375,
+      "learning_rate": 4.4444444444444447e-05,
+      "logits/chosen": -0.6987114548683167,
+      "logits/rejected": -0.7058257460594177,
+      "logps/chosen": -15.217192649841309,
+      "logps/rejected": -22.54599380493164,
+      "loss": 1.2511308193206787,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6868589520454407,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12514850497245789,
+      "rewards/margins": 0.4101864695549011,
+      "rewards/rejected": -0.5353350043296814,
+      "step": 10,
+      "train_speed(iter/s)": 0.007962
+    },
+    {
+      "epoch": 0.024395314990643843,
+      "grad_norm": 6.01246976852417,
+      "learning_rate": 4.888888888888889e-05,
+      "logits/chosen": -0.6994076371192932,
+      "logits/rejected": -0.7003622055053711,
+      "logps/chosen": -14.472052574157715,
+      "logps/rejected": -23.425853729248047,
+      "loss": 1.284260630607605,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6495041251182556,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21374526619911194,
+      "rewards/margins": 0.31260305643081665,
+      "rewards/rejected": -0.5263482928276062,
+      "step": 11,
+      "train_speed(iter/s)": 0.007969
+    },
+    {
+      "epoch": 0.026613070898884192,
+      "grad_norm": 6.6873040199279785,
+      "learning_rate": 5.333333333333333e-05,
+      "logits/chosen": -0.719663143157959,
+      "logits/rejected": -0.7418795824050903,
+      "logps/chosen": -15.615825653076172,
+      "logps/rejected": -24.63580322265625,
+      "loss": 1.4747008085250854,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.847057044506073,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2930046319961548,
+      "rewards/margins": 0.3898415267467499,
+      "rewards/rejected": -0.682846188545227,
+      "step": 12,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.02883082680712454,
+      "grad_norm": 4.942108631134033,
+      "learning_rate": 5.7777777777777776e-05,
+      "logits/chosen": -0.6910253167152405,
+      "logits/rejected": -0.7100927829742432,
+      "logps/chosen": -10.659884452819824,
+      "logps/rejected": -25.604801177978516,
+      "loss": 1.0991154909133911,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.529512882232666,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1485268771648407,
+      "rewards/margins": 0.43985775113105774,
+      "rewards/rejected": -0.5883846282958984,
+      "step": 13,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.03104858271536489,
+      "grad_norm": 8.013643264770508,
+      "learning_rate": 6.222222222222222e-05,
+      "logits/chosen": -0.6786947846412659,
+      "logits/rejected": -0.7003186345100403,
+      "logps/chosen": -8.749262809753418,
+      "logps/rejected": -22.5197811126709,
+      "loss": 1.3776031732559204,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7835733294487,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08169429004192352,
+      "rewards/margins": 0.344772070646286,
+      "rewards/rejected": -0.42646634578704834,
+      "step": 14,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.033266338623605236,
+      "grad_norm": 5.5514817237854,
+      "learning_rate": 6.666666666666667e-05,
+      "logits/chosen": -0.703814685344696,
+      "logits/rejected": -0.7373018264770508,
+      "logps/chosen": -6.65080451965332,
+      "logps/rejected": -23.22998809814453,
+      "loss": 1.1897517442703247,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.626986026763916,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03274273872375488,
+      "rewards/margins": 0.42112234234809875,
+      "rewards/rejected": -0.38837966322898865,
+      "step": 15,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.03548409453184559,
+      "grad_norm": 3.7225890159606934,
+      "learning_rate": 7.111111111111112e-05,
+      "logits/chosen": -0.6734000444412231,
+      "logits/rejected": -0.6864615678787231,
+      "logps/chosen": -11.966102600097656,
+      "logps/rejected": -23.13942527770996,
+      "loss": 1.2046136856079102,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5588831901550293,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07107432186603546,
+      "rewards/margins": 0.2272331863641739,
+      "rewards/rejected": -0.29830753803253174,
+      "step": 16,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.037701850440085935,
+      "grad_norm": 3.068159580230713,
+      "learning_rate": 7.555555555555556e-05,
+      "logits/chosen": -0.6323463916778564,
+      "logits/rejected": -0.6556217670440674,
+      "logps/chosen": -7.376963138580322,
+      "logps/rejected": -23.585092544555664,
+      "loss": 0.990348756313324,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.377433717250824,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09706290066242218,
+      "rewards/margins": 0.31319916248321533,
+      "rewards/rejected": -0.21613627672195435,
+      "step": 17,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.03991960634832629,
+      "grad_norm": 2.306462526321411,
+      "learning_rate": 8e-05,
+      "logits/chosen": -0.652298092842102,
+      "logits/rejected": -0.6973198056221008,
+      "logps/chosen": -4.506044864654541,
+      "logps/rejected": -23.26398468017578,
+      "loss": 0.8916468024253845,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35007452964782715,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.22034089267253876,
+      "rewards/margins": 0.45051315426826477,
+      "rewards/rejected": -0.23017224669456482,
+      "step": 18,
+      "train_speed(iter/s)": 0.008047
+    },
+    {
+      "epoch": 0.042137362256566634,
+      "grad_norm": 2.24660325050354,
+      "learning_rate": 8.444444444444444e-05,
+      "logits/chosen": -0.6684755086898804,
+      "logits/rejected": -0.6921530365943909,
+      "logps/chosen": -5.159789562225342,
+      "logps/rejected": -20.33713722229004,
+      "loss": 0.9385991096496582,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3643404245376587,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1963498443365097,
+      "rewards/margins": 0.30785509943962097,
+      "rewards/rejected": -0.11150527000427246,
+      "step": 19,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.044355118164806986,
+      "grad_norm": 4.343178749084473,
+      "learning_rate": 8.888888888888889e-05,
+      "logits/chosen": -0.64969801902771,
+      "logits/rejected": -0.6867151856422424,
+      "logps/chosen": -8.542341232299805,
+      "logps/rejected": -27.74468421936035,
+      "loss": 1.159504771232605,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5642345547676086,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.17812781035900116,
+      "rewards/margins": 0.28209057450294495,
+      "rewards/rejected": -0.10396274924278259,
+      "step": 20,
+      "train_speed(iter/s)": 0.008063
+    },
+    {
+      "epoch": 0.04657287407304733,
+      "grad_norm": 2.5781965255737305,
+      "learning_rate": 9.333333333333334e-05,
+      "logits/chosen": -0.7248848676681519,
+      "logits/rejected": -0.7462906837463379,
+      "logps/chosen": -8.212335586547852,
+      "logps/rejected": -15.989672660827637,
+      "loss": 1.174903392791748,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5117508172988892,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.13524048030376434,
+      "rewards/margins": 0.12396176159381866,
+      "rewards/rejected": 0.011278722435235977,
+      "step": 21,
+      "train_speed(iter/s)": 0.008072
+    },
+    {
+      "epoch": 0.048790629981287685,
+      "grad_norm": 2.095536947250366,
+      "learning_rate": 9.777777777777778e-05,
+      "logits/chosen": -0.618206262588501,
+      "logits/rejected": -0.6151365041732788,
+      "logps/chosen": -8.146368026733398,
+      "logps/rejected": -15.488615036010742,
+      "loss": 1.011189579963684,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4133646488189697,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.2263016253709793,
+      "rewards/margins": 0.25505900382995605,
+      "rewards/rejected": -0.02875736728310585,
+      "step": 22,
+      "train_speed(iter/s)": 0.008074
+    },
+    {
+      "epoch": 0.05100838588952803,
+      "grad_norm": 2.504054307937622,
+      "learning_rate": 0.00010222222222222222,
+      "logits/chosen": -0.6510525345802307,
+      "logits/rejected": -0.664685845375061,
+      "logps/chosen": -8.884237289428711,
+      "logps/rejected": -20.657085418701172,
+      "loss": 1.1501473188400269,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5572206974029541,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1401275396347046,
+      "rewards/margins": 0.2846068739891052,
+      "rewards/rejected": -0.14447934925556183,
+      "step": 23,
+      "train_speed(iter/s)": 0.008064
+    },
+    {
+      "epoch": 0.053226141797768384,
+      "grad_norm": 2.6294727325439453,
+      "learning_rate": 0.00010666666666666667,
+      "logits/chosen": -0.6562620997428894,
+      "logits/rejected": -0.6580171585083008,
+      "logps/chosen": -11.128717422485352,
+      "logps/rejected": -19.960248947143555,
+      "loss": 1.025831699371338,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35331588983535767,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05753737688064575,
+      "rewards/margins": 0.09378618001937866,
+      "rewards/rejected": -0.03624879568815231,
+      "step": 24,
+      "train_speed(iter/s)": 0.008068
+    },
+    {
+      "epoch": 0.05544389770600873,
+      "grad_norm": 4.113434314727783,
+      "learning_rate": 0.00011111111111111112,
+      "logits/chosen": -0.7034849524497986,
+      "logits/rejected": -0.703572154045105,
+      "logps/chosen": -12.834946632385254,
+      "logps/rejected": -22.007543563842773,
+      "loss": 1.219478726387024,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.56423419713974,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.04055144265294075,
+      "rewards/margins": 0.14624251425266266,
+      "rewards/rejected": -0.10569106787443161,
+      "step": 25,
+      "train_speed(iter/s)": 0.008066
+    },
+    {
+      "epoch": 0.05766165361424908,
+      "grad_norm": 4.309391498565674,
+      "learning_rate": 0.00011555555555555555,
+      "logits/chosen": -0.662431001663208,
+      "logits/rejected": -0.684702455997467,
+      "logps/chosen": -11.661463737487793,
+      "logps/rejected": -24.191795349121094,
+      "loss": 1.1345590353012085,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5379157662391663,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.027302712202072144,
+      "rewards/margins": 0.2985782027244568,
+      "rewards/rejected": -0.27127546072006226,
+      "step": 26,
+      "train_speed(iter/s)": 0.008061
+    },
+    {
+      "epoch": 0.05987940952248943,
+      "grad_norm": 4.278309345245361,
+      "learning_rate": 0.00012,
+      "logits/chosen": -0.7126907706260681,
+      "logits/rejected": -0.7174281477928162,
+      "logps/chosen": -11.30073070526123,
+      "logps/rejected": -19.983867645263672,
+      "loss": 1.1452325582504272,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5350662469863892,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.15527383983135223,
+      "rewards/margins": 0.2666708827018738,
+      "rewards/rejected": -0.11139706522226334,
+      "step": 27,
+      "train_speed(iter/s)": 0.008058
+    },
+    {
+      "epoch": 0.06209716543072978,
+      "grad_norm": 3.8597252368927,
+      "learning_rate": 0.00012444444444444444,
+      "logits/chosen": -0.7358130216598511,
+      "logits/rejected": -0.7531285285949707,
+      "logps/chosen": -6.426143169403076,
+      "logps/rejected": -24.873878479003906,
+      "loss": 0.8902767896652222,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37025153636932373,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17672136425971985,
+      "rewards/margins": 0.4878941774368286,
+      "rewards/rejected": -0.3111727833747864,
+      "step": 28,
+      "train_speed(iter/s)": 0.008062
+    },
+    {
+      "epoch": 0.06431492133897013,
+      "grad_norm": 5.0088348388671875,
+      "learning_rate": 0.00012888888888888892,
+      "logits/chosen": -0.6740167140960693,
+      "logits/rejected": -0.7026465535163879,
+      "logps/chosen": -6.703609466552734,
+      "logps/rejected": -27.67329978942871,
+      "loss": 1.0634881258010864,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5235491991043091,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11447454988956451,
+      "rewards/margins": 0.4801509380340576,
+      "rewards/rejected": -0.3656763732433319,
+      "step": 29,
+      "train_speed(iter/s)": 0.008064
+    },
+    {
+      "epoch": 0.06653267724721047,
+      "grad_norm": 7.510077953338623,
+      "learning_rate": 0.00013333333333333334,
+      "logits/chosen": -0.649000883102417,
+      "logits/rejected": -0.6501227021217346,
+      "logps/chosen": -9.731987953186035,
+      "logps/rejected": -22.024688720703125,
+      "loss": 1.4005825519561768,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7944965362548828,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0400068461894989,
+      "rewards/margins": 0.3603759706020355,
+      "rewards/rejected": -0.3203691244125366,
+      "step": 30,
+      "train_speed(iter/s)": 0.00806
+    },
+    {
+      "epoch": 0.06875043315545083,
+      "grad_norm": 2.7688963413238525,
+      "learning_rate": 0.0001377777777777778,
+      "logits/chosen": -0.683768093585968,
+      "logits/rejected": -0.6991692781448364,
+      "logps/chosen": -9.37608528137207,
+      "logps/rejected": -19.639158248901367,
+      "loss": 1.0507181882858276,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45388615131378174,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11749469488859177,
+      "rewards/margins": 0.371566504240036,
+      "rewards/rejected": -0.25407183170318604,
+      "step": 31,
+      "train_speed(iter/s)": 0.00806
+    },
+    {
+      "epoch": 0.07096818906369118,
+      "grad_norm": 2.550858974456787,
+      "learning_rate": 0.00014222222222222224,
+      "logits/chosen": -0.6819652318954468,
+      "logits/rejected": -0.6938492059707642,
+      "logps/chosen": -12.82135009765625,
+      "logps/rejected": -32.83695602416992,
+      "loss": 1.083910346031189,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5153654217720032,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08128099143505096,
+      "rewards/margins": 0.3762062191963196,
+      "rewards/rejected": -0.2949252128601074,
+      "step": 32,
+      "train_speed(iter/s)": 0.008067
+    },
+    {
+      "epoch": 0.07318594497193152,
+      "grad_norm": 2.6052372455596924,
+      "learning_rate": 0.00014666666666666666,
+      "logits/chosen": -0.6425620317459106,
+      "logits/rejected": -0.6855881214141846,
+      "logps/chosen": -5.170282363891602,
+      "logps/rejected": -25.80876922607422,
+      "loss": 0.9529789090156555,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.420663058757782,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.21988970041275024,
+      "rewards/margins": 0.48745402693748474,
+      "rewards/rejected": -0.2675642967224121,
+      "step": 33,
+      "train_speed(iter/s)": 0.008068
+    },
+    {
+      "epoch": 0.07540370088017187,
+      "grad_norm": 3.4502556324005127,
+      "learning_rate": 0.0001511111111111111,
+      "logits/chosen": -0.6484149694442749,
+      "logits/rejected": -0.6727481484413147,
+      "logps/chosen": -6.394935607910156,
+      "logps/rejected": -20.673736572265625,
+      "loss": 0.9740100502967834,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39780640602111816,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.23207196593284607,
+      "rewards/margins": 0.35720905661582947,
+      "rewards/rejected": -0.12513704597949982,
+      "step": 34,
+      "train_speed(iter/s)": 0.008066
+    },
+    {
+      "epoch": 0.07762145678841223,
+      "grad_norm": 2.5214908123016357,
+      "learning_rate": 0.00015555555555555556,
+      "logits/chosen": -0.6334148049354553,
+      "logits/rejected": -0.6357681751251221,
+      "logps/chosen": -7.275469779968262,
+      "logps/rejected": -26.607770919799805,
+      "loss": 0.8099223375320435,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2854778468608856,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.12199296802282333,
+      "rewards/margins": 0.5918037295341492,
+      "rewards/rejected": -0.4698107838630676,
+      "step": 35,
+      "train_speed(iter/s)": 0.008065
+    },
+    {
+      "epoch": 0.07983921269665258,
+      "grad_norm": 4.882220268249512,
+      "learning_rate": 0.00016,
+      "logits/chosen": -0.6802562475204468,
+      "logits/rejected": -0.7064938545227051,
+      "logps/chosen": -15.904359817504883,
+      "logps/rejected": -29.350811004638672,
+      "loss": 1.5268551111221313,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.9425684809684753,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.18609833717346191,
+      "rewards/margins": 0.4418560266494751,
+      "rewards/rejected": -0.627954363822937,
+      "step": 36,
+      "train_speed(iter/s)": 0.008068
+    },
+    {
+      "epoch": 0.08205696860489292,
+      "grad_norm": 3.0210578441619873,
+      "learning_rate": 0.00016444444444444444,
+      "logits/chosen": -0.7036482095718384,
+      "logits/rejected": -0.7044983506202698,
+      "logps/chosen": -14.613797187805176,
+      "logps/rejected": -30.51740837097168,
+      "loss": 1.017217993736267,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5247446298599243,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.03162863850593567,
+      "rewards/margins": 0.6317514181137085,
+      "rewards/rejected": -0.6633801460266113,
+      "step": 37,
+      "train_speed(iter/s)": 0.008059
+    },
+    {
+      "epoch": 0.08427472451313327,
+      "grad_norm": 3.098156213760376,
+      "learning_rate": 0.00016888888888888889,
+      "logits/chosen": -0.6714186072349548,
+      "logits/rejected": -0.6812422871589661,
+      "logps/chosen": -16.367725372314453,
+      "logps/rejected": -28.301345825195312,
+      "loss": 1.2068523168563843,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6946620345115662,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.015942862257361412,
+      "rewards/margins": 0.6814810633659363,
+      "rewards/rejected": -0.6655381917953491,
+      "step": 38,
+      "train_speed(iter/s)": 0.00807
+    },
+    {
+      "epoch": 0.08649248042137363,
+      "grad_norm": 3.322134256362915,
+      "learning_rate": 0.00017333333333333334,
+      "logits/chosen": -0.6880184412002563,
+      "logits/rejected": -0.7056170701980591,
+      "logps/chosen": -6.932201385498047,
+      "logps/rejected": -23.875953674316406,
+      "loss": 0.8887794613838196,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.32947438955307007,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.09907499700784683,
+      "rewards/margins": 0.4684091806411743,
+      "rewards/rejected": -0.3693341612815857,
+      "step": 39,
+      "train_speed(iter/s)": 0.008055
+    },
+    {
+      "epoch": 0.08871023632961397,
+      "grad_norm": 3.0120160579681396,
+      "learning_rate": 0.00017777777777777779,
+      "logits/chosen": -0.6974343061447144,
+      "logits/rejected": -0.7076331377029419,
+      "logps/chosen": -10.623748779296875,
+      "logps/rejected": -26.470630645751953,
+      "loss": 1.1136667728424072,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.532155454158783,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0889241099357605,
+      "rewards/margins": 0.455722838640213,
+      "rewards/rejected": -0.3667986989021301,
+      "step": 40,
+      "train_speed(iter/s)": 0.008055
+    },
+    {
+      "epoch": 0.09092799223785432,
+      "grad_norm": 2.5109620094299316,
+      "learning_rate": 0.00018222222222222224,
+      "logits/chosen": -0.7382675409317017,
+      "logits/rejected": -0.7507362961769104,
+      "logps/chosen": -8.733287811279297,
+      "logps/rejected": -22.5925350189209,
+      "loss": 0.8756768703460693,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3629510700702667,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2706006169319153,
+      "rewards/margins": 0.6100682616233826,
+      "rewards/rejected": -0.3394676446914673,
+      "step": 41,
+      "train_speed(iter/s)": 0.008049
+    },
+    {
+      "epoch": 0.09314574814609466,
+      "grad_norm": 3.9691097736358643,
+      "learning_rate": 0.0001866666666666667,
+      "logits/chosen": -0.7684446573257446,
+      "logits/rejected": -0.787712037563324,
+      "logps/chosen": -11.393584251403809,
+      "logps/rejected": -30.219039916992188,
+      "loss": 1.211122751235962,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6706364750862122,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.007796842604875565,
+      "rewards/margins": 0.7110621333122253,
+      "rewards/rejected": -0.7188589572906494,
+      "step": 42,
+      "train_speed(iter/s)": 0.008051
+    },
+    {
+      "epoch": 0.09536350405433502,
+      "grad_norm": 2.963203191757202,
+      "learning_rate": 0.00019111111111111114,
+      "logits/chosen": -0.6818578839302063,
+      "logits/rejected": -0.6889616250991821,
+      "logps/chosen": -12.270942687988281,
+      "logps/rejected": -27.153846740722656,
+      "loss": 0.9894061088562012,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4868725836277008,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.012282166630029678,
+      "rewards/margins": 0.6492558121681213,
+      "rewards/rejected": -0.6369736194610596,
+      "step": 43,
+      "train_speed(iter/s)": 0.008043
+    },
+    {
+      "epoch": 0.09758125996257537,
+      "grad_norm": 3.587053060531616,
+      "learning_rate": 0.00019555555555555556,
+      "logits/chosen": -0.7400009632110596,
+      "logits/rejected": -0.7725372314453125,
+      "logps/chosen": -12.326873779296875,
+      "logps/rejected": -22.454561233520508,
+      "loss": 1.0897188186645508,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5316156148910522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.11403915286064148,
+      "rewards/margins": 0.4852198362350464,
+      "rewards/rejected": -0.3711806833744049,
+      "step": 44,
+      "train_speed(iter/s)": 0.00805
+    },
+    {
+      "epoch": 0.09979901587081572,
+      "grad_norm": 2.648092269897461,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.7456753849983215,
+      "logits/rejected": -0.7561041116714478,
+      "logps/chosen": -7.531292915344238,
+      "logps/rejected": -21.980024337768555,
+      "loss": 0.9317559003829956,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3929036855697632,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.10482822358608246,
+      "rewards/margins": 0.6396186351776123,
+      "rewards/rejected": -0.5347905158996582,
+      "step": 45,
+      "train_speed(iter/s)": 0.008052
+    },
+    {
+      "epoch": 0.10201677177905606,
+      "grad_norm": 3.2653682231903076,
+      "learning_rate": 0.0001999993249483057,
+      "logits/chosen": -0.7328958511352539,
+      "logits/rejected": -0.7547612190246582,
+      "logps/chosen": -11.5140380859375,
+      "logps/rejected": -22.490699768066406,
+      "loss": 1.031274676322937,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5204803943634033,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.19196264445781708,
+      "rewards/margins": 0.651879072189331,
+      "rewards/rejected": -0.45991644263267517,
+      "step": 46,
+      "train_speed(iter/s)": 0.008051
+    },
+    {
+      "epoch": 0.10423452768729642,
+      "grad_norm": 5.505344867706299,
+      "learning_rate": 0.0001999972998023366,
+      "logits/chosen": -0.7554954290390015,
+      "logits/rejected": -0.7810638546943665,
+      "logps/chosen": -5.766447067260742,
+      "logps/rejected": -19.390804290771484,
+      "loss": 1.2154221534729004,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7068721055984497,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2990206182003021,
+      "rewards/margins": 0.5781352519989014,
+      "rewards/rejected": -0.27911463379859924,
+      "step": 47,
+      "train_speed(iter/s)": 0.008058
+    },
+    {
+      "epoch": 0.10645228359553677,
+      "grad_norm": 2.649653911590576,
+      "learning_rate": 0.00019999392458943432,
+      "logits/chosen": -0.6805804967880249,
+      "logits/rejected": -0.7093008756637573,
+      "logps/chosen": -14.792132377624512,
+      "logps/rejected": -24.17580795288086,
+      "loss": 1.1033304929733276,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5765480995178223,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2228904664516449,
+      "rewards/margins": 0.643200695514679,
+      "rewards/rejected": -0.42031019926071167,
+      "step": 48,
+      "train_speed(iter/s)": 0.008059
+    },
+    {
+      "epoch": 0.10867003950377711,
+      "grad_norm": 3.4825599193573,
+      "learning_rate": 0.00019998919935516768,
+      "logits/chosen": -0.6554335951805115,
+      "logits/rejected": -0.6713435053825378,
+      "logps/chosen": -10.660847663879395,
+      "logps/rejected": -35.06268310546875,
+      "loss": 1.0064078569412231,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6083590388298035,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3527636229991913,
+      "rewards/margins": 0.9422526359558105,
+      "rewards/rejected": -0.5894889831542969,
+      "step": 49,
+      "train_speed(iter/s)": 0.008061
+    },
+    {
+      "epoch": 0.11088779541201746,
+      "grad_norm": 2.771404504776001,
+      "learning_rate": 0.00019998312416333227,
+      "logits/chosen": -0.6812981367111206,
+      "logits/rejected": -0.7410666942596436,
+      "logps/chosen": -7.697882652282715,
+      "logps/rejected": -44.32658386230469,
+      "loss": 0.7727497816085815,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4560812711715698,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10594843327999115,
+      "rewards/margins": 1.4077725410461426,
+      "rewards/rejected": -1.3018239736557007,
+      "step": 50,
+      "train_speed(iter/s)": 0.008057
+    },
+    {
+      "epoch": 0.11310555132025782,
+      "grad_norm": 2.724386215209961,
+      "learning_rate": 0.00019997569909594947,
+      "logits/chosen": -0.6333714723587036,
+      "logits/rejected": -0.6679136753082275,
+      "logps/chosen": -5.1590166091918945,
+      "logps/rejected": -28.71851348876953,
+      "loss": 0.6836625933647156,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.27343082427978516,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.15617597103118896,
+      "rewards/margins": 1.0674690008163452,
+      "rewards/rejected": -0.9112929701805115,
+      "step": 51,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.11532330722849816,
+      "grad_norm": 3.1469614505767822,
+      "learning_rate": 0.00019996692425326533,
+      "logits/chosen": -0.6757405996322632,
+      "logits/rejected": -0.7173976898193359,
+      "logps/chosen": -5.490239143371582,
+      "logps/rejected": -35.709468841552734,
+      "loss": 0.6690353751182556,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3624095916748047,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2558611333370209,
+      "rewards/margins": 1.5283734798431396,
+      "rewards/rejected": -1.272512435913086,
+      "step": 52,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.11754106313673851,
+      "grad_norm": 3.79186749458313,
+      "learning_rate": 0.0001999567997537493,
+      "logits/chosen": -0.7215967774391174,
+      "logits/rejected": -0.733575165271759,
+      "logps/chosen": -14.72846794128418,
+      "logps/rejected": -33.80124282836914,
+      "loss": 0.8965668678283691,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4457498788833618,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06722234189510345,
+      "rewards/margins": 1.353319764137268,
+      "rewards/rejected": -1.286097526550293,
+      "step": 53,
+      "train_speed(iter/s)": 0.008032
+    },
+    {
+      "epoch": 0.11975881904497886,
+      "grad_norm": 4.82743501663208,
+      "learning_rate": 0.00019994532573409262,
+      "logits/chosen": -0.7556912899017334,
+      "logits/rejected": -0.777891993522644,
+      "logps/chosen": -8.923099517822266,
+      "logps/rejected": -26.59355926513672,
+      "loss": 0.9261423945426941,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5014331936836243,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2390597015619278,
+      "rewards/margins": 1.2192004919052124,
+      "rewards/rejected": -0.9801408648490906,
+      "step": 54,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.12197657495321922,
+      "grad_norm": 5.3560004234313965,
+      "learning_rate": 0.00019993250234920636,
+      "logits/chosen": -0.8043528199195862,
+      "logits/rejected": -0.8303660154342651,
+      "logps/chosen": -8.159523010253906,
+      "logps/rejected": -33.00286865234375,
+      "loss": 0.9617294073104858,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5782371759414673,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.04396045207977295,
+      "rewards/margins": 1.489919900894165,
+      "rewards/rejected": -1.4459596872329712,
+      "step": 55,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.12419433086145956,
+      "grad_norm": 5.173131465911865,
+      "learning_rate": 0.00019991832977221956,
+      "logits/chosen": -0.7803252339363098,
+      "logits/rejected": -0.8187588453292847,
+      "logps/chosen": -10.553321838378906,
+      "logps/rejected": -36.212623596191406,
+      "loss": 0.8786243796348572,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5231858491897583,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.05092973634600639,
+      "rewards/margins": 1.722278356552124,
+      "rewards/rejected": -1.6713483333587646,
+      "step": 56,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.12641208676969992,
+      "grad_norm": 8.218033790588379,
+      "learning_rate": 0.0001999028081944766,
+      "logits/chosen": -0.8003982305526733,
+      "logits/rejected": -0.8230556845664978,
+      "logps/chosen": -9.953974723815918,
+      "logps/rejected": -38.88124465942383,
+      "loss": 0.9577436447143555,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5776119232177734,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04882112890481949,
+      "rewards/margins": 1.8343088626861572,
+      "rewards/rejected": -1.8831300735473633,
+      "step": 57,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.12862984267794025,
+      "grad_norm": 3.3079514503479004,
+      "learning_rate": 0.00019988593782553483,
+      "logits/chosen": -0.7837247252464294,
+      "logits/rejected": -0.8169953227043152,
+      "logps/chosen": -6.463255882263184,
+      "logps/rejected": -35.57067108154297,
+      "loss": 0.6752747893333435,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37694644927978516,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2689625918865204,
+      "rewards/margins": 1.9062916040420532,
+      "rewards/rejected": -1.6373289823532104,
+      "step": 58,
+      "train_speed(iter/s)": 0.008042
+    },
+    {
+      "epoch": 0.1308475985861806,
+      "grad_norm": 6.0413665771484375,
+      "learning_rate": 0.00019986771889316172,
+      "logits/chosen": -0.7406100034713745,
+      "logits/rejected": -0.7850071787834167,
+      "logps/chosen": -13.76762580871582,
+      "logps/rejected": -43.040740966796875,
+      "loss": 1.0095659494400024,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6776630282402039,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.015833202749490738,
+      "rewards/margins": 1.8320919275283813,
+      "rewards/rejected": -1.8479251861572266,
+      "step": 59,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.13306535449442095,
+      "grad_norm": 4.190580368041992,
+      "learning_rate": 0.00019984815164333163,
+      "logits/chosen": -0.7999930381774902,
+      "logits/rejected": -0.8120794296264648,
+      "logps/chosen": -7.115518569946289,
+      "logps/rejected": -34.076663970947266,
+      "loss": 1.0062350034713745,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5622979402542114,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.14373064041137695,
+      "rewards/margins": 1.5221604108810425,
+      "rewards/rejected": -1.378429889678955,
+      "step": 60,
+      "train_speed(iter/s)": 0.008032
+    },
+    {
+      "epoch": 0.1352831104026613,
+      "grad_norm": 3.536224842071533,
+      "learning_rate": 0.00019982723634022272,
+      "logits/chosen": -0.7172912955284119,
+      "logits/rejected": -0.7351298928260803,
+      "logps/chosen": -11.005714416503906,
+      "logps/rejected": -33.29360580444336,
+      "loss": 0.7737724184989929,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3809564411640167,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19083550572395325,
+      "rewards/margins": 1.2791187763214111,
+      "rewards/rejected": -1.0882835388183594,
+      "step": 61,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.13750086631090166,
+      "grad_norm": 4.615377426147461,
+      "learning_rate": 0.00019980497326621316,
+      "logits/chosen": -0.7390002012252808,
+      "logits/rejected": -0.7629216909408569,
+      "logps/chosen": -14.71272087097168,
+      "logps/rejected": -28.800384521484375,
+      "loss": 1.2719998359680176,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6139569282531738,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0792509913444519,
+      "rewards/margins": 0.693626344203949,
+      "rewards/rejected": -0.6143754124641418,
+      "step": 62,
+      "train_speed(iter/s)": 0.008031
+    },
+    {
+      "epoch": 0.139718622219142,
+      "grad_norm": 4.907950401306152,
+      "learning_rate": 0.00019978136272187747,
+      "logits/chosen": -0.757483720779419,
+      "logits/rejected": -0.7904630899429321,
+      "logps/chosen": -7.143397331237793,
+      "logps/rejected": -37.529693603515625,
+      "loss": 0.8591395020484924,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.47260788083076477,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03792594373226166,
+      "rewards/margins": 1.2297474145889282,
+      "rewards/rejected": -1.1918214559555054,
+      "step": 63,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.14193637812738236,
+      "grad_norm": 9.14727783203125,
+      "learning_rate": 0.00019975640502598244,
+      "logits/chosen": -0.7742553949356079,
+      "logits/rejected": -0.8133993148803711,
+      "logps/chosen": -10.656136512756348,
+      "logps/rejected": -45.71231460571289,
+      "loss": 0.9479065537452698,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6450760364532471,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07653652131557465,
+      "rewards/margins": 2.4191930294036865,
+      "rewards/rejected": -2.3426566123962402,
+      "step": 64,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.14415413403562272,
+      "grad_norm": 6.157256603240967,
+      "learning_rate": 0.00019973010051548275,
+      "logits/chosen": -0.7872298955917358,
+      "logits/rejected": -0.8282527923583984,
+      "logps/chosen": -8.776991844177246,
+      "logps/rejected": -43.008079528808594,
+      "loss": 0.7078882455825806,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.43169137835502625,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08513045310974121,
+      "rewards/margins": 2.5409374237060547,
+      "rewards/rejected": -2.4558072090148926,
+      "step": 65,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.14637188994386305,
+      "grad_norm": 2.469653844833374,
+      "learning_rate": 0.0001997024495455165,
+      "logits/chosen": -0.7316989898681641,
+      "logits/rejected": -0.7609662413597107,
+      "logps/chosen": -9.443560600280762,
+      "logps/rejected": -41.85050964355469,
+      "loss": 0.6616421937942505,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.381237268447876,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.05163441598415375,
+      "rewards/margins": 2.0620944499969482,
+      "rewards/rejected": -2.010460138320923,
+      "step": 66,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.1485896458521034,
+      "grad_norm": 5.567859172821045,
+      "learning_rate": 0.00019967345248940034,
+      "logits/chosen": -0.695087730884552,
+      "logits/rejected": -0.708895206451416,
+      "logps/chosen": -14.36052417755127,
+      "logps/rejected": -31.9357852935791,
+      "loss": 1.0636613368988037,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6741958260536194,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.0757778137922287,
+      "rewards/margins": 1.4089328050613403,
+      "rewards/rejected": -1.3331550359725952,
+      "step": 67,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.15080740176034374,
+      "grad_norm": 2.6372759342193604,
+      "learning_rate": 0.00019964310973862455,
+      "logits/chosen": -0.7357401847839355,
+      "logits/rejected": -0.7486222982406616,
+      "logps/chosen": -6.373124122619629,
+      "logps/rejected": -39.36447525024414,
+      "loss": 0.7257212996482849,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.36459508538246155,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18388086557388306,
+      "rewards/margins": 1.8258121013641357,
+      "rewards/rejected": -1.641931414604187,
+      "step": 68,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.1530251576685841,
+      "grad_norm": 5.665070056915283,
+      "learning_rate": 0.00019961142170284762,
+      "logits/chosen": -0.7021870017051697,
+      "logits/rejected": -0.7211427688598633,
+      "logps/chosen": -12.284502983093262,
+      "logps/rejected": -30.56876564025879,
+      "loss": 1.1223974227905273,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.653534471988678,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.0864773616194725,
+      "rewards/margins": 1.4587070941925049,
+      "rewards/rejected": -1.3722295761108398,
+      "step": 69,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.15524291357682446,
+      "grad_norm": 4.089859485626221,
+      "learning_rate": 0.00019957838880989078,
+      "logits/chosen": -0.6898348331451416,
+      "logits/rejected": -0.6964607834815979,
+      "logps/chosen": -14.947086334228516,
+      "logps/rejected": -34.29672622680664,
+      "loss": 1.0402615070343018,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.540707528591156,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.036627963185310364,
+      "rewards/margins": 1.189173698425293,
+      "rewards/rejected": -1.2258018255233765,
+      "step": 70,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.1574606694850648,
+      "grad_norm": 3.818058729171753,
+      "learning_rate": 0.00019954401150573222,
+      "logits/chosen": -0.6947743892669678,
+      "logits/rejected": -0.7197954654693604,
+      "logps/chosen": -6.173555850982666,
+      "logps/rejected": -28.507305145263672,
+      "loss": 0.8432615399360657,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4707663655281067,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.432411789894104,
+      "rewards/margins": 1.5417673587799072,
+      "rewards/rejected": -1.1093555688858032,
+      "step": 71,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.15967842539330515,
+      "grad_norm": 4.713813304901123,
+      "learning_rate": 0.00019950829025450114,
+      "logits/chosen": -0.6722406148910522,
+      "logits/rejected": -0.6857435703277588,
+      "logps/chosen": -7.766820430755615,
+      "logps/rejected": -26.340253829956055,
+      "loss": 0.9169723391532898,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5056720972061157,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1677010953426361,
+      "rewards/margins": 1.1372748613357544,
+      "rewards/rejected": -0.9695737361907959,
+      "step": 72,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.1618961813015455,
+      "grad_norm": 3.4665582180023193,
+      "learning_rate": 0.00019947122553847136,
+      "logits/chosen": -0.7526758909225464,
+      "logits/rejected": -0.7613145112991333,
+      "logps/chosen": -14.30706787109375,
+      "logps/rejected": -30.180295944213867,
+      "loss": 0.8724768161773682,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.41890856623649597,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1295187771320343,
+      "rewards/margins": 1.2079225778579712,
+      "rewards/rejected": -1.0784037113189697,
+      "step": 73,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.16411393720978584,
+      "grad_norm": 12.609716415405273,
+      "learning_rate": 0.0001994328178580548,
+      "logits/chosen": -0.7560169696807861,
+      "logits/rejected": -0.7651617527008057,
+      "logps/chosen": -8.213447570800781,
+      "logps/rejected": -43.658180236816406,
+      "loss": 0.9154157042503357,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6314950585365295,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.061887748539447784,
+      "rewards/margins": 2.2252354621887207,
+      "rewards/rejected": -2.1633477210998535,
+      "step": 74,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.1663316931180262,
+      "grad_norm": 12.707162857055664,
+      "learning_rate": 0.00019939306773179497,
+      "logits/chosen": -0.7599114775657654,
+      "logits/rejected": -0.7735819816589355,
+      "logps/chosen": -7.509787082672119,
+      "logps/rejected": -52.7489013671875,
+      "loss": 0.7145227193832397,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30735087394714355,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.028781913220882416,
+      "rewards/margins": 2.859144926071167,
+      "rewards/rejected": -2.8303627967834473,
+      "step": 75,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.16854944902626653,
+      "grad_norm": 4.863827705383301,
+      "learning_rate": 0.00019935197569635954,
+      "logits/chosen": -0.7566101551055908,
+      "logits/rejected": -0.7977889180183411,
+      "logps/chosen": -8.841687202453613,
+      "logps/rejected": -49.88290786743164,
+      "loss": 0.7746098637580872,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.488862007856369,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13911329209804535,
+      "rewards/margins": 2.4493186473846436,
+      "rewards/rejected": -2.3102054595947266,
+      "step": 76,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.1707672049345069,
+      "grad_norm": 2.638216972351074,
+      "learning_rate": 0.00019930954230653355,
+      "logits/chosen": -0.7588043808937073,
+      "logits/rejected": -0.7855987548828125,
+      "logps/chosen": -4.888318061828613,
+      "logps/rejected": -46.317176818847656,
+      "loss": 0.5076995491981506,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2771949768066406,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2532336711883545,
+      "rewards/margins": 2.56557035446167,
+      "rewards/rejected": -2.3123364448547363,
+      "step": 77,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.17298496084274725,
+      "grad_norm": 3.485717296600342,
+      "learning_rate": 0.00019926576813521164,
+      "logits/chosen": -0.8214580416679382,
+      "logits/rejected": -0.8605206608772278,
+      "logps/chosen": -8.181400299072266,
+      "logps/rejected": -42.75120544433594,
+      "loss": 0.7162066698074341,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.41232535243034363,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1498756855726242,
+      "rewards/margins": 2.346996784210205,
+      "rewards/rejected": -2.1971211433410645,
+      "step": 78,
+      "train_speed(iter/s)": 0.008024
+    },
+    {
+      "epoch": 0.17520271675098759,
+      "grad_norm": 3.2950491905212402,
+      "learning_rate": 0.00019922065377339036,
+      "logits/chosen": -0.8543375730514526,
+      "logits/rejected": -0.8765980005264282,
+      "logps/chosen": -15.666474342346191,
+      "logps/rejected": -36.56987380981445,
+      "loss": 0.8590094447135925,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5429059863090515,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2669352889060974,
+      "rewards/margins": 1.803323745727539,
+      "rewards/rejected": -1.5363885164260864,
+      "step": 79,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.17742047265922795,
+      "grad_norm": 4.1412506103515625,
+      "learning_rate": 0.00019917419983016025,
+      "logits/chosen": -0.8439186215400696,
+      "logits/rejected": -0.9001189470291138,
+      "logps/chosen": -10.366392135620117,
+      "logps/rejected": -49.56153106689453,
+      "loss": 0.9545846581459045,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7163697481155396,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22070294618606567,
+      "rewards/margins": 2.2959794998168945,
+      "rewards/rejected": -2.0752763748168945,
+      "step": 80,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.1796382285674683,
+      "grad_norm": 2.809291362762451,
+      "learning_rate": 0.00019912640693269752,
+      "logits/chosen": -0.7752825021743774,
+      "logits/rejected": -0.8082448244094849,
+      "logps/chosen": -6.980072021484375,
+      "logps/rejected": -49.13923645019531,
+      "loss": 0.7785353660583496,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4162166714668274,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3260034918785095,
+      "rewards/margins": 2.5372085571289062,
+      "rewards/rejected": -2.211205005645752,
+      "step": 81,
+      "train_speed(iter/s)": 0.008021
+    },
+    {
+      "epoch": 0.18185598447570864,
+      "grad_norm": 5.757961750030518,
+      "learning_rate": 0.0001990772757262558,
+      "logits/chosen": -0.8326177597045898,
+      "logits/rejected": -0.8457602262496948,
+      "logps/chosen": -14.738750457763672,
+      "logps/rejected": -50.59590530395508,
+      "loss": 1.1126980781555176,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6883702874183655,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.017612110823392868,
+      "rewards/margins": 3.0471317768096924,
+      "rewards/rejected": -3.02951979637146,
+      "step": 82,
+      "train_speed(iter/s)": 0.008024
+    },
+    {
+      "epoch": 0.184073740383949,
+      "grad_norm": 43.46441650390625,
+      "learning_rate": 0.00019902680687415705,
+      "logits/chosen": -0.9140123128890991,
+      "logits/rejected": -0.9116270542144775,
+      "logps/chosen": -23.103540420532227,
+      "logps/rejected": -48.863033294677734,
+      "loss": 2.1918702125549316,
+      "memory(GiB)": 46.1,
+      "nll_loss": 1.8151426315307617,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8213595747947693,
+      "rewards/margins": 2.689864158630371,
+      "rewards/rejected": -3.511223554611206,
+      "step": 83,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.18629149629218933,
+      "grad_norm": 7.742120265960693,
+      "learning_rate": 0.000198975001057783,
+      "logits/chosen": -0.8376122713088989,
+      "logits/rejected": -0.8708577752113342,
+      "logps/chosen": -12.746369361877441,
+      "logps/rejected": -67.55133819580078,
+      "loss": 0.7703056335449219,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4422933757305145,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.010235786437988281,
+      "rewards/margins": 4.232974529266357,
+      "rewards/rejected": -4.222738742828369,
+      "step": 84,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.1885092522004297,
+      "grad_norm": 9.322490692138672,
+      "learning_rate": 0.00019892185897656578,
+      "logits/chosen": -0.8576998710632324,
+      "logits/rejected": -0.8946700692176819,
+      "logps/chosen": -16.83005142211914,
+      "logps/rejected": -67.54959106445312,
+      "loss": 1.138474464416504,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.8976783156394958,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.3624171316623688,
+      "rewards/margins": 4.450318813323975,
+      "rewards/rejected": -4.812736511230469,
+      "step": 85,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.19072700810867005,
+      "grad_norm": 3.996424913406372,
+      "learning_rate": 0.00019886738134797843,
+      "logits/chosen": -0.8325670957565308,
+      "logits/rejected": -0.9040736556053162,
+      "logps/chosen": -9.318441390991211,
+      "logps/rejected": -49.33755111694336,
+      "loss": 0.8532267808914185,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5119913220405579,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10566120594739914,
+      "rewards/margins": 2.421614170074463,
+      "rewards/rejected": -2.315953254699707,
+      "step": 86,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.19294476401691038,
+      "grad_norm": 7.568336009979248,
+      "learning_rate": 0.00019881156890752517,
+      "logits/chosen": -0.7875989079475403,
+      "logits/rejected": -0.8255676627159119,
+      "logps/chosen": -10.209391593933105,
+      "logps/rejected": -38.12163162231445,
+      "loss": 1.1016978025436401,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5710477828979492,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.12029634416103363,
+      "rewards/margins": 1.949970006942749,
+      "rewards/rejected": -2.0702662467956543,
+      "step": 87,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.19516251992515074,
+      "grad_norm": 4.460707187652588,
+      "learning_rate": 0.00019875442240873173,
+      "logits/chosen": -0.7924566268920898,
+      "logits/rejected": -0.7932747602462769,
+      "logps/chosen": -10.550544738769531,
+      "logps/rejected": -29.39406967163086,
+      "loss": 0.8096880912780762,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42373886704444885,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3358071446418762,
+      "rewards/margins": 1.8472529649734497,
+      "rewards/rejected": -1.5114457607269287,
+      "step": 88,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.1973802758333911,
+      "grad_norm": 3.9834723472595215,
+      "learning_rate": 0.0001986959426231349,
+      "logits/chosen": -0.7446599006652832,
+      "logits/rejected": -0.7426899075508118,
+      "logps/chosen": -12.930614471435547,
+      "logps/rejected": -26.922080993652344,
+      "loss": 1.132792592048645,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5469169616699219,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.048162102699279785,
+      "rewards/margins": 0.6297942996025085,
+      "rewards/rejected": -0.6779564619064331,
+      "step": 89,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.19959803174163143,
+      "grad_norm": 4.092418193817139,
+      "learning_rate": 0.00019863613034027224,
+      "logits/chosen": -0.7061215043067932,
+      "logits/rejected": -0.7569677233695984,
+      "logps/chosen": -6.195008754730225,
+      "logps/rejected": -34.00957489013672,
+      "loss": 0.8494804501533508,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4875006675720215,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.26478877663612366,
+      "rewards/margins": 1.2916200160980225,
+      "rewards/rejected": -1.0268312692642212,
+      "step": 90,
+      "train_speed(iter/s)": 0.008024
+    },
+    {
+      "epoch": 0.2018157876498718,
+      "grad_norm": 3.3873367309570312,
+      "learning_rate": 0.00019857498636767143,
+      "logits/chosen": -0.6566476225852966,
+      "logits/rejected": -0.7009319067001343,
+      "logps/chosen": -6.160096645355225,
+      "logps/rejected": -30.111385345458984,
+      "loss": 0.8439165949821472,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5105728507041931,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.41078829765319824,
+      "rewards/margins": 1.2888054847717285,
+      "rewards/rejected": -0.8780173063278198,
+      "step": 91,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.20403354355811212,
+      "grad_norm": 2.58686900138855,
+      "learning_rate": 0.0001985125115308393,
+      "logits/chosen": -0.7298556566238403,
+      "logits/rejected": -0.7364020347595215,
+      "logps/chosen": -9.868298530578613,
+      "logps/rejected": -22.635440826416016,
+      "loss": 0.8916825652122498,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.466416597366333,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.39984455704689026,
+      "rewards/margins": 0.953176736831665,
+      "rewards/rejected": -0.5533321499824524,
+      "step": 92,
+      "train_speed(iter/s)": 0.008021
+    },
+    {
+      "epoch": 0.20625129946635248,
+      "grad_norm": 2.869520902633667,
+      "learning_rate": 0.00019844870667325073,
+      "logits/chosen": -0.6459857821464539,
+      "logits/rejected": -0.6821721792221069,
+      "logps/chosen": -10.646278381347656,
+      "logps/rejected": -33.94523620605469,
+      "loss": 0.8635712265968323,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40875259041786194,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.17814505100250244,
+      "rewards/margins": 1.1260223388671875,
+      "rewards/rejected": -0.9478773474693298,
+      "step": 93,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.20846905537459284,
+      "grad_norm": 2.5596885681152344,
+      "learning_rate": 0.00019838357265633728,
+      "logits/chosen": -0.6933214068412781,
+      "logits/rejected": -0.7294182777404785,
+      "logps/chosen": -9.423470497131348,
+      "logps/rejected": -28.46645164489746,
+      "loss": 0.8668123483657837,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4432825744152069,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2616448402404785,
+      "rewards/margins": 1.0607826709747314,
+      "rewards/rejected": -0.7991377711296082,
+      "step": 94,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.21068681128283318,
+      "grad_norm": 2.987144947052002,
+      "learning_rate": 0.0001983171103594755,
+      "logits/chosen": -0.7186170816421509,
+      "logits/rejected": -0.7554275989532471,
+      "logps/chosen": -13.879290580749512,
+      "logps/rejected": -41.82307434082031,
+      "loss": 0.8854122161865234,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5749840140342712,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.34967276453971863,
+      "rewards/margins": 2.095144510269165,
+      "rewards/rejected": -1.7454715967178345,
+      "step": 95,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.21290456719107353,
+      "grad_norm": 4.350345134735107,
+      "learning_rate": 0.00019824932067997515,
+      "logits/chosen": -0.6538293361663818,
+      "logits/rejected": -0.7083633542060852,
+      "logps/chosen": -13.237122535705566,
+      "logps/rejected": -43.02968215942383,
+      "loss": 1.1358224153518677,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7784836292266846,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17013956606388092,
+      "rewards/margins": 2.1158745288848877,
+      "rewards/rejected": -1.945734977722168,
+      "step": 96,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.2151223230993139,
+      "grad_norm": 10.145339965820312,
+      "learning_rate": 0.00019818020453306697,
+      "logits/chosen": -0.7033687829971313,
+      "logits/rejected": -0.7348278760910034,
+      "logps/chosen": -9.687533378601074,
+      "logps/rejected": -38.41090774536133,
+      "loss": 0.9101305603981018,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6460333466529846,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22131364047527313,
+      "rewards/margins": 2.366105079650879,
+      "rewards/rejected": -2.1447913646698,
+      "step": 97,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.21734007900755423,
+      "grad_norm": 4.888674736022949,
+      "learning_rate": 0.00019810976285189038,
+      "logits/chosen": -0.6517484784126282,
+      "logits/rejected": -0.6647759079933167,
+      "logps/chosen": -11.991264343261719,
+      "logps/rejected": -40.05374526977539,
+      "loss": 0.7888365983963013,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5127142667770386,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08162616193294525,
+      "rewards/margins": 2.0730180740356445,
+      "rewards/rejected": -1.9913920164108276,
+      "step": 98,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.2195578349157946,
+      "grad_norm": 2.9053163528442383,
+      "learning_rate": 0.00019803799658748094,
+      "logits/chosen": -0.6322453618049622,
+      "logits/rejected": -0.6195716261863708,
+      "logps/chosen": -11.029327392578125,
+      "logps/rejected": -27.67604637145996,
+      "loss": 0.8408295512199402,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44112899899482727,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.332018107175827,
+      "rewards/margins": 1.5282344818115234,
+      "rewards/rejected": -1.196216344833374,
+      "step": 99,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.22177559082403492,
+      "grad_norm": 3.9380838871002197,
+      "learning_rate": 0.0001979649067087574,
+      "logits/chosen": -0.6178653240203857,
+      "logits/rejected": -0.6780340075492859,
+      "logps/chosen": -6.1165595054626465,
+      "logps/rejected": -48.394866943359375,
+      "loss": 0.6825374960899353,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3889180123806,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.35112470388412476,
+      "rewards/margins": 2.020500421524048,
+      "rewards/rejected": -1.6693755388259888,
+      "step": 100,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.22399334673227528,
+      "grad_norm": 4.091344356536865,
+      "learning_rate": 0.0001978904942025087,
+      "logits/chosen": -0.5362146496772766,
+      "logits/rejected": -0.5435670018196106,
+      "logps/chosen": -10.14964485168457,
+      "logps/rejected": -33.93182373046875,
+      "loss": 0.8511705994606018,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4244406819343567,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.0012584757059812546,
+      "rewards/margins": 1.5515294075012207,
+      "rewards/rejected": -1.5527877807617188,
+      "step": 101,
+      "train_speed(iter/s)": 0.008013
+    },
+    {
+      "epoch": 0.22621110264051564,
+      "grad_norm": 3.6777844429016113,
+      "learning_rate": 0.00019781476007338058,
+      "logits/chosen": -0.519984245300293,
+      "logits/rejected": -0.5302433371543884,
+      "logps/chosen": -12.393686294555664,
+      "logps/rejected": -28.21156883239746,
+      "loss": 0.9213598966598511,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5426865816116333,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.25866204500198364,
+      "rewards/margins": 1.387097954750061,
+      "rewards/rejected": -1.1284359693527222,
+      "step": 102,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.22842885854875597,
+      "grad_norm": 2.3189024925231934,
+      "learning_rate": 0.0001977377053438621,
+      "logits/chosen": -0.6001867651939392,
+      "logits/rejected": -0.632231593132019,
+      "logps/chosen": -7.773558616638184,
+      "logps/rejected": -34.698673248291016,
+      "loss": 0.7014553546905518,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3608202338218689,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3325607180595398,
+      "rewards/margins": 1.3831188678741455,
+      "rewards/rejected": -1.0505582094192505,
+      "step": 103,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.23064661445699633,
+      "grad_norm": 3.2740068435668945,
+      "learning_rate": 0.0001976593310542718,
+      "logits/chosen": -0.578256368637085,
+      "logits/rejected": -0.5909325480461121,
+      "logps/chosen": -6.1796793937683105,
+      "logps/rejected": -30.154891967773438,
+      "loss": 0.8033835887908936,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39110761880874634,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1742677092552185,
+      "rewards/margins": 1.1370247602462769,
+      "rewards/rejected": -0.9627571702003479,
+      "step": 104,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.2328643703652367,
+      "grad_norm": 3.2135934829711914,
+      "learning_rate": 0.00019757963826274357,
+      "logits/chosen": -0.5837016105651855,
+      "logits/rejected": -0.588310182094574,
+      "logps/chosen": -9.729472160339355,
+      "logps/rejected": -34.4300422668457,
+      "loss": 0.8487012386322021,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37863489985466003,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.037823110818862915,
+      "rewards/margins": 1.5622057914733887,
+      "rewards/rejected": -1.5243828296661377,
+      "step": 105,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.23508212627347702,
+      "grad_norm": 3.867928981781006,
+      "learning_rate": 0.00019749862804521254,
+      "logits/chosen": -0.5833920836448669,
+      "logits/rejected": -0.5896369218826294,
+      "logps/chosen": -14.619677543640137,
+      "logps/rejected": -39.831119537353516,
+      "loss": 1.0550124645233154,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6425492763519287,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.002796528860926628,
+      "rewards/margins": 1.8236141204833984,
+      "rewards/rejected": -1.8208175897598267,
+      "step": 106,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.23729988218171738,
+      "grad_norm": 4.911788463592529,
+      "learning_rate": 0.00019741630149540035,
+      "logits/chosen": -0.6418726444244385,
+      "logits/rejected": -0.6708973050117493,
+      "logps/chosen": -8.22497272491455,
+      "logps/rejected": -45.075836181640625,
+      "loss": 0.7518470287322998,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4536428451538086,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.12467997521162033,
+      "rewards/margins": 2.3994452953338623,
+      "rewards/rejected": -2.524125337600708,
+      "step": 107,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.2395176380899577,
+      "grad_norm": 4.135776519775391,
+      "learning_rate": 0.0001973326597248006,
+      "logits/chosen": -0.6478183269500732,
+      "logits/rejected": -0.6533663272857666,
+      "logps/chosen": -6.512712001800537,
+      "logps/rejected": -40.08877944946289,
+      "loss": 0.690629243850708,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3907169997692108,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.21786418557167053,
+      "rewards/margins": 2.6031618118286133,
+      "rewards/rejected": -2.3852977752685547,
+      "step": 108,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.24173539399819807,
+      "grad_norm": 4.422867298126221,
+      "learning_rate": 0.00019724770386266363,
+      "logits/chosen": -0.6119152307510376,
+      "logits/rejected": -0.6426375508308411,
+      "logps/chosen": -7.145269870758057,
+      "logps/rejected": -49.40720748901367,
+      "loss": 0.6389705538749695,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3778725564479828,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0635518729686737,
+      "rewards/margins": 2.8675644397735596,
+      "rewards/rejected": -2.8040125370025635,
+      "step": 109,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.24395314990643843,
+      "grad_norm": 6.416353702545166,
+      "learning_rate": 0.0001971614350559814,
+      "logits/chosen": -0.5623615384101868,
+      "logits/rejected": -0.5931446552276611,
+      "logps/chosen": -7.337302207946777,
+      "logps/rejected": -48.94050979614258,
+      "loss": 0.5740457773208618,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42228132486343384,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6646949052810669,
+      "rewards/margins": 3.210062026977539,
+      "rewards/rejected": -2.545367479324341,
+      "step": 110,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.24617090581467876,
+      "grad_norm": 2.363133668899536,
+      "learning_rate": 0.000197073854469472,
+      "logits/chosen": -0.6107865571975708,
+      "logits/rejected": -0.6239950060844421,
+      "logps/chosen": -10.528251647949219,
+      "logps/rejected": -37.559669494628906,
+      "loss": 0.7298096418380737,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4291239082813263,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.32069048285484314,
+      "rewards/margins": 2.222153663635254,
+      "rewards/rejected": -1.9014630317687988,
+      "step": 111,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.24838866172291912,
+      "grad_norm": 4.300288677215576,
+      "learning_rate": 0.00019698496328556396,
+      "logits/chosen": -0.6013592481613159,
+      "logits/rejected": -0.6298633217811584,
+      "logps/chosen": -12.510583877563477,
+      "logps/rejected": -36.04363250732422,
+      "loss": 0.9174787998199463,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5793477296829224,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19869008660316467,
+      "rewards/margins": 1.6629492044448853,
+      "rewards/rejected": -1.4642590284347534,
+      "step": 112,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.2506064176311595,
+      "grad_norm": 7.3313493728637695,
+      "learning_rate": 0.00019689476270438004,
+      "logits/chosen": -0.5540329217910767,
+      "logits/rejected": -0.5571173429489136,
+      "logps/chosen": -8.091080665588379,
+      "logps/rejected": -27.752933502197266,
+      "loss": 0.8949199914932251,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4742899537086487,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1788148581981659,
+      "rewards/margins": 1.2291390895843506,
+      "rewards/rejected": -1.0503242015838623,
+      "step": 113,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.25282417353939984,
+      "grad_norm": 2.6731913089752197,
+      "learning_rate": 0.0001968032539437215,
+      "logits/chosen": -0.5931655764579773,
+      "logits/rejected": -0.6056005358695984,
+      "logps/chosen": -5.9029951095581055,
+      "logps/rejected": -40.255130767822266,
+      "loss": 0.6467551589012146,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4438669979572296,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24888494610786438,
+      "rewards/margins": 2.4213175773620605,
+      "rewards/rejected": -2.1724328994750977,
+      "step": 114,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.25504192944764015,
+      "grad_norm": 2.6930527687072754,
+      "learning_rate": 0.0001967104382390511,
+      "logits/chosen": -0.6651092767715454,
+      "logits/rejected": -0.6895506978034973,
+      "logps/chosen": -9.578208923339844,
+      "logps/rejected": -34.64287185668945,
+      "loss": 0.7224618792533875,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4180094599723816,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3409712314605713,
+      "rewards/margins": 2.1104307174682617,
+      "rewards/rejected": -1.7694597244262695,
+      "step": 115,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.2572596853558805,
+      "grad_norm": 3.387476682662964,
+      "learning_rate": 0.00019661631684347685,
+      "logits/chosen": -0.6467074155807495,
+      "logits/rejected": -0.6696929335594177,
+      "logps/chosen": -12.390437126159668,
+      "logps/rejected": -44.5852165222168,
+      "loss": 0.9567733407020569,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5903316736221313,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06747841835021973,
+      "rewards/margins": 1.779471755027771,
+      "rewards/rejected": -1.7119932174682617,
+      "step": 116,
+      "train_speed(iter/s)": 0.00801
+    },
+    {
+      "epoch": 0.25947744126412087,
+      "grad_norm": 3.1305816173553467,
+      "learning_rate": 0.00019652089102773488,
+      "logits/chosen": -0.6367936730384827,
+      "logits/rejected": -0.6425185203552246,
+      "logps/chosen": -11.601859092712402,
+      "logps/rejected": -36.16029739379883,
+      "loss": 0.9545689821243286,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5791404247283936,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.139948308467865,
+      "rewards/margins": 1.7838351726531982,
+      "rewards/rejected": -1.6438865661621094,
+      "step": 117,
+      "train_speed(iter/s)": 0.008009
+    },
+    {
+      "epoch": 0.2616951971723612,
+      "grad_norm": 5.935551166534424,
+      "learning_rate": 0.0001964241620801724,
+      "logits/chosen": -0.7171432375907898,
+      "logits/rejected": -0.7515658140182495,
+      "logps/chosen": -12.206275939941406,
+      "logps/rejected": -41.78958511352539,
+      "loss": 1.0742188692092896,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6272014379501343,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12931828200817108,
+      "rewards/margins": 1.775916576385498,
+      "rewards/rejected": -1.905234932899475,
+      "step": 118,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.2639129530806016,
+      "grad_norm": 3.182480812072754,
+      "learning_rate": 0.0001963261313067302,
+      "logits/chosen": -0.6834742426872253,
+      "logits/rejected": -0.7226250171661377,
+      "logps/chosen": -6.070950984954834,
+      "logps/rejected": -34.18709182739258,
+      "loss": 0.8980444073677063,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4420132637023926,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2397366762161255,
+      "rewards/margins": 1.7652424573898315,
+      "rewards/rejected": -1.5255059003829956,
+      "step": 119,
+      "train_speed(iter/s)": 0.008013
+    },
+    {
+      "epoch": 0.2661307089888419,
+      "grad_norm": 4.235217094421387,
+      "learning_rate": 0.00019622680003092503,
+      "logits/chosen": -0.7337202429771423,
+      "logits/rejected": -0.7412666082382202,
+      "logps/chosen": -13.724693298339844,
+      "logps/rejected": -46.71962356567383,
+      "loss": 0.9812621474266052,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6411746144294739,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.09581395983695984,
+      "rewards/margins": 2.239417552947998,
+      "rewards/rejected": -2.335231304168701,
+      "step": 120,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.26834846489708225,
+      "grad_norm": 2.509409189224243,
+      "learning_rate": 0.0001961261695938319,
+      "logits/chosen": -0.6983414888381958,
+      "logits/rejected": -0.7302532196044922,
+      "logps/chosen": -12.647697448730469,
+      "logps/rejected": -47.85969161987305,
+      "loss": 0.7407037019729614,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4768805503845215,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3689803183078766,
+      "rewards/margins": 3.114835023880005,
+      "rewards/rejected": -2.745854377746582,
+      "step": 121,
+      "train_speed(iter/s)": 0.008014
+    },
+    {
+      "epoch": 0.2705662208053226,
+      "grad_norm": 3.401812791824341,
+      "learning_rate": 0.00019602424135406569,
+      "logits/chosen": -0.7398380041122437,
+      "logits/rejected": -0.7770130634307861,
+      "logps/chosen": -9.498916625976562,
+      "logps/rejected": -52.57138442993164,
+      "loss": 0.6471344828605652,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.46904683113098145,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18257564306259155,
+      "rewards/margins": 3.0506017208099365,
+      "rewards/rejected": -2.868025779724121,
+      "step": 122,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.27278397671356297,
+      "grad_norm": 3.7279372215270996,
+      "learning_rate": 0.00019592101668776298,
+      "logits/chosen": -0.7581114768981934,
+      "logits/rejected": -0.7886280417442322,
+      "logps/chosen": -8.954914093017578,
+      "logps/rejected": -49.436580657958984,
+      "loss": 0.7794913649559021,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42245572805404663,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24546337127685547,
+      "rewards/margins": 2.368910789489746,
+      "rewards/rejected": -2.1234471797943115,
+      "step": 123,
+      "train_speed(iter/s)": 0.008011
+    },
+    {
+      "epoch": 0.27500173262180333,
+      "grad_norm": 2.763056993484497,
+      "learning_rate": 0.00019581649698856358,
+      "logits/chosen": -0.754249632358551,
+      "logits/rejected": -0.7789746522903442,
+      "logps/chosen": -6.999545097351074,
+      "logps/rejected": -54.994590759277344,
+      "loss": 0.49618056416511536,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30507829785346985,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30616074800491333,
+      "rewards/margins": 3.148125171661377,
+      "rewards/rejected": -2.8419644832611084,
+      "step": 124,
+      "train_speed(iter/s)": 0.008007
+    },
+    {
+      "epoch": 0.2772194885300437,
+      "grad_norm": 4.055456638336182,
+      "learning_rate": 0.00019571068366759143,
+      "logits/chosen": -0.7367149591445923,
+      "logits/rejected": -0.7724103331565857,
+      "logps/chosen": -7.0940070152282715,
+      "logps/rejected": -38.532779693603516,
+      "loss": 0.8412638306617737,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5069564580917358,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26311713457107544,
+      "rewards/margins": 2.167259693145752,
+      "rewards/rejected": -1.9041424989700317,
+      "step": 125,
+      "train_speed(iter/s)": 0.008011
+    },
+    {
+      "epoch": 0.279437244438284,
+      "grad_norm": 7.62572717666626,
+      "learning_rate": 0.00019560357815343577,
+      "logits/chosen": -0.7477836608886719,
+      "logits/rejected": -0.771050214767456,
+      "logps/chosen": -9.567910194396973,
+      "logps/rejected": -41.98735809326172,
+      "loss": 0.6985211968421936,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3839423358440399,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3293745815753937,
+      "rewards/margins": 2.38517165184021,
+      "rewards/rejected": -2.0557968616485596,
+      "step": 126,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.28165500034652435,
+      "grad_norm": 9.083622932434082,
+      "learning_rate": 0.0001954951818921318,
+      "logits/chosen": -0.6845836639404297,
+      "logits/rejected": -0.7126042246818542,
+      "logps/chosen": -10.922572135925293,
+      "logps/rejected": -50.7385368347168,
+      "loss": 0.7883904576301575,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45509275794029236,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08339966833591461,
+      "rewards/margins": 2.7709593772888184,
+      "rewards/rejected": -2.6875598430633545,
+      "step": 127,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.2838727562547647,
+      "grad_norm": 6.918692111968994,
+      "learning_rate": 0.00019538549634714108,
+      "logits/chosen": -0.7463397979736328,
+      "logits/rejected": -0.7875937819480896,
+      "logps/chosen": -6.1263532638549805,
+      "logps/rejected": -56.90166091918945,
+      "loss": 0.5629154443740845,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3863481283187866,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3725871443748474,
+      "rewards/margins": 3.788067579269409,
+      "rewards/rejected": -3.415480375289917,
+      "step": 128,
+      "train_speed(iter/s)": 0.008013
+    },
+    {
+      "epoch": 0.2860905121630051,
+      "grad_norm": 3.020493507385254,
+      "learning_rate": 0.0001952745229993319,
+      "logits/chosen": -0.6778846383094788,
+      "logits/rejected": -0.7200884819030762,
+      "logps/chosen": -8.347317695617676,
+      "logps/rejected": -45.266326904296875,
+      "loss": 0.6461578011512756,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4276276230812073,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32448530197143555,
+      "rewards/margins": 2.904148578643799,
+      "rewards/rejected": -2.5796632766723633,
+      "step": 129,
+      "train_speed(iter/s)": 0.008013
+    },
+    {
+      "epoch": 0.28830826807124543,
+      "grad_norm": 6.105565071105957,
+      "learning_rate": 0.0001951622633469592,
+      "logits/chosen": -0.6739642024040222,
+      "logits/rejected": -0.7229712009429932,
+      "logps/chosen": -9.595758438110352,
+      "logps/rejected": -61.285579681396484,
+      "loss": 0.524199366569519,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3470918536186218,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26431241631507874,
+      "rewards/margins": 3.8313231468200684,
+      "rewards/rejected": -3.5670104026794434,
+      "step": 130,
+      "train_speed(iter/s)": 0.00801
+    },
+    {
+      "epoch": 0.29052602397948574,
+      "grad_norm": 3.1043567657470703,
+      "learning_rate": 0.0001950487189056443,
+      "logits/chosen": -0.725166916847229,
+      "logits/rejected": -0.7790873646736145,
+      "logps/chosen": -6.4268879890441895,
+      "logps/rejected": -45.298736572265625,
+      "loss": 0.7616266012191772,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4726749062538147,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20096376538276672,
+      "rewards/margins": 2.216933250427246,
+      "rewards/rejected": -2.0159695148468018,
+      "step": 131,
+      "train_speed(iter/s)": 0.008009
+    },
+    {
+      "epoch": 0.2927437798877261,
+      "grad_norm": 4.7475457191467285,
+      "learning_rate": 0.00019493389120835462,
+      "logits/chosen": -0.7168976068496704,
+      "logits/rejected": -0.7184516191482544,
+      "logps/chosen": -8.480973243713379,
+      "logps/rejected": -39.217987060546875,
+      "loss": 0.6954407691955566,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3810206949710846,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1063593327999115,
+      "rewards/margins": 2.2143924236297607,
+      "rewards/rejected": -2.1080331802368164,
+      "step": 132,
+      "train_speed(iter/s)": 0.008009
+    },
+    {
+      "epoch": 0.29496153579596646,
+      "grad_norm": 2.6074578762054443,
+      "learning_rate": 0.0001948177818053827,
+      "logits/chosen": -0.7467566132545471,
+      "logits/rejected": -0.7593668699264526,
+      "logps/chosen": -5.696949005126953,
+      "logps/rejected": -39.8399543762207,
+      "loss": 0.5127253532409668,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2950189709663391,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.28634998202323914,
+      "rewards/margins": 2.5809803009033203,
+      "rewards/rejected": -2.2946300506591797,
+      "step": 133,
+      "train_speed(iter/s)": 0.008008
+    },
+    {
+      "epoch": 0.2971792917042068,
+      "grad_norm": 2.2834794521331787,
+      "learning_rate": 0.0001947003922643256,
+      "logits/chosen": -0.6837459802627563,
+      "logits/rejected": -0.7182618975639343,
+      "logps/chosen": -6.691904067993164,
+      "logps/rejected": -50.19198226928711,
+      "loss": 0.4669988751411438,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2525062561035156,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3388081192970276,
+      "rewards/margins": 2.9852235317230225,
+      "rewards/rejected": -2.6464154720306396,
+      "step": 134,
+      "train_speed(iter/s)": 0.008008
+    },
+    {
+      "epoch": 0.2993970476124472,
+      "grad_norm": 3.200366497039795,
+      "learning_rate": 0.00019458172417006347,
+      "logits/chosen": -0.7646272778511047,
+      "logits/rejected": -0.7724561095237732,
+      "logps/chosen": -11.684989929199219,
+      "logps/rejected": -54.008689880371094,
+      "loss": 0.6327359676361084,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4540945291519165,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0852358415722847,
+      "rewards/margins": 3.4554994106292725,
+      "rewards/rejected": -3.3702638149261475,
+      "step": 135,
+      "train_speed(iter/s)": 0.008008
+    },
+    {
+      "epoch": 0.3016148035206875,
+      "grad_norm": 3.3365910053253174,
+      "learning_rate": 0.0001944617791247383,
+      "logits/chosen": -0.7103350162506104,
+      "logits/rejected": -0.7452230453491211,
+      "logps/chosen": -8.785733222961426,
+      "logps/rejected": -54.68450927734375,
+      "loss": 0.5810406804084778,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34376901388168335,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.027914080768823624,
+      "rewards/margins": 3.3302230834960938,
+      "rewards/rejected": -3.302309036254883,
+      "step": 136,
+      "train_speed(iter/s)": 0.008009
+    },
+    {
+      "epoch": 0.30383255942892784,
+      "grad_norm": 2.9257876873016357,
+      "learning_rate": 0.00019434055874773215,
+      "logits/chosen": -0.8141021132469177,
+      "logits/rejected": -0.8318851590156555,
+      "logps/chosen": -9.164875984191895,
+      "logps/rejected": -67.25471496582031,
+      "loss": 0.5762239098548889,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.38338595628738403,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13336460292339325,
+      "rewards/margins": 4.796402454376221,
+      "rewards/rejected": -4.66303825378418,
+      "step": 137,
+      "train_speed(iter/s)": 0.008011
+    },
+    {
+      "epoch": 0.3060503153371682,
+      "grad_norm": 11.465249061584473,
+      "learning_rate": 0.00019421806467564544,
+      "logits/chosen": -0.8505054712295532,
+      "logits/rejected": -0.8561327457427979,
+      "logps/chosen": -14.665526390075684,
+      "logps/rejected": -62.366455078125,
+      "loss": 1.0324087142944336,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.8431021571159363,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.13415789604187012,
+      "rewards/margins": 4.457357883453369,
+      "rewards/rejected": -4.591516017913818,
+      "step": 138,
+      "train_speed(iter/s)": 0.008009
+    },
+    {
+      "epoch": 0.30826807124540856,
+      "grad_norm": 9.11307144165039,
+      "learning_rate": 0.00019409429856227485,
+      "logits/chosen": -0.8643976449966431,
+      "logits/rejected": -0.8899680376052856,
+      "logps/chosen": -15.65466022491455,
+      "logps/rejected": -67.30322265625,
+      "loss": 0.9621648192405701,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6606032848358154,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.6521536111831665,
+      "rewards/margins": 4.181577205657959,
+      "rewards/rejected": -4.833731174468994,
+      "step": 139,
+      "train_speed(iter/s)": 0.008006
+    },
+    {
+      "epoch": 0.3104858271536489,
+      "grad_norm": 4.7639479637146,
+      "learning_rate": 0.00019396926207859084,
+      "logits/chosen": -0.801943302154541,
+      "logits/rejected": -0.8345421552658081,
+      "logps/chosen": -12.861966133117676,
+      "logps/rejected": -66.0527114868164,
+      "loss": 0.8016439080238342,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5596946477890015,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.06389661133289337,
+      "rewards/margins": 4.517420291900635,
+      "rewards/rejected": -4.581316947937012,
+      "step": 140,
+      "train_speed(iter/s)": 0.008006
+    },
+    {
+      "epoch": 0.3127035830618892,
+      "grad_norm": 2.60500431060791,
+      "learning_rate": 0.00019384295691271522,
+      "logits/chosen": -0.7634690999984741,
+      "logits/rejected": -0.7890049815177917,
+      "logps/chosen": -14.464373588562012,
+      "logps/rejected": -45.3537712097168,
+      "loss": 0.7553504109382629,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4683433771133423,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1130230501294136,
+      "rewards/margins": 2.7340469360351562,
+      "rewards/rejected": -2.6210238933563232,
+      "step": 141,
+      "train_speed(iter/s)": 0.008008
+    },
+    {
+      "epoch": 0.3149213389701296,
+      "grad_norm": 14.524946212768555,
+      "learning_rate": 0.0001937153847698983,
+      "logits/chosen": -0.7227780818939209,
+      "logits/rejected": -0.7420411705970764,
+      "logps/chosen": -15.700582504272461,
+      "logps/rejected": -41.36809539794922,
+      "loss": 1.1037310361862183,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6305118799209595,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0189147237688303,
+      "rewards/margins": 2.218311071395874,
+      "rewards/rejected": -2.1993966102600098,
+      "step": 142,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.31713909487836994,
+      "grad_norm": 2.8219783306121826,
+      "learning_rate": 0.00019358654737249592,
+      "logits/chosen": -0.6690309643745422,
+      "logits/rejected": -0.7185367345809937,
+      "logps/chosen": -14.496091842651367,
+      "logps/rejected": -48.948814392089844,
+      "loss": 0.785977840423584,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.508230984210968,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3897581100463867,
+      "rewards/margins": 2.4535489082336426,
+      "rewards/rejected": -2.063790798187256,
+      "step": 143,
+      "train_speed(iter/s)": 0.008013
+    },
+    {
+      "epoch": 0.3193568507866103,
+      "grad_norm": 3.476428270339966,
+      "learning_rate": 0.0001934564464599461,
+      "logits/chosen": -0.6897503137588501,
+      "logits/rejected": -0.7208132743835449,
+      "logps/chosen": -7.115019798278809,
+      "logps/rejected": -35.45432662963867,
+      "loss": 0.6814590692520142,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3637453615665436,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2670714855194092,
+      "rewards/margins": 1.9641859531402588,
+      "rewards/rejected": -1.6971145868301392,
+      "step": 144,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.32157460669485066,
+      "grad_norm": 4.858798503875732,
+      "learning_rate": 0.0001933250837887457,
+      "logits/chosen": -0.6452350616455078,
+      "logits/rejected": -0.6644193530082703,
+      "logps/chosen": -11.701910018920898,
+      "logps/rejected": -29.85957908630371,
+      "loss": 1.0518276691436768,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5833199620246887,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1761394739151001,
+      "rewards/margins": 0.9052659869194031,
+      "rewards/rejected": -0.7291264533996582,
+      "step": 145,
+      "train_speed(iter/s)": 0.008017
+    },
+    {
+      "epoch": 0.323792362603091,
+      "grad_norm": 2.92645001411438,
+      "learning_rate": 0.00019319246113242664,
+      "logits/chosen": -0.6735572814941406,
+      "logits/rejected": -0.6972106099128723,
+      "logps/chosen": -9.01904296875,
+      "logps/rejected": -23.485305786132812,
+      "loss": 0.8951749205589294,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5108327269554138,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28073376417160034,
+      "rewards/margins": 1.0756409168243408,
+      "rewards/rejected": -0.7949072122573853,
+      "step": 146,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.3260101185113313,
+      "grad_norm": 3.2365665435791016,
+      "learning_rate": 0.00019305858028153186,
+      "logits/chosen": -0.7025231719017029,
+      "logits/rejected": -0.7322810888290405,
+      "logps/chosen": -8.566629409790039,
+      "logps/rejected": -40.44590377807617,
+      "loss": 0.7701326012611389,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42253950238227844,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22694279253482819,
+      "rewards/margins": 1.8927738666534424,
+      "rewards/rejected": -1.6658309698104858,
+      "step": 147,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.3282278744195717,
+      "grad_norm": 3.0658862590789795,
+      "learning_rate": 0.00019292344304359124,
+      "logits/chosen": -0.6364326477050781,
+      "logits/rejected": -0.6599412560462952,
+      "logps/chosen": -9.841287612915039,
+      "logps/rejected": -29.358394622802734,
+      "loss": 0.7886962294578552,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40780922770500183,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2544412314891815,
+      "rewards/margins": 1.3426122665405273,
+      "rewards/rejected": -1.088171124458313,
+      "step": 148,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.33044563032781205,
+      "grad_norm": 12.607818603515625,
+      "learning_rate": 0.00019278705124309723,
+      "logits/chosen": -0.6786209344863892,
+      "logits/rejected": -0.7097568511962891,
+      "logps/chosen": -5.964594841003418,
+      "logps/rejected": -53.86275863647461,
+      "loss": 0.5599427223205566,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40204113721847534,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21791748702526093,
+      "rewards/margins": 3.074826240539551,
+      "rewards/rejected": -2.8569085597991943,
+      "step": 149,
+      "train_speed(iter/s)": 0.008021
+    },
+    {
+      "epoch": 0.3326633862360524,
+      "grad_norm": 2.428448438644409,
+      "learning_rate": 0.00019264940672148018,
+      "logits/chosen": -0.7238892316818237,
+      "logits/rejected": -0.7602605819702148,
+      "logps/chosen": -10.805157661437988,
+      "logps/rejected": -51.04298400878906,
+      "loss": 0.6146640777587891,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.313792884349823,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.05205656588077545,
+      "rewards/margins": 2.0433130264282227,
+      "rewards/rejected": -2.095369577407837,
+      "step": 150,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.33488114214429276,
+      "grad_norm": 3.576777458190918,
+      "learning_rate": 0.0001925105113370834,
+      "logits/chosen": -0.734486997127533,
+      "logits/rejected": -0.7705245614051819,
+      "logps/chosen": -9.474486351013184,
+      "logps/rejected": -41.55698776245117,
+      "loss": 0.8946757316589355,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5271129608154297,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17018195986747742,
+      "rewards/margins": 2.1353487968444824,
+      "rewards/rejected": -1.9651670455932617,
+      "step": 151,
+      "train_speed(iter/s)": 0.008012
+    },
+    {
+      "epoch": 0.33709889805253307,
+      "grad_norm": 13.971368789672852,
+      "learning_rate": 0.00019237036696513818,
+      "logits/chosen": -0.7242223620414734,
+      "logits/rejected": -0.7463615536689758,
+      "logps/chosen": -12.119179725646973,
+      "logps/rejected": -39.10074996948242,
+      "loss": 1.3560465574264526,
+      "memory(GiB)": 46.1,
+      "nll_loss": 1.0073633193969727,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06672754883766174,
+      "rewards/margins": 1.9847290515899658,
+      "rewards/rejected": -1.9180015325546265,
+      "step": 152,
+      "train_speed(iter/s)": 0.008014
+    },
+    {
+      "epoch": 0.33931665396077343,
+      "grad_norm": 3.170781135559082,
+      "learning_rate": 0.00019222897549773848,
+      "logits/chosen": -0.7781574130058289,
+      "logits/rejected": -0.7878897190093994,
+      "logps/chosen": -9.464417457580566,
+      "logps/rejected": -42.03047180175781,
+      "loss": 0.7055684328079224,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3885788023471832,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.05404181033372879,
+      "rewards/margins": 2.449146270751953,
+      "rewards/rejected": -2.39510440826416,
+      "step": 153,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.3415344098690138,
+      "grad_norm": 2.7523646354675293,
+      "learning_rate": 0.00019208633884381526,
+      "logits/chosen": -0.7232592701911926,
+      "logits/rejected": -0.7529851198196411,
+      "logps/chosen": -7.77958345413208,
+      "logps/rejected": -44.71973419189453,
+      "loss": 0.6493588089942932,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42470622062683105,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.388906866312027,
+      "rewards/margins": 2.8554818630218506,
+      "rewards/rejected": -2.4665746688842773,
+      "step": 154,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.34375216577725415,
+      "grad_norm": 20.620134353637695,
+      "learning_rate": 0.0001919424589291108,
+      "logits/chosen": -0.7193765044212341,
+      "logits/rejected": -0.7572377920150757,
+      "logps/chosen": -11.307958602905273,
+      "logps/rejected": -51.39036178588867,
+      "loss": 1.0872689485549927,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.8757498860359192,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19819001853466034,
+      "rewards/margins": 2.9048383235931396,
+      "rewards/rejected": -2.706648111343384,
+      "step": 155,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.3459699216854945,
+      "grad_norm": 14.992989540100098,
+      "learning_rate": 0.0001917973376961527,
+      "logits/chosen": -0.7125108242034912,
+      "logits/rejected": -0.7249738574028015,
+      "logps/chosen": -5.366758346557617,
+      "logps/rejected": -33.353816986083984,
+      "loss": 0.7495843768119812,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4479677975177765,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20626316964626312,
+      "rewards/margins": 1.818756341934204,
+      "rewards/rejected": -1.6124933958053589,
+      "step": 156,
+      "train_speed(iter/s)": 0.008015
+    },
+    {
+      "epoch": 0.3481876775937348,
+      "grad_norm": 2.707430362701416,
+      "learning_rate": 0.0001916509771042277,
+      "logits/chosen": -0.7225325703620911,
+      "logits/rejected": -0.7751076221466064,
+      "logps/chosen": -7.558128356933594,
+      "logps/rejected": -59.62013626098633,
+      "loss": 0.3974663317203522,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2714276611804962,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11370375007390976,
+      "rewards/margins": 3.6732826232910156,
+      "rewards/rejected": -3.5595788955688477,
+      "step": 157,
+      "train_speed(iter/s)": 0.008016
+    },
+    {
+      "epoch": 0.35040543350197517,
+      "grad_norm": 8.831613540649414,
+      "learning_rate": 0.0001915033791293551,
+      "logits/chosen": -0.6869341731071472,
+      "logits/rejected": -0.7035802006721497,
+      "logps/chosen": -11.216403007507324,
+      "logps/rejected": -38.74596405029297,
+      "loss": 0.7746688723564148,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45386001467704773,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.166388601064682,
+      "rewards/margins": 1.9721925258636475,
+      "rewards/rejected": -1.8058040142059326,
+      "step": 158,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.35262318941021553,
+      "grad_norm": 2.9480724334716797,
+      "learning_rate": 0.0001913545457642601,
+      "logits/chosen": -0.6564315557479858,
+      "logits/rejected": -0.6884806752204895,
+      "logps/chosen": -12.068757057189941,
+      "logps/rejected": -44.580867767333984,
+      "loss": 0.7749238610267639,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5597700476646423,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.36497005820274353,
+      "rewards/margins": 2.6146392822265625,
+      "rewards/rejected": -2.249669075012207,
+      "step": 159,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.3548409453184559,
+      "grad_norm": 2.8434133529663086,
+      "learning_rate": 0.00019120447901834706,
+      "logits/chosen": -0.7489297389984131,
+      "logits/rejected": -0.777446985244751,
+      "logps/chosen": -8.937162399291992,
+      "logps/rejected": -36.274837493896484,
+      "loss": 0.885219395160675,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.46223700046539307,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03733991086483002,
+      "rewards/margins": 1.7854254245758057,
+      "rewards/rejected": -1.7480854988098145,
+      "step": 160,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.35705870122669625,
+      "grad_norm": 3.0293800830841064,
+      "learning_rate": 0.0001910531809176722,
+      "logits/chosen": -0.7100561261177063,
+      "logits/rejected": -0.7253575921058655,
+      "logps/chosen": -11.041877746582031,
+      "logps/rejected": -45.55367660522461,
+      "loss": 0.7278915643692017,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4644385874271393,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07243353128433228,
+      "rewards/margins": 2.7052664756774902,
+      "rewards/rejected": -2.6328327655792236,
+      "step": 161,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.3592764571349366,
+      "grad_norm": 4.234739780426025,
+      "learning_rate": 0.00019090065350491626,
+      "logits/chosen": -0.7882344126701355,
+      "logits/rejected": -0.8183198571205139,
+      "logps/chosen": -8.469147682189941,
+      "logps/rejected": -49.76445007324219,
+      "loss": 0.9398418664932251,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6014910936355591,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.029901552945375443,
+      "rewards/margins": 2.6962218284606934,
+      "rewards/rejected": -2.6663200855255127,
+      "step": 162,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.3614942130431769,
+      "grad_norm": 3.7181763648986816,
+      "learning_rate": 0.00019074689883935705,
+      "logits/chosen": -0.833709716796875,
+      "logits/rejected": -0.8633042573928833,
+      "logps/chosen": -9.177538871765137,
+      "logps/rejected": -48.86741638183594,
+      "loss": 0.6912304759025574,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4258120059967041,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2436945140361786,
+      "rewards/margins": 3.05234432220459,
+      "rewards/rejected": -2.808649778366089,
+      "step": 163,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.3637119689514173,
+      "grad_norm": 3.1451849937438965,
+      "learning_rate": 0.00019059191899684154,
+      "logits/chosen": -0.8144320845603943,
+      "logits/rejected": -0.8437734842300415,
+      "logps/chosen": -11.074281692504883,
+      "logps/rejected": -45.48798370361328,
+      "loss": 0.6197302937507629,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3811179995536804,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.15162959694862366,
+      "rewards/margins": 2.7688112258911133,
+      "rewards/rejected": -2.6171817779541016,
+      "step": 164,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.36592972485965763,
+      "grad_norm": 4.043789386749268,
+      "learning_rate": 0.00019043571606975777,
+      "logits/chosen": -0.7649494409561157,
+      "logits/rejected": -0.7870344519615173,
+      "logps/chosen": -15.020736694335938,
+      "logps/rejected": -43.62554931640625,
+      "loss": 1.026597261428833,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6712150573730469,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06518545746803284,
+      "rewards/margins": 2.171983480453491,
+      "rewards/rejected": -2.106797933578491,
+      "step": 165,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.368147480767898,
+      "grad_norm": 3.109821081161499,
+      "learning_rate": 0.00019027829216700678,
+      "logits/chosen": -0.7875198721885681,
+      "logits/rejected": -0.8325733542442322,
+      "logps/chosen": -9.139036178588867,
+      "logps/rejected": -56.36309051513672,
+      "loss": 0.6341462731361389,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.451867938041687,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23630189895629883,
+      "rewards/margins": 3.3503074645996094,
+      "rewards/rejected": -3.1140055656433105,
+      "step": 166,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.37036523667613835,
+      "grad_norm": 2.6877574920654297,
+      "learning_rate": 0.00019011964941397405,
+      "logits/chosen": -0.7355461120605469,
+      "logits/rejected": -0.7912125587463379,
+      "logps/chosen": -7.943358421325684,
+      "logps/rejected": -54.66022872924805,
+      "loss": 0.6476521492004395,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4549351930618286,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3065571188926697,
+      "rewards/margins": 3.0600643157958984,
+      "rewards/rejected": -2.753507614135742,
+      "step": 167,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.37258299258437866,
+      "grad_norm": 2.4588301181793213,
+      "learning_rate": 0.0001899597899525007,
+      "logits/chosen": -0.7490441799163818,
+      "logits/rejected": -0.7880818843841553,
+      "logps/chosen": -5.591465473175049,
+      "logps/rejected": -50.0340690612793,
+      "loss": 0.5648373961448669,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3865860402584076,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4422666132450104,
+      "rewards/margins": 3.342369318008423,
+      "rewards/rejected": -2.9001028537750244,
+      "step": 168,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.374800748492619,
+      "grad_norm": 1.955082893371582,
+      "learning_rate": 0.0001897987159408548,
+      "logits/chosen": -0.7648451328277588,
+      "logits/rejected": -0.7907530665397644,
+      "logps/chosen": -4.5870208740234375,
+      "logps/rejected": -48.017208099365234,
+      "loss": 0.44469985365867615,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.24593624472618103,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.260402113199234,
+      "rewards/margins": 3.37278151512146,
+      "rewards/rejected": -3.112379312515259,
+      "step": 169,
+      "train_speed(iter/s)": 0.008031
+    },
+    {
+      "epoch": 0.3770185044008594,
+      "grad_norm": 4.219427585601807,
+      "learning_rate": 0.00018963642955370201,
+      "logits/chosen": -0.7837445139884949,
+      "logits/rejected": -0.8009445667266846,
+      "logps/chosen": -12.525315284729004,
+      "logps/rejected": -44.30619430541992,
+      "loss": 0.9331468939781189,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5888757109642029,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.15850315988063812,
+      "rewards/margins": 2.3680062294006348,
+      "rewards/rejected": -2.5265092849731445,
+      "step": 170,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.37923626030909974,
+      "grad_norm": 3.6719424724578857,
+      "learning_rate": 0.00018947293298207635,
+      "logits/chosen": -0.6751590967178345,
+      "logits/rejected": -0.7674037218093872,
+      "logps/chosen": -9.650009155273438,
+      "logps/rejected": -63.452606201171875,
+      "loss": 0.7752348184585571,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6017739772796631,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13403475284576416,
+      "rewards/margins": 3.4565603733062744,
+      "rewards/rejected": -3.3225257396698,
+      "step": 171,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.3814540162173401,
+      "grad_norm": 3.4011969566345215,
+      "learning_rate": 0.00018930822843335056,
+      "logits/chosen": -0.704400360584259,
+      "logits/rejected": -0.7298606634140015,
+      "logps/chosen": -8.851949691772461,
+      "logps/rejected": -45.742679595947266,
+      "loss": 0.7604430317878723,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5082038044929504,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08242911100387573,
+      "rewards/margins": 2.7466654777526855,
+      "rewards/rejected": -2.664236307144165,
+      "step": 172,
+      "train_speed(iter/s)": 0.008032
+    },
+    {
+      "epoch": 0.3836717721255804,
+      "grad_norm": 3.709373712539673,
+      "learning_rate": 0.00018914231813120635,
+      "logits/chosen": -0.7612942457199097,
+      "logits/rejected": -0.7619401216506958,
+      "logps/chosen": -14.791999816894531,
+      "logps/rejected": -39.538509368896484,
+      "loss": 0.8681156635284424,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5611632466316223,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0010512899607419968,
+      "rewards/margins": 2.186850070953369,
+      "rewards/rejected": -2.187901496887207,
+      "step": 173,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.38588952803382076,
+      "grad_norm": 2.616783380508423,
+      "learning_rate": 0.00018897520431560434,
+      "logits/chosen": -0.7609409689903259,
+      "logits/rejected": -0.7808184623718262,
+      "logps/chosen": -5.7523698806762695,
+      "logps/rejected": -45.965145111083984,
+      "loss": 0.5051267743110657,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2791001796722412,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.376434326171875,
+      "rewards/margins": 2.61958909034729,
+      "rewards/rejected": -2.243154764175415,
+      "step": 174,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.3881072839420611,
+      "grad_norm": 2.9114208221435547,
+      "learning_rate": 0.00018880688924275378,
+      "logits/chosen": -0.6863377094268799,
+      "logits/rejected": -0.7190892696380615,
+      "logps/chosen": -4.734968662261963,
+      "logps/rejected": -39.10625076293945,
+      "loss": 0.6738218069076538,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40073657035827637,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2266547530889511,
+      "rewards/margins": 2.0988874435424805,
+      "rewards/rejected": -1.8722326755523682,
+      "step": 175,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.3903250398503015,
+      "grad_norm": 6.051851272583008,
+      "learning_rate": 0.00018863737518508218,
+      "logits/chosen": -0.7415744066238403,
+      "logits/rejected": -0.773291826248169,
+      "logps/chosen": -9.922603607177734,
+      "logps/rejected": -46.90787887573242,
+      "loss": 0.76039057970047,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4856896996498108,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12430408596992493,
+      "rewards/margins": 2.302933692932129,
+      "rewards/rejected": -2.1786296367645264,
+      "step": 176,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.39254279575854184,
+      "grad_norm": 2.987578868865967,
+      "learning_rate": 0.0001884666644312046,
+      "logits/chosen": -0.7061051726341248,
+      "logits/rejected": -0.7338166832923889,
+      "logps/chosen": -15.18138599395752,
+      "logps/rejected": -43.154415130615234,
+      "loss": 0.7650496363639832,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5320225954055786,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2508482336997986,
+      "rewards/margins": 2.6881744861602783,
+      "rewards/rejected": -2.437325954437256,
+      "step": 177,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.3947605516667822,
+      "grad_norm": 2.5779054164886475,
+      "learning_rate": 0.00018829475928589271,
+      "logits/chosen": -0.7169279456138611,
+      "logits/rejected": -0.7530863285064697,
+      "logps/chosen": -7.394716262817383,
+      "logps/rejected": -49.48583221435547,
+      "loss": 0.45556652545928955,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30026838183403015,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38607895374298096,
+      "rewards/margins": 3.1158361434936523,
+      "rewards/rejected": -2.729757070541382,
+      "step": 178,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.3969783075750225,
+      "grad_norm": 3.9343268871307373,
+      "learning_rate": 0.00018812166207004367,
+      "logits/chosen": -0.6970735788345337,
+      "logits/rejected": -0.724778413772583,
+      "logps/chosen": -4.960330009460449,
+      "logps/rejected": -49.74527359008789,
+      "loss": 0.5463108420372009,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3669103980064392,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27587890625,
+      "rewards/margins": 3.2249057292938232,
+      "rewards/rejected": -2.9490268230438232,
+      "step": 179,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.39919606348326286,
+      "grad_norm": 2.298149824142456,
+      "learning_rate": 0.0001879473751206489,
+      "logits/chosen": -0.7472426295280457,
+      "logits/rejected": -0.7810541987419128,
+      "logps/chosen": -5.616783142089844,
+      "logps/rejected": -51.225318908691406,
+      "loss": 0.5331763625144958,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3780348300933838,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3602917790412903,
+      "rewards/margins": 3.3162002563476562,
+      "rewards/rejected": -2.9559082984924316,
+      "step": 180,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.4014138193915032,
+      "grad_norm": 3.723597764968872,
+      "learning_rate": 0.00018777190079076242,
+      "logits/chosen": -0.7926808595657349,
+      "logits/rejected": -0.8186173439025879,
+      "logps/chosen": -3.8802855014801025,
+      "logps/rejected": -52.186126708984375,
+      "loss": 0.5307185649871826,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3474883437156677,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3934330642223358,
+      "rewards/margins": 3.725450038909912,
+      "rewards/rejected": -3.332016706466675,
+      "step": 181,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.4036315752997436,
+      "grad_norm": 7.891061305999756,
+      "learning_rate": 0.00018759524144946904,
+      "logits/chosen": -0.7886751294136047,
+      "logits/rejected": -0.8069500923156738,
+      "logps/chosen": -11.522048950195312,
+      "logps/rejected": -51.2484130859375,
+      "loss": 0.785020112991333,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.41102084517478943,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10307621210813522,
+      "rewards/margins": 3.4609313011169434,
+      "rewards/rejected": -3.3578553199768066,
+      "step": 182,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.40584933120798394,
+      "grad_norm": 12.93850040435791,
+      "learning_rate": 0.00018741739948185257,
+      "logits/chosen": -0.8120619058609009,
+      "logits/rejected": -0.852086067199707,
+      "logps/chosen": -6.879787445068359,
+      "logps/rejected": -65.4516830444336,
+      "loss": 0.8547239303588867,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7350503206253052,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0880105122923851,
+      "rewards/margins": 4.625463962554932,
+      "rewards/rejected": -4.5374531745910645,
+      "step": 183,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.40806708711622425,
+      "grad_norm": 3.9282493591308594,
+      "learning_rate": 0.0001872383772889634,
+      "logits/chosen": -0.8234286308288574,
+      "logits/rejected": -0.8348793983459473,
+      "logps/chosen": -9.335253715515137,
+      "logps/rejected": -44.39677047729492,
+      "loss": 0.7768021821975708,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5781885981559753,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2790551781654358,
+      "rewards/margins": 3.196932077407837,
+      "rewards/rejected": -2.917877197265625,
+      "step": 184,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.4102848430244646,
+      "grad_norm": 15.93510913848877,
+      "learning_rate": 0.00018705817728778624,
+      "logits/chosen": -0.8253858685493469,
+      "logits/rejected": -0.8627069592475891,
+      "logps/chosen": -9.565252304077148,
+      "logps/rejected": -53.15513610839844,
+      "loss": 0.8498061895370483,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6343203186988831,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21805116534233093,
+      "rewards/margins": 3.4257686138153076,
+      "rewards/rejected": -3.207717180252075,
+      "step": 185,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.41250259893270497,
+      "grad_norm": 15.02178955078125,
+      "learning_rate": 0.00018687680191120743,
+      "logits/chosen": -0.8546393513679504,
+      "logits/rejected": -0.9091347455978394,
+      "logps/chosen": -4.4741058349609375,
+      "logps/rejected": -63.36324691772461,
+      "loss": 1.0827393531799316,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7935152649879456,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1945604830980301,
+      "rewards/margins": 3.8709142208099365,
+      "rewards/rejected": -3.676353693008423,
+      "step": 186,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.4147203548409453,
+      "grad_norm": 2.5898377895355225,
+      "learning_rate": 0.00018669425360798205,
+      "logits/chosen": -0.8482339382171631,
+      "logits/rejected": -0.9088505506515503,
+      "logps/chosen": -5.740197658538818,
+      "logps/rejected": -51.71613311767578,
+      "loss": 0.5818274021148682,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3660181760787964,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2397884875535965,
+      "rewards/margins": 2.8303396701812744,
+      "rewards/rejected": -2.5905513763427734,
+      "step": 187,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.4169381107491857,
+      "grad_norm": 5.367336750030518,
+      "learning_rate": 0.00018651053484270095,
+      "logits/chosen": -0.7865353226661682,
+      "logits/rejected": -0.8548979759216309,
+      "logps/chosen": -7.190474987030029,
+      "logps/rejected": -60.156734466552734,
+      "loss": 0.5877411365509033,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39522460103034973,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21685907244682312,
+      "rewards/margins": 3.5003058910369873,
+      "rewards/rejected": -3.283446788787842,
+      "step": 188,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.419155866657426,
+      "grad_norm": 2.8252148628234863,
+      "learning_rate": 0.00018632564809575742,
+      "logits/chosen": -0.9151175022125244,
+      "logits/rejected": -0.956945538520813,
+      "logps/chosen": -11.193031311035156,
+      "logps/rejected": -50.2137451171875,
+      "loss": 0.6273083090782166,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44245627522468567,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.5056411027908325,
+      "rewards/margins": 3.342442512512207,
+      "rewards/rejected": -2.836801290512085,
+      "step": 189,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.42137362256566635,
+      "grad_norm": 5.62361478805542,
+      "learning_rate": 0.00018613959586331362,
+      "logits/chosen": -0.957817792892456,
+      "logits/rejected": -1.0089643001556396,
+      "logps/chosen": -11.406503677368164,
+      "logps/rejected": -59.45309829711914,
+      "loss": 0.69350266456604,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4786533713340759,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26705771684646606,
+      "rewards/margins": 3.8952856063842773,
+      "rewards/rejected": -3.628227949142456,
+      "step": 190,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.4235913784739067,
+      "grad_norm": 6.09588098526001,
+      "learning_rate": 0.00018595238065726713,
+      "logits/chosen": -0.9361623525619507,
+      "logits/rejected": -0.9750722646713257,
+      "logps/chosen": -11.82553482055664,
+      "logps/rejected": -59.44593811035156,
+      "loss": 0.8459053039550781,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5596847534179688,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2538452744483948,
+      "rewards/margins": 3.787569999694824,
+      "rewards/rejected": -3.533724784851074,
+      "step": 191,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.42580913438214707,
+      "grad_norm": 5.49157190322876,
+      "learning_rate": 0.00018576400500521672,
+      "logits/chosen": -1.1267294883728027,
+      "logits/rejected": -1.157508373260498,
+      "logps/chosen": -9.233796119689941,
+      "logps/rejected": -65.15520477294922,
+      "loss": 0.7781911492347717,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5769992470741272,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04808952659368515,
+      "rewards/margins": 4.439774513244629,
+      "rewards/rejected": -4.487864017486572,
+      "step": 192,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.42802689029038743,
+      "grad_norm": 2.235313653945923,
+      "learning_rate": 0.0001855744714504285,
+      "logits/chosen": -1.0502121448516846,
+      "logits/rejected": -1.0979869365692139,
+      "logps/chosen": -7.734978675842285,
+      "logps/rejected": -61.93341064453125,
+      "loss": 0.5888234972953796,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4166957437992096,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4335370361804962,
+      "rewards/margins": 4.3844313621521,
+      "rewards/rejected": -3.9508938789367676,
+      "step": 193,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.4302446461986278,
+      "grad_norm": 6.296457767486572,
+      "learning_rate": 0.0001853837825518014,
+      "logits/chosen": -1.0367488861083984,
+      "logits/rejected": -1.058951497077942,
+      "logps/chosen": -12.308891296386719,
+      "logps/rejected": -56.541419982910156,
+      "loss": 0.9084363579750061,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6487494707107544,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.29248520731925964,
+      "rewards/margins": 3.4791908264160156,
+      "rewards/rejected": -3.7716763019561768,
+      "step": 194,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.4324624021068681,
+      "grad_norm": 4.8967132568359375,
+      "learning_rate": 0.00018519194088383273,
+      "logits/chosen": -1.0082939863204956,
+      "logits/rejected": -1.0576558113098145,
+      "logps/chosen": -9.070019721984863,
+      "logps/rejected": -54.84600067138672,
+      "loss": 0.8942745327949524,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5133223533630371,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05057057738304138,
+      "rewards/margins": 3.1192855834960938,
+      "rewards/rejected": -3.0687148571014404,
+      "step": 195,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.43468015801510845,
+      "grad_norm": 6.081669807434082,
+      "learning_rate": 0.00018499894903658328,
+      "logits/chosen": -0.907891035079956,
+      "logits/rejected": -0.9242041110992432,
+      "logps/chosen": -9.73530101776123,
+      "logps/rejected": -37.79449462890625,
+      "loss": 0.67271488904953,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2690475881099701,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10105058550834656,
+      "rewards/margins": 1.979455828666687,
+      "rewards/rejected": -1.878405213356018,
+      "step": 196,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.4368979139233488,
+      "grad_norm": 3.40010666847229,
+      "learning_rate": 0.0001848048096156426,
+      "logits/chosen": -0.8272972106933594,
+      "logits/rejected": -0.9016137719154358,
+      "logps/chosen": -6.67510461807251,
+      "logps/rejected": -48.94219970703125,
+      "loss": 0.6460402607917786,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3624219000339508,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13816983997821808,
+      "rewards/margins": 3.1119472980499268,
+      "rewards/rejected": -2.9737775325775146,
+      "step": 197,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.4391156698315892,
+      "grad_norm": 3.359297037124634,
+      "learning_rate": 0.00018460952524209355,
+      "logits/chosen": -0.877692699432373,
+      "logits/rejected": -0.9115339517593384,
+      "logps/chosen": -11.648417472839355,
+      "logps/rejected": -37.90315246582031,
+      "loss": 0.8015753626823425,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5033032894134521,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2795218527317047,
+      "rewards/margins": 2.2155895233154297,
+      "rewards/rejected": -1.9360675811767578,
+      "step": 198,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.44133342573982953,
+      "grad_norm": 2.250875234603882,
+      "learning_rate": 0.00018441309855247708,
+      "logits/chosen": -0.787319004535675,
+      "logits/rejected": -0.8128381371498108,
+      "logps/chosen": -11.259406089782715,
+      "logps/rejected": -48.81473922729492,
+      "loss": 0.6350412964820862,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3753763735294342,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.42155084013938904,
+      "rewards/margins": 2.7902255058288574,
+      "rewards/rejected": -2.3686749935150146,
+      "step": 199,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.44355118164806984,
+      "grad_norm": 3.3108201026916504,
+      "learning_rate": 0.00018421553219875658,
+      "logits/chosen": -0.8686717748641968,
+      "logits/rejected": -0.8734286427497864,
+      "logps/chosen": -13.712151527404785,
+      "logps/rejected": -52.73145294189453,
+      "loss": 0.6721420288085938,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4126514196395874,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.028258126229047775,
+      "rewards/margins": 2.731067180633545,
+      "rewards/rejected": -2.7028088569641113,
+      "step": 200,
+      "train_speed(iter/s)": 0.008025
+    },
+    {
+      "epoch": 0.4457689375563102,
+      "grad_norm": 4.97205114364624,
+      "learning_rate": 0.00018401682884828212,
+      "logits/chosen": -0.7808172106742859,
+      "logits/rejected": -0.7950178980827332,
+      "logps/chosen": -11.171830177307129,
+      "logps/rejected": -40.51845932006836,
+      "loss": 0.6930698156356812,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4298695921897888,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20948606729507446,
+      "rewards/margins": 2.620697498321533,
+      "rewards/rejected": -2.4112114906311035,
+      "step": 201,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.44798669346455056,
+      "grad_norm": 5.371218204498291,
+      "learning_rate": 0.00018381699118375427,
+      "logits/chosen": -0.7809624075889587,
+      "logits/rejected": -0.8142802715301514,
+      "logps/chosen": -5.608348846435547,
+      "logps/rejected": -55.663177490234375,
+      "loss": 0.4353767931461334,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2700658440589905,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20935064554214478,
+      "rewards/margins": 3.5973005294799805,
+      "rewards/rejected": -3.3879499435424805,
+      "step": 202,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.4502044493727909,
+      "grad_norm": 16.89154815673828,
+      "learning_rate": 0.00018361602190318822,
+      "logits/chosen": -0.7608879208564758,
+      "logits/rejected": -0.7771663665771484,
+      "logps/chosen": -10.431235313415527,
+      "logps/rejected": -37.032108306884766,
+      "loss": 0.9102643728256226,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.635382354259491,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2574206590652466,
+      "rewards/margins": 2.4064812660217285,
+      "rewards/rejected": -2.1490604877471924,
+      "step": 203,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.4524222052810313,
+      "grad_norm": 6.729372978210449,
+      "learning_rate": 0.000183413923719877,
+      "logits/chosen": -0.7215007543563843,
+      "logits/rejected": -0.758901059627533,
+      "logps/chosen": -8.58769416809082,
+      "logps/rejected": -39.22972869873047,
+      "loss": 0.8954074382781982,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6078510284423828,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.25617051124572754,
+      "rewards/margins": 2.248603343963623,
+      "rewards/rejected": -1.992432713508606,
+      "step": 204,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.4546399611892716,
+      "grad_norm": 3.6512980461120605,
+      "learning_rate": 0.00018321069936235503,
+      "logits/chosen": -0.6804372668266296,
+      "logits/rejected": -0.718124508857727,
+      "logps/chosen": -8.9603853225708,
+      "logps/rejected": -47.75540542602539,
+      "loss": 0.6032508611679077,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.41573473811149597,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4219416379928589,
+      "rewards/margins": 3.0125231742858887,
+      "rewards/rejected": -2.5905814170837402,
+      "step": 205,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.45685771709751194,
+      "grad_norm": 7.076963424682617,
+      "learning_rate": 0.00018300635157436125,
+      "logits/chosen": -0.7391089797019958,
+      "logits/rejected": -0.745570182800293,
+      "logps/chosen": -4.518499851226807,
+      "logps/rejected": -34.36119079589844,
+      "loss": 0.6407385468482971,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3392406404018402,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.31493237614631653,
+      "rewards/margins": 2.2406697273254395,
+      "rewards/rejected": -1.9257375001907349,
+      "step": 206,
+      "train_speed(iter/s)": 0.008018
+    },
+    {
+      "epoch": 0.4590754730057523,
+      "grad_norm": 2.3934824466705322,
+      "learning_rate": 0.00018280088311480201,
+      "logits/chosen": -0.7122136354446411,
+      "logits/rejected": -0.7361628413200378,
+      "logps/chosen": -6.559683322906494,
+      "logps/rejected": -39.97085189819336,
+      "loss": 0.6831241846084595,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3898596167564392,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3751702308654785,
+      "rewards/margins": 2.362565279006958,
+      "rewards/rejected": -1.987395167350769,
+      "step": 207,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.46129322891399266,
+      "grad_norm": 2.4925472736358643,
+      "learning_rate": 0.00018259429675771403,
+      "logits/chosen": -0.6431427001953125,
+      "logits/rejected": -0.6792439818382263,
+      "logps/chosen": -6.0603485107421875,
+      "logps/rejected": -43.98711395263672,
+      "loss": 0.6350247859954834,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40350115299224854,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4135381281375885,
+      "rewards/margins": 2.4943490028381348,
+      "rewards/rejected": -2.0808112621307373,
+      "step": 208,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.463510984822233,
+      "grad_norm": 2.020061492919922,
+      "learning_rate": 0.0001823865952922267,
+      "logits/chosen": -0.7055493593215942,
+      "logits/rejected": -0.7343183755874634,
+      "logps/chosen": -6.9183878898620605,
+      "logps/rejected": -37.08061599731445,
+      "loss": 0.5601203441619873,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3676638901233673,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.38555678725242615,
+      "rewards/margins": 2.058412790298462,
+      "rewards/rejected": -1.6728558540344238,
+      "step": 209,
+      "train_speed(iter/s)": 0.008019
+    },
+    {
+      "epoch": 0.4657287407304734,
+      "grad_norm": 2.5581166744232178,
+      "learning_rate": 0.0001821777815225245,
+      "logits/chosen": -0.6911201477050781,
+      "logits/rejected": -0.7565470337867737,
+      "logps/chosen": -7.890411376953125,
+      "logps/rejected": -48.7196159362793,
+      "loss": 0.6602129340171814,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4272047281265259,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27426618337631226,
+      "rewards/margins": 2.3485374450683594,
+      "rewards/rejected": -2.0742714405059814,
+      "step": 210,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.4679464966387137,
+      "grad_norm": 3.0122952461242676,
+      "learning_rate": 0.00018196785826780928,
+      "logits/chosen": -0.6988839507102966,
+      "logits/rejected": -0.7252997159957886,
+      "logps/chosen": -11.326674461364746,
+      "logps/rejected": -38.2576904296875,
+      "loss": 0.7591457962989807,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.48633021116256714,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3746309280395508,
+      "rewards/margins": 2.2771949768066406,
+      "rewards/rejected": -1.9025641679763794,
+      "step": 211,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.47016425254695404,
+      "grad_norm": 2.265782356262207,
+      "learning_rate": 0.000181756828362262,
+      "logits/chosen": -0.7268755435943604,
+      "logits/rejected": -0.7466446161270142,
+      "logps/chosen": -7.725147724151611,
+      "logps/rejected": -51.996009826660156,
+      "loss": 0.4454570412635803,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2833636403083801,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36816588044166565,
+      "rewards/margins": 3.318744659423828,
+      "rewards/rejected": -2.9505786895751953,
+      "step": 212,
+      "train_speed(iter/s)": 0.008021
+    },
+    {
+      "epoch": 0.4723820084551944,
+      "grad_norm": 2.547436237335205,
+      "learning_rate": 0.00018154469465500448,
+      "logits/chosen": -0.7705230712890625,
+      "logits/rejected": -0.7850083112716675,
+      "logps/chosen": -9.060789108276367,
+      "logps/rejected": -43.8983154296875,
+      "loss": 0.6688026189804077,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44372719526290894,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3685440421104431,
+      "rewards/margins": 2.777890920639038,
+      "rewards/rejected": -2.40934681892395,
+      "step": 213,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.47459976436343476,
+      "grad_norm": 5.478801250457764,
+      "learning_rate": 0.00018133146001006117,
+      "logits/chosen": -0.7916015982627869,
+      "logits/rejected": -0.8327130675315857,
+      "logps/chosen": -12.882250785827637,
+      "logps/rejected": -54.540977478027344,
+      "loss": 0.7713211178779602,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6156861782073975,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4459156095981598,
+      "rewards/margins": 3.8908560276031494,
+      "rewards/rejected": -3.4449405670166016,
+      "step": 214,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.4768175202716751,
+      "grad_norm": 9.98646354675293,
+      "learning_rate": 0.00018111712730632022,
+      "logits/chosen": -0.8455632328987122,
+      "logits/rejected": -0.8789849877357483,
+      "logps/chosen": -6.6688761711120605,
+      "logps/rejected": -67.25550079345703,
+      "loss": 0.4042738378047943,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.27104586362838745,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2038407027721405,
+      "rewards/margins": 4.787556171417236,
+      "rewards/rejected": -4.583715438842773,
+      "step": 215,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.4790352761799154,
+      "grad_norm": 8.007806777954102,
+      "learning_rate": 0.00018090169943749476,
+      "logits/chosen": -0.8711842894554138,
+      "logits/rejected": -0.8995304703712463,
+      "logps/chosen": -9.905319213867188,
+      "logps/rejected": -61.446109771728516,
+      "loss": 0.7026961445808411,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5331434011459351,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16945196688175201,
+      "rewards/margins": 4.315336227416992,
+      "rewards/rejected": -4.1458845138549805,
+      "step": 216,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.4812530320881558,
+      "grad_norm": 2.730963706970215,
+      "learning_rate": 0.0001806851793120837,
+      "logits/chosen": -0.8149902820587158,
+      "logits/rejected": -0.8820565342903137,
+      "logps/chosen": -9.960224151611328,
+      "logps/rejected": -72.3023681640625,
+      "loss": 0.5872762799263,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44592928886413574,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16795015335083008,
+      "rewards/margins": 5.117605686187744,
+      "rewards/rejected": -4.949655055999756,
+      "step": 217,
+      "train_speed(iter/s)": 0.008024
+    },
+    {
+      "epoch": 0.48347078799639615,
+      "grad_norm": 16.840694427490234,
+      "learning_rate": 0.00018046756985333256,
+      "logits/chosen": -0.8040902018547058,
+      "logits/rejected": -0.8409407138824463,
+      "logps/chosen": -9.597344398498535,
+      "logps/rejected": -60.84199523925781,
+      "loss": 0.6265321373939514,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39480292797088623,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07195545732975006,
+      "rewards/margins": 4.253119468688965,
+      "rewards/rejected": -4.181163787841797,
+      "step": 218,
+      "train_speed(iter/s)": 0.008023
+    },
+    {
+      "epoch": 0.4856885439046365,
+      "grad_norm": 11.527325630187988,
+      "learning_rate": 0.0001802488739991941,
+      "logits/chosen": -0.8545507192611694,
+      "logits/rejected": -0.8696693181991577,
+      "logps/chosen": -8.045662879943848,
+      "logps/rejected": -52.85993194580078,
+      "loss": 1.0190614461898804,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6636497974395752,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.0182030089199543,
+      "rewards/margins": 3.632762908935547,
+      "rewards/rejected": -3.614560127258301,
+      "step": 219,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.48790629981287686,
+      "grad_norm": 2.8976073265075684,
+      "learning_rate": 0.00018002909470228842,
+      "logits/chosen": -0.8644663691520691,
+      "logits/rejected": -0.873846709728241,
+      "logps/chosen": -9.438857078552246,
+      "logps/rejected": -60.52991485595703,
+      "loss": 0.5179789066314697,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.365613728761673,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08784684538841248,
+      "rewards/margins": 4.254735469818115,
+      "rewards/rejected": -4.16688871383667,
+      "step": 220,
+      "train_speed(iter/s)": 0.008021
+    },
+    {
+      "epoch": 0.49012405572111717,
+      "grad_norm": 3.299361228942871,
+      "learning_rate": 0.00017980823492986324,
+      "logits/chosen": -0.8386255502700806,
+      "logits/rejected": -0.8992589712142944,
+      "logps/chosen": -8.689769744873047,
+      "logps/rejected": -60.96467590332031,
+      "loss": 0.618039608001709,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5055224299430847,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2881576120853424,
+      "rewards/margins": 4.106770992279053,
+      "rewards/rejected": -3.8186135292053223,
+      "step": 221,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.49234181162935753,
+      "grad_norm": 4.465257167816162,
+      "learning_rate": 0.00017958629766375386,
+      "logits/chosen": -0.8478450179100037,
+      "logits/rejected": -0.9085181355476379,
+      "logps/chosen": -6.204191207885742,
+      "logps/rejected": -56.054569244384766,
+      "loss": 0.47616931796073914,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.31598034501075745,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.258948415517807,
+      "rewards/margins": 3.6201562881469727,
+      "rewards/rejected": -3.3612074851989746,
+      "step": 222,
+      "train_speed(iter/s)": 0.00802
+    },
+    {
+      "epoch": 0.4945595675375979,
+      "grad_norm": 2.8880817890167236,
+      "learning_rate": 0.0001793632859003428,
+      "logits/chosen": -0.8514662384986877,
+      "logits/rejected": -0.8807088732719421,
+      "logps/chosen": -10.981437683105469,
+      "logps/rejected": -55.400272369384766,
+      "loss": 0.6770332455635071,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5153285264968872,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.30325403809547424,
+      "rewards/margins": 3.651240110397339,
+      "rewards/rejected": -3.3479862213134766,
+      "step": 223,
+      "train_speed(iter/s)": 0.008022
+    },
+    {
+      "epoch": 0.49677732344583825,
+      "grad_norm": 3.5312280654907227,
+      "learning_rate": 0.00017913920265051947,
+      "logits/chosen": -0.8069252371788025,
+      "logits/rejected": -0.8705534934997559,
+      "logps/chosen": -8.313520431518555,
+      "logps/rejected": -58.24019241333008,
+      "loss": 0.6806789636611938,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4746111035346985,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10646762698888779,
+      "rewards/margins": 3.66450834274292,
+      "rewards/rejected": -3.5580403804779053,
+      "step": 224,
+      "train_speed(iter/s)": 0.008024
+    },
+    {
+      "epoch": 0.4989950793540786,
+      "grad_norm": 3.3835299015045166,
+      "learning_rate": 0.00017891405093963938,
+      "logits/chosen": -0.830254077911377,
+      "logits/rejected": -0.861682653427124,
+      "logps/chosen": -10.002070426940918,
+      "logps/rejected": -47.45036315917969,
+      "loss": 0.6337284445762634,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4381062090396881,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26333028078079224,
+      "rewards/margins": 3.0331172943115234,
+      "rewards/rejected": -2.769787073135376,
+      "step": 225,
+      "train_speed(iter/s)": 0.008026
+    },
+    {
+      "epoch": 0.501212835262319,
+      "grad_norm": 3.5198752880096436,
+      "learning_rate": 0.00017868783380748342,
+      "logits/chosen": -0.8473120331764221,
+      "logits/rejected": -0.8709195852279663,
+      "logps/chosen": -13.000303268432617,
+      "logps/rejected": -42.49477767944336,
+      "loss": 0.6278812885284424,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4056437909603119,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.39378899335861206,
+      "rewards/margins": 2.7199783325195312,
+      "rewards/rejected": -2.3261890411376953,
+      "step": 226,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.5034305911705593,
+      "grad_norm": 3.3003716468811035,
+      "learning_rate": 0.00017846055430821678,
+      "logits/chosen": -0.8059426546096802,
+      "logits/rejected": -0.8080525398254395,
+      "logps/chosen": -9.30306339263916,
+      "logps/rejected": -40.58232116699219,
+      "loss": 0.6896506547927856,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37519916892051697,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3133179843425751,
+      "rewards/margins": 2.731419324874878,
+      "rewards/rejected": -2.4181013107299805,
+      "step": 227,
+      "train_speed(iter/s)": 0.008027
+    },
+    {
+      "epoch": 0.5056483470787997,
+      "grad_norm": 1.7876431941986084,
+      "learning_rate": 0.00017823221551034764,
+      "logits/chosen": -0.8154786229133606,
+      "logits/rejected": -0.8503978848457336,
+      "logps/chosen": -5.010475158691406,
+      "logps/rejected": -48.111228942871094,
+      "loss": 0.4449242055416107,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2939717471599579,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3824361264705658,
+      "rewards/margins": 3.612694501876831,
+      "rewards/rejected": -3.2302584648132324,
+      "step": 228,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.5078661029870399,
+      "grad_norm": 2.703659772872925,
+      "learning_rate": 0.00017800282049668594,
+      "logits/chosen": -0.8498862981796265,
+      "logits/rejected": -0.917186975479126,
+      "logps/chosen": -7.1461615562438965,
+      "logps/rejected": -59.192298889160156,
+      "loss": 0.4826613962650299,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3588803708553314,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3578621447086334,
+      "rewards/margins": 3.88301944732666,
+      "rewards/rejected": -3.5251574516296387,
+      "step": 229,
+      "train_speed(iter/s)": 0.008028
+    },
+    {
+      "epoch": 0.5100838588952803,
+      "grad_norm": 5.917115688323975,
+      "learning_rate": 0.0001777723723643014,
+      "logits/chosen": -0.8150835037231445,
+      "logits/rejected": -0.8721492290496826,
+      "logps/chosen": -15.683819770812988,
+      "logps/rejected": -61.561241149902344,
+      "loss": 0.9741629958152771,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7258660197257996,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.29861539602279663,
+      "rewards/margins": 3.3912651538848877,
+      "rewards/rejected": -3.0926499366760254,
+      "step": 230,
+      "train_speed(iter/s)": 0.008029
+    },
+    {
+      "epoch": 0.5123016148035207,
+      "grad_norm": 2.1567375659942627,
+      "learning_rate": 0.00017754087422448215,
+      "logits/chosen": -0.8800138235092163,
+      "logits/rejected": -0.9136852622032166,
+      "logps/chosen": -9.487836837768555,
+      "logps/rejected": -61.10226821899414,
+      "loss": 0.5113418102264404,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34745144844055176,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.20129577815532684,
+      "rewards/margins": 4.3714752197265625,
+      "rewards/rejected": -4.170179843902588,
+      "step": 231,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.514519370711761,
+      "grad_norm": 4.626194000244141,
+      "learning_rate": 0.0001773083292026924,
+      "logits/chosen": -0.9154278039932251,
+      "logits/rejected": -0.9261500239372253,
+      "logps/chosen": -7.649507522583008,
+      "logps/rejected": -66.75289154052734,
+      "loss": 0.5653941631317139,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.36132049560546875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16542571783065796,
+      "rewards/margins": 5.1535139083862305,
+      "rewards/rejected": -4.988088130950928,
+      "step": 232,
+      "train_speed(iter/s)": 0.00803
+    },
+    {
+      "epoch": 0.5167371266200014,
+      "grad_norm": 2.771819591522217,
+      "learning_rate": 0.00017707474043853041,
+      "logits/chosen": -0.8962085247039795,
+      "logits/rejected": -0.9393426179885864,
+      "logps/chosen": -4.859526634216309,
+      "logps/rejected": -70.9289779663086,
+      "loss": 0.528771698474884,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.32141542434692383,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14139828085899353,
+      "rewards/margins": 5.118078708648682,
+      "rewards/rejected": -4.976680278778076,
+      "step": 233,
+      "train_speed(iter/s)": 0.008032
+    },
+    {
+      "epoch": 0.5189548825282417,
+      "grad_norm": 2.016167402267456,
+      "learning_rate": 0.00017684011108568592,
+      "logits/chosen": -0.89229416847229,
+      "logits/rejected": -0.9399268627166748,
+      "logps/chosen": -9.858261108398438,
+      "logps/rejected": -73.9881591796875,
+      "loss": 0.46442869305610657,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3368317484855652,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33955106139183044,
+      "rewards/margins": 5.430692672729492,
+      "rewards/rejected": -5.091141223907471,
+      "step": 234,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5211726384364821,
+      "grad_norm": 2.207803726196289,
+      "learning_rate": 0.0001766044443118978,
+      "logits/chosen": -0.8679851293563843,
+      "logits/rejected": -0.9161220192909241,
+      "logps/chosen": -8.69020938873291,
+      "logps/rejected": -57.00996780395508,
+      "loss": 0.5931269526481628,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4199146032333374,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28541409969329834,
+      "rewards/margins": 3.884521245956421,
+      "rewards/rejected": -3.599107503890991,
+      "step": 235,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.5233903943447225,
+      "grad_norm": 3.842848062515259,
+      "learning_rate": 0.00017636774329891122,
+      "logits/chosen": -0.8388394713401794,
+      "logits/rejected": -0.9103097915649414,
+      "logps/chosen": -9.551056861877441,
+      "logps/rejected": -68.45164489746094,
+      "loss": 0.617426872253418,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4130822420120239,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2819046080112457,
+      "rewards/margins": 4.378241062164307,
+      "rewards/rejected": -4.096336841583252,
+      "step": 236,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5256081502529628,
+      "grad_norm": 2.8949806690216064,
+      "learning_rate": 0.00017613001124243446,
+      "logits/chosen": -0.8529986143112183,
+      "logits/rejected": -0.9170435667037964,
+      "logps/chosen": -8.742960929870605,
+      "logps/rejected": -70.04315948486328,
+      "loss": 0.5013477802276611,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3726488947868347,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19001907110214233,
+      "rewards/margins": 4.866143226623535,
+      "rewards/rejected": -4.676124572753906,
+      "step": 237,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5278259061612032,
+      "grad_norm": 2.255075454711914,
+      "learning_rate": 0.00017589125135209616,
+      "logits/chosen": -0.8355762958526611,
+      "logits/rejected": -0.9108222126960754,
+      "logps/chosen": -7.4976043701171875,
+      "logps/rejected": -79.64945983886719,
+      "loss": 0.5185106992721558,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4060248136520386,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4182281494140625,
+      "rewards/margins": 5.220707416534424,
+      "rewards/rejected": -4.802480220794678,
+      "step": 238,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5300436620694435,
+      "grad_norm": 5.300905704498291,
+      "learning_rate": 0.00017565146685140167,
+      "logits/chosen": -0.9112374782562256,
+      "logits/rejected": -0.9718393683433533,
+      "logps/chosen": -5.972594261169434,
+      "logps/rejected": -67.36833190917969,
+      "loss": 0.4636631906032562,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3712203800678253,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3854513168334961,
+      "rewards/margins": 4.805893421173096,
+      "rewards/rejected": -4.420441627502441,
+      "step": 239,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5322614179776838,
+      "grad_norm": 3.5167770385742188,
+      "learning_rate": 0.00017541066097768963,
+      "logits/chosen": -0.9272933006286621,
+      "logits/rejected": -0.9693167209625244,
+      "logps/chosen": -8.705961227416992,
+      "logps/rejected": -68.90886688232422,
+      "loss": 0.5253361463546753,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3944278061389923,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2587865889072418,
+      "rewards/margins": 5.000888824462891,
+      "rewards/rejected": -4.742102146148682,
+      "step": 240,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.5344791738859241,
+      "grad_norm": 2.322474241256714,
+      "learning_rate": 0.00017516883698208836,
+      "logits/chosen": -0.9203046560287476,
+      "logits/rejected": -0.9854355454444885,
+      "logps/chosen": -6.003438949584961,
+      "logps/rejected": -89.39216613769531,
+      "loss": 0.35788312554359436,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.26435545086860657,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.31243598461151123,
+      "rewards/margins": 6.715966701507568,
+      "rewards/rejected": -6.403530597686768,
+      "step": 241,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.5366969297941645,
+      "grad_norm": 2.527200937271118,
+      "learning_rate": 0.00017492599812947174,
+      "logits/chosen": -0.9331361651420593,
+      "logits/rejected": -0.9946306347846985,
+      "logps/chosen": -8.94567584991455,
+      "logps/rejected": -69.54719543457031,
+      "loss": 0.5303357839584351,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3786414563655853,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3045698404312134,
+      "rewards/margins": 4.875159740447998,
+      "rewards/rejected": -4.570590496063232,
+      "step": 242,
+      "train_speed(iter/s)": 0.008033
+    },
+    {
+      "epoch": 0.5389146857024049,
+      "grad_norm": 3.39489483833313,
+      "learning_rate": 0.0001746821476984154,
+      "logits/chosen": -1.0111048221588135,
+      "logits/rejected": -1.0517295598983765,
+      "logps/chosen": -7.177391052246094,
+      "logps/rejected": -78.28370666503906,
+      "loss": 0.4529821276664734,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.26994505524635315,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.38182204961776733,
+      "rewards/margins": 5.7827043533325195,
+      "rewards/rejected": -5.400881767272949,
+      "step": 243,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.5411324416106452,
+      "grad_norm": 2.4717650413513184,
+      "learning_rate": 0.00017443728898115226,
+      "logits/chosen": -0.880812406539917,
+      "logits/rejected": -0.9722001552581787,
+      "logps/chosen": -6.132741451263428,
+      "logps/rejected": -88.99906158447266,
+      "loss": 0.475276380777359,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3436709940433502,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3613809645175934,
+      "rewards/margins": 6.569655895233154,
+      "rewards/rejected": -6.2082743644714355,
+      "step": 244,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.5433501975188856,
+      "grad_norm": 4.479578495025635,
+      "learning_rate": 0.00017419142528352817,
+      "logits/chosen": -0.960247278213501,
+      "logits/rejected": -1.024147868156433,
+      "logps/chosen": -10.586898803710938,
+      "logps/rejected": -76.42874145507812,
+      "loss": 0.6297825574874878,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4393399953842163,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31354814767837524,
+      "rewards/margins": 5.518167018890381,
+      "rewards/rejected": -5.204617977142334,
+      "step": 245,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.5455679534271259,
+      "grad_norm": 7.8160719871521,
+      "learning_rate": 0.00017394455992495722,
+      "logits/chosen": -0.9498053789138794,
+      "logits/rejected": -0.9790096879005432,
+      "logps/chosen": -15.403467178344727,
+      "logps/rejected": -63.19144058227539,
+      "loss": 0.8869673013687134,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.684601366519928,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4287480115890503,
+      "rewards/margins": 4.687766075134277,
+      "rewards/rejected": -4.2590179443359375,
+      "step": 246,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.5477857093353663,
+      "grad_norm": 4.335257530212402,
+      "learning_rate": 0.00017369669623837702,
+      "logits/chosen": -0.9585328102111816,
+      "logits/rejected": -0.969738781452179,
+      "logps/chosen": -7.035947322845459,
+      "logps/rejected": -55.91766357421875,
+      "loss": 0.5887908935546875,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.467349648475647,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3129452168941498,
+      "rewards/margins": 4.274552345275879,
+      "rewards/rejected": -3.961606979370117,
+      "step": 247,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.5500034652436067,
+      "grad_norm": 4.468607425689697,
+      "learning_rate": 0.00017344783757020356,
+      "logits/chosen": -0.9219909906387329,
+      "logits/rejected": -0.9567832350730896,
+      "logps/chosen": -6.767885208129883,
+      "logps/rejected": -64.09295654296875,
+      "loss": 0.570389986038208,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3981003165245056,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21861377358436584,
+      "rewards/margins": 4.632497310638428,
+      "rewards/rejected": -4.413884162902832,
+      "step": 248,
+      "train_speed(iter/s)": 0.00804
+    },
+    {
+      "epoch": 0.552221221151847,
+      "grad_norm": 3.9139609336853027,
+      "learning_rate": 0.00017319798728028619,
+      "logits/chosen": -0.8682211637496948,
+      "logits/rejected": -0.9467407464981079,
+      "logps/chosen": -7.962907791137695,
+      "logps/rejected": -57.789955139160156,
+      "loss": 0.6136176586151123,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.36956262588500977,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25562986731529236,
+      "rewards/margins": 3.6893582344055176,
+      "rewards/rejected": -3.4337282180786133,
+      "step": 249,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.5544389770600874,
+      "grad_norm": 3.867413282394409,
+      "learning_rate": 0.0001729471487418621,
+      "logits/chosen": -0.9048889875411987,
+      "logits/rejected": -0.9317964315414429,
+      "logps/chosen": -8.489683151245117,
+      "logps/rejected": -35.35362243652344,
+      "loss": 0.6919417977333069,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.38164252042770386,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.31191277503967285,
+      "rewards/margins": 2.2607834339141846,
+      "rewards/rejected": -1.9488706588745117,
+      "step": 250,
+      "train_speed(iter/s)": 0.00804
+    },
+    {
+      "epoch": 0.5566567329683276,
+      "grad_norm": 2.298877239227295,
+      "learning_rate": 0.00017269532534151092,
+      "logits/chosen": -0.7615423202514648,
+      "logits/rejected": -0.8214331865310669,
+      "logps/chosen": -4.562041282653809,
+      "logps/rejected": -50.8994140625,
+      "loss": 0.4953613579273224,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3353947401046753,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3833675682544708,
+      "rewards/margins": 3.5597290992736816,
+      "rewards/rejected": -3.176361560821533,
+      "step": 251,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.558874488876568,
+      "grad_norm": 2.6716578006744385,
+      "learning_rate": 0.00017244252047910892,
+      "logits/chosen": -0.7982797622680664,
+      "logits/rejected": -0.8429900407791138,
+      "logps/chosen": -7.085336685180664,
+      "logps/rejected": -56.750762939453125,
+      "loss": 0.6702145934104919,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39718589186668396,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08545264601707458,
+      "rewards/margins": 3.4916038513183594,
+      "rewards/rejected": -3.406151056289673,
+      "step": 252,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.5610922447848083,
+      "grad_norm": 25.809972763061523,
+      "learning_rate": 0.0001721887375677831,
+      "logits/chosen": -0.8431349396705627,
+      "logits/rejected": -0.8907182216644287,
+      "logps/chosen": -9.1442289352417,
+      "logps/rejected": -49.07356643676758,
+      "loss": 0.6953955888748169,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4963495135307312,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.40793710947036743,
+      "rewards/margins": 3.5371923446655273,
+      "rewards/rejected": -3.129255533218384,
+      "step": 253,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.5633100006930487,
+      "grad_norm": 3.9957404136657715,
+      "learning_rate": 0.0001719339800338651,
+      "logits/chosen": -0.8641637563705444,
+      "logits/rejected": -0.9326637983322144,
+      "logps/chosen": -6.770094394683838,
+      "logps/rejected": -52.5345458984375,
+      "loss": 0.5996891260147095,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39112862944602966,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.35435986518859863,
+      "rewards/margins": 3.2783637046813965,
+      "rewards/rejected": -2.9240036010742188,
+      "step": 254,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.5655277566012891,
+      "grad_norm": 2.1859161853790283,
+      "learning_rate": 0.00017167825131684513,
+      "logits/chosen": -0.8427779674530029,
+      "logits/rejected": -0.8935760855674744,
+      "logps/chosen": -6.190837383270264,
+      "logps/rejected": -64.83207702636719,
+      "loss": 0.41889873147010803,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34747910499572754,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39933833479881287,
+      "rewards/margins": 4.655778408050537,
+      "rewards/rejected": -4.256440162658691,
+      "step": 255,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.5677455125095294,
+      "grad_norm": 6.577281951904297,
+      "learning_rate": 0.00017142155486932518,
+      "logits/chosen": -0.9050501585006714,
+      "logits/rejected": -0.9711922407150269,
+      "logps/chosen": -5.258976936340332,
+      "logps/rejected": -66.69300842285156,
+      "loss": 0.5538998246192932,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.36948734521865845,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23015204071998596,
+      "rewards/margins": 4.450538158416748,
+      "rewards/rejected": -4.220386505126953,
+      "step": 256,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.5699632684177698,
+      "grad_norm": 7.422839641571045,
+      "learning_rate": 0.00017116389415697272,
+      "logits/chosen": -0.8993850946426392,
+      "logits/rejected": -0.9391924738883972,
+      "logps/chosen": -7.405189514160156,
+      "logps/rejected": -80.06634521484375,
+      "loss": 0.45386412739753723,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3150070011615753,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22320273518562317,
+      "rewards/margins": 6.410494804382324,
+      "rewards/rejected": -6.187292098999023,
+      "step": 257,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.5721810243260101,
+      "grad_norm": 2.2811427116394043,
+      "learning_rate": 0.00017090527265847374,
+      "logits/chosen": -0.9498234987258911,
+      "logits/rejected": -1.023263692855835,
+      "logps/chosen": -7.831456184387207,
+      "logps/rejected": -94.71128845214844,
+      "loss": 0.4886764883995056,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37856870889663696,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3619069457054138,
+      "rewards/margins": 7.0146894454956055,
+      "rewards/rejected": -6.652782440185547,
+      "step": 258,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.5743987802342505,
+      "grad_norm": 13.671805381774902,
+      "learning_rate": 0.00017064569386548585,
+      "logits/chosen": -0.9374775290489197,
+      "logits/rejected": -0.9560403227806091,
+      "logps/chosen": -15.280105590820312,
+      "logps/rejected": -59.25236129760742,
+      "loss": 1.0413533449172974,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7767413854598999,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.21862581372261047,
+      "rewards/margins": 4.168983459472656,
+      "rewards/rejected": -4.387609958648682,
+      "step": 259,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.5766165361424909,
+      "grad_norm": 20.491348266601562,
+      "learning_rate": 0.00017038516128259115,
+      "logits/chosen": -0.9478408098220825,
+      "logits/rejected": -0.9793086051940918,
+      "logps/chosen": -5.960484504699707,
+      "logps/rejected": -58.90625762939453,
+      "loss": 0.8467053174972534,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6261860728263855,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22471851110458374,
+      "rewards/margins": 4.334632873535156,
+      "rewards/rejected": -4.109914302825928,
+      "step": 260,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.5788342920507311,
+      "grad_norm": 4.862920761108398,
+      "learning_rate": 0.00017012367842724887,
+      "logits/chosen": -0.8967673778533936,
+      "logits/rejected": -0.922260582447052,
+      "logps/chosen": -8.773322105407715,
+      "logps/rejected": -41.337547302246094,
+      "loss": 0.5787839293479919,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39914894104003906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3420258164405823,
+      "rewards/margins": 2.8629865646362305,
+      "rewards/rejected": -2.520960807800293,
+      "step": 261,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.5810520479589715,
+      "grad_norm": 6.606571674346924,
+      "learning_rate": 0.0001698612488297479,
+      "logits/chosen": -0.8431069254875183,
+      "logits/rejected": -0.8718072772026062,
+      "logps/chosen": -4.862249374389648,
+      "logps/rejected": -69.5229721069336,
+      "loss": 0.48749804496765137,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.298774778842926,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2969810962677002,
+      "rewards/margins": 5.211779594421387,
+      "rewards/rejected": -4.914798736572266,
+      "step": 262,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.5832698038672118,
+      "grad_norm": 8.771227836608887,
+      "learning_rate": 0.0001695978760331591,
+      "logits/chosen": -0.844187319278717,
+      "logits/rejected": -0.8691195249557495,
+      "logps/chosen": -10.602951049804688,
+      "logps/rejected": -55.179969787597656,
+      "loss": 0.7893258929252625,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5229781866073608,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.20470717549324036,
+      "rewards/margins": 3.2157349586486816,
+      "rewards/rejected": -3.0110273361206055,
+      "step": 263,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.5854875597754522,
+      "grad_norm": 4.490575313568115,
+      "learning_rate": 0.00016933356359328757,
+      "logits/chosen": -0.8540911674499512,
+      "logits/rejected": -0.8924884796142578,
+      "logps/chosen": -11.638439178466797,
+      "logps/rejected": -56.00303649902344,
+      "loss": 0.802644670009613,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.46404212713241577,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.22095105051994324,
+      "rewards/margins": 3.449429512023926,
+      "rewards/rejected": -3.228478193283081,
+      "step": 264,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.5877053156836926,
+      "grad_norm": 2.0575592517852783,
+      "learning_rate": 0.00016906831507862443,
+      "logits/chosen": -0.879247784614563,
+      "logits/rejected": -0.9168909788131714,
+      "logps/chosen": -8.618342399597168,
+      "logps/rejected": -58.41561508178711,
+      "loss": 0.5658751726150513,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.424156129360199,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.47101202607154846,
+      "rewards/margins": 4.365726470947266,
+      "rewards/rejected": -3.89471435546875,
+      "step": 265,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.5899230715919329,
+      "grad_norm": 5.172542095184326,
+      "learning_rate": 0.00016880213407029899,
+      "logits/chosen": -0.8254646062850952,
+      "logits/rejected": -0.8460347652435303,
+      "logps/chosen": -13.859678268432617,
+      "logps/rejected": -62.08647537231445,
+      "loss": 0.5818012952804565,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42503443360328674,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.44672632217407227,
+      "rewards/margins": 4.058588981628418,
+      "rewards/rejected": -3.6118626594543457,
+      "step": 266,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.5921408275001733,
+      "grad_norm": 6.763800144195557,
+      "learning_rate": 0.00016853502416203,
+      "logits/chosen": -0.8868339657783508,
+      "logits/rejected": -0.9178940653800964,
+      "logps/chosen": -10.896245002746582,
+      "logps/rejected": -62.17267608642578,
+      "loss": 0.8170678019523621,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5326193571090698,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13541792333126068,
+      "rewards/margins": 4.492279529571533,
+      "rewards/rejected": -4.356861591339111,
+      "step": 267,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.5943585834084136,
+      "grad_norm": 1.9904747009277344,
+      "learning_rate": 0.00016826698896007733,
+      "logits/chosen": -0.9039366841316223,
+      "logits/rejected": -0.9352925419807434,
+      "logps/chosen": -11.314581871032715,
+      "logps/rejected": -61.60304260253906,
+      "loss": 0.42121097445487976,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3151026964187622,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3598387837409973,
+      "rewards/margins": 4.653321266174316,
+      "rewards/rejected": -4.293482780456543,
+      "step": 268,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.596576339316654,
+      "grad_norm": 5.165607452392578,
+      "learning_rate": 0.0001679980320831934,
+      "logits/chosen": -0.8385277986526489,
+      "logits/rejected": -0.8792409300804138,
+      "logps/chosen": -8.309085845947266,
+      "logps/rejected": -64.71354675292969,
+      "loss": 0.5031875371932983,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3636503219604492,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24522201716899872,
+      "rewards/margins": 4.567659854888916,
+      "rewards/rejected": -4.322437763214111,
+      "step": 269,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.5987940952248944,
+      "grad_norm": 4.2751874923706055,
+      "learning_rate": 0.00016772815716257412,
+      "logits/chosen": -0.9187635779380798,
+      "logits/rejected": -0.9605762958526611,
+      "logps/chosen": -8.409623146057129,
+      "logps/rejected": -52.862247467041016,
+      "loss": 0.6836012005805969,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44153153896331787,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.39110705256462097,
+      "rewards/margins": 3.830803871154785,
+      "rewards/rejected": -3.439696788787842,
+      "step": 270,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.6010118511331347,
+      "grad_norm": 4.208390712738037,
+      "learning_rate": 0.0001674573678418099,
+      "logits/chosen": -0.8860863447189331,
+      "logits/rejected": -0.9442291855812073,
+      "logps/chosen": -4.314522743225098,
+      "logps/rejected": -57.75453567504883,
+      "loss": 0.5371519923210144,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.33125901222229004,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22049468755722046,
+      "rewards/margins": 4.082085132598877,
+      "rewards/rejected": -3.8615903854370117,
+      "step": 271,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.603229607041375,
+      "grad_norm": 4.579020977020264,
+      "learning_rate": 0.00016718566777683655,
+      "logits/chosen": -0.8948354721069336,
+      "logits/rejected": -0.9177219271659851,
+      "logps/chosen": -14.298483848571777,
+      "logps/rejected": -62.74816131591797,
+      "loss": 0.6376166939735413,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5089208483695984,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32330945134162903,
+      "rewards/margins": 4.685830593109131,
+      "rewards/rejected": -4.362521171569824,
+      "step": 272,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6054473629496153,
+      "grad_norm": 1.6271522045135498,
+      "learning_rate": 0.00016691306063588583,
+      "logits/chosen": -0.8025689125061035,
+      "logits/rejected": -0.8775883316993713,
+      "logps/chosen": -6.190898418426514,
+      "logps/rejected": -67.3564453125,
+      "loss": 0.512791633605957,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.33794161677360535,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36618149280548096,
+      "rewards/margins": 4.5750837326049805,
+      "rewards/rejected": -4.208902359008789,
+      "step": 273,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.6076651188578557,
+      "grad_norm": 1.6816551685333252,
+      "learning_rate": 0.00016663955009943603,
+      "logits/chosen": -0.8147826790809631,
+      "logits/rejected": -0.8742930889129639,
+      "logps/chosen": -3.890578508377075,
+      "logps/rejected": -65.23680114746094,
+      "loss": 0.3021196126937866,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.15443052351474762,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26169687509536743,
+      "rewards/margins": 4.66456413269043,
+      "rewards/rejected": -4.402867317199707,
+      "step": 274,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.609882874766096,
+      "grad_norm": 5.99432373046875,
+      "learning_rate": 0.00016636513986016213,
+      "logits/chosen": -0.7995700836181641,
+      "logits/rejected": -0.8312158584594727,
+      "logps/chosen": -8.787904739379883,
+      "logps/rejected": -66.17198181152344,
+      "loss": 0.6568697690963745,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4868704080581665,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.334846168756485,
+      "rewards/margins": 4.610203742980957,
+      "rewards/rejected": -4.275357246398926,
+      "step": 275,
+      "train_speed(iter/s)": 0.008041
+    },
+    {
+      "epoch": 0.6121006306743364,
+      "grad_norm": 1.89693021774292,
+      "learning_rate": 0.00016608983362288612,
+      "logits/chosen": -0.7742518782615662,
+      "logits/rejected": -0.8116195797920227,
+      "logps/chosen": -8.038908958435059,
+      "logps/rejected": -50.42762756347656,
+      "loss": 0.6119199991226196,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.41143566370010376,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38594791293144226,
+      "rewards/margins": 3.199566125869751,
+      "rewards/rejected": -2.8136179447174072,
+      "step": 276,
+      "train_speed(iter/s)": 0.00804
+    },
+    {
+      "epoch": 0.6143183865825768,
+      "grad_norm": 7.4511284828186035,
+      "learning_rate": 0.00016581363510452684,
+      "logits/chosen": -0.7676944136619568,
+      "logits/rejected": -0.7938407063484192,
+      "logps/chosen": -9.055317878723145,
+      "logps/rejected": -58.31163787841797,
+      "loss": 0.6325221061706543,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42814794182777405,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3626343905925751,
+      "rewards/margins": 4.097815036773682,
+      "rewards/rejected": -3.735180377960205,
+      "step": 277,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6165361424908171,
+      "grad_norm": 8.191842079162598,
+      "learning_rate": 0.00016553654803404974,
+      "logits/chosen": -0.8072691559791565,
+      "logits/rejected": -0.8492940068244934,
+      "logps/chosen": -9.613892555236816,
+      "logps/rejected": -63.351585388183594,
+      "loss": 0.7843741178512573,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.44085100293159485,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12112342566251755,
+      "rewards/margins": 4.584871292114258,
+      "rewards/rejected": -4.463747978210449,
+      "step": 278,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6187538983990575,
+      "grad_norm": 1.822716236114502,
+      "learning_rate": 0.00016525857615241687,
+      "logits/chosen": -0.7915133237838745,
+      "logits/rejected": -0.834713876247406,
+      "logps/chosen": -8.906228065490723,
+      "logps/rejected": -65.07171630859375,
+      "loss": 0.4892042279243469,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34095320105552673,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.35025283694267273,
+      "rewards/margins": 4.735526084899902,
+      "rewards/rejected": -4.385272979736328,
+      "step": 279,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.6209716543072978,
+      "grad_norm": 2.0796329975128174,
+      "learning_rate": 0.000164979723212536,
+      "logits/chosen": -0.820456326007843,
+      "logits/rejected": -0.8396943807601929,
+      "logps/chosen": -8.370631217956543,
+      "logps/rejected": -63.127323150634766,
+      "loss": 0.6209460496902466,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45356690883636475,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28135380148887634,
+      "rewards/margins": 4.626897811889648,
+      "rewards/rejected": -4.34554386138916,
+      "step": 280,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6231894102155382,
+      "grad_norm": 5.120301246643066,
+      "learning_rate": 0.00016469999297921004,
+      "logits/chosen": -0.8289170265197754,
+      "logits/rejected": -0.857169508934021,
+      "logps/chosen": -8.919156074523926,
+      "logps/rejected": -68.47624206542969,
+      "loss": 0.6525104641914368,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.544549822807312,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32334885001182556,
+      "rewards/margins": 5.191709995269775,
+      "rewards/rejected": -4.868360996246338,
+      "step": 281,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6254071661237784,
+      "grad_norm": 3.569572687149048,
+      "learning_rate": 0.00016441938922908645,
+      "logits/chosen": -0.8497668504714966,
+      "logits/rejected": -0.8852089047431946,
+      "logps/chosen": -8.614233016967773,
+      "logps/rejected": -86.08775329589844,
+      "loss": 0.39198628067970276,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.29619306325912476,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2805593013763428,
+      "rewards/margins": 6.5137038230896,
+      "rewards/rejected": -6.2331438064575195,
+      "step": 282,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.6276249220320188,
+      "grad_norm": 3.6795778274536133,
+      "learning_rate": 0.0001641379157506059,
+      "logits/chosen": -0.8261008262634277,
+      "logits/rejected": -0.8573625087738037,
+      "logps/chosen": -8.30790901184082,
+      "logps/rejected": -68.17735290527344,
+      "loss": 0.4799172878265381,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3408132791519165,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.45212632417678833,
+      "rewards/margins": 5.448065757751465,
+      "rewards/rejected": -4.995939254760742,
+      "step": 283,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.6298426779402592,
+      "grad_norm": 4.690124034881592,
+      "learning_rate": 0.00016385557634395137,
+      "logits/chosen": -0.7986090183258057,
+      "logits/rejected": -0.8487676978111267,
+      "logps/chosen": -11.636720657348633,
+      "logps/rejected": -72.38658142089844,
+      "loss": 0.6701381206512451,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3881206512451172,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22570770978927612,
+      "rewards/margins": 4.7588019371032715,
+      "rewards/rejected": -4.5330939292907715,
+      "step": 284,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6320604338484995,
+      "grad_norm": 7.493976593017578,
+      "learning_rate": 0.00016357237482099684,
+      "logits/chosen": -0.8304284811019897,
+      "logits/rejected": -0.8699071407318115,
+      "logps/chosen": -5.870106220245361,
+      "logps/rejected": -76.31659698486328,
+      "loss": 0.5334377884864807,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39371258020401,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.32191622257232666,
+      "rewards/margins": 5.466432094573975,
+      "rewards/rejected": -5.1445159912109375,
+      "step": 285,
+      "train_speed(iter/s)": 0.008039
+    },
+    {
+      "epoch": 0.6342781897567399,
+      "grad_norm": 3.969316005706787,
+      "learning_rate": 0.00016328831500525554,
+      "logits/chosen": -0.8328202366828918,
+      "logits/rejected": -0.8629224896430969,
+      "logps/chosen": -6.880237102508545,
+      "logps/rejected": -74.27322387695312,
+      "loss": 0.5944677591323853,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3375623822212219,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2438332587480545,
+      "rewards/margins": 5.4755988121032715,
+      "rewards/rejected": -5.231765270233154,
+      "step": 286,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.6364959456649802,
+      "grad_norm": 7.615932464599609,
+      "learning_rate": 0.00016300340073182877,
+      "logits/chosen": -0.8713105916976929,
+      "logits/rejected": -0.9103774428367615,
+      "logps/chosen": -8.833354949951172,
+      "logps/rejected": -71.67160034179688,
+      "loss": 0.5350167751312256,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3725404441356659,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16415414214134216,
+      "rewards/margins": 5.522144317626953,
+      "rewards/rejected": -5.35798978805542,
+      "step": 287,
+      "train_speed(iter/s)": 0.008038
+    },
+    {
+      "epoch": 0.6387137015732206,
+      "grad_norm": 1.877372145652771,
+      "learning_rate": 0.0001627176358473537,
+      "logits/chosen": -0.7970234751701355,
+      "logits/rejected": -0.8734095692634583,
+      "logps/chosen": -6.636631011962891,
+      "logps/rejected": -77.04444122314453,
+      "loss": 0.4367782473564148,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3150050640106201,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2868354022502899,
+      "rewards/margins": 5.483615875244141,
+      "rewards/rejected": -5.196780204772949,
+      "step": 288,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.640931457481461,
+      "grad_norm": 1.9805866479873657,
+      "learning_rate": 0.00016243102420995182,
+      "logits/chosen": -0.9195335507392883,
+      "logits/rejected": -0.9563324451446533,
+      "logps/chosen": -9.391670227050781,
+      "logps/rejected": -74.65196228027344,
+      "loss": 0.4557032883167267,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3854813575744629,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2575431764125824,
+      "rewards/margins": 5.831593990325928,
+      "rewards/rejected": -5.5740509033203125,
+      "step": 289,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.6431492133897013,
+      "grad_norm": 1.8995643854141235,
+      "learning_rate": 0.00016214356968917648,
+      "logits/chosen": -0.8684622645378113,
+      "logits/rejected": -0.9041600823402405,
+      "logps/chosen": -6.005845069885254,
+      "logps/rejected": -63.756404876708984,
+      "loss": 0.5114766359329224,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34650737047195435,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3093988299369812,
+      "rewards/margins": 3.883380174636841,
+      "rewards/rejected": -3.573981285095215,
+      "step": 290,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.6453669692979417,
+      "grad_norm": 2.626262903213501,
+      "learning_rate": 0.00016185527616596095,
+      "logits/chosen": -0.8445376753807068,
+      "logits/rejected": -0.9054903388023376,
+      "logps/chosen": -8.279337882995605,
+      "logps/rejected": -67.75119018554688,
+      "loss": 0.5382622480392456,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37347856163978577,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31158941984176636,
+      "rewards/margins": 5.205071926116943,
+      "rewards/rejected": -4.893482208251953,
+      "step": 291,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.647584725206182,
+      "grad_norm": 1.6392858028411865,
+      "learning_rate": 0.0001615661475325658,
+      "logits/chosen": -0.9225890636444092,
+      "logits/rejected": -0.961870551109314,
+      "logps/chosen": -8.307012557983398,
+      "logps/rejected": -68.97980499267578,
+      "loss": 0.4374139606952667,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30878713726997375,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4022150933742523,
+      "rewards/margins": 4.990863800048828,
+      "rewards/rejected": -4.588648319244385,
+      "step": 292,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.6498024811144223,
+      "grad_norm": 2.5469810962677,
+      "learning_rate": 0.0001612761876925266,
+      "logits/chosen": -0.8949599266052246,
+      "logits/rejected": -0.9571384191513062,
+      "logps/chosen": -9.247197151184082,
+      "logps/rejected": -69.23826599121094,
+      "loss": 0.569223165512085,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.38202205300331116,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.41862952709198,
+      "rewards/margins": 4.359453201293945,
+      "rewards/rejected": -3.940823554992676,
+      "step": 293,
+      "train_speed(iter/s)": 0.008036
+    },
+    {
+      "epoch": 0.6520202370226627,
+      "grad_norm": 1.9419584274291992,
+      "learning_rate": 0.00016098540056060093,
+      "logits/chosen": -0.9679895043373108,
+      "logits/rejected": -1.007946491241455,
+      "logps/chosen": -9.110626220703125,
+      "logps/rejected": -64.00426483154297,
+      "loss": 0.6166866421699524,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4243084490299225,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3954381048679352,
+      "rewards/margins": 3.941873073577881,
+      "rewards/rejected": -3.5464346408843994,
+      "step": 294,
+      "train_speed(iter/s)": 0.008037
+    },
+    {
+      "epoch": 0.654237992930903,
+      "grad_norm": 2.5056684017181396,
+      "learning_rate": 0.00016069379006271566,
+      "logits/chosen": -0.9131721258163452,
+      "logits/rejected": -0.9555903673171997,
+      "logps/chosen": -7.171519756317139,
+      "logps/rejected": -65.33702087402344,
+      "loss": 0.5050148963928223,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35454410314559937,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3253967761993408,
+      "rewards/margins": 4.702347278594971,
+      "rewards/rejected": -4.376949787139893,
+      "step": 295,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.6564557488391434,
+      "grad_norm": 1.6568937301635742,
+      "learning_rate": 0.0001604013601359141,
+      "logits/chosen": -0.9838509559631348,
+      "logits/rejected": -1.0402789115905762,
+      "logps/chosen": -3.7990381717681885,
+      "logps/rejected": -71.07801055908203,
+      "loss": 0.3659912049770355,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.25355616211891174,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24511104822158813,
+      "rewards/margins": 5.158456325531006,
+      "rewards/rejected": -4.9133453369140625,
+      "step": 296,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.6586735047473837,
+      "grad_norm": 3.638634443283081,
+      "learning_rate": 0.00016010811472830252,
+      "logits/chosen": -1.039500117301941,
+      "logits/rejected": -1.0798026323318481,
+      "logps/chosen": -6.155025482177734,
+      "logps/rejected": -55.543113708496094,
+      "loss": 0.533197283744812,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.36974427103996277,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4299376308917999,
+      "rewards/margins": 4.183603763580322,
+      "rewards/rejected": -3.7536661624908447,
+      "step": 297,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.6608912606556241,
+      "grad_norm": 1.9530096054077148,
+      "learning_rate": 0.00015981405779899715,
+      "logits/chosen": -1.0155575275421143,
+      "logits/rejected": -1.0633889436721802,
+      "logps/chosen": -9.130773544311523,
+      "logps/rejected": -62.94365692138672,
+      "loss": 0.47470739483833313,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3274574279785156,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2257828563451767,
+      "rewards/margins": 4.259277820587158,
+      "rewards/rejected": -4.03349494934082,
+      "step": 298,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.6631090165638645,
+      "grad_norm": 2.4985907077789307,
+      "learning_rate": 0.0001595191933180705,
+      "logits/chosen": -1.0449122190475464,
+      "logits/rejected": -1.0995962619781494,
+      "logps/chosen": -5.003297805786133,
+      "logps/rejected": -76.33629608154297,
+      "loss": 0.4325657784938812,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2864378094673157,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1750936508178711,
+      "rewards/margins": 5.564494609832764,
+      "rewards/rejected": -5.389400482177734,
+      "step": 299,
+      "train_speed(iter/s)": 0.008035
+    },
+    {
+      "epoch": 0.6653267724721048,
+      "grad_norm": 4.527393817901611,
+      "learning_rate": 0.00015922352526649803,
+      "logits/chosen": -1.013086199760437,
+      "logits/rejected": -1.0818321704864502,
+      "logps/chosen": -7.39998197555542,
+      "logps/rejected": -79.36940002441406,
+      "loss": 0.5117042660713196,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35834765434265137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22214177250862122,
+      "rewards/margins": 5.567633152008057,
+      "rewards/rejected": -5.345491409301758,
+      "step": 300,
+      "train_speed(iter/s)": 0.008034
+    },
+    {
+      "epoch": 0.6653267724721048,
+      "eval_logits/chosen": -1.051897406578064,
+      "eval_logits/rejected": -1.1186614036560059,
+      "eval_logps/chosen": -5.492368698120117,
+      "eval_logps/rejected": -83.7062759399414,
+      "eval_loss": 0.41139093041419983,
+      "eval_nll_loss": 0.3018193244934082,
+      "eval_rewards/accuracies": 0.9655172228813171,
+      "eval_rewards/chosen": 0.31715357303619385,
+      "eval_rewards/margins": 6.428208351135254,
+      "eval_rewards/rejected": -6.111053466796875,
+      "eval_runtime": 221.1454,
+      "eval_samples_per_second": 0.656,
+      "eval_steps_per_second": 0.656,
+      "step": 300
+    },
+    {
+      "epoch": 0.6675445283803452,
+      "grad_norm": 2.912181854248047,
+      "learning_rate": 0.00015892705763610398,
+      "logits/chosen": -1.0837887525558472,
+      "logits/rejected": -1.1028895378112793,
+      "logps/chosen": -11.03648853302002,
+      "logps/rejected": -74.05093383789062,
+      "loss": 0.5985814929008484,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.39898425340652466,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3419343829154968,
+      "rewards/margins": 5.46358060836792,
+      "rewards/rejected": -5.121645927429199,
+      "step": 301,
+      "train_speed(iter/s)": 0.007984
+    },
+    {
+      "epoch": 0.6697622842885855,
+      "grad_norm": 6.333323001861572,
+      "learning_rate": 0.000158629794429508,
+      "logits/chosen": -1.1112759113311768,
+      "logits/rejected": -1.1460927724838257,
+      "logps/chosen": -7.317679405212402,
+      "logps/rejected": -69.5050048828125,
+      "loss": 0.65586918592453,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.37288913130760193,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.0826641321182251,
+      "rewards/margins": 5.427277088165283,
+      "rewards/rejected": -5.344613075256348,
+      "step": 302,
+      "train_speed(iter/s)": 0.007984
+    },
+    {
+      "epoch": 0.6719800401968259,
+      "grad_norm": 15.77000904083252,
+      "learning_rate": 0.00015833173966007066,
+      "logits/chosen": -1.1002230644226074,
+      "logits/rejected": -1.1550010442733765,
+      "logps/chosen": -6.355401992797852,
+      "logps/rejected": -96.39167022705078,
+      "loss": 0.5191652774810791,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.359164834022522,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.48963892459869385,
+      "rewards/margins": 8.402400970458984,
+      "rewards/rejected": -7.912761688232422,
+      "step": 303,
+      "train_speed(iter/s)": 0.007984
+    },
+    {
+      "epoch": 0.6741977961050661,
+      "grad_norm": 2.923369884490967,
+      "learning_rate": 0.00015803289735183952,
+      "logits/chosen": -1.0692856311798096,
+      "logits/rejected": -1.1164627075195312,
+      "logps/chosen": -11.489358901977539,
+      "logps/rejected": -65.28047943115234,
+      "loss": 0.6315346360206604,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5478549599647522,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.20820873975753784,
+      "rewards/margins": 5.051858425140381,
+      "rewards/rejected": -4.843649864196777,
+      "step": 304,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6764155520133065,
+      "grad_norm": 4.412753105163574,
+      "learning_rate": 0.00015773327153949465,
+      "logits/chosen": -1.123039960861206,
+      "logits/rejected": -1.1700791120529175,
+      "logps/chosen": -6.8377275466918945,
+      "logps/rejected": -71.43708801269531,
+      "loss": 0.6268560886383057,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4594406187534332,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25862306356430054,
+      "rewards/margins": 5.758920192718506,
+      "rewards/rejected": -5.500297546386719,
+      "step": 305,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6786333079215469,
+      "grad_norm": 4.7620530128479,
+      "learning_rate": 0.00015743286626829437,
+      "logits/chosen": -1.0160408020019531,
+      "logits/rejected": -1.062739610671997,
+      "logps/chosen": -9.769542694091797,
+      "logps/rejected": -62.2449836730957,
+      "loss": 0.9608262777328491,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.7778749465942383,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.25323987007141113,
+      "rewards/margins": 4.6640191078186035,
+      "rewards/rejected": -4.410778999328613,
+      "step": 306,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6808510638297872,
+      "grad_norm": 5.518799304962158,
+      "learning_rate": 0.0001571316855940204,
+      "logits/chosen": -1.0274622440338135,
+      "logits/rejected": -1.0647941827774048,
+      "logps/chosen": -12.439924240112305,
+      "logps/rejected": -77.76188659667969,
+      "loss": 0.5631053447723389,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.26418688893318176,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.08976743370294571,
+      "rewards/margins": 5.368993282318115,
+      "rewards/rejected": -5.279226303100586,
+      "step": 307,
+      "train_speed(iter/s)": 0.007986
+    },
+    {
+      "epoch": 0.6830688197380276,
+      "grad_norm": 3.4025986194610596,
+      "learning_rate": 0.00015682973358292323,
+      "logits/chosen": -0.9364197850227356,
+      "logits/rejected": -0.9693114757537842,
+      "logps/chosen": -6.508739471435547,
+      "logps/rejected": -64.5718994140625,
+      "loss": 0.4292842745780945,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30177903175354004,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.355379581451416,
+      "rewards/margins": 4.948610305786133,
+      "rewards/rejected": -4.593230724334717,
+      "step": 308,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6852865756462679,
+      "grad_norm": 5.073081016540527,
+      "learning_rate": 0.0001565270143116672,
+      "logits/chosen": -0.9697186350822449,
+      "logits/rejected": -1.0311468839645386,
+      "logps/chosen": -10.48011302947998,
+      "logps/rejected": -68.85475158691406,
+      "loss": 0.63325434923172,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4581890106201172,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10327774286270142,
+      "rewards/margins": 4.557413101196289,
+      "rewards/rejected": -4.454134941101074,
+      "step": 309,
+      "train_speed(iter/s)": 0.007984
+    },
+    {
+      "epoch": 0.6875043315545083,
+      "grad_norm": 3.0755183696746826,
+      "learning_rate": 0.00015622353186727544,
+      "logits/chosen": -0.8866985440254211,
+      "logits/rejected": -0.9442430138587952,
+      "logps/chosen": -11.030970573425293,
+      "logps/rejected": -63.98460388183594,
+      "loss": 0.650689423084259,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45447224378585815,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23037654161453247,
+      "rewards/margins": 4.3462677001953125,
+      "rewards/rejected": -4.115890979766846,
+      "step": 310,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6897220874627487,
+      "grad_norm": 1.9100881814956665,
+      "learning_rate": 0.0001559192903470747,
+      "logits/chosen": -0.8884042501449585,
+      "logits/rejected": -0.9456340074539185,
+      "logps/chosen": -4.9610724449157715,
+      "logps/rejected": -53.072811126708984,
+      "loss": 0.4617457985877991,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.28007352352142334,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.36447668075561523,
+      "rewards/margins": 3.690591335296631,
+      "rewards/rejected": -3.3261144161224365,
+      "step": 311,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.691939843370989,
+      "grad_norm": 2.4238641262054443,
+      "learning_rate": 0.00015561429385864005,
+      "logits/chosen": -0.908905565738678,
+      "logits/rejected": -0.9534474611282349,
+      "logps/chosen": -8.066839218139648,
+      "logps/rejected": -45.6877326965332,
+      "loss": 0.7352919578552246,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5458393692970276,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3332860767841339,
+      "rewards/margins": 3.4314332008361816,
+      "rewards/rejected": -3.098147392272949,
+      "step": 312,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.6941575992792294,
+      "grad_norm": 2.4582879543304443,
+      "learning_rate": 0.00015530854651973948,
+      "logits/chosen": -0.9047088623046875,
+      "logits/rejected": -0.9368931651115417,
+      "logps/chosen": -11.2597017288208,
+      "logps/rejected": -52.92218017578125,
+      "loss": 0.674723207950592,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.49418923258781433,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41948604583740234,
+      "rewards/margins": 3.426848888397217,
+      "rewards/rejected": -3.0073628425598145,
+      "step": 313,
+      "train_speed(iter/s)": 0.007986
+    },
+    {
+      "epoch": 0.6963753551874696,
+      "grad_norm": 2.3463730812072754,
+      "learning_rate": 0.00015500205245827812,
+      "logits/chosen": -0.9688816666603088,
+      "logits/rejected": -1.001115322113037,
+      "logps/chosen": -7.497335433959961,
+      "logps/rejected": -52.40256118774414,
+      "loss": 0.5067635774612427,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3210291862487793,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2679075002670288,
+      "rewards/margins": 3.5391669273376465,
+      "rewards/rejected": -3.2712595462799072,
+      "step": 314,
+      "train_speed(iter/s)": 0.007985
+    },
+    {
+      "epoch": 0.69859311109571,
+      "grad_norm": 1.4639019966125488,
+      "learning_rate": 0.00015469481581224272,
+      "logits/chosen": -0.9862826466560364,
+      "logits/rejected": -1.0402218103408813,
+      "logps/chosen": -11.151278495788574,
+      "logps/rejected": -63.666648864746094,
+      "loss": 0.542059063911438,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42706289887428284,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5953376293182373,
+      "rewards/margins": 4.478287696838379,
+      "rewards/rejected": -3.8829500675201416,
+      "step": 315,
+      "train_speed(iter/s)": 0.007986
+    },
+    {
+      "epoch": 0.7008108670039503,
+      "grad_norm": 1.0964159965515137,
+      "learning_rate": 0.00015438684072964555,
+      "logits/chosen": -0.9509701728820801,
+      "logits/rejected": -1.0120131969451904,
+      "logps/chosen": -3.282318115234375,
+      "logps/rejected": -72.4108657836914,
+      "loss": 0.23854827880859375,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.15861773490905762,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3274027109146118,
+      "rewards/margins": 5.171144008636475,
+      "rewards/rejected": -4.8437418937683105,
+      "step": 316,
+      "train_speed(iter/s)": 0.007987
+    },
+    {
+      "epoch": 0.7030286229121907,
+      "grad_norm": 3.6565847396850586,
+      "learning_rate": 0.00015407813136846877,
+      "logits/chosen": -1.0520319938659668,
+      "logits/rejected": -1.0684181451797485,
+      "logps/chosen": -6.481374740600586,
+      "logps/rejected": -50.84836959838867,
+      "loss": 0.6093093752861023,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4111699163913727,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2717169523239136,
+      "rewards/margins": 3.6485066413879395,
+      "rewards/rejected": -3.3767895698547363,
+      "step": 317,
+      "train_speed(iter/s)": 0.007987
+    },
+    {
+      "epoch": 0.7052463788204311,
+      "grad_norm": 1.8486236333847046,
+      "learning_rate": 0.00015376869189660783,
+      "logits/chosen": -0.9767893552780151,
+      "logits/rejected": -1.0699020624160767,
+      "logps/chosen": -5.003157138824463,
+      "logps/rejected": -61.613548278808594,
+      "loss": 0.38953274488449097,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2612164616584778,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32774925231933594,
+      "rewards/margins": 4.157557010650635,
+      "rewards/rejected": -3.829807758331299,
+      "step": 318,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7074641347286714,
+      "grad_norm": 2.2850542068481445,
+      "learning_rate": 0.00015345852649181556,
+      "logits/chosen": -1.0067814588546753,
+      "logits/rejected": -1.055223822593689,
+      "logps/chosen": -9.41201114654541,
+      "logps/rejected": -69.31196594238281,
+      "loss": 0.47490939497947693,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.344195693731308,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.29239606857299805,
+      "rewards/margins": 5.051109790802002,
+      "rewards/rejected": -4.7587127685546875,
+      "step": 319,
+      "train_speed(iter/s)": 0.007989
+    },
+    {
+      "epoch": 0.7096818906369118,
+      "grad_norm": 2.4990978240966797,
+      "learning_rate": 0.0001531476393416456,
+      "logits/chosen": -1.0527924299240112,
+      "logits/rejected": -1.0691872835159302,
+      "logps/chosen": -8.629197120666504,
+      "logps/rejected": -52.35713195800781,
+      "loss": 0.5260642766952515,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3726637661457062,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.46037670969963074,
+      "rewards/margins": 3.889458656311035,
+      "rewards/rejected": -3.429081916809082,
+      "step": 320,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7118996465451521,
+      "grad_norm": 2.756251096725464,
+      "learning_rate": 0.0001528360346433959,
+      "logits/chosen": -1.0590455532073975,
+      "logits/rejected": -1.1003897190093994,
+      "logps/chosen": -9.220525741577148,
+      "logps/rejected": -66.90814971923828,
+      "loss": 0.5508866906166077,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.43441012501716614,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4031120538711548,
+      "rewards/margins": 5.307968616485596,
+      "rewards/rejected": -4.904856204986572,
+      "step": 321,
+      "train_speed(iter/s)": 0.007989
+    },
+    {
+      "epoch": 0.7141174024533925,
+      "grad_norm": 8.120321273803711,
+      "learning_rate": 0.00015252371660405203,
+      "logits/chosen": -1.045861005783081,
+      "logits/rejected": -1.072197437286377,
+      "logps/chosen": -8.183548927307129,
+      "logps/rejected": -55.87236404418945,
+      "loss": 0.5517191290855408,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4383443593978882,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5038261413574219,
+      "rewards/margins": 4.238821983337402,
+      "rewards/rejected": -3.7349960803985596,
+      "step": 322,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7163351583616329,
+      "grad_norm": 5.439812660217285,
+      "learning_rate": 0.00015221068944023047,
+      "logits/chosen": -0.9942280650138855,
+      "logits/rejected": -1.0908329486846924,
+      "logps/chosen": -4.599414348602295,
+      "logps/rejected": -86.45954895019531,
+      "loss": 0.5564896464347839,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3748651444911957,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21072176098823547,
+      "rewards/margins": 6.380655288696289,
+      "rewards/rejected": -6.169933319091797,
+      "step": 323,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7185529142698732,
+      "grad_norm": 2.908318519592285,
+      "learning_rate": 0.00015189695737812152,
+      "logits/chosen": -1.0733102560043335,
+      "logits/rejected": -1.11064875125885,
+      "logps/chosen": -7.096078872680664,
+      "logps/rejected": -77.15171813964844,
+      "loss": 0.41845327615737915,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3608825206756592,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2574445605278015,
+      "rewards/margins": 5.614250659942627,
+      "rewards/rejected": -5.356806755065918,
+      "step": 324,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7207706701781135,
+      "grad_norm": 2.001560688018799,
+      "learning_rate": 0.00015158252465343242,
+      "logits/chosen": -1.0531553030014038,
+      "logits/rejected": -1.0946694612503052,
+      "logps/chosen": -8.788086891174316,
+      "logps/rejected": -50.299198150634766,
+      "loss": 0.5263574123382568,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35159769654273987,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39179500937461853,
+      "rewards/margins": 3.5899038314819336,
+      "rewards/rejected": -3.198108673095703,
+      "step": 325,
+      "train_speed(iter/s)": 0.007989
+    },
+    {
+      "epoch": 0.7229884260863538,
+      "grad_norm": 2.0563783645629883,
+      "learning_rate": 0.00015126739551132997,
+      "logits/chosen": -0.9845147132873535,
+      "logits/rejected": -1.024277687072754,
+      "logps/chosen": -6.742694854736328,
+      "logps/rejected": -66.78865051269531,
+      "loss": 0.5805354714393616,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4498131275177002,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.45866793394088745,
+      "rewards/margins": 4.89275598526001,
+      "rewards/rejected": -4.434087753295898,
+      "step": 326,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7252061819945942,
+      "grad_norm": 2.4432246685028076,
+      "learning_rate": 0.00015095157420638348,
+      "logits/chosen": -0.982561469078064,
+      "logits/rejected": -1.0583231449127197,
+      "logps/chosen": -3.3377718925476074,
+      "logps/rejected": -73.46272277832031,
+      "loss": 0.3824958801269531,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.25690242648124695,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2936461567878723,
+      "rewards/margins": 5.495882511138916,
+      "rewards/rejected": -5.202236652374268,
+      "step": 327,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7274239379028345,
+      "grad_norm": 3.9595139026641846,
+      "learning_rate": 0.00015063506500250708,
+      "logits/chosen": -1.0502831935882568,
+      "logits/rejected": -1.1004114151000977,
+      "logps/chosen": -8.691965103149414,
+      "logps/rejected": -71.41580200195312,
+      "loss": 0.44681987166404724,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.31214770674705505,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22634059190750122,
+      "rewards/margins": 5.629345893859863,
+      "rewards/rejected": -5.403005599975586,
+      "step": 328,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7296416938110749,
+      "grad_norm": 3.618638038635254,
+      "learning_rate": 0.0001503178721729022,
+      "logits/chosen": -1.021295428276062,
+      "logits/rejected": -1.098530888557434,
+      "logps/chosen": -8.527656555175781,
+      "logps/rejected": -69.10211944580078,
+      "loss": 0.6168586015701294,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5225605964660645,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.42750608921051025,
+      "rewards/margins": 5.383771896362305,
+      "rewards/rejected": -4.956265449523926,
+      "step": 329,
+      "train_speed(iter/s)": 0.007987
+    },
+    {
+      "epoch": 0.7318594497193153,
+      "grad_norm": 17.75547218322754,
+      "learning_rate": 0.00015000000000000001,
+      "logits/chosen": -1.089531660079956,
+      "logits/rejected": -1.162044644355774,
+      "logps/chosen": -7.145535469055176,
+      "logps/rejected": -97.1467514038086,
+      "loss": 0.8052725195884705,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.6069628596305847,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.16202522814273834,
+      "rewards/margins": 7.54561185836792,
+      "rewards/rejected": -7.383586883544922,
+      "step": 330,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7340772056275556,
+      "grad_norm": 7.715478420257568,
+      "learning_rate": 0.0001496814527754035,
+      "logits/chosen": -1.021614670753479,
+      "logits/rejected": -1.1059943437576294,
+      "logps/chosen": -4.589797496795654,
+      "logps/rejected": -98.64924621582031,
+      "loss": 0.4247751235961914,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2338293045759201,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14829346537590027,
+      "rewards/margins": 7.475074768066406,
+      "rewards/rejected": -7.326781272888184,
+      "step": 331,
+      "train_speed(iter/s)": 0.007987
+    },
+    {
+      "epoch": 0.736294961535796,
+      "grad_norm": 5.703808784484863,
+      "learning_rate": 0.00014936223479982953,
+      "logits/chosen": -1.0938533544540405,
+      "logits/rejected": -1.1311089992523193,
+      "logps/chosen": -10.840590476989746,
+      "logps/rejected": -83.04175567626953,
+      "loss": 1.1042178869247437,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.72503662109375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.027026668190956116,
+      "rewards/margins": 6.462612628936768,
+      "rewards/rejected": -6.435585975646973,
+      "step": 332,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7385127174440363,
+      "grad_norm": 3.51871657371521,
+      "learning_rate": 0.00014904235038305083,
+      "logits/chosen": -1.0023188591003418,
+      "logits/rejected": -1.0812650918960571,
+      "logps/chosen": -9.990187644958496,
+      "logps/rejected": -82.44390869140625,
+      "loss": 0.4421953558921814,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3568181097507477,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3243933320045471,
+      "rewards/margins": 6.271053791046143,
+      "rewards/rejected": -5.94666051864624,
+      "step": 333,
+      "train_speed(iter/s)": 0.007988
+    },
+    {
+      "epoch": 0.7407304733522767,
+      "grad_norm": 2.359330654144287,
+      "learning_rate": 0.00014872180384383773,
+      "logits/chosen": -1.0005170106887817,
+      "logits/rejected": -1.0614187717437744,
+      "logps/chosen": -4.916748046875,
+      "logps/rejected": -84.43028259277344,
+      "loss": 0.5017070770263672,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.25003811717033386,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.17200681567192078,
+      "rewards/margins": 5.694801330566406,
+      "rewards/rejected": -5.522794246673584,
+      "step": 334,
+      "train_speed(iter/s)": 0.007987
+    },
+    {
+      "epoch": 0.7429482292605171,
+      "grad_norm": 2.417720317840576,
+      "learning_rate": 0.0001484005995098999,
+      "logits/chosen": -0.9173535108566284,
+      "logits/rejected": -0.9738480448722839,
+      "logps/chosen": -7.978065490722656,
+      "logps/rejected": -52.23478317260742,
+      "loss": 0.5884492993354797,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3843919336795807,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.414502888917923,
+      "rewards/margins": 3.7098348140716553,
+      "rewards/rejected": -3.2953319549560547,
+      "step": 335,
+      "train_speed(iter/s)": 0.007989
+    },
+    {
+      "epoch": 0.7451659851687573,
+      "grad_norm": 5.175177097320557,
+      "learning_rate": 0.00014807874171782795,
+      "logits/chosen": -0.9095063209533691,
+      "logits/rejected": -0.9394166469573975,
+      "logps/chosen": -5.037604808807373,
+      "logps/rejected": -48.952796936035156,
+      "loss": 0.7686702013015747,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5135473608970642,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2529716193675995,
+      "rewards/margins": 3.3015551567077637,
+      "rewards/rejected": -3.0485832691192627,
+      "step": 336,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7473837410769977,
+      "grad_norm": 1.4395772218704224,
+      "learning_rate": 0.00014775623481303487,
+      "logits/chosen": -0.8478690981864929,
+      "logits/rejected": -0.9579203128814697,
+      "logps/chosen": -3.3176069259643555,
+      "logps/rejected": -75.16621398925781,
+      "loss": 0.25232070684432983,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.18015892803668976,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33292415738105774,
+      "rewards/margins": 5.303890705108643,
+      "rewards/rejected": -4.970966339111328,
+      "step": 337,
+      "train_speed(iter/s)": 0.007991
+    },
+    {
+      "epoch": 0.749601496985238,
+      "grad_norm": 4.6426920890808105,
+      "learning_rate": 0.0001474330831496973,
+      "logits/chosen": -0.8614159822463989,
+      "logits/rejected": -0.8955745697021484,
+      "logps/chosen": -7.800498962402344,
+      "logps/rejected": -50.34724807739258,
+      "loss": 0.5634305477142334,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3574722111225128,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1773032397031784,
+      "rewards/margins": 3.574413776397705,
+      "rewards/rejected": -3.3971107006073,
+      "step": 338,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.7518192528934784,
+      "grad_norm": 2.0889782905578613,
+      "learning_rate": 0.00014710929109069674,
+      "logits/chosen": -0.8648290038108826,
+      "logits/rejected": -0.9016939997673035,
+      "logps/chosen": -9.325538635253906,
+      "logps/rejected": -54.02799987792969,
+      "loss": 0.5422332882881165,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.38347160816192627,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4468119144439697,
+      "rewards/margins": 3.857147693634033,
+      "rewards/rejected": -3.4103355407714844,
+      "step": 339,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.7540370088017188,
+      "grad_norm": 1.8795205354690552,
+      "learning_rate": 0.0001467848630075608,
+      "logits/chosen": -0.8260731101036072,
+      "logits/rejected": -0.8987483382225037,
+      "logps/chosen": -8.172541618347168,
+      "logps/rejected": -68.80128479003906,
+      "loss": 0.42829784750938416,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.33098679780960083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.463642418384552,
+      "rewards/margins": 4.9933061599731445,
+      "rewards/rejected": -4.5296630859375,
+      "step": 340,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.7562547647099591,
+      "grad_norm": 1.8088308572769165,
+      "learning_rate": 0.00014645980328040401,
+      "logits/chosen": -0.83375483751297,
+      "logits/rejected": -0.9051945805549622,
+      "logps/chosen": -8.962587356567383,
+      "logps/rejected": -55.46277618408203,
+      "loss": 0.5223706364631653,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3683018684387207,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4433327615261078,
+      "rewards/margins": 4.030201435089111,
+      "rewards/rejected": -3.5868687629699707,
+      "step": 341,
+      "train_speed(iter/s)": 0.007994
+    },
+    {
+      "epoch": 0.7584725206181995,
+      "grad_norm": 1.6160343885421753,
+      "learning_rate": 0.0001461341162978688,
+      "logits/chosen": -0.8449936509132385,
+      "logits/rejected": -0.921272873878479,
+      "logps/chosen": -7.773403167724609,
+      "logps/rejected": -82.05076599121094,
+      "loss": 0.3855322301387787,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3014349937438965,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3078088164329529,
+      "rewards/margins": 5.843523025512695,
+      "rewards/rejected": -5.535714626312256,
+      "step": 342,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7606902765264398,
+      "grad_norm": 2.3332436084747314,
+      "learning_rate": 0.0001458078064570661,
+      "logits/chosen": -0.865425705909729,
+      "logits/rejected": -0.9039071202278137,
+      "logps/chosen": -6.671085834503174,
+      "logps/rejected": -74.30247497558594,
+      "loss": 0.4341945946216583,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3516058623790741,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6183019876480103,
+      "rewards/margins": 5.456951141357422,
+      "rewards/rejected": -4.838649749755859,
+      "step": 343,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7629080324346802,
+      "grad_norm": 2.2075419425964355,
+      "learning_rate": 0.00014548087816351616,
+      "logits/chosen": -0.8886226415634155,
+      "logits/rejected": -0.9319683313369751,
+      "logps/chosen": -7.503270626068115,
+      "logps/rejected": -69.36604309082031,
+      "loss": 0.5470494627952576,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40474218130111694,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2907697558403015,
+      "rewards/margins": 5.133359909057617,
+      "rewards/rejected": -4.84259033203125,
+      "step": 344,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7651257883429206,
+      "grad_norm": 5.5458197593688965,
+      "learning_rate": 0.00014515333583108896,
+      "logits/chosen": -0.8889970779418945,
+      "logits/rejected": -0.9374210834503174,
+      "logps/chosen": -6.389466285705566,
+      "logps/rejected": -64.43991088867188,
+      "loss": 0.5072943568229675,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3807336986064911,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6497054100036621,
+      "rewards/margins": 5.243303298950195,
+      "rewards/rejected": -4.593598365783691,
+      "step": 345,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7673435442511608,
+      "grad_norm": 3.809356212615967,
+      "learning_rate": 0.00014482518388194457,
+      "logits/chosen": -0.850013792514801,
+      "logits/rejected": -0.8974230289459229,
+      "logps/chosen": -5.620848655700684,
+      "logps/rejected": -72.27090454101562,
+      "loss": 0.6183085441589355,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.444873571395874,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.4231736660003662,
+      "rewards/margins": 5.374911308288574,
+      "rewards/rejected": -4.951737403869629,
+      "step": 346,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7695613001594012,
+      "grad_norm": 5.957520961761475,
+      "learning_rate": 0.00014449642674647367,
+      "logits/chosen": -0.903602659702301,
+      "logits/rejected": -0.9699824452400208,
+      "logps/chosen": -6.9516706466674805,
+      "logps/rejected": -66.93140411376953,
+      "loss": 0.4030936360359192,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.27591440081596375,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5947475433349609,
+      "rewards/margins": 4.9736480712890625,
+      "rewards/rejected": -4.378900527954102,
+      "step": 347,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.7717790560676415,
+      "grad_norm": 7.984817981719971,
+      "learning_rate": 0.0001441670688632374,
+      "logits/chosen": -0.9703332185745239,
+      "logits/rejected": -0.9999107718467712,
+      "logps/chosen": -13.621354103088379,
+      "logps/rejected": -54.89396667480469,
+      "loss": 0.8680185079574585,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.472337007522583,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.08581960201263428,
+      "rewards/margins": 3.7399158477783203,
+      "rewards/rejected": -3.6540961265563965,
+      "step": 348,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.7739968119758819,
+      "grad_norm": 3.5452890396118164,
+      "learning_rate": 0.00014383711467890774,
+      "logits/chosen": -0.9808631539344788,
+      "logits/rejected": -1.0241118669509888,
+      "logps/chosen": -6.2970123291015625,
+      "logps/rejected": -57.30949783325195,
+      "loss": 0.5067135691642761,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.34234529733657837,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4528099000453949,
+      "rewards/margins": 4.415470123291016,
+      "rewards/rejected": -3.962660789489746,
+      "step": 349,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7762145678841222,
+      "grad_norm": 2.270117998123169,
+      "learning_rate": 0.00014350656864820733,
+      "logits/chosen": -0.8902170062065125,
+      "logits/rejected": -0.9376388192176819,
+      "logps/chosen": -6.128102779388428,
+      "logps/rejected": -84.18601989746094,
+      "loss": 0.3887496292591095,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.29120758175849915,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.410897433757782,
+      "rewards/margins": 6.373892784118652,
+      "rewards/rejected": -5.9629950523376465,
+      "step": 350,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.7784323237923626,
+      "grad_norm": 1.537272334098816,
+      "learning_rate": 0.00014317543523384928,
+      "logits/chosen": -0.8935893177986145,
+      "logits/rejected": -0.9689089059829712,
+      "logps/chosen": -9.015190124511719,
+      "logps/rejected": -72.697265625,
+      "loss": 0.3616684079170227,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.26947712898254395,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5364948511123657,
+      "rewards/margins": 5.098753929138184,
+      "rewards/rejected": -4.562259197235107,
+      "step": 351,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.780650079700603,
+      "grad_norm": 8.092968940734863,
+      "learning_rate": 0.00014284371890647713,
+      "logits/chosen": -0.9558290243148804,
+      "logits/rejected": -0.9736660718917847,
+      "logps/chosen": -9.879434585571289,
+      "logps/rejected": -67.37289428710938,
+      "loss": 0.5441817045211792,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4073256552219391,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2747573256492615,
+      "rewards/margins": 4.777169227600098,
+      "rewards/rejected": -4.502411842346191,
+      "step": 352,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.7828678356088433,
+      "grad_norm": 2.3354103565216064,
+      "learning_rate": 0.00014251142414460415,
+      "logits/chosen": -0.9495961666107178,
+      "logits/rejected": -1.0119011402130127,
+      "logps/chosen": -7.465116500854492,
+      "logps/rejected": -69.65184020996094,
+      "loss": 0.40892308950424194,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.31010234355926514,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29755479097366333,
+      "rewards/margins": 5.3398756980896,
+      "rewards/rejected": -5.042320728302002,
+      "step": 353,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.7850855915170837,
+      "grad_norm": 3.8496086597442627,
+      "learning_rate": 0.00014217855543455322,
+      "logits/chosen": -0.9601128101348877,
+      "logits/rejected": -0.9981716275215149,
+      "logps/chosen": -6.954006195068359,
+      "logps/rejected": -58.822914123535156,
+      "loss": 0.6071369051933289,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4900137186050415,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.032600052654743195,
+      "rewards/margins": 4.149467945098877,
+      "rewards/rejected": -4.116868019104004,
+      "step": 354,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.787303347425324,
+      "grad_norm": 1.835015058517456,
+      "learning_rate": 0.00014184511727039612,
+      "logits/chosen": -1.031874656677246,
+      "logits/rejected": -1.0789393186569214,
+      "logps/chosen": -4.562386989593506,
+      "logps/rejected": -66.75709533691406,
+      "loss": 0.35645246505737305,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2355533242225647,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36652734875679016,
+      "rewards/margins": 4.739514350891113,
+      "rewards/rejected": -4.372987747192383,
+      "step": 355,
+      "train_speed(iter/s)": 0.007991
+    },
+    {
+      "epoch": 0.7895211033335644,
+      "grad_norm": 9.796154022216797,
+      "learning_rate": 0.00014151111415389273,
+      "logits/chosen": -0.9901955127716064,
+      "logits/rejected": -1.027661919593811,
+      "logps/chosen": -5.177408218383789,
+      "logps/rejected": -83.82344055175781,
+      "loss": 0.6649367809295654,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5719815492630005,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32677945494651794,
+      "rewards/margins": 6.3124799728393555,
+      "rewards/rejected": -5.985701084136963,
+      "step": 356,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7917388592418046,
+      "grad_norm": 4.476024150848389,
+      "learning_rate": 0.0001411765505944305,
+      "logits/chosen": -0.9815511703491211,
+      "logits/rejected": -1.0286136865615845,
+      "logps/chosen": -9.594337463378906,
+      "logps/rejected": -65.8331069946289,
+      "loss": 0.6891093850135803,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4483054280281067,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20106908679008484,
+      "rewards/margins": 4.649129867553711,
+      "rewards/rejected": -4.448061466217041,
+      "step": 357,
+      "train_speed(iter/s)": 0.007991
+    },
+    {
+      "epoch": 0.793956615150045,
+      "grad_norm": 6.72329568862915,
+      "learning_rate": 0.00014084143110896336,
+      "logits/chosen": -0.9816399216651917,
+      "logits/rejected": -1.0880467891693115,
+      "logps/chosen": -10.076456069946289,
+      "logps/rejected": -96.55436706542969,
+      "loss": 0.5759286284446716,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.42951053380966187,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3208717703819275,
+      "rewards/margins": 6.648129463195801,
+      "rewards/rejected": -6.3272576332092285,
+      "step": 358,
+      "train_speed(iter/s)": 0.00799
+    },
+    {
+      "epoch": 0.7961743710582854,
+      "grad_norm": 1.4733343124389648,
+      "learning_rate": 0.00014050576022195084,
+      "logits/chosen": -0.9938861131668091,
+      "logits/rejected": -1.1024624109268188,
+      "logps/chosen": -5.6730475425720215,
+      "logps/rejected": -96.65937805175781,
+      "loss": 0.3570317029953003,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3157106339931488,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35878437757492065,
+      "rewards/margins": 7.850948333740234,
+      "rewards/rejected": -7.49216365814209,
+      "step": 359,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.7983921269665257,
+      "grad_norm": 6.765986919403076,
+      "learning_rate": 0.00014016954246529696,
+      "logits/chosen": -0.9971696138381958,
+      "logits/rejected": -1.03761887550354,
+      "logps/chosen": -7.510034561157227,
+      "logps/rejected": -73.65198516845703,
+      "loss": 0.46261534094810486,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3362881541252136,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.27089494466781616,
+      "rewards/margins": 5.386101245880127,
+      "rewards/rejected": -5.115205764770508,
+      "step": 360,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.8006098828747661,
+      "grad_norm": 7.017738342285156,
+      "learning_rate": 0.00013983278237828905,
+      "logits/chosen": -0.9837071299552917,
+      "logits/rejected": -1.0181175470352173,
+      "logps/chosen": -8.63487434387207,
+      "logps/rejected": -83.47882843017578,
+      "loss": 0.44132688641548157,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3716186285018921,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41110920906066895,
+      "rewards/margins": 6.7674946784973145,
+      "rewards/rejected": -6.356384754180908,
+      "step": 361,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.8028276387830064,
+      "grad_norm": 8.974201202392578,
+      "learning_rate": 0.00013949548450753645,
+      "logits/chosen": -0.9519540071487427,
+      "logits/rejected": -1.0129084587097168,
+      "logps/chosen": -5.2814507484436035,
+      "logps/rejected": -72.11774444580078,
+      "loss": 0.5307304859161377,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4160841703414917,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4031183123588562,
+      "rewards/margins": 5.740135669708252,
+      "rewards/rejected": -5.337017059326172,
+      "step": 362,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.8050453946912468,
+      "grad_norm": 1.2695608139038086,
+      "learning_rate": 0.00013915765340690917,
+      "logits/chosen": -1.0254669189453125,
+      "logits/rejected": -1.0584160089492798,
+      "logps/chosen": -4.022584915161133,
+      "logps/rejected": -76.5419692993164,
+      "loss": 0.25830894708633423,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.15799354016780853,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4456513822078705,
+      "rewards/margins": 6.39218807220459,
+      "rewards/rejected": -5.946537494659424,
+      "step": 363,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.8072631505994872,
+      "grad_norm": 3.4139556884765625,
+      "learning_rate": 0.00013881929363747627,
+      "logits/chosen": -0.9054716229438782,
+      "logits/rejected": -0.9829826951026917,
+      "logps/chosen": -6.469770908355713,
+      "logps/rejected": -81.60964965820312,
+      "loss": 0.3816515803337097,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2315388023853302,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23071371018886566,
+      "rewards/margins": 5.678869724273682,
+      "rewards/rejected": -5.448155403137207,
+      "step": 364,
+      "train_speed(iter/s)": 0.007992
+    },
+    {
+      "epoch": 0.8094809065077275,
+      "grad_norm": 2.8562862873077393,
+      "learning_rate": 0.00013848040976744457,
+      "logits/chosen": -0.9915538430213928,
+      "logits/rejected": -1.0249900817871094,
+      "logps/chosen": -11.345012664794922,
+      "logps/rejected": -60.174583435058594,
+      "loss": 0.6162142753601074,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5174874663352966,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4874635338783264,
+      "rewards/margins": 4.736081600189209,
+      "rewards/rejected": -4.248618125915527,
+      "step": 365,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.8116986624159679,
+      "grad_norm": 11.424062728881836,
+      "learning_rate": 0.0001381410063720966,
+      "logits/chosen": -0.885972261428833,
+      "logits/rejected": -0.9483691453933716,
+      "logps/chosen": -6.95948600769043,
+      "logps/rejected": -67.90422821044922,
+      "loss": 0.48116442561149597,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3793320059776306,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4664710462093353,
+      "rewards/margins": 5.4724249839782715,
+      "rewards/rejected": -5.005953788757324,
+      "step": 366,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.8139164183242082,
+      "grad_norm": 2.13146710395813,
+      "learning_rate": 0.00013780108803372916,
+      "logits/chosen": -0.9675875902175903,
+      "logits/rejected": -1.0174977779388428,
+      "logps/chosen": -2.7178149223327637,
+      "logps/rejected": -57.41184616088867,
+      "loss": 0.2717885971069336,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.17145709693431854,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41647231578826904,
+      "rewards/margins": 4.505980968475342,
+      "rewards/rejected": -4.089508533477783,
+      "step": 367,
+      "train_speed(iter/s)": 0.007993
+    },
+    {
+      "epoch": 0.8161341742324485,
+      "grad_norm": 2.014505386352539,
+      "learning_rate": 0.00013746065934159123,
+      "logits/chosen": -0.8749208450317383,
+      "logits/rejected": -0.9097785949707031,
+      "logps/chosen": -11.590734481811523,
+      "logps/rejected": -61.093849182128906,
+      "loss": 0.4811657667160034,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4215659201145172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5911797285079956,
+      "rewards/margins": 4.6354546546936035,
+      "rewards/rejected": -4.044274806976318,
+      "step": 368,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.8183519301406889,
+      "grad_norm": 6.679834365844727,
+      "learning_rate": 0.00013711972489182208,
+      "logits/chosen": -0.9587826728820801,
+      "logits/rejected": -0.9994091391563416,
+      "logps/chosen": -4.139106273651123,
+      "logps/rejected": -51.25725555419922,
+      "loss": 0.5976202487945557,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45815497636795044,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4207558035850525,
+      "rewards/margins": 4.1789679527282715,
+      "rewards/rejected": -3.7582123279571533,
+      "step": 369,
+      "train_speed(iter/s)": 0.007995
+    },
+    {
+      "epoch": 0.8205696860489292,
+      "grad_norm": 4.250732421875,
+      "learning_rate": 0.00013677828928738934,
+      "logits/chosen": -0.8436453938484192,
+      "logits/rejected": -0.9403105974197388,
+      "logps/chosen": -8.053564071655273,
+      "logps/rejected": -84.89209747314453,
+      "loss": 0.6519731283187866,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5838944911956787,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6696658134460449,
+      "rewards/margins": 7.059585094451904,
+      "rewards/rejected": -6.389919757843018,
+      "step": 370,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8227874419571696,
+      "grad_norm": 3.9205992221832275,
+      "learning_rate": 0.0001364363571380266,
+      "logits/chosen": -0.955507755279541,
+      "logits/rejected": -1.0599405765533447,
+      "logps/chosen": -7.400586128234863,
+      "logps/rejected": -74.92298889160156,
+      "loss": 0.5989251732826233,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5146618485450745,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4105219542980194,
+      "rewards/margins": 5.604883670806885,
+      "rewards/rejected": -5.194361686706543,
+      "step": 371,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8250051978654099,
+      "grad_norm": 4.548377990722656,
+      "learning_rate": 0.0001360939330601715,
+      "logits/chosen": -0.9952760934829712,
+      "logits/rejected": -1.0298787355422974,
+      "logps/chosen": -6.161623477935791,
+      "logps/rejected": -78.73953247070312,
+      "loss": 0.5819877982139587,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4920639097690582,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07609675824642181,
+      "rewards/margins": 5.800478935241699,
+      "rewards/rejected": -5.724382400512695,
+      "step": 372,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8272229537736503,
+      "grad_norm": 1.8827123641967773,
+      "learning_rate": 0.000135751021676903,
+      "logits/chosen": -0.8674598336219788,
+      "logits/rejected": -0.9275334477424622,
+      "logps/chosen": -5.713299751281738,
+      "logps/rejected": -87.57781219482422,
+      "loss": 0.379931777715683,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3211643397808075,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21036267280578613,
+      "rewards/margins": 6.820375442504883,
+      "rewards/rejected": -6.610012531280518,
+      "step": 373,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8294407096818907,
+      "grad_norm": 3.5420541763305664,
+      "learning_rate": 0.00013540762761787937,
+      "logits/chosen": -1.0363105535507202,
+      "logits/rejected": -1.0776422023773193,
+      "logps/chosen": -13.957275390625,
+      "logps/rejected": -76.94481658935547,
+      "loss": 0.5567349195480347,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.45086780190467834,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28840070962905884,
+      "rewards/margins": 5.7179765701293945,
+      "rewards/rejected": -5.4295759201049805,
+      "step": 374,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.831658465590131,
+      "grad_norm": 1.7102012634277344,
+      "learning_rate": 0.00013506375551927547,
+      "logits/chosen": -0.9630483388900757,
+      "logits/rejected": -1.0484064817428589,
+      "logps/chosen": -8.863858222961426,
+      "logps/rejected": -87.95539093017578,
+      "loss": 0.48169463872909546,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3565702438354492,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33859264850616455,
+      "rewards/margins": 6.4991984367370605,
+      "rewards/rejected": -6.160606384277344,
+      "step": 375,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8338762214983714,
+      "grad_norm": 5.707740306854248,
+      "learning_rate": 0.00013471941002372005,
+      "logits/chosen": -0.9741328954696655,
+      "logits/rejected": -1.0323518514633179,
+      "logps/chosen": -6.637020111083984,
+      "logps/rejected": -93.5245132446289,
+      "loss": 0.38472601771354675,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.310046911239624,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3390035629272461,
+      "rewards/margins": 7.584478855133057,
+      "rewards/rejected": -7.245475769042969,
+      "step": 376,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8360939774066117,
+      "grad_norm": 18.117982864379883,
+      "learning_rate": 0.00013437459578023343,
+      "logits/chosen": -0.9638034105300903,
+      "logits/rejected": -1.0537152290344238,
+      "logps/chosen": -9.010321617126465,
+      "logps/rejected": -106.76775360107422,
+      "loss": 0.5627694725990295,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5133727788925171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2734406590461731,
+      "rewards/margins": 8.917407989501953,
+      "rewards/rejected": -8.643967628479004,
+      "step": 377,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.838311733314852,
+      "grad_norm": 10.60085678100586,
+      "learning_rate": 0.00013402931744416433,
+      "logits/chosen": -0.9089164137840271,
+      "logits/rejected": -0.9688107967376709,
+      "logps/chosen": -10.777193069458008,
+      "logps/rejected": -84.58053588867188,
+      "loss": 0.6482276916503906,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5216525793075562,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.347297728061676,
+      "rewards/margins": 6.164276123046875,
+      "rewards/rejected": -5.816978454589844,
+      "step": 378,
+      "train_speed(iter/s)": 0.007998
+    },
+    {
+      "epoch": 0.8405294892230923,
+      "grad_norm": 6.294896602630615,
+      "learning_rate": 0.00013368357967712726,
+      "logits/chosen": -0.962710976600647,
+      "logits/rejected": -1.038513422012329,
+      "logps/chosen": -10.020147323608398,
+      "logps/rejected": -88.00511169433594,
+      "loss": 0.6880609393119812,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.5630819797515869,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3197646141052246,
+      "rewards/margins": 6.758570671081543,
+      "rewards/rejected": -6.438806533813477,
+      "step": 379,
+      "train_speed(iter/s)": 0.007998
+    },
+    {
+      "epoch": 0.8427472451313327,
+      "grad_norm": 4.594664096832275,
+      "learning_rate": 0.00013333738714693956,
+      "logits/chosen": -0.9401661157608032,
+      "logits/rejected": -0.9714579582214355,
+      "logps/chosen": -9.314223289489746,
+      "logps/rejected": -65.65319061279297,
+      "loss": 0.5762234330177307,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.49913567304611206,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4756416380405426,
+      "rewards/margins": 5.191895961761475,
+      "rewards/rejected": -4.716253757476807,
+      "step": 380,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8449650010395731,
+      "grad_norm": 9.826415061950684,
+      "learning_rate": 0.0001329907445275583,
+      "logits/chosen": -0.8897924423217773,
+      "logits/rejected": -0.9633165597915649,
+      "logps/chosen": -12.073375701904297,
+      "logps/rejected": -73.02918243408203,
+      "loss": 0.912665069103241,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.656974196434021,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3072448968887329,
+      "rewards/margins": 4.9169487953186035,
+      "rewards/rejected": -4.609704494476318,
+      "step": 381,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8471827569478134,
+      "grad_norm": 2.968611001968384,
+      "learning_rate": 0.00013264365649901723,
+      "logits/chosen": -0.9875434637069702,
+      "logits/rejected": -1.0535120964050293,
+      "logps/chosen": -6.4254536628723145,
+      "logps/rejected": -93.53910064697266,
+      "loss": 0.4106255769729614,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30968165397644043,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5462108850479126,
+      "rewards/margins": 7.555147647857666,
+      "rewards/rejected": -7.008937358856201,
+      "step": 382,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8494005128560538,
+      "grad_norm": 2.7569844722747803,
+      "learning_rate": 0.00013229612774736359,
+      "logits/chosen": -0.9040098786354065,
+      "logits/rejected": -0.911220908164978,
+      "logps/chosen": -11.235910415649414,
+      "logps/rejected": -46.86307144165039,
+      "loss": 0.6170834898948669,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40689852833747864,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.34967976808547974,
+      "rewards/margins": 3.505056858062744,
+      "rewards/rejected": -3.155377149581909,
+      "step": 383,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8516182687642941,
+      "grad_norm": 1.343916654586792,
+      "learning_rate": 0.0001319481629645948,
+      "logits/chosen": -0.8736822009086609,
+      "logits/rejected": -0.9404423832893372,
+      "logps/chosen": -1.9940882921218872,
+      "logps/rejected": -73.57098388671875,
+      "loss": 0.19009298086166382,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.15470954775810242,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34637439250946045,
+      "rewards/margins": 5.987215518951416,
+      "rewards/rejected": -5.640841007232666,
+      "step": 384,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8538360246725345,
+      "grad_norm": 3.7166006565093994,
+      "learning_rate": 0.00013159976684859527,
+      "logits/chosen": -0.9331986308097839,
+      "logits/rejected": -0.9687966108322144,
+      "logps/chosen": -4.263778209686279,
+      "logps/rejected": -58.46650314331055,
+      "loss": 0.6141018867492676,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3756692707538605,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18798117339611053,
+      "rewards/margins": 4.480571746826172,
+      "rewards/rejected": -4.292590141296387,
+      "step": 385,
+      "train_speed(iter/s)": 0.007996
+    },
+    {
+      "epoch": 0.8560537805807749,
+      "grad_norm": 2.7633893489837646,
+      "learning_rate": 0.00013125094410307265,
+      "logits/chosen": -0.8766888976097107,
+      "logits/rejected": -0.9078878164291382,
+      "logps/chosen": -11.883410453796387,
+      "logps/rejected": -59.319793701171875,
+      "loss": 0.6583555340766907,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4740571677684784,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31328243017196655,
+      "rewards/margins": 4.362224578857422,
+      "rewards/rejected": -4.0489420890808105,
+      "step": 386,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8582715364890152,
+      "grad_norm": 1.7285346984863281,
+      "learning_rate": 0.00013090169943749476,
+      "logits/chosen": -0.9330658912658691,
+      "logits/rejected": -0.9578065872192383,
+      "logps/chosen": -8.566822052001953,
+      "logps/rejected": -59.75345993041992,
+      "loss": 0.4198892116546631,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.2942117750644684,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5056283473968506,
+      "rewards/margins": 4.707999229431152,
+      "rewards/rejected": -4.202371120452881,
+      "step": 387,
+      "train_speed(iter/s)": 0.007997
+    },
+    {
+      "epoch": 0.8604892923972556,
+      "grad_norm": 3.06683349609375,
+      "learning_rate": 0.0001305520375670256,
+      "logits/chosen": -0.8998062014579773,
+      "logits/rejected": -0.9751926064491272,
+      "logps/chosen": -6.561751365661621,
+      "logps/rejected": -78.16478729248047,
+      "loss": 0.4542686939239502,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3592434525489807,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.416767954826355,
+      "rewards/margins": 6.343855381011963,
+      "rewards/rejected": -5.927087783813477,
+      "step": 388,
+      "train_speed(iter/s)": 0.007998
+    },
+    {
+      "epoch": 0.8627070483054958,
+      "grad_norm": 2.2208168506622314,
+      "learning_rate": 0.0001302019632124619,
+      "logits/chosen": -0.9485350847244263,
+      "logits/rejected": -1.0033212900161743,
+      "logps/chosen": -4.072909355163574,
+      "logps/rejected": -65.76300048828125,
+      "loss": 0.42102301120758057,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.26937127113342285,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5535694360733032,
+      "rewards/margins": 5.2655839920043945,
+      "rewards/rejected": -4.712014198303223,
+      "step": 389,
+      "train_speed(iter/s)": 0.007998
+    },
+    {
+      "epoch": 0.8649248042137362,
+      "grad_norm": 1.9687930345535278,
+      "learning_rate": 0.00012985148110016947,
+      "logits/chosen": -1.0221461057662964,
+      "logits/rejected": -1.0833958387374878,
+      "logps/chosen": -5.025339603424072,
+      "logps/rejected": -82.09105682373047,
+      "loss": 0.46604421734809875,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3294580578804016,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3730618953704834,
+      "rewards/margins": 6.059564590454102,
+      "rewards/rejected": -5.6865034103393555,
+      "step": 390,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8671425601219765,
+      "grad_norm": 3.648024559020996,
+      "learning_rate": 0.0001295005959620191,
+      "logits/chosen": -1.051031231880188,
+      "logits/rejected": -1.0930334329605103,
+      "logps/chosen": -8.805713653564453,
+      "logps/rejected": -72.9891357421875,
+      "loss": 0.5263576507568359,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.35741278529167175,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3289564549922943,
+      "rewards/margins": 5.768798351287842,
+      "rewards/rejected": -5.439842224121094,
+      "step": 391,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8693603160302169,
+      "grad_norm": 3.0183286666870117,
+      "learning_rate": 0.00012914931253532304,
+      "logits/chosen": -1.0728670358657837,
+      "logits/rejected": -1.1561932563781738,
+      "logps/chosen": -7.894364833831787,
+      "logps/rejected": -91.58610534667969,
+      "loss": 0.5266696214675903,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4139808118343353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32448941469192505,
+      "rewards/margins": 6.884645462036133,
+      "rewards/rejected": -6.560155868530273,
+      "step": 392,
+      "train_speed(iter/s)": 0.008
+    },
+    {
+      "epoch": 0.8715780719384573,
+      "grad_norm": 6.086115837097168,
+      "learning_rate": 0.00012879763556277062,
+      "logits/chosen": -1.1569675207138062,
+      "logits/rejected": -1.1982351541519165,
+      "logps/chosen": -8.720643997192383,
+      "logps/rejected": -99.62559509277344,
+      "loss": 0.45403456687927246,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.40008455514907837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6170443296432495,
+      "rewards/margins": 8.753840446472168,
+      "rewards/rejected": -8.136796951293945,
+      "step": 393,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8737958278466976,
+      "grad_norm": 1.8687723875045776,
+      "learning_rate": 0.0001284455697923646,
+      "logits/chosen": -1.1504981517791748,
+      "logits/rejected": -1.225466012954712,
+      "logps/chosen": -6.932313442230225,
+      "logps/rejected": -97.39447784423828,
+      "loss": 0.40548500418663025,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.30781060457229614,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5330286622047424,
+      "rewards/margins": 7.959634780883789,
+      "rewards/rejected": -7.426605701446533,
+      "step": 394,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.876013583754938,
+      "grad_norm": 1.7781997919082642,
+      "learning_rate": 0.00012809311997735696,
+      "logits/chosen": -1.1534217596054077,
+      "logits/rejected": -1.2358542680740356,
+      "logps/chosen": -8.768854141235352,
+      "logps/rejected": -91.94288635253906,
+      "loss": 0.4044226408004761,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3020440340042114,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46018874645233154,
+      "rewards/margins": 7.929361343383789,
+      "rewards/rejected": -7.469172477722168,
+      "step": 395,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8782313396631783,
+      "grad_norm": 12.74526309967041,
+      "learning_rate": 0.00012774029087618446,
+      "logits/chosen": -1.2151590585708618,
+      "logits/rejected": -1.251021385192871,
+      "logps/chosen": -10.305855751037598,
+      "logps/rejected": -91.78905487060547,
+      "loss": 0.6720870137214661,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.4713570773601532,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16427800059318542,
+      "rewards/margins": 7.4839887619018555,
+      "rewards/rejected": -7.319710731506348,
+      "step": 396,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8804490955714187,
+      "grad_norm": 2.1509110927581787,
+      "learning_rate": 0.00012738708725240484,
+      "logits/chosen": -1.2225698232650757,
+      "logits/rejected": -1.2627900838851929,
+      "logps/chosen": -11.55994987487793,
+      "logps/rejected": -98.7964096069336,
+      "loss": 0.5669252276420593,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3946492075920105,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3987804055213928,
+      "rewards/margins": 8.50069522857666,
+      "rewards/rejected": -8.10191535949707,
+      "step": 397,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8826668514796591,
+      "grad_norm": 2.0327556133270264,
+      "learning_rate": 0.0001270335138746322,
+      "logits/chosen": -1.1845993995666504,
+      "logits/rejected": -1.2103253602981567,
+      "logps/chosen": -6.5673980712890625,
+      "logps/rejected": -69.4446792602539,
+      "loss": 0.5495756268501282,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3730127215385437,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29667022824287415,
+      "rewards/margins": 5.815788745880127,
+      "rewards/rejected": -5.519117832183838,
+      "step": 398,
+      "train_speed(iter/s)": 0.007999
+    },
+    {
+      "epoch": 0.8848846073878994,
+      "grad_norm": 5.390183925628662,
+      "learning_rate": 0.00012667957551647262,
+      "logits/chosen": -1.2288250923156738,
+      "logits/rejected": -1.2931530475616455,
+      "logps/chosen": -5.809046268463135,
+      "logps/rejected": -112.88240051269531,
+      "loss": 0.3403761684894562,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.21371078491210938,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.43278729915618896,
+      "rewards/margins": 9.281527519226074,
+      "rewards/rejected": -8.848739624023438,
+      "step": 399,
+      "train_speed(iter/s)": 0.008
+    },
+    {
+      "epoch": 0.8871023632961397,
+      "grad_norm": 3.3119659423828125,
+      "learning_rate": 0.00012632527695645993,
+      "logits/chosen": -1.2474465370178223,
+      "logits/rejected": -1.3153529167175293,
+      "logps/chosen": -5.87565803527832,
+      "logps/rejected": -95.62116241455078,
+      "loss": 0.43962395191192627,
+      "memory(GiB)": 46.1,
+      "nll_loss": 0.3828383982181549,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4066924750804901,
+      "rewards/margins": 7.693698883056641,
+      "rewards/rejected": -7.287006378173828,
+      "step": 400,
+      "train_speed(iter/s)": 0.008
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 900,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1370776801606369e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}