diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5578800557880056,
+  "eval_steps": 300,
+  "global_step": 250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0022315202231520223,
+      "grad_norm": 11.248553276062012,
+      "learning_rate": 4.444444444444445e-06,
+      "logits/chosen": -0.48349103331565857,
+      "logits/rejected": -0.652603030204773,
+      "logps/chosen": -8.859519004821777,
+      "logps/rejected": -16.016435623168945,
+      "loss": 1.3468515872955322,
+      "memory(GiB)": 33.34,
+      "nll_loss": 0.6537042856216431,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.010678
+    },
+    {
+      "epoch": 0.004463040446304045,
+      "grad_norm": 8.97808837890625,
+      "learning_rate": 8.88888888888889e-06,
+      "logits/chosen": -0.5174899101257324,
+      "logits/rejected": -0.6262180805206299,
+      "logps/chosen": -9.287378311157227,
+      "logps/rejected": -13.981985092163086,
+      "loss": 1.290852665901184,
+      "memory(GiB)": 36.2,
+      "nll_loss": 0.5977055430412292,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.010974
+    },
+    {
+      "epoch": 0.0066945606694560665,
+      "grad_norm": 7.36937141418457,
+      "learning_rate": 1.3333333333333333e-05,
+      "logits/chosen": -0.42612141370773315,
+      "logits/rejected": -0.6118584871292114,
+      "logps/chosen": -6.56147575378418,
+      "logps/rejected": -16.162681579589844,
+      "loss": 1.1864153146743774,
+      "memory(GiB)": 36.2,
+      "nll_loss": 0.492550790309906,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0009138870518654585,
+      "rewards/margins": -0.0012306026183068752,
+      "rewards/rejected": 0.00031671510078012943,
+      "step": 3,
+      "train_speed(iter/s)": 0.011052
+    },
+    {
+      "epoch": 0.00892608089260809,
+      "grad_norm": 7.918132305145264,
+      "learning_rate": 1.777777777777778e-05,
+      "logits/chosen": -0.5244420170783997,
+      "logits/rejected": -0.6519989371299744,
+      "logps/chosen": -7.461406707763672,
+      "logps/rejected": -17.006786346435547,
+      "loss": 1.3234074115753174,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.6342031359672546,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0032776433508843184,
+      "rewards/margins": 0.008089437149465084,
+      "rewards/rejected": -0.004811794031411409,
+      "step": 4,
+      "train_speed(iter/s)": 0.011094
+    },
+    {
+      "epoch": 0.011157601115760111,
+      "grad_norm": 9.900442123413086,
+      "learning_rate": 2.2222222222222223e-05,
+      "logits/chosen": -0.5073186159133911,
+      "logits/rejected": -0.6049282550811768,
+      "logps/chosen": -7.057607650756836,
+      "logps/rejected": -15.16832447052002,
+      "loss": 1.071779489517212,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.40041351318359375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.022623565047979355,
+      "rewards/margins": 0.04467733949422836,
+      "rewards/rejected": -0.02205377072095871,
+      "step": 5,
+      "train_speed(iter/s)": 0.011121
+    },
+    {
+      "epoch": 0.013389121338912133,
+      "grad_norm": 5.12174129486084,
+      "learning_rate": 2.6666666666666667e-05,
+      "logits/chosen": -0.48164039850234985,
+      "logits/rejected": -0.5844419002532959,
+      "logps/chosen": -6.2985005378723145,
+      "logps/rejected": -16.83810043334961,
+      "loss": 1.2731884717941284,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.600845992565155,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.010373853147029877,
+      "rewards/margins": 0.046181634068489075,
+      "rewards/rejected": -0.0358077809214592,
+      "step": 6,
+      "train_speed(iter/s)": 0.011143
+    },
+    {
+      "epoch": 0.015620641562064157,
+      "grad_norm": 4.1249213218688965,
+      "learning_rate": 3.111111111111111e-05,
+      "logits/chosen": -0.5566272735595703,
+      "logits/rejected": -0.6051127910614014,
+      "logps/chosen": -5.424515724182129,
+      "logps/rejected": -11.893548011779785,
+      "loss": 1.1159045696258545,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.46413302421569824,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.009653191082179546,
+      "rewards/margins": 0.11444520205259323,
+      "rewards/rejected": -0.10479200631380081,
+      "step": 7,
+      "train_speed(iter/s)": 0.011163
+    },
+    {
+      "epoch": 0.01785216178521618,
+      "grad_norm": 3.8928415775299072,
+      "learning_rate": 3.555555555555556e-05,
+      "logits/chosen": -0.5597310662269592,
+      "logits/rejected": -0.6341894268989563,
+      "logps/chosen": -6.350505352020264,
+      "logps/rejected": -20.239299774169922,
+      "loss": 0.8606890439987183,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.3352409899234772,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.018700765445828438,
+      "rewards/margins": 0.419380784034729,
+      "rewards/rejected": -0.4380815923213959,
+      "step": 8,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.0200836820083682,
+      "grad_norm": 7.264344692230225,
+      "learning_rate": 4e-05,
+      "logits/chosen": -0.500182569026947,
+      "logits/rejected": -0.595187246799469,
+      "logps/chosen": -16.194358825683594,
+      "logps/rejected": -20.5479793548584,
+      "loss": 1.6191558837890625,
+      "memory(GiB)": 39.28,
+      "nll_loss": 1.0435733795166016,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.21446219086647034,
+      "rewards/margins": 0.40107572078704834,
+      "rewards/rejected": -0.6155378818511963,
+      "step": 9,
+      "train_speed(iter/s)": 0.011188
+    },
+    {
+      "epoch": 0.022315202231520222,
+      "grad_norm": 7.542423725128174,
+      "learning_rate": 4.4444444444444447e-05,
+      "logits/chosen": -0.41380318999290466,
+      "logits/rejected": -0.5418319702148438,
+      "logps/chosen": -10.30124568939209,
+      "logps/rejected": -26.6314697265625,
+      "loss": 1.2661710977554321,
+      "memory(GiB)": 39.28,
+      "nll_loss": 0.6692220568656921,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.29620301723480225,
+      "rewards/margins": 0.5043031573295593,
+      "rewards/rejected": -0.8005062341690063,
+      "step": 10,
+      "train_speed(iter/s)": 0.01119
+    },
+    {
+      "epoch": 0.024546722454672244,
+      "grad_norm": 12.055508613586426,
+      "learning_rate": 4.888888888888889e-05,
+      "logits/chosen": -0.45050907135009766,
+      "logits/rejected": -0.5843921899795532,
+      "logps/chosen": -15.530860900878906,
+      "logps/rejected": -27.80389976501465,
+      "loss": 1.6187087297439575,
+      "memory(GiB)": 39.28,
+      "nll_loss": 1.0532186031341553,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.47808465361595154,
+      "rewards/margins": 0.41364309191703796,
+      "rewards/rejected": -0.8917278051376343,
+      "step": 11,
+      "train_speed(iter/s)": 0.011193
+    },
+    {
+      "epoch": 0.026778242677824266,
+      "grad_norm": 6.556560516357422,
+      "learning_rate": 5.333333333333333e-05,
+      "logits/chosen": -0.4449861943721771,
+      "logits/rejected": -0.5602853894233704,
+      "logps/chosen": -10.715709686279297,
+      "logps/rejected": -27.810680389404297,
+      "loss": 1.2998032569885254,
+      "memory(GiB)": 42.68,
+      "nll_loss": 0.6746751070022583,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.33532828092575073,
+      "rewards/margins": 0.39097556471824646,
+      "rewards/rejected": -0.7263038754463196,
+      "step": 12,
+      "train_speed(iter/s)": 0.011193
+    },
+    {
+      "epoch": 0.02900976290097629,
+      "grad_norm": 6.815319061279297,
+      "learning_rate": 5.7777777777777776e-05,
+      "logits/chosen": -0.45461201667785645,
+      "logits/rejected": -0.5696772933006287,
+      "logps/chosen": -8.947586059570312,
+      "logps/rejected": -23.452707290649414,
+      "loss": 1.347914218902588,
+      "memory(GiB)": 42.68,
+      "nll_loss": 0.7918389439582825,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14807622134685516,
+      "rewards/margins": 0.4436675012111664,
+      "rewards/rejected": -0.5917437076568604,
+      "step": 13,
+      "train_speed(iter/s)": 0.011196
+    },
+    {
+      "epoch": 0.031241283124128313,
+      "grad_norm": 4.015015125274658,
+      "learning_rate": 6.222222222222222e-05,
+      "logits/chosen": -0.4277312457561493,
+      "logits/rejected": -0.5162039995193481,
+      "logps/chosen": -7.859683513641357,
+      "logps/rejected": -25.14496612548828,
+      "loss": 1.1709389686584473,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.5424311757087708,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09304673224687576,
+      "rewards/margins": 0.27104437351226807,
+      "rewards/rejected": -0.36409106850624084,
+      "step": 14,
+      "train_speed(iter/s)": 0.011191
+    },
+    {
+      "epoch": 0.03347280334728033,
+      "grad_norm": 2.5992424488067627,
+      "learning_rate": 6.666666666666667e-05,
+      "logits/chosen": -0.4692430794239044,
+      "logits/rejected": -0.5111399292945862,
+      "logps/chosen": -8.574230194091797,
+      "logps/rejected": -16.565174102783203,
+      "loss": 1.1323829889297485,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.5113993287086487,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08413823693990707,
+      "rewards/margins": 0.19407325983047485,
+      "rewards/rejected": -0.27821147441864014,
+      "step": 15,
+      "train_speed(iter/s)": 0.011195
+    },
+    {
+      "epoch": 0.03570432357043236,
+      "grad_norm": 2.479030132293701,
+      "learning_rate": 7.111111111111112e-05,
+      "logits/chosen": -0.38820019364356995,
+      "logits/rejected": -0.5241715908050537,
+      "logps/chosen": -4.694685935974121,
+      "logps/rejected": -19.62047576904297,
+      "loss": 1.052955985069275,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.47422000765800476,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06994247436523438,
+      "rewards/margins": 0.2729727327823639,
+      "rewards/rejected": -0.2030302733182907,
+      "step": 16,
+      "train_speed(iter/s)": 0.011196
+    },
+    {
+      "epoch": 0.03793584379358438,
+      "grad_norm": 2.1875240802764893,
+      "learning_rate": 7.555555555555556e-05,
+      "logits/chosen": -0.4483625292778015,
+      "logits/rejected": -0.5458305478096008,
+      "logps/chosen": -9.014262199401855,
+      "logps/rejected": -18.748149871826172,
+      "loss": 1.2149709463119507,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.6185500621795654,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1215842068195343,
+      "rewards/margins": 0.24843284487724304,
+      "rewards/rejected": -0.12684863805770874,
+      "step": 17,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.0401673640167364,
+      "grad_norm": 2.639186382293701,
+      "learning_rate": 8e-05,
+      "logits/chosen": -0.4394489824771881,
+      "logits/rejected": -0.48552829027175903,
+      "logps/chosen": -9.954276084899902,
+      "logps/rejected": -14.681390762329102,
+      "loss": 1.1648714542388916,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.46223676204681396,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.011617322452366352,
+      "rewards/margins": 0.034979358315467834,
+      "rewards/rejected": -0.02336202934384346,
+      "step": 18,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.042398884239888426,
+      "grad_norm": 2.351567029953003,
+      "learning_rate": 8.444444444444444e-05,
+      "logits/chosen": -0.4882590174674988,
+      "logits/rejected": -0.5300821661949158,
+      "logps/chosen": -7.595667362213135,
+      "logps/rejected": -17.29962921142578,
+      "loss": 1.1524200439453125,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.5075826644897461,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.06894361972808838,
+      "rewards/margins": 0.13017994165420532,
+      "rewards/rejected": -0.061236318200826645,
+      "step": 19,
+      "train_speed(iter/s)": 0.011201
+    },
+    {
+      "epoch": 0.044630404463040445,
+      "grad_norm": 2.8206121921539307,
+      "learning_rate": 8.888888888888889e-05,
+      "logits/chosen": -0.4782502055168152,
+      "logits/rejected": -0.5565172433853149,
+      "logps/chosen": -6.507530212402344,
+      "logps/rejected": -13.604747772216797,
+      "loss": 1.089160442352295,
+      "memory(GiB)": 46.13,
+      "nll_loss": 0.45874831080436707,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.06730557233095169,
+      "rewards/margins": 0.18410882353782654,
+      "rewards/rejected": -0.11680323630571365,
+      "step": 20,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.04686192468619247,
+      "grad_norm": 2.452335834503174,
+      "learning_rate": 9.333333333333334e-05,
+      "logits/chosen": -0.4106925129890442,
+      "logits/rejected": -0.48896345496177673,
+      "logps/chosen": -7.722554683685303,
+      "logps/rejected": -22.094852447509766,
+      "loss": 1.153997540473938,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5091268420219421,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.060057930648326874,
+      "rewards/margins": 0.16068722307682037,
+      "rewards/rejected": -0.1006293073296547,
+      "step": 21,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.04909344490934449,
+      "grad_norm": 2.3324975967407227,
+      "learning_rate": 9.777777777777778e-05,
+      "logits/chosen": -0.4366399943828583,
+      "logits/rejected": -0.4600733816623688,
+      "logps/chosen": -9.774506568908691,
+      "logps/rejected": -14.850456237792969,
+      "loss": 1.185971975326538,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5413804650306702,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08716820925474167,
+      "rewards/margins": 0.17242644727230072,
+      "rewards/rejected": -0.08525823801755905,
+      "step": 22,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.051324965132496514,
+      "grad_norm": 2.7291500568389893,
+      "learning_rate": 0.00010222222222222222,
+      "logits/chosen": -0.4168204665184021,
+      "logits/rejected": -0.5373830795288086,
+      "logps/chosen": -7.675040245056152,
+      "logps/rejected": -20.635356903076172,
+      "loss": 1.1511971950531006,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4784548282623291,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.05419651046395302,
+      "rewards/margins": 0.0959930419921875,
+      "rewards/rejected": -0.04179652780294418,
+      "step": 23,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.05355648535564853,
+      "grad_norm": 2.6357204914093018,
+      "learning_rate": 0.00010666666666666667,
+      "logits/chosen": -0.39612719416618347,
+      "logits/rejected": -0.4547193944454193,
+      "logps/chosen": -9.427398681640625,
+      "logps/rejected": -20.611848831176758,
+      "loss": 1.0459527969360352,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.42900872230529785,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0577767938375473,
+      "rewards/margins": 0.24017955362796783,
+      "rewards/rejected": -0.18240275979042053,
+      "step": 24,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.05578800557880056,
+      "grad_norm": 2.7458126544952393,
+      "learning_rate": 0.00011111111111111112,
+      "logits/chosen": -0.3978196978569031,
+      "logits/rejected": -0.4898090958595276,
+      "logps/chosen": -9.801487922668457,
+      "logps/rejected": -21.7921199798584,
+      "loss": 1.1577973365783691,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5382578372955322,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.03896612673997879,
+      "rewards/margins": 0.24017201364040375,
+      "rewards/rejected": -0.20120586454868317,
+      "step": 25,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.05801952580195258,
+      "grad_norm": 4.185514450073242,
+      "learning_rate": 0.00011555555555555555,
+      "logits/chosen": -0.43978849053382874,
+      "logits/rejected": -0.5358645915985107,
+      "logps/chosen": -4.664649963378906,
+      "logps/rejected": -18.568632125854492,
+      "loss": 0.9911558032035828,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.39514851570129395,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.01693350076675415,
+      "rewards/margins": 0.28997230529785156,
+      "rewards/rejected": -0.3069058060646057,
+      "step": 26,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.0602510460251046,
+      "grad_norm": 2.983151912689209,
+      "learning_rate": 0.00012,
+      "logits/chosen": -0.46957868337631226,
+      "logits/rejected": -0.5086190700531006,
+      "logps/chosen": -14.16710376739502,
+      "logps/rejected": -15.19424819946289,
+      "loss": 1.3119632005691528,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.732680082321167,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1205766424536705,
+      "rewards/margins": 0.2763415277004242,
+      "rewards/rejected": -0.1557648628950119,
+      "step": 27,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.06248256624825663,
+      "grad_norm": 3.7991929054260254,
+      "learning_rate": 0.00012444444444444444,
+      "logits/chosen": -0.3287929892539978,
+      "logits/rejected": -0.46299535036087036,
+      "logps/chosen": -6.527912616729736,
+      "logps/rejected": -23.983104705810547,
+      "loss": 1.0033739805221558,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.44460976123809814,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07841134071350098,
+      "rewards/margins": 0.3698306679725647,
+      "rewards/rejected": -0.29141926765441895,
+      "step": 28,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.06471408647140865,
+      "grad_norm": 3.0063183307647705,
+      "learning_rate": 0.00012888888888888892,
+      "logits/chosen": -0.3735123574733734,
+      "logits/rejected": -0.458845853805542,
+      "logps/chosen": -7.325934886932373,
+      "logps/rejected": -21.65030288696289,
+      "loss": 1.0098869800567627,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.477042555809021,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.025919266045093536,
+      "rewards/margins": 0.47939103841781616,
+      "rewards/rejected": -0.4534717798233032,
+      "step": 29,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.06694560669456066,
+      "grad_norm": 2.6957406997680664,
+      "learning_rate": 0.00013333333333333334,
+      "logits/chosen": -0.3790513873100281,
+      "logits/rejected": -0.4815320372581482,
+      "logps/chosen": -6.8280110359191895,
+      "logps/rejected": -22.987621307373047,
+      "loss": 1.0349267721176147,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5045362114906311,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09961891919374466,
+      "rewards/margins": 0.4102502763271332,
+      "rewards/rejected": -0.3106313645839691,
+      "step": 30,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.06917712691771269,
+      "grad_norm": 2.7142391204833984,
+      "learning_rate": 0.0001377777777777778,
+      "logits/chosen": -0.36308056116104126,
+      "logits/rejected": -0.44216084480285645,
+      "logps/chosen": -5.850837230682373,
+      "logps/rejected": -21.365463256835938,
+      "loss": 0.9850523471832275,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.43448853492736816,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.0431375615298748,
+      "rewards/margins": 0.3983423113822937,
+      "rewards/rejected": -0.3552047610282898,
+      "step": 31,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.07140864714086471,
+      "grad_norm": 3.897137403488159,
+      "learning_rate": 0.00014222222222222224,
+      "logits/chosen": -0.3979527950286865,
+      "logits/rejected": -0.43778184056282043,
+      "logps/chosen": -10.285924911499023,
+      "logps/rejected": -18.7073974609375,
+      "loss": 1.1233854293823242,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5153944492340088,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.026699109002947807,
+      "rewards/margins": 0.23945647478103638,
+      "rewards/rejected": -0.21275737881660461,
+      "step": 32,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.07364016736401674,
+      "grad_norm": 3.93380069732666,
+      "learning_rate": 0.00014666666666666666,
+      "logits/chosen": -0.2758394777774811,
+      "logits/rejected": -0.37545740604400635,
+      "logps/chosen": -12.159762382507324,
+      "logps/rejected": -24.926610946655273,
+      "loss": 1.2487003803253174,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6138952970504761,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09166956692934036,
+      "rewards/margins": 0.2808037996292114,
+      "rewards/rejected": -0.3724733591079712,
+      "step": 33,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.07587168758716877,
+      "grad_norm": 3.21878981590271,
+      "learning_rate": 0.0001511111111111111,
+      "logits/chosen": -0.24420826137065887,
+      "logits/rejected": -0.38765522837638855,
+      "logps/chosen": -6.041399955749512,
+      "logps/rejected": -22.503984451293945,
+      "loss": 0.9895752668380737,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.44009482860565186,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.010253368876874447,
+      "rewards/margins": 0.5208209753036499,
+      "rewards/rejected": -0.5105676054954529,
+      "step": 34,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.07810320781032078,
+      "grad_norm": 5.1504316329956055,
+      "learning_rate": 0.00015555555555555556,
+      "logits/chosen": -0.2664889693260193,
+      "logits/rejected": -0.40212178230285645,
+      "logps/chosen": -5.9132609367370605,
+      "logps/rejected": -24.527639389038086,
+      "loss": 0.8355879187583923,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4006969928741455,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.03980417177081108,
+      "rewards/margins": 0.8774286508560181,
+      "rewards/rejected": -0.9172328114509583,
+      "step": 35,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.0803347280334728,
+      "grad_norm": 3.388197660446167,
+      "learning_rate": 0.00016,
+      "logits/chosen": -0.25354304909706116,
+      "logits/rejected": -0.33330878615379333,
+      "logps/chosen": -9.807186126708984,
+      "logps/rejected": -22.963045120239258,
+      "loss": 1.1024540662765503,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6454060673713684,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.001722879707813263,
+      "rewards/margins": 0.8297844529151917,
+      "rewards/rejected": -0.8315073847770691,
+      "step": 36,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.08256624825662483,
+      "grad_norm": 2.596273899078369,
+      "learning_rate": 0.00016444444444444444,
+      "logits/chosen": -0.21093566715717316,
+      "logits/rejected": -0.3617200553417206,
+      "logps/chosen": -2.8380072116851807,
+      "logps/rejected": -30.42251968383789,
+      "loss": 0.6882003545761108,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.2877930700778961,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06296033412218094,
+      "rewards/margins": 1.0582321882247925,
+      "rewards/rejected": -0.9952719211578369,
+      "step": 37,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.08479776847977685,
+      "grad_norm": 2.358452796936035,
+      "learning_rate": 0.00016888888888888889,
+      "logits/chosen": -0.2989475429058075,
+      "logits/rejected": -0.37469911575317383,
+      "logps/chosen": -6.1970534324646,
+      "logps/rejected": -19.100358963012695,
+      "loss": 0.8981406092643738,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3934318423271179,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.002394435927271843,
+      "rewards/margins": 0.6755657196044922,
+      "rewards/rejected": -0.6731712818145752,
+      "step": 38,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.08702928870292886,
+      "grad_norm": 2.9299051761627197,
+      "learning_rate": 0.00017333333333333334,
+      "logits/chosen": -0.2796667516231537,
+      "logits/rejected": -0.34129300713539124,
+      "logps/chosen": -8.498173713684082,
+      "logps/rejected": -26.264965057373047,
+      "loss": 1.012364387512207,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.586083173751831,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07207094132900238,
+      "rewards/margins": 1.1341111660003662,
+      "rewards/rejected": -1.0620403289794922,
+      "step": 39,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.08926080892608089,
+      "grad_norm": 3.547189474105835,
+      "learning_rate": 0.00017777777777777779,
+      "logits/chosen": -0.28194305300712585,
+      "logits/rejected": -0.33060967922210693,
+      "logps/chosen": -7.982609748840332,
+      "logps/rejected": -20.187652587890625,
+      "loss": 0.9403265118598938,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.42783123254776,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.002833280712366104,
+      "rewards/margins": 0.7690448760986328,
+      "rewards/rejected": -0.7662116289138794,
+      "step": 40,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.09149232914923291,
+      "grad_norm": 3.4967706203460693,
+      "learning_rate": 0.00018222222222222224,
+      "logits/chosen": -0.21270643174648285,
+      "logits/rejected": -0.3004887104034424,
+      "logps/chosen": -5.226779937744141,
+      "logps/rejected": -22.208778381347656,
+      "loss": 0.8620257377624512,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3914262056350708,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.018529795110225677,
+      "rewards/margins": 0.9286664128303528,
+      "rewards/rejected": -0.9101365804672241,
+      "step": 41,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.09372384937238494,
+      "grad_norm": 6.100057601928711,
+      "learning_rate": 0.0001866666666666667,
+      "logits/chosen": -0.29974937438964844,
+      "logits/rejected": -0.36353808641433716,
+      "logps/chosen": -10.446137428283691,
+      "logps/rejected": -22.973119735717773,
+      "loss": 1.2477580308914185,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6882976293563843,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2526606023311615,
+      "rewards/margins": 0.5461662411689758,
+      "rewards/rejected": -0.7988268136978149,
+      "step": 42,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.09595536959553697,
+      "grad_norm": 4.625721454620361,
+      "learning_rate": 0.00019111111111111114,
+      "logits/chosen": -0.21875600516796112,
+      "logits/rejected": -0.3691503405570984,
+      "logps/chosen": -8.029109001159668,
+      "logps/rejected": -33.986671447753906,
+      "loss": 0.9927695989608765,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6190873384475708,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.10942590236663818,
+      "rewards/margins": 1.7278107404708862,
+      "rewards/rejected": -1.837236762046814,
+      "step": 43,
+      "train_speed(iter/s)": 0.011215
+    },
+    {
+      "epoch": 0.09818688981868898,
+      "grad_norm": 4.994227886199951,
+      "learning_rate": 0.00019555555555555556,
+      "logits/chosen": -0.25192195177078247,
+      "logits/rejected": -0.32522761821746826,
+      "logps/chosen": -9.805305480957031,
+      "logps/rejected": -34.161895751953125,
+      "loss": 0.9997942447662354,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6020480990409851,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.30676746368408203,
+      "rewards/margins": 1.7450387477874756,
+      "rewards/rejected": -2.0518062114715576,
+      "step": 44,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.100418410041841,
+      "grad_norm": 4.717008590698242,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.33617067337036133,
+      "logits/rejected": -0.35846105217933655,
+      "logps/chosen": -9.175792694091797,
+      "logps/rejected": -21.536231994628906,
+      "loss": 1.014683485031128,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5040097236633301,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.014129618182778358,
+      "rewards/margins": 0.7111825942993164,
+      "rewards/rejected": -0.6970530152320862,
+      "step": 45,
+      "train_speed(iter/s)": 0.011215
+    },
+    {
+      "epoch": 0.10264993026499303,
+      "grad_norm": 3.089895248413086,
+      "learning_rate": 0.0001999993185874369,
+      "logits/chosen": -0.3643554747104645,
+      "logits/rejected": -0.3828020393848419,
+      "logps/chosen": -11.385210990905762,
+      "logps/rejected": -27.30405044555664,
+      "loss": 0.9333115220069885,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5031883120536804,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08416246622800827,
+      "rewards/margins": 1.1319228410720825,
+      "rewards/rejected": -1.216085433959961,
+      "step": 46,
+      "train_speed(iter/s)": 0.011215
+    },
+    {
+      "epoch": 0.10488145048814505,
+      "grad_norm": 4.295483589172363,
+      "learning_rate": 0.00019999727435903407,
+      "logits/chosen": -0.3375833332538605,
+      "logits/rejected": -0.44685542583465576,
+      "logps/chosen": -4.36940336227417,
+      "logps/rejected": -21.149555206298828,
+      "loss": 0.959221601486206,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.42982059717178345,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15477100014686584,
+      "rewards/margins": 0.5941969752311707,
+      "rewards/rejected": -0.4394259452819824,
+      "step": 47,
+      "train_speed(iter/s)": 0.011216
+    },
+    {
+      "epoch": 0.10711297071129706,
+      "grad_norm": 2.4132261276245117,
+      "learning_rate": 0.0001999938673426507,
+      "logits/chosen": -0.3103625178337097,
+      "logits/rejected": -0.4320617616176605,
+      "logps/chosen": -5.777155876159668,
+      "logps/rejected": -20.897228240966797,
+      "loss": 0.9094122648239136,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.41829684376716614,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18217116594314575,
+      "rewards/margins": 0.714988112449646,
+      "rewards/rejected": -0.5328169465065002,
+      "step": 48,
+      "train_speed(iter/s)": 0.011216
+    },
+    {
+      "epoch": 0.10934449093444909,
+      "grad_norm": 2.716282367706299,
+      "learning_rate": 0.00019998909758471852,
+      "logits/chosen": -0.3802068829536438,
+      "logits/rejected": -0.4424472749233246,
+      "logps/chosen": -7.844504356384277,
+      "logps/rejected": -21.755281448364258,
+      "loss": 1.0127230882644653,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5032894611358643,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07639233767986298,
+      "rewards/margins": 0.79319167137146,
+      "rewards/rejected": -0.7167993187904358,
+      "step": 49,
+      "train_speed(iter/s)": 0.011217
+    },
+    {
+      "epoch": 0.11157601115760112,
+      "grad_norm": 3.5755205154418945,
+      "learning_rate": 0.000199982965150241,
+      "logits/chosen": -0.32785969972610474,
+      "logits/rejected": -0.384295791387558,
+      "logps/chosen": -12.152947425842285,
+      "logps/rejected": -21.354625701904297,
+      "loss": 1.1883673667907715,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6465156674385071,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11994989961385727,
+      "rewards/margins": 0.5093130469322205,
+      "rewards/rejected": -0.6292628645896912,
+      "step": 50,
+      "train_speed(iter/s)": 0.011217
+    },
+    {
+      "epoch": 0.11380753138075314,
+      "grad_norm": 2.3085532188415527,
+      "learning_rate": 0.00019997547012279244,
+      "logits/chosen": -0.3017635941505432,
+      "logits/rejected": -0.4112682342529297,
+      "logps/chosen": -6.112311363220215,
+      "logps/rejected": -27.533615112304688,
+      "loss": 0.8378341794013977,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4290671646595001,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1803205907344818,
+      "rewards/margins": 0.9729282855987549,
+      "rewards/rejected": -0.7926077246665955,
+      "step": 51,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.11603905160390517,
+      "grad_norm": 2.699073076248169,
+      "learning_rate": 0.00019996661260451704,
+      "logits/chosen": -0.30682653188705444,
+      "logits/rejected": -0.3987538814544678,
+      "logps/chosen": -9.152729034423828,
+      "logps/rejected": -34.165122985839844,
+      "loss": 0.8945070505142212,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5094670653343201,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.06719347834587097,
+      "rewards/margins": 1.5018631219863892,
+      "rewards/rejected": -1.569056510925293,
+      "step": 52,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.11827057182705718,
+      "grad_norm": 2.7043395042419434,
+      "learning_rate": 0.00019995639271612722,
+      "logits/chosen": -0.3258576989173889,
+      "logits/rejected": -0.43161430954933167,
+      "logps/chosen": -9.04993724822998,
+      "logps/rejected": -29.138545989990234,
+      "loss": 0.8318122029304504,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3884216248989105,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.015111252665519714,
+      "rewards/margins": 1.0627268552780151,
+      "rewards/rejected": -1.0778381824493408,
+      "step": 53,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.1205020920502092,
+      "grad_norm": 5.87897253036499,
+      "learning_rate": 0.00019994481059690223,
+      "logits/chosen": -0.36754727363586426,
+      "logits/rejected": -0.4476523995399475,
+      "logps/chosen": -10.40526008605957,
+      "logps/rejected": -32.984188079833984,
+      "loss": 1.1996067762374878,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7492297887802124,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.08549878001213074,
+      "rewards/margins": 1.944332242012024,
+      "rewards/rejected": -2.0298309326171875,
+      "step": 54,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.12273361227336123,
+      "grad_norm": 2.4889438152313232,
+      "learning_rate": 0.00019993186640468604,
+      "logits/chosen": -0.28098922967910767,
+      "logits/rejected": -0.3998360335826874,
+      "logps/chosen": -11.18938159942627,
+      "logps/rejected": -33.78412628173828,
+      "loss": 0.97004234790802,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.532280445098877,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.10216796398162842,
+      "rewards/margins": 1.2508039474487305,
+      "rewards/rejected": -1.3529717922210693,
+      "step": 55,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.12496513249651325,
+      "grad_norm": 3.1122195720672607,
+      "learning_rate": 0.0001999175603158854,
+      "logits/chosen": -0.25420328974723816,
+      "logits/rejected": -0.40874406695365906,
+      "logps/chosen": -9.440892219543457,
+      "logps/rejected": -36.343971252441406,
+      "loss": 0.8295825123786926,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4795616567134857,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.014746717177331448,
+      "rewards/margins": 1.4991542100906372,
+      "rewards/rejected": -1.4844075441360474,
+      "step": 56,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.12719665271966527,
+      "grad_norm": 3.5490951538085938,
+      "learning_rate": 0.0001999018925254673,
+      "logits/chosen": -0.3385506272315979,
+      "logits/rejected": -0.45762842893600464,
+      "logps/chosen": -9.239278793334961,
+      "logps/rejected": -32.54361343383789,
+      "loss": 0.9066818952560425,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5777646899223328,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.03782232105731964,
+      "rewards/margins": 1.4800399541854858,
+      "rewards/rejected": -1.5178624391555786,
+      "step": 57,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.1294281729428173,
+      "grad_norm": 3.0512146949768066,
+      "learning_rate": 0.00019988486324695628,
+      "logits/chosen": -0.3186691403388977,
+      "logits/rejected": -0.4108823835849762,
+      "logps/chosen": -13.795214653015137,
+      "logps/rejected": -34.02068328857422,
+      "loss": 0.8977184295654297,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.49673789739608765,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.021573714911937714,
+      "rewards/margins": 1.2825357913970947,
+      "rewards/rejected": -1.3041096925735474,
+      "step": 58,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.13165969316596932,
+      "grad_norm": 3.224839925765991,
+      "learning_rate": 0.00019986647271243163,
+      "logits/chosen": -0.379732221364975,
+      "logits/rejected": -0.4699477553367615,
+      "logps/chosen": -6.0904154777526855,
+      "logps/rejected": -22.338991165161133,
+      "loss": 0.921177327632904,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.41462287306785583,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.054278142750263214,
+      "rewards/margins": 0.6603462100028992,
+      "rewards/rejected": -0.7146244049072266,
+      "step": 59,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.13389121338912133,
+      "grad_norm": 3.088550567626953,
+      "learning_rate": 0.00019984672117252423,
+      "logits/chosen": -0.3191598057746887,
+      "logits/rejected": -0.4799092411994934,
+      "logps/chosen": -8.916747093200684,
+      "logps/rejected": -38.1026611328125,
+      "loss": 0.9097569584846497,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5499361753463745,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.09366529434919357,
+      "rewards/margins": 1.7305129766464233,
+      "rewards/rejected": -1.6368474960327148,
+      "step": 60,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.13612273361227337,
+      "grad_norm": 3.0570552349090576,
+      "learning_rate": 0.00019982560889641296,
+      "logits/chosen": -0.34574997425079346,
+      "logits/rejected": -0.5139629244804382,
+      "logps/chosen": -3.300583600997925,
+      "logps/rejected": -31.543636322021484,
+      "loss": 0.6459969878196716,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.29164713621139526,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10295584052801132,
+      "rewards/margins": 1.4123960733413696,
+      "rewards/rejected": -1.3094402551651,
+      "step": 61,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.13835425383542538,
+      "grad_norm": 4.404062747955322,
+      "learning_rate": 0.00019980313617182127,
+      "logits/chosen": -0.3277460038661957,
+      "logits/rejected": -0.5349987745285034,
+      "logps/chosen": -6.637196063995361,
+      "logps/rejected": -34.88399124145508,
+      "loss": 0.9588042497634888,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5974500179290771,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.10418260097503662,
+      "rewards/margins": 1.5509659051895142,
+      "rewards/rejected": -1.6551483869552612,
+      "step": 62,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.14058577405857742,
+      "grad_norm": 7.597557067871094,
+      "learning_rate": 0.00019977930330501308,
+      "logits/chosen": -0.40753576159477234,
+      "logits/rejected": -0.5000228881835938,
+      "logps/chosen": -9.73190975189209,
+      "logps/rejected": -31.304035186767578,
+      "loss": 1.1911669969558716,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6490563154220581,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.18217454850673676,
+      "rewards/margins": 1.1403659582138062,
+      "rewards/rejected": -1.322540521621704,
+      "step": 63,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.14281729428172943,
+      "grad_norm": 5.447345733642578,
+      "learning_rate": 0.0001997541106207887,
+      "logits/chosen": -0.4495885968208313,
+      "logits/rejected": -0.521405816078186,
+      "logps/chosen": -7.186385631561279,
+      "logps/rejected": -29.914262771606445,
+      "loss": 0.9763236045837402,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5955564379692078,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.02932646870613098,
+      "rewards/margins": 1.4675655364990234,
+      "rewards/rejected": -1.496891975402832,
+      "step": 64,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.14504881450488144,
+      "grad_norm": 4.937906742095947,
+      "learning_rate": 0.0001997275584624803,
+      "logits/chosen": -0.3159691393375397,
+      "logits/rejected": -0.48153895139694214,
+      "logps/chosen": -7.167924404144287,
+      "logps/rejected": -33.78512954711914,
+      "loss": 0.9605119228363037,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5666901469230652,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06481783092021942,
+      "rewards/margins": 1.3504029512405396,
+      "rewards/rejected": -1.2855849266052246,
+      "step": 65,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.14728033472803348,
+      "grad_norm": 4.266327857971191,
+      "learning_rate": 0.00019969964719194745,
+      "logits/chosen": -0.4321137070655823,
+      "logits/rejected": -0.5152468681335449,
+      "logps/chosen": -8.84105110168457,
+      "logps/rejected": -15.863529205322266,
+      "loss": 1.171226978302002,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5947294235229492,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.039367154240608215,
+      "rewards/margins": 0.543604850769043,
+      "rewards/rejected": -0.504237711429596,
+      "step": 66,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.1495118549511855,
+      "grad_norm": 3.343055009841919,
+      "learning_rate": 0.0001996703771895719,
+      "logits/chosen": -0.5153965353965759,
+      "logits/rejected": -0.5072118043899536,
+      "logps/chosen": -13.175134658813477,
+      "logps/rejected": -18.902141571044922,
+      "loss": 1.3753248453140259,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6952039003372192,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.00693463534116745,
+      "rewards/margins": 0.5273423194885254,
+      "rewards/rejected": -0.5204076766967773,
+      "step": 67,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.15174337517433753,
+      "grad_norm": 2.8605148792266846,
+      "learning_rate": 0.00019963974885425266,
+      "logits/chosen": -0.394383043050766,
+      "logits/rejected": -0.4663364887237549,
+      "logps/chosen": -7.570481300354004,
+      "logps/rejected": -18.867816925048828,
+      "loss": 1.048596739768982,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.447956383228302,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.06730639934539795,
+      "rewards/margins": 0.42720258235931396,
+      "rewards/rejected": -0.359896183013916,
+      "step": 68,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.15397489539748954,
+      "grad_norm": 2.4179482460021973,
+      "learning_rate": 0.0001996077626034003,
+      "logits/chosen": -0.43737077713012695,
+      "logits/rejected": -0.49085646867752075,
+      "logps/chosen": -11.430294036865234,
+      "logps/rejected": -16.858463287353516,
+      "loss": 1.1468638181686401,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5987557172775269,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.13983695209026337,
+      "rewards/margins": 0.43572792410850525,
+      "rewards/rejected": -0.2958909869194031,
+      "step": 69,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.15620641562064155,
+      "grad_norm": 2.619527578353882,
+      "learning_rate": 0.00019957441887293156,
+      "logits/chosen": -0.3810432255268097,
+      "logits/rejected": -0.4588402509689331,
+      "logps/chosen": -7.18558931350708,
+      "logps/rejected": -27.03573989868164,
+      "loss": 0.9460563659667969,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4206099212169647,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.027686119079589844,
+      "rewards/margins": 0.6544371843338013,
+      "rewards/rejected": -0.6267510652542114,
+      "step": 70,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.1584379358437936,
+      "grad_norm": 2.098484754562378,
+      "learning_rate": 0.0001995397181172631,
+      "logits/chosen": -0.38461002707481384,
+      "logits/rejected": -0.5058427453041077,
+      "logps/chosen": -4.015483856201172,
+      "logps/rejected": -28.747943878173828,
+      "loss": 0.784480094909668,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36208027601242065,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08316721022129059,
+      "rewards/margins": 1.0465381145477295,
+      "rewards/rejected": -0.9633709788322449,
+      "step": 71,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.1606694560669456,
+      "grad_norm": 2.3822543621063232,
+      "learning_rate": 0.0001995036608093056,
+      "logits/chosen": -0.4334784150123596,
+      "logits/rejected": -0.4664575457572937,
+      "logps/chosen": -8.978615760803223,
+      "logps/rejected": -20.211299896240234,
+      "loss": 0.8698700070381165,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3961753249168396,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.03191111981868744,
+      "rewards/margins": 0.8400025963783264,
+      "rewards/rejected": -0.8080915808677673,
+      "step": 72,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.16290097629009762,
+      "grad_norm": 2.3012869358062744,
+      "learning_rate": 0.00019946624744045704,
+      "logits/chosen": -0.3998824954032898,
+      "logits/rejected": -0.4553842544555664,
+      "logps/chosen": -11.310371398925781,
+      "logps/rejected": -30.51799964904785,
+      "loss": 0.9168710708618164,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40945351123809814,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09965535998344421,
+      "rewards/margins": 1.1768431663513184,
+      "rewards/rejected": -1.2764984369277954,
+      "step": 73,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.16513249651324965,
+      "grad_norm": 2.779480218887329,
+      "learning_rate": 0.00019942747852059632,
+      "logits/chosen": -0.3825194835662842,
+      "logits/rejected": -0.4605410397052765,
+      "logps/chosen": -9.17955493927002,
+      "logps/rejected": -35.66179656982422,
+      "loss": 0.8414367437362671,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.39000657200813293,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.10101831704378128,
+      "rewards/margins": 1.6313400268554688,
+      "rewards/rejected": -1.732358455657959,
+      "step": 74,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.16736401673640167,
+      "grad_norm": 2.800877571105957,
+      "learning_rate": 0.00019938735457807592,
+      "logits/chosen": -0.358651727437973,
+      "logits/rejected": -0.43520039319992065,
+      "logps/chosen": -10.250865936279297,
+      "logps/rejected": -31.788986206054688,
+      "loss": 0.8999710083007812,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4516286253929138,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.00027565285563468933,
+      "rewards/margins": 1.1217522621154785,
+      "rewards/rejected": -1.1220279932022095,
+      "step": 75,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.1695955369595537,
+      "grad_norm": 3.3984220027923584,
+      "learning_rate": 0.00019934587615971506,
+      "logits/chosen": -0.3334260582923889,
+      "logits/rejected": -0.4145124852657318,
+      "logps/chosen": -6.258616924285889,
+      "logps/rejected": -29.351835250854492,
+      "loss": 0.9294677376747131,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4573590159416199,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.005807792767882347,
+      "rewards/margins": 0.9444352388381958,
+      "rewards/rejected": -0.9386276006698608,
+      "step": 76,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.17182705718270572,
+      "grad_norm": 2.565323829650879,
+      "learning_rate": 0.00019930304383079204,
+      "logits/chosen": -0.3684268593788147,
+      "logits/rejected": -0.4622272551059723,
+      "logps/chosen": -8.288152694702148,
+      "logps/rejected": -27.87519645690918,
+      "loss": 0.9020035266876221,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4591652750968933,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.04557492583990097,
+      "rewards/margins": 1.3201135396957397,
+      "rewards/rejected": -1.2745387554168701,
+      "step": 77,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.17405857740585773,
+      "grad_norm": 3.3687551021575928,
+      "learning_rate": 0.00019925885817503663,
+      "logits/chosen": -0.3821089267730713,
+      "logits/rejected": -0.4754073917865753,
+      "logps/chosen": -5.0986528396606445,
+      "logps/rejected": -25.687240600585938,
+      "loss": 0.8081955909729004,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4151981770992279,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06252564489841461,
+      "rewards/margins": 1.102398157119751,
+      "rewards/rejected": -1.039872407913208,
+      "step": 78,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.17629009762900977,
+      "grad_norm": 3.3963940143585205,
+      "learning_rate": 0.00019921331979462197,
+      "logits/chosen": -0.2969811260700226,
+      "logits/rejected": -0.43268370628356934,
+      "logps/chosen": -5.748870372772217,
+      "logps/rejected": -27.06174659729004,
+      "loss": 0.8259724378585815,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.39037975668907166,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07963411509990692,
+      "rewards/margins": 0.9517691135406494,
+      "rewards/rejected": -0.8721350431442261,
+      "step": 79,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.17852161785216178,
+      "grad_norm": 4.5683393478393555,
+      "learning_rate": 0.00019916642931015662,
+      "logits/chosen": -0.38007351756095886,
+      "logits/rejected": -0.44829314947128296,
+      "logps/chosen": -6.661276817321777,
+      "logps/rejected": -23.286151885986328,
+      "loss": 1.0330675840377808,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5717910528182983,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.01540664117783308,
+      "rewards/margins": 0.9885216951370239,
+      "rewards/rejected": -0.9731149673461914,
+      "step": 80,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.18075313807531382,
+      "grad_norm": 2.7448818683624268,
+      "learning_rate": 0.00019911818736067586,
+      "logits/chosen": -0.362295538187027,
+      "logits/rejected": -0.4660547375679016,
+      "logps/chosen": -4.499999046325684,
+      "logps/rejected": -21.89557647705078,
+      "loss": 0.7531627416610718,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40978550910949707,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23986083269119263,
+      "rewards/margins": 1.2353352308273315,
+      "rewards/rejected": -0.9954744577407837,
+      "step": 81,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.18298465829846583,
+      "grad_norm": 2.189898729324341,
+      "learning_rate": 0.00019906859460363307,
+      "logits/chosen": -0.34852471947669983,
+      "logits/rejected": -0.4147579073905945,
+      "logps/chosen": -7.054965972900391,
+      "logps/rejected": -27.020904541015625,
+      "loss": 0.6932824850082397,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40176427364349365,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19743630290031433,
+      "rewards/margins": 1.7189998626708984,
+      "rewards/rejected": -1.5215637683868408,
+      "step": 82,
+      "train_speed(iter/s)": 0.011183
+    },
+    {
+      "epoch": 0.18521617852161784,
+      "grad_norm": 6.675446510314941,
+      "learning_rate": 0.00019901765171489086,
+      "logits/chosen": -0.2617488205432892,
+      "logits/rejected": -0.42673808336257935,
+      "logps/chosen": -10.860867500305176,
+      "logps/rejected": -47.94599533081055,
+      "loss": 1.1189333200454712,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7764029502868652,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33700618147850037,
+      "rewards/margins": 2.8077898025512695,
+      "rewards/rejected": -3.144796133041382,
+      "step": 83,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.18744769874476988,
+      "grad_norm": 3.3365304470062256,
+      "learning_rate": 0.0001989653593887117,
+      "logits/chosen": -0.33305802941322327,
+      "logits/rejected": -0.41463255882263184,
+      "logps/chosen": -6.076395034790039,
+      "logps/rejected": -32.26319885253906,
+      "loss": 0.7819843292236328,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4539300501346588,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10270927101373672,
+      "rewards/margins": 2.0022926330566406,
+      "rewards/rejected": -1.899583339691162,
+      "step": 84,
+      "train_speed(iter/s)": 0.011185
+    },
+    {
+      "epoch": 0.1896792189679219,
+      "grad_norm": 3.774372100830078,
+      "learning_rate": 0.00019891171833774854,
+      "logits/chosen": -0.2869430482387543,
+      "logits/rejected": -0.38458919525146484,
+      "logps/chosen": -9.53564453125,
+      "logps/rejected": -44.279476165771484,
+      "loss": 0.9625971913337708,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5546936392784119,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.1114846020936966,
+      "rewards/margins": 2.2966198921203613,
+      "rewards/rejected": -2.408104419708252,
+      "step": 85,
+      "train_speed(iter/s)": 0.011185
+    },
+    {
+      "epoch": 0.19191073919107393,
+      "grad_norm": 7.161458969116211,
+      "learning_rate": 0.00019885672929303508,
+      "logits/chosen": -0.2910900115966797,
+      "logits/rejected": -0.4121524691581726,
+      "logps/chosen": -10.86208724975586,
+      "logps/rejected": -43.04856491088867,
+      "loss": 0.8533170819282532,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.49533799290657043,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.0742935836315155,
+      "rewards/margins": 2.4323437213897705,
+      "rewards/rejected": -2.5066375732421875,
+      "step": 86,
+      "train_speed(iter/s)": 0.011185
+    },
+    {
+      "epoch": 0.19414225941422594,
+      "grad_norm": 3.097945213317871,
+      "learning_rate": 0.00019880039300397591,
+      "logits/chosen": -0.38776522874832153,
+      "logits/rejected": -0.4260600805282593,
+      "logps/chosen": -7.062775135040283,
+      "logps/rejected": -30.766754150390625,
+      "loss": 0.8838493227958679,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5182939767837524,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07534455507993698,
+      "rewards/margins": 1.623365044593811,
+      "rewards/rejected": -1.5480204820632935,
+      "step": 87,
+      "train_speed(iter/s)": 0.011186
+    },
+    {
+      "epoch": 0.19637377963737795,
+      "grad_norm": 2.158613443374634,
+      "learning_rate": 0.00019874271023833604,
+      "logits/chosen": -0.3456321060657501,
+      "logits/rejected": -0.43574386835098267,
+      "logps/chosen": -7.558653831481934,
+      "logps/rejected": -30.61342430114746,
+      "loss": 0.7866738438606262,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4206286072731018,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15493661165237427,
+      "rewards/margins": 1.5608376264572144,
+      "rewards/rejected": -1.4059010744094849,
+      "step": 88,
+      "train_speed(iter/s)": 0.011187
+    },
+    {
+      "epoch": 0.19860529986053,
+      "grad_norm": 3.826850414276123,
+      "learning_rate": 0.00019868368178223075,
+      "logits/chosen": -0.3609853982925415,
+      "logits/rejected": -0.4561440646648407,
+      "logps/chosen": -9.480233192443848,
+      "logps/rejected": -24.056900024414062,
+      "loss": 1.0715264081954956,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5571523904800415,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.019891131669282913,
+      "rewards/margins": 0.8333019018173218,
+      "rewards/rejected": -0.8531929850578308,
+      "step": 89,
+      "train_speed(iter/s)": 0.011188
+    },
+    {
+      "epoch": 0.200836820083682,
+      "grad_norm": 2.5776290893554688,
+      "learning_rate": 0.00019862330844011466,
+      "logits/chosen": -0.39898231625556946,
+      "logits/rejected": -0.5255773663520813,
+      "logps/chosen": -6.868825435638428,
+      "logps/rejected": -27.24391746520996,
+      "loss": 0.9221024513244629,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5272523760795593,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.16320021450519562,
+      "rewards/margins": 1.3196171522140503,
+      "rewards/rejected": -1.156416893005371,
+      "step": 90,
+      "train_speed(iter/s)": 0.011188
+    },
+    {
+      "epoch": 0.20306834030683404,
+      "grad_norm": 2.8630950450897217,
+      "learning_rate": 0.00019856159103477086,
+      "logits/chosen": -0.38147053122520447,
+      "logits/rejected": -0.49192503094673157,
+      "logps/chosen": -4.319600582122803,
+      "logps/rejected": -29.042064666748047,
+      "loss": 0.6743653416633606,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.2815118730068207,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06610685586929321,
+      "rewards/margins": 1.2929189205169678,
+      "rewards/rejected": -1.2268121242523193,
+      "step": 91,
+      "train_speed(iter/s)": 0.011189
+    },
+    {
+      "epoch": 0.20529986052998606,
+      "grad_norm": 2.3517754077911377,
+      "learning_rate": 0.00019849853040729962,
+      "logits/chosen": -0.3899337947368622,
+      "logits/rejected": -0.512879490852356,
+      "logps/chosen": -3.6735916137695312,
+      "logps/rejected": -38.82426452636719,
+      "loss": 0.575823962688446,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.27289441227912903,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.014910156838595867,
+      "rewards/margins": 2.080000638961792,
+      "rewards/rejected": -2.0650904178619385,
+      "step": 92,
+      "train_speed(iter/s)": 0.011189
+    },
+    {
+      "epoch": 0.20753138075313807,
+      "grad_norm": 7.1787543296813965,
+      "learning_rate": 0.00019843412741710705,
+      "logits/chosen": -0.4152039885520935,
+      "logits/rejected": -0.5459296107292175,
+      "logps/chosen": -9.160662651062012,
+      "logps/rejected": -33.41329574584961,
+      "loss": 0.9777874946594238,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5870996713638306,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.06245291978120804,
+      "rewards/margins": 1.6734349727630615,
+      "rewards/rejected": -1.7358877658843994,
+      "step": 93,
+      "train_speed(iter/s)": 0.01119
+    },
+    {
+      "epoch": 0.2097629009762901,
+      "grad_norm": 6.694753646850586,
+      "learning_rate": 0.00019836838294189327,
+      "logits/chosen": -0.2928682565689087,
+      "logits/rejected": -0.5464470386505127,
+      "logps/chosen": -4.742891788482666,
+      "logps/rejected": -48.63407516479492,
+      "loss": 0.6743770241737366,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4215732514858246,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.016849342733621597,
+      "rewards/margins": 2.5316309928894043,
+      "rewards/rejected": -2.548480272293091,
+      "step": 94,
+      "train_speed(iter/s)": 0.011189
+    },
+    {
+      "epoch": 0.21199442119944212,
+      "grad_norm": 3.5523533821105957,
+      "learning_rate": 0.00019830129787764048,
+      "logits/chosen": -0.43435996770858765,
+      "logits/rejected": -0.5884309411048889,
+      "logps/chosen": -7.528013229370117,
+      "logps/rejected": -40.399600982666016,
+      "loss": 0.655762255191803,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36153632402420044,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.16746671497821808,
+      "rewards/margins": 2.1469664573669434,
+      "rewards/rejected": -2.3144330978393555,
+      "step": 95,
+      "train_speed(iter/s)": 0.011189
+    },
+    {
+      "epoch": 0.21422594142259413,
+      "grad_norm": 3.652083396911621,
+      "learning_rate": 0.00019823287313860087,
+      "logits/chosen": -0.39443954825401306,
+      "logits/rejected": -0.5311736464500427,
+      "logps/chosen": -10.99143123626709,
+      "logps/rejected": -34.9860725402832,
+      "loss": 1.1298080682754517,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6828101873397827,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.050042107701301575,
+      "rewards/margins": 1.6755130290985107,
+      "rewards/rejected": -1.725555181503296,
+      "step": 96,
+      "train_speed(iter/s)": 0.011189
+    },
+    {
+      "epoch": 0.21645746164574617,
+      "grad_norm": 3.373784065246582,
+      "learning_rate": 0.00019816310965728388,
+      "logits/chosen": -0.4609702229499817,
+      "logits/rejected": -0.534326434135437,
+      "logps/chosen": -9.729392051696777,
+      "logps/rejected": -26.942209243774414,
+      "loss": 0.970382571220398,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4684171676635742,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.06363824754953384,
+      "rewards/margins": 1.244420051574707,
+      "rewards/rejected": -1.308058261871338,
+      "step": 97,
+      "train_speed(iter/s)": 0.01119
+    },
+    {
+      "epoch": 0.21868898186889818,
+      "grad_norm": 2.9718940258026123,
+      "learning_rate": 0.00019809200838444383,
+      "logits/chosen": -0.4923046827316284,
+      "logits/rejected": -0.5684210062026978,
+      "logps/chosen": -7.5212883949279785,
+      "logps/rejected": -26.84278678894043,
+      "loss": 0.7905946373939514,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4011859893798828,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1688103973865509,
+      "rewards/margins": 1.1646181344985962,
+      "rewards/rejected": -0.9958077669143677,
+      "step": 98,
+      "train_speed(iter/s)": 0.011191
+    },
+    {
+      "epoch": 0.22092050209205022,
+      "grad_norm": 4.091674327850342,
+      "learning_rate": 0.0001980195702890667,
+      "logits/chosen": -0.41893988847732544,
+      "logits/rejected": -0.5414128303527832,
+      "logps/chosen": -7.620563507080078,
+      "logps/rejected": -34.5891227722168,
+      "loss": 0.8988033533096313,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4130387604236603,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.0692744329571724,
+      "rewards/margins": 1.639555811882019,
+      "rewards/rejected": -1.7088301181793213,
+      "step": 99,
+      "train_speed(iter/s)": 0.011191
+    },
+    {
+      "epoch": 0.22315202231520223,
+      "grad_norm": 4.4936113357543945,
+      "learning_rate": 0.00019794579635835704,
+      "logits/chosen": -0.4513593912124634,
+      "logits/rejected": -0.5386379361152649,
+      "logps/chosen": -7.520744323730469,
+      "logps/rejected": -21.09442710876465,
+      "loss": 1.0648761987686157,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.551341712474823,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.06751161813735962,
+      "rewards/margins": 0.5798344016075134,
+      "rewards/rejected": -0.5123227834701538,
+      "step": 100,
+      "train_speed(iter/s)": 0.011192
+    },
+    {
+      "epoch": 0.22538354253835424,
+      "grad_norm": 3.2135934829711914,
+      "learning_rate": 0.00019787068759772458,
+      "logits/chosen": -0.30221307277679443,
+      "logits/rejected": -0.4815448820590973,
+      "logps/chosen": -5.885772705078125,
+      "logps/rejected": -42.119300842285156,
+      "loss": 0.7016221880912781,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3754323422908783,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.016318581998348236,
+      "rewards/margins": 1.9625664949417114,
+      "rewards/rejected": -1.9462475776672363,
+      "step": 101,
+      "train_speed(iter/s)": 0.011169
+    },
+    {
+      "epoch": 0.22761506276150628,
+      "grad_norm": 2.6394968032836914,
+      "learning_rate": 0.0001977942450307703,
+      "logits/chosen": -0.4444178342819214,
+      "logits/rejected": -0.550942599773407,
+      "logps/chosen": -2.515904188156128,
+      "logps/rejected": -24.57438087463379,
+      "loss": 0.5894904136657715,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.2374342381954193,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13210514187812805,
+      "rewards/margins": 1.279597520828247,
+      "rewards/rejected": -1.1474921703338623,
+      "step": 102,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.2298465829846583,
+      "grad_norm": 3.511199712753296,
+      "learning_rate": 0.00019771646969927276,
+      "logits/chosen": -0.37454891204833984,
+      "logits/rejected": -0.5178055763244629,
+      "logps/chosen": -11.968851089477539,
+      "logps/rejected": -43.35523986816406,
+      "loss": 0.9422297477722168,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.643075704574585,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.14428094029426575,
+      "rewards/margins": 2.009899377822876,
+      "rewards/rejected": -2.1541800498962402,
+      "step": 103,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.23207810320781033,
+      "grad_norm": 3.9855709075927734,
+      "learning_rate": 0.00019763736266317373,
+      "logits/chosen": -0.4065098166465759,
+      "logits/rejected": -0.4758867025375366,
+      "logps/chosen": -10.250575065612793,
+      "logps/rejected": -30.894758224487305,
+      "loss": 1.0436267852783203,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6723781228065491,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.054463766515254974,
+      "rewards/margins": 1.513323187828064,
+      "rewards/rejected": -1.4588593244552612,
+      "step": 104,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.23430962343096234,
+      "grad_norm": 2.9467523097991943,
+      "learning_rate": 0.00019755692500056377,
+      "logits/chosen": -0.3900934159755707,
+      "logits/rejected": -0.47925370931625366,
+      "logps/chosen": -7.484468460083008,
+      "logps/rejected": -43.62319564819336,
+      "loss": 0.7645498514175415,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4648410677909851,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.10762156546115875,
+      "rewards/margins": 2.406116247177124,
+      "rewards/rejected": -2.513737916946411,
+      "step": 105,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.23654114365411436,
+      "grad_norm": 4.7432637214660645,
+      "learning_rate": 0.0001974751578076675,
+      "logits/chosen": -0.4480246603488922,
+      "logits/rejected": -0.5329591631889343,
+      "logps/chosen": -8.911030769348145,
+      "logps/rejected": -36.37174987792969,
+      "loss": 1.0600045919418335,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6955873966217041,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.00720573216676712,
+      "rewards/margins": 2.03155517578125,
+      "rewards/rejected": -2.0243494510650635,
+      "step": 106,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.2387726638772664,
+      "grad_norm": 5.058660984039307,
+      "learning_rate": 0.0001973920621988288,
+      "logits/chosen": -0.3936890661716461,
+      "logits/rejected": -0.4737499952316284,
+      "logps/chosen": -9.114482879638672,
+      "logps/rejected": -38.9593505859375,
+      "loss": 0.9397415518760681,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.619577169418335,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1711510568857193,
+      "rewards/margins": 2.256584882736206,
+      "rewards/rejected": -2.4277358055114746,
+      "step": 107,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.2410041841004184,
+      "grad_norm": 5.576498985290527,
+      "learning_rate": 0.00019730763930649557,
+      "logits/chosen": -0.3989205062389374,
+      "logits/rejected": -0.5156173706054688,
+      "logps/chosen": -7.493094444274902,
+      "logps/rejected": -38.66194152832031,
+      "loss": 1.1445399522781372,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7500584125518799,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.078619584441185,
+      "rewards/margins": 2.2690536975860596,
+      "rewards/rejected": -2.347673177719116,
+      "step": 108,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.24323570432357045,
+      "grad_norm": 5.548501491546631,
+      "learning_rate": 0.0001972218902812041,
+      "logits/chosen": -0.4444655776023865,
+      "logits/rejected": -0.5106954574584961,
+      "logps/chosen": -10.32758903503418,
+      "logps/rejected": -29.883926391601562,
+      "loss": 1.1218242645263672,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7221024036407471,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.10507391393184662,
+      "rewards/margins": 1.7042087316513062,
+      "rewards/rejected": -1.8092825412750244,
+      "step": 109,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.24546722454672246,
+      "grad_norm": 4.309199333190918,
+      "learning_rate": 0.0001971348162915637,
+      "logits/chosen": -0.3248450756072998,
+      "logits/rejected": -0.4501338601112366,
+      "logps/chosen": -12.836463928222656,
+      "logps/rejected": -46.953514099121094,
+      "loss": 1.0559560060501099,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6316272020339966,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.07269488275051117,
+      "rewards/margins": 1.5467530488967896,
+      "rewards/rejected": -1.6194478273391724,
+      "step": 110,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.24769874476987447,
+      "grad_norm": 3.218020439147949,
+      "learning_rate": 0.0001970464185242406,
+      "logits/chosen": -0.39539581537246704,
+      "logits/rejected": -0.48628556728363037,
+      "logps/chosen": -8.726967811584473,
+      "logps/rejected": -29.990245819091797,
+      "loss": 0.9488601088523865,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6031092405319214,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.023079559206962585,
+      "rewards/margins": 1.3200984001159668,
+      "rewards/rejected": -1.3431780338287354,
+      "step": 111,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.2499302649930265,
+      "grad_norm": 2.6869122982025146,
+      "learning_rate": 0.00019695669818394176,
+      "logits/chosen": -0.41482657194137573,
+      "logits/rejected": -0.4805757999420166,
+      "logps/chosen": -6.431066036224365,
+      "logps/rejected": -25.98788833618164,
+      "loss": 0.8483121991157532,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.37977075576782227,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08659300208091736,
+      "rewards/margins": 0.9510751962661743,
+      "rewards/rejected": -0.8644822835922241,
+      "step": 112,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.25216178521617855,
+      "grad_norm": 2.551693916320801,
+      "learning_rate": 0.00019686565649339852,
+      "logits/chosen": -0.3926604092121124,
+      "logits/rejected": -0.4559895694255829,
+      "logps/chosen": -7.078885078430176,
+      "logps/rejected": -27.05070686340332,
+      "loss": 0.8254140019416809,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4410409927368164,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.20977970957756042,
+      "rewards/margins": 1.3954874277114868,
+      "rewards/rejected": -1.1857078075408936,
+      "step": 113,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.25439330543933053,
+      "grad_norm": 2.873304843902588,
+      "learning_rate": 0.0001967732946933499,
+      "logits/chosen": -0.40831753611564636,
+      "logits/rejected": -0.43265292048454285,
+      "logps/chosen": -9.373550415039062,
+      "logps/rejected": -18.696136474609375,
+      "loss": 1.1380702257156372,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5557818412780762,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.030941195785999298,
+      "rewards/margins": 0.6564021110534668,
+      "rewards/rejected": -0.6873433589935303,
+      "step": 114,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.25662482566248257,
+      "grad_norm": 2.8062174320220947,
+      "learning_rate": 0.00019667961404252573,
+      "logits/chosen": -0.39943259954452515,
+      "logits/rejected": -0.4698421061038971,
+      "logps/chosen": -6.604292869567871,
+      "logps/rejected": -27.13275909423828,
+      "loss": 0.904656708240509,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40879639983177185,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.06587034463882446,
+      "rewards/margins": 1.2184947729110718,
+      "rewards/rejected": -1.1526243686676025,
+      "step": 115,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.2588563458856346,
+      "grad_norm": 2.269819974899292,
+      "learning_rate": 0.00019658461581762948,
+      "logits/chosen": -0.37201863527297974,
+      "logits/rejected": -0.49889877438545227,
+      "logps/chosen": -5.184394836425781,
+      "logps/rejected": -33.83866500854492,
+      "loss": 0.6697641015052795,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3163735568523407,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12605488300323486,
+      "rewards/margins": 1.7490179538726807,
+      "rewards/rejected": -1.6229631900787354,
+      "step": 116,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.2610878661087866,
+      "grad_norm": 2.150320291519165,
+      "learning_rate": 0.0001964883013133208,
+      "logits/chosen": -0.3641276955604553,
+      "logits/rejected": -0.46924924850463867,
+      "logps/chosen": -6.894084930419922,
+      "logps/rejected": -30.72964096069336,
+      "loss": 0.7019263505935669,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3518219292163849,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11152325570583344,
+      "rewards/margins": 1.5766544342041016,
+      "rewards/rejected": -1.465131163597107,
+      "step": 117,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.26331938633193863,
+      "grad_norm": 1.940995693206787,
+      "learning_rate": 0.00019639067184219796,
+      "logits/chosen": -0.4233744740486145,
+      "logits/rejected": -0.49646085500717163,
+      "logps/chosen": -2.5980770587921143,
+      "logps/rejected": -22.459583282470703,
+      "loss": 0.5635585784912109,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.23038321733474731,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20914426445960999,
+      "rewards/margins": 1.3377940654754639,
+      "rewards/rejected": -1.1286499500274658,
+      "step": 118,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.26555090655509067,
+      "grad_norm": 4.13163423538208,
+      "learning_rate": 0.00019629172873477995,
+      "logits/chosen": -0.43275851011276245,
+      "logits/rejected": -0.5510309934616089,
+      "logps/chosen": -7.118757724761963,
+      "logps/rejected": -33.80887222290039,
+      "loss": 0.9356746077537537,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5518943071365356,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.06265924125909805,
+      "rewards/margins": 1.4980642795562744,
+      "rewards/rejected": -1.5607234239578247,
+      "step": 119,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.26778242677824265,
+      "grad_norm": 4.10536003112793,
+      "learning_rate": 0.00019619147333948823,
+      "logits/chosen": -0.3524538576602936,
+      "logits/rejected": -0.5940331220626831,
+      "logps/chosen": -3.5720577239990234,
+      "logps/rejected": -37.1466064453125,
+      "loss": 0.6390153169631958,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.33055761456489563,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.012503441423177719,
+      "rewards/margins": 1.8064663410186768,
+      "rewards/rejected": -1.818969964981079,
+      "step": 120,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.2700139470013947,
+      "grad_norm": 3.469205856323242,
+      "learning_rate": 0.00019608990702262862,
+      "logits/chosen": -0.43667861819267273,
+      "logits/rejected": -0.5170078873634338,
+      "logps/chosen": -10.792959213256836,
+      "logps/rejected": -33.0848503112793,
+      "loss": 0.915253758430481,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5493744015693665,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0627615749835968,
+      "rewards/margins": 1.9415191411972046,
+      "rewards/rejected": -2.0042805671691895,
+      "step": 121,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.27224546722454673,
+      "grad_norm": 3.509106159210205,
+      "learning_rate": 0.0001959870311683723,
+      "logits/chosen": -0.42921414971351624,
+      "logits/rejected": -0.5258873105049133,
+      "logps/chosen": -11.711372375488281,
+      "logps/rejected": -33.04314041137695,
+      "loss": 0.928779125213623,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5003984570503235,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.20121777057647705,
+      "rewards/margins": 1.2945905923843384,
+      "rewards/rejected": -1.4958083629608154,
+      "step": 122,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.2744769874476987,
+      "grad_norm": 3.868866443634033,
+      "learning_rate": 0.00019588284717873738,
+      "logits/chosen": -0.47573891282081604,
+      "logits/rejected": -0.5573767423629761,
+      "logps/chosen": -11.404234886169434,
+      "logps/rejected": -44.85041427612305,
+      "loss": 0.868504524230957,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5618444085121155,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06724707782268524,
+      "rewards/margins": 2.6722822189331055,
+      "rewards/rejected": -2.7395291328430176,
+      "step": 123,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.27670850767085076,
+      "grad_norm": 2.392876625061035,
+      "learning_rate": 0.00019577735647356928,
+      "logits/chosen": -0.4367309510707855,
+      "logits/rejected": -0.5319121479988098,
+      "logps/chosen": -6.822817802429199,
+      "logps/rejected": -31.786712646484375,
+      "loss": 0.7306314706802368,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40483999252319336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14910897612571716,
+      "rewards/margins": 1.7431702613830566,
+      "rewards/rejected": -1.594061255455017,
+      "step": 124,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.2789400278940028,
+      "grad_norm": 3.1055872440338135,
+      "learning_rate": 0.00019567056049052203,
+      "logits/chosen": -0.34188365936279297,
+      "logits/rejected": -0.5163190364837646,
+      "logps/chosen": -4.810876369476318,
+      "logps/rejected": -44.724796295166016,
+      "loss": 0.5886633396148682,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.33786720037460327,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.056971918791532516,
+      "rewards/margins": 2.44872784614563,
+      "rewards/rejected": -2.391756296157837,
+      "step": 125,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.28117154811715483,
+      "grad_norm": 5.503366947174072,
+      "learning_rate": 0.00019556246068503795,
+      "logits/chosen": -0.46696051955223083,
+      "logits/rejected": -0.5219815969467163,
+      "logps/chosen": -12.945513725280762,
+      "logps/rejected": -39.0778923034668,
+      "loss": 0.9357361197471619,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6219462752342224,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.06543703377246857,
+      "rewards/margins": 2.5177855491638184,
+      "rewards/rejected": -2.5832223892211914,
+      "step": 126,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.2834030683403068,
+      "grad_norm": 4.39600133895874,
+      "learning_rate": 0.00019545305853032848,
+      "logits/chosen": -0.40930306911468506,
+      "logits/rejected": -0.5165880918502808,
+      "logps/chosen": -8.137343406677246,
+      "logps/rejected": -36.05834197998047,
+      "loss": 0.9704120755195618,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6009897589683533,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06638351082801819,
+      "rewards/margins": 1.870111107826233,
+      "rewards/rejected": -1.9364944696426392,
+      "step": 127,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.28563458856345886,
+      "grad_norm": 2.2071218490600586,
+      "learning_rate": 0.0001953423555173536,
+      "logits/chosen": -0.3733062446117401,
+      "logits/rejected": -0.5163143277168274,
+      "logps/chosen": -5.431221008300781,
+      "logps/rejected": -50.760562896728516,
+      "loss": 0.5296469926834106,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.33633342385292053,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02796844020485878,
+      "rewards/margins": 2.690657377243042,
+      "rewards/rejected": -2.662688732147217,
+      "step": 128,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.2878661087866109,
+      "grad_norm": 3.5191235542297363,
+      "learning_rate": 0.0001952303531548018,
+      "logits/chosen": -0.36135807633399963,
+      "logits/rejected": -0.46837830543518066,
+      "logps/chosen": -5.7369561195373535,
+      "logps/rejected": -40.44853591918945,
+      "loss": 0.591960072517395,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.2807152271270752,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08742895722389221,
+      "rewards/margins": 2.5908262729644775,
+      "rewards/rejected": -2.503397226333618,
+      "step": 129,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.2900976290097629,
+      "grad_norm": 3.575482130050659,
+      "learning_rate": 0.00019511705296906945,
+      "logits/chosen": -0.3611457347869873,
+      "logits/rejected": -0.503516435623169,
+      "logps/chosen": -5.223241329193115,
+      "logps/rejected": -37.931541442871094,
+      "loss": 0.534879207611084,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.29983922839164734,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0478663444519043,
+      "rewards/margins": 2.279771327972412,
+      "rewards/rejected": -2.231905221939087,
+      "step": 130,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.2923291492329149,
+      "grad_norm": 3.9460039138793945,
+      "learning_rate": 0.00019500245650423987,
+      "logits/chosen": -0.3660318851470947,
+      "logits/rejected": -0.4796612560749054,
+      "logps/chosen": -11.665671348571777,
+      "logps/rejected": -49.753868103027344,
+      "loss": 0.9670472145080566,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7097752690315247,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.1927420198917389,
+      "rewards/margins": 3.0559210777282715,
+      "rewards/rejected": -3.2486634254455566,
+      "step": 131,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.29456066945606696,
+      "grad_norm": 6.069462776184082,
+      "learning_rate": 0.00019488656532206256,
+      "logits/chosen": -0.32610565423965454,
+      "logits/rejected": -0.4974173605442047,
+      "logps/chosen": -6.3772382736206055,
+      "logps/rejected": -48.19947814941406,
+      "loss": 0.575228214263916,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3116891086101532,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06776230037212372,
+      "rewards/margins": 2.7837750911712646,
+      "rewards/rejected": -2.8515372276306152,
+      "step": 132,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.29679218967921894,
+      "grad_norm": 8.746431350708008,
+      "learning_rate": 0.00019476938100193166,
+      "logits/chosen": -0.3666072189807892,
+      "logits/rejected": -0.4251011908054352,
+      "logps/chosen": -11.349700927734375,
+      "logps/rejected": -45.43285369873047,
+      "loss": 0.9895598292350769,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6815392971038818,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.039151158183813095,
+      "rewards/margins": 2.9056763648986816,
+      "rewards/rejected": -2.9448275566101074,
+      "step": 133,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.299023709902371,
+      "grad_norm": 2.3924574851989746,
+      "learning_rate": 0.00019465090514086448,
+      "logits/chosen": -0.3149105906486511,
+      "logits/rejected": -0.4992123246192932,
+      "logps/chosen": -6.0071845054626465,
+      "logps/rejected": -48.10839080810547,
+      "loss": 0.524428129196167,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.34304654598236084,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.037540286779403687,
+      "rewards/margins": 2.7853667736053467,
+      "rewards/rejected": -2.74782657623291,
+      "step": 134,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.301255230125523,
+      "grad_norm": 5.539307594299316,
+      "learning_rate": 0.00019453113935347983,
+      "logits/chosen": -0.31063827872276306,
+      "logits/rejected": -0.4415431618690491,
+      "logps/chosen": -9.355257987976074,
+      "logps/rejected": -32.0480842590332,
+      "loss": 0.9339735507965088,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5960800647735596,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02493138611316681,
+      "rewards/margins": 1.7281333208084106,
+      "rewards/rejected": -1.753064751625061,
+      "step": 135,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.30348675034867506,
+      "grad_norm": 2.773390531539917,
+      "learning_rate": 0.000194410085271976,
+      "logits/chosen": -0.32995009422302246,
+      "logits/rejected": -0.47322651743888855,
+      "logps/chosen": -7.31096076965332,
+      "logps/rejected": -39.725521087646484,
+      "loss": 0.7643193006515503,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4867798089981079,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06533154845237732,
+      "rewards/margins": 2.221080780029297,
+      "rewards/rejected": -2.1557490825653076,
+      "step": 136,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.30571827057182704,
+      "grad_norm": 3.071211099624634,
+      "learning_rate": 0.00019428774454610843,
+      "logits/chosen": -0.3505553901195526,
+      "logits/rejected": -0.45163339376449585,
+      "logps/chosen": -7.793257713317871,
+      "logps/rejected": -34.78621292114258,
+      "loss": 0.726679265499115,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4261872172355652,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.06730077415704727,
+      "rewards/margins": 2.1191506385803223,
+      "rewards/rejected": -2.0518500804901123,
+      "step": 137,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.3079497907949791,
+      "grad_norm": 2.8342206478118896,
+      "learning_rate": 0.00019416411884316722,
+      "logits/chosen": -0.23897941410541534,
+      "logits/rejected": -0.4108068645000458,
+      "logps/chosen": -5.271795272827148,
+      "logps/rejected": -33.89106750488281,
+      "loss": 0.6566328406333923,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3124980330467224,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06509758532047272,
+      "rewards/margins": 1.5957400798797607,
+      "rewards/rejected": -1.5306426286697388,
+      "step": 138,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.3101813110181311,
+      "grad_norm": 2.3552298545837402,
+      "learning_rate": 0.00019403920984795454,
+      "logits/chosen": -0.22743001580238342,
+      "logits/rejected": -0.3986920714378357,
+      "logps/chosen": -4.359319686889648,
+      "logps/rejected": -34.49266815185547,
+      "loss": 0.7009214162826538,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.37971875071525574,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1354263424873352,
+      "rewards/margins": 1.9385743141174316,
+      "rewards/rejected": -1.8031479120254517,
+      "step": 139,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.3124128312412831,
+      "grad_norm": 3.9349329471588135,
+      "learning_rate": 0.00019391301926276156,
+      "logits/chosen": -0.25204330682754517,
+      "logits/rejected": -0.40545159578323364,
+      "logps/chosen": -4.7128987312316895,
+      "logps/rejected": -39.89329147338867,
+      "loss": 0.7504162192344666,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.41977450251579285,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.04398459568619728,
+      "rewards/margins": 2.3563807010650635,
+      "rewards/rejected": -2.400365114212036,
+      "step": 140,
+      "train_speed(iter/s)": 0.011182
+    },
+    {
+      "epoch": 0.31464435146443515,
+      "grad_norm": 2.3725013732910156,
+      "learning_rate": 0.00019378554880734527,
+      "logits/chosen": -0.25585880875587463,
+      "logits/rejected": -0.40883561968803406,
+      "logps/chosen": -5.454152584075928,
+      "logps/rejected": -47.51259994506836,
+      "loss": 0.6222885251045227,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3938242197036743,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.09286467730998993,
+      "rewards/margins": 3.4271411895751953,
+      "rewards/rejected": -3.3342764377593994,
+      "step": 141,
+      "train_speed(iter/s)": 0.011183
+    },
+    {
+      "epoch": 0.3168758716875872,
+      "grad_norm": 3.706570863723755,
+      "learning_rate": 0.00019365680021890506,
+      "logits/chosen": -0.26097267866134644,
+      "logits/rejected": -0.3484109938144684,
+      "logps/chosen": -8.48320198059082,
+      "logps/rejected": -31.970016479492188,
+      "loss": 1.0012670755386353,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5183718204498291,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14288514852523804,
+      "rewards/margins": 1.521733045578003,
+      "rewards/rejected": -1.6646180152893066,
+      "step": 142,
+      "train_speed(iter/s)": 0.011183
+    },
+    {
+      "epoch": 0.31910739191073917,
+      "grad_norm": 4.2724609375,
+      "learning_rate": 0.0001935267752520591,
+      "logits/chosen": -0.294653058052063,
+      "logits/rejected": -0.40346813201904297,
+      "logps/chosen": -9.981515884399414,
+      "logps/rejected": -44.07274627685547,
+      "loss": 0.8689901828765869,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5607314705848694,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2522781491279602,
+      "rewards/margins": 2.5947649478912354,
+      "rewards/rejected": -2.847043037414551,
+      "step": 143,
+      "train_speed(iter/s)": 0.011183
+    },
+    {
+      "epoch": 0.3213389121338912,
+      "grad_norm": 6.414367198944092,
+      "learning_rate": 0.00019339547567882024,
+      "logits/chosen": -0.20774677395820618,
+      "logits/rejected": -0.3707664906978607,
+      "logps/chosen": -9.347281455993652,
+      "logps/rejected": -48.28213882446289,
+      "loss": 0.9361674189567566,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.667937159538269,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2894432842731476,
+      "rewards/margins": 3.465684175491333,
+      "rewards/rejected": -3.1762406826019287,
+      "step": 144,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.32357043235704325,
+      "grad_norm": 4.899329662322998,
+      "learning_rate": 0.0001932629032885721,
+      "logits/chosen": -0.26781731843948364,
+      "logits/rejected": -0.36117565631866455,
+      "logps/chosen": -12.092925071716309,
+      "logps/rejected": -45.867366790771484,
+      "loss": 0.9857743382453918,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6580345630645752,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3960377275943756,
+      "rewards/margins": 2.526942729949951,
+      "rewards/rejected": -2.9229800701141357,
+      "step": 145,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.32580195258019523,
+      "grad_norm": 2.351534843444824,
+      "learning_rate": 0.0001931290598880445,
+      "logits/chosen": -0.21401235461235046,
+      "logits/rejected": -0.3749268651008606,
+      "logps/chosen": -6.150831699371338,
+      "logps/rejected": -38.348854064941406,
+      "loss": 0.6613858342170715,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4117865562438965,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14331001043319702,
+      "rewards/margins": 2.4033961296081543,
+      "rewards/rejected": -2.2600862979888916,
+      "step": 146,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.32803347280334727,
+      "grad_norm": 3.732591152191162,
+      "learning_rate": 0.00019299394730128895,
+      "logits/chosen": -0.2112840712070465,
+      "logits/rejected": -0.29343438148498535,
+      "logps/chosen": -12.490756034851074,
+      "logps/rejected": -47.45105743408203,
+      "loss": 0.8992500901222229,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5636640787124634,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10838784277439117,
+      "rewards/margins": 2.9080140590667725,
+      "rewards/rejected": -3.016401767730713,
+      "step": 147,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.3302649930264993,
+      "grad_norm": 3.371856927871704,
+      "learning_rate": 0.00019285756736965367,
+      "logits/chosen": -0.2626963257789612,
+      "logits/rejected": -0.3471822142601013,
+      "logps/chosen": -7.939330101013184,
+      "logps/rejected": -30.135757446289062,
+      "loss": 0.9493183493614197,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5397716760635376,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.016782023012638092,
+      "rewards/margins": 1.6466865539550781,
+      "rewards/rejected": -1.6299041509628296,
+      "step": 148,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.33249651324965135,
+      "grad_norm": 2.4161276817321777,
+      "learning_rate": 0.00019271992195175875,
+      "logits/chosen": -0.3086288869380951,
+      "logits/rejected": -0.41329652070999146,
+      "logps/chosen": -8.446435928344727,
+      "logps/rejected": -36.33450698852539,
+      "loss": 0.9108085632324219,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5454064607620239,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.148118793964386,
+      "rewards/margins": 2.209937810897827,
+      "rewards/rejected": -2.061818838119507,
+      "step": 149,
+      "train_speed(iter/s)": 0.011185
+    },
+    {
+      "epoch": 0.33472803347280333,
+      "grad_norm": 3.4835684299468994,
+      "learning_rate": 0.00019258101292347042,
+      "logits/chosen": -0.22059586644172668,
+      "logits/rejected": -0.3747985363006592,
+      "logps/chosen": -8.812771797180176,
+      "logps/rejected": -44.0800895690918,
+      "loss": 0.7637215852737427,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5271515250205994,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.041512295603752136,
+      "rewards/margins": 2.4112486839294434,
+      "rewards/rejected": -2.369736433029175,
+      "step": 150,
+      "train_speed(iter/s)": 0.011185
+    },
+    {
+      "epoch": 0.33695955369595537,
+      "grad_norm": 2.394838333129883,
+      "learning_rate": 0.00019244084217787588,
+      "logits/chosen": -0.24826237559318542,
+      "logits/rejected": -0.3791409730911255,
+      "logps/chosen": -8.393878936767578,
+      "logps/rejected": -45.125953674316406,
+      "loss": 0.7294331192970276,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4361075460910797,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18614013493061066,
+      "rewards/margins": 2.6040918827056885,
+      "rewards/rejected": -2.417951822280884,
+      "step": 151,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.3391910739191074,
+      "grad_norm": 2.946138620376587,
+      "learning_rate": 0.00019229941162525726,
+      "logits/chosen": -0.2787875831127167,
+      "logits/rejected": -0.3120588958263397,
+      "logps/chosen": -9.058300018310547,
+      "logps/rejected": -30.770740509033203,
+      "loss": 0.9096699357032776,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.46970880031585693,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.08315091580152512,
+      "rewards/margins": 1.5154523849487305,
+      "rewards/rejected": -1.5986034870147705,
+      "step": 152,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.3414225941422594,
+      "grad_norm": 2.37446928024292,
+      "learning_rate": 0.00019215672319306565,
+      "logits/chosen": -0.29989758133888245,
+      "logits/rejected": -0.372408926486969,
+      "logps/chosen": -6.495873928070068,
+      "logps/rejected": -32.42622375488281,
+      "loss": 0.7089872360229492,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3819705545902252,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0852634608745575,
+      "rewards/margins": 1.8665060997009277,
+      "rewards/rejected": -1.7812427282333374,
+      "step": 153,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.34365411436541143,
+      "grad_norm": 1.6320723295211792,
+      "learning_rate": 0.00019201277882589487,
+      "logits/chosen": -0.2685769498348236,
+      "logits/rejected": -0.4338504374027252,
+      "logps/chosen": -4.207334995269775,
+      "logps/rejected": -54.69655990600586,
+      "loss": 0.4186314344406128,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.23857831954956055,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.104047991335392,
+      "rewards/margins": 3.6357991695404053,
+      "rewards/rejected": -3.5317511558532715,
+      "step": 154,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.3458856345885635,
+      "grad_norm": 2.5706071853637695,
+      "learning_rate": 0.00019186758048545498,
+      "logits/chosen": -0.31499671936035156,
+      "logits/rejected": -0.4452478885650635,
+      "logps/chosen": -8.811934471130371,
+      "logps/rejected": -41.911869049072266,
+      "loss": 0.8359880447387695,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5718236565589905,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.00384414941072464,
+      "rewards/margins": 2.2188119888305664,
+      "rewards/rejected": -2.214968204498291,
+      "step": 155,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.34811715481171546,
+      "grad_norm": 3.741011619567871,
+      "learning_rate": 0.00019172113015054532,
+      "logits/chosen": -0.42364075779914856,
+      "logits/rejected": -0.48594316840171814,
+      "logps/chosen": -9.2898588180542,
+      "logps/rejected": -39.12040710449219,
+      "loss": 0.7001189589500427,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.43081939220428467,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08180733025074005,
+      "rewards/margins": 2.411517381668091,
+      "rewards/rejected": -2.329710006713867,
+      "step": 156,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.3503486750348675,
+      "grad_norm": 2.2846014499664307,
+      "learning_rate": 0.00019157342981702792,
+      "logits/chosen": -0.34239232540130615,
+      "logits/rejected": -0.5085087418556213,
+      "logps/chosen": -3.7471375465393066,
+      "logps/rejected": -39.35340118408203,
+      "loss": 0.537229597568512,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.32565146684646606,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18335413932800293,
+      "rewards/margins": 2.308189868927002,
+      "rewards/rejected": -2.124835968017578,
+      "step": 157,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.35258019525801954,
+      "grad_norm": 2.8782355785369873,
+      "learning_rate": 0.0001914244814978001,
+      "logits/chosen": -0.36600109934806824,
+      "logits/rejected": -0.4807679057121277,
+      "logps/chosen": -12.43294906616211,
+      "logps/rejected": -36.17433547973633,
+      "loss": 0.7901278734207153,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5272985100746155,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03357819840312004,
+      "rewards/margins": 2.113076686859131,
+      "rewards/rejected": -2.079498291015625,
+      "step": 158,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.3548117154811716,
+      "grad_norm": 4.173655033111572,
+      "learning_rate": 0.00019127428722276685,
+      "logits/chosen": -0.29084256291389465,
+      "logits/rejected": -0.49927467107772827,
+      "logps/chosen": -8.242868423461914,
+      "logps/rejected": -54.8849983215332,
+      "loss": 0.7512383460998535,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5454739928245544,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.08500596880912781,
+      "rewards/margins": 3.6066102981567383,
+      "rewards/rejected": -3.521604299545288,
+      "step": 159,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.35704323570432356,
+      "grad_norm": 2.355539083480835,
+      "learning_rate": 0.0001911228490388136,
+      "logits/chosen": -0.3650870621204376,
+      "logits/rejected": -0.5237394571304321,
+      "logps/chosen": -10.184951782226562,
+      "logps/rejected": -48.1030387878418,
+      "loss": 0.7325472831726074,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5253479480743408,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15602552890777588,
+      "rewards/margins": 3.2045021057128906,
+      "rewards/rejected": -3.048476457595825,
+      "step": 160,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.3592747559274756,
+      "grad_norm": 4.184498310089111,
+      "learning_rate": 0.00019097016900977788,
+      "logits/chosen": -0.41671380400657654,
+      "logits/rejected": -0.5081248879432678,
+      "logps/chosen": -10.973024368286133,
+      "logps/rejected": -47.46335983276367,
+      "loss": 0.5701618194580078,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3158893883228302,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09667229652404785,
+      "rewards/margins": 3.3210151195526123,
+      "rewards/rejected": -3.2243423461914062,
+      "step": 161,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.36150627615062764,
+      "grad_norm": 4.150604724884033,
+      "learning_rate": 0.00019081624921642157,
+      "logits/chosen": -0.36136606335639954,
+      "logits/rejected": -0.5436823964118958,
+      "logps/chosen": -7.6842756271362305,
+      "logps/rejected": -45.09693908691406,
+      "loss": 0.7053879499435425,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4340396523475647,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07871361076831818,
+      "rewards/margins": 2.7361950874328613,
+      "rewards/rejected": -2.6574814319610596,
+      "step": 162,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3637377963737796,
+      "grad_norm": 3.704453706741333,
+      "learning_rate": 0.00019066109175640224,
+      "logits/chosen": -0.40510815382003784,
+      "logits/rejected": -0.4312695860862732,
+      "logps/chosen": -12.371721267700195,
+      "logps/rejected": -50.284141540527344,
+      "loss": 0.7488542795181274,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4703201949596405,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1792062222957611,
+      "rewards/margins": 3.446262836456299,
+      "rewards/rejected": -3.267056941986084,
+      "step": 163,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.36596931659693166,
+      "grad_norm": 2.7603952884674072,
+      "learning_rate": 0.00019050469874424476,
+      "logits/chosen": -0.32453101873397827,
+      "logits/rejected": -0.4822920560836792,
+      "logps/chosen": -8.248761177062988,
+      "logps/rejected": -45.44584655761719,
+      "loss": 0.8071178197860718,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4910924434661865,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.050887446850538254,
+      "rewards/margins": 3.054812431335449,
+      "rewards/rejected": -3.00392484664917,
+      "step": 164,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3682008368200837,
+      "grad_norm": 5.73638916015625,
+      "learning_rate": 0.0001903470723113124,
+      "logits/chosen": -0.37211766839027405,
+      "logits/rejected": -0.36960119009017944,
+      "logps/chosen": -13.764412879943848,
+      "logps/rejected": -28.488418579101562,
+      "loss": 1.0636341571807861,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5649504661560059,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17467395961284637,
+      "rewards/margins": 1.285156488418579,
+      "rewards/rejected": -1.459830403327942,
+      "step": 165,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3704323570432357,
+      "grad_norm": 3.937652587890625,
+      "learning_rate": 0.00019018821460577777,
+      "logits/chosen": -0.2960231602191925,
+      "logits/rejected": -0.4136447012424469,
+      "logps/chosen": -12.963543891906738,
+      "logps/rejected": -43.19397735595703,
+      "loss": 0.8952102065086365,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5876767039299011,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3215300142765045,
+      "rewards/margins": 2.2675259113311768,
+      "rewards/rejected": -2.5890560150146484,
+      "step": 166,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3726638772663877,
+      "grad_norm": 2.5968947410583496,
+      "learning_rate": 0.00019002812779259364,
+      "logits/chosen": -0.31316545605659485,
+      "logits/rejected": -0.510556161403656,
+      "logps/chosen": -6.747772216796875,
+      "logps/rejected": -37.75751876831055,
+      "loss": 0.7592855095863342,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.47158926725387573,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.17958350479602814,
+      "rewards/margins": 2.317811965942383,
+      "rewards/rejected": -2.138228416442871,
+      "step": 167,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.37489539748953976,
+      "grad_norm": 9.810136795043945,
+      "learning_rate": 0.00018986681405346322,
+      "logits/chosen": -0.28982365131378174,
+      "logits/rejected": -0.48794350028038025,
+      "logps/chosen": -9.523417472839355,
+      "logps/rejected": -53.576263427734375,
+      "loss": 0.8609976172447205,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.55437833070755,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.018897701054811478,
+      "rewards/margins": 3.141695499420166,
+      "rewards/rejected": -3.160593032836914,
+      "step": 168,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.37712691771269174,
+      "grad_norm": 3.2620904445648193,
+      "learning_rate": 0.00018970427558681082,
+      "logits/chosen": -0.2701128125190735,
+      "logits/rejected": -0.5124972462654114,
+      "logps/chosen": -7.385637283325195,
+      "logps/rejected": -66.33370971679688,
+      "loss": 0.595870852470398,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4602699279785156,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2253667265176773,
+      "rewards/margins": 4.247678756713867,
+      "rewards/rejected": -4.022312164306641,
+      "step": 169,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3793584379358438,
+      "grad_norm": 2.7175841331481934,
+      "learning_rate": 0.0001895405146077514,
+      "logits/chosen": -0.3605695366859436,
+      "logits/rejected": -0.521330714225769,
+      "logps/chosen": -5.586010456085205,
+      "logps/rejected": -52.13179397583008,
+      "loss": 0.652187705039978,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.32753562927246094,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04342030733823776,
+      "rewards/margins": 3.0307912826538086,
+      "rewards/rejected": -3.074211597442627,
+      "step": 170,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.3815899581589958,
+      "grad_norm": 3.162261486053467,
+      "learning_rate": 0.00018937553334806077,
+      "logits/chosen": -0.29991668462753296,
+      "logits/rejected": -0.4432262182235718,
+      "logps/chosen": -9.342361450195312,
+      "logps/rejected": -48.89967346191406,
+      "loss": 0.7870311737060547,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.543519139289856,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10350081324577332,
+      "rewards/margins": 3.252532720565796,
+      "rewards/rejected": -3.149031639099121,
+      "step": 171,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.38382147838214786,
+      "grad_norm": 11.82007122039795,
+      "learning_rate": 0.000189209334056145,
+      "logits/chosen": -0.3029250502586365,
+      "logits/rejected": -0.4914131164550781,
+      "logps/chosen": -5.792529582977295,
+      "logps/rejected": -52.44511413574219,
+      "loss": 0.6816909313201904,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5198902487754822,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.058606795966625214,
+      "rewards/margins": 3.3576695919036865,
+      "rewards/rejected": -3.299063205718994,
+      "step": 172,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.38605299860529985,
+      "grad_norm": 2.9290287494659424,
+      "learning_rate": 0.00018904191899700977,
+      "logits/chosen": -0.40706759691238403,
+      "logits/rejected": -0.45858079195022583,
+      "logps/chosen": -6.328141212463379,
+      "logps/rejected": -34.812583923339844,
+      "loss": 0.7009620666503906,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.44378477334976196,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.12429780513048172,
+      "rewards/margins": 2.033411979675293,
+      "rewards/rejected": -1.909114122390747,
+      "step": 173,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.3882845188284519,
+      "grad_norm": 3.7593777179718018,
+      "learning_rate": 0.0001888732904522296,
+      "logits/chosen": -0.380102276802063,
+      "logits/rejected": -0.4688776731491089,
+      "logps/chosen": -5.4205002784729,
+      "logps/rejected": -42.66455078125,
+      "loss": 0.705348014831543,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.48004987835884094,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.09237709641456604,
+      "rewards/margins": 2.7925949096679688,
+      "rewards/rejected": -2.7002179622650146,
+      "step": 174,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.3905160390516039,
+      "grad_norm": 4.404231548309326,
+      "learning_rate": 0.00018870345071991663,
+      "logits/chosen": -0.42475855350494385,
+      "logits/rejected": -0.49599534273147583,
+      "logps/chosen": -4.999125957489014,
+      "logps/rejected": -31.932191848754883,
+      "loss": 0.5820786356925964,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3192511200904846,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.01989058405160904,
+      "rewards/margins": 1.8925269842147827,
+      "rewards/rejected": -1.8726365566253662,
+      "step": 175,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.3927475592747559,
+      "grad_norm": 4.2940239906311035,
+      "learning_rate": 0.00018853240211468944,
+      "logits/chosen": -0.393295019865036,
+      "logits/rejected": -0.4561651647090912,
+      "logps/chosen": -12.356813430786133,
+      "logps/rejected": -33.95934295654297,
+      "loss": 0.9481363892555237,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5598008036613464,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.053550563752651215,
+      "rewards/margins": 1.7341490983963013,
+      "rewards/rejected": -1.680598497390747,
+      "step": 176,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.39497907949790795,
+      "grad_norm": 5.875354766845703,
+      "learning_rate": 0.0001883601469676414,
+      "logits/chosen": -0.36429429054260254,
+      "logits/rejected": -0.5053366422653198,
+      "logps/chosen": -4.228557586669922,
+      "logps/rejected": -41.48766326904297,
+      "loss": 0.5440742373466492,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.34625735878944397,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.046092353761196136,
+      "rewards/margins": 2.599119186401367,
+      "rewards/rejected": -2.5530271530151367,
+      "step": 177,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.39721059972106,
+      "grad_norm": 2.555330753326416,
+      "learning_rate": 0.00018818668762630888,
+      "logits/chosen": -0.40784329175949097,
+      "logits/rejected": -0.5250139236450195,
+      "logps/chosen": -6.325516700744629,
+      "logps/rejected": -37.756141662597656,
+      "loss": 0.5609665513038635,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36217477917671204,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06898089498281479,
+      "rewards/margins": 2.456758499145508,
+      "rewards/rejected": -2.38777756690979,
+      "step": 178,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.39944211994421197,
+      "grad_norm": 2.5570666790008545,
+      "learning_rate": 0.00018801202645463947,
+      "logits/chosen": -0.49009108543395996,
+      "logits/rejected": -0.5421456694602966,
+      "logps/chosen": -7.800501823425293,
+      "logps/rejected": -31.90920639038086,
+      "loss": 0.7761667966842651,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5564302206039429,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22300021350383759,
+      "rewards/margins": 2.3164584636688232,
+      "rewards/rejected": -2.093458414077759,
+      "step": 179,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.401673640167364,
+      "grad_norm": 3.062514066696167,
+      "learning_rate": 0.00018783616583295943,
+      "logits/chosen": -0.3902238607406616,
+      "logits/rejected": -0.5166201591491699,
+      "logps/chosen": -5.271551132202148,
+      "logps/rejected": -43.41081237792969,
+      "loss": 0.586236834526062,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3715082108974457,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10219533741474152,
+      "rewards/margins": 2.8101999759674072,
+      "rewards/rejected": -2.7080047130584717,
+      "step": 180,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.40390516039051605,
+      "grad_norm": 2.8764405250549316,
+      "learning_rate": 0.0001876591081579415,
+      "logits/chosen": -0.4361119270324707,
+      "logits/rejected": -0.5355557203292847,
+      "logps/chosen": -5.954665184020996,
+      "logps/rejected": -40.49996566772461,
+      "loss": 0.6423499584197998,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3932062089443207,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.11792459338903427,
+      "rewards/margins": 2.8236496448516846,
+      "rewards/rejected": -2.7057251930236816,
+      "step": 181,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.4061366806136681,
+      "grad_norm": 3.4861021041870117,
+      "learning_rate": 0.0001874808558425722,
+      "logits/chosen": -0.44822514057159424,
+      "logits/rejected": -0.5208377838134766,
+      "logps/chosen": -8.507841110229492,
+      "logps/rejected": -38.145362854003906,
+      "loss": 0.7365833520889282,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.45295223593711853,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18383249640464783,
+      "rewards/margins": 2.3255980014801025,
+      "rewards/rejected": -2.141765594482422,
+      "step": 182,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.40836820083682007,
+      "grad_norm": 3.0177836418151855,
+      "learning_rate": 0.00018730141131611882,
+      "logits/chosen": -0.4113612771034241,
+      "logits/rejected": -0.5111112594604492,
+      "logps/chosen": -7.983984470367432,
+      "logps/rejected": -39.916988372802734,
+      "loss": 0.7822529077529907,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.48471909761428833,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.010233555920422077,
+      "rewards/margins": 2.3506059646606445,
+      "rewards/rejected": -2.340372323989868,
+      "step": 183,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.4105997210599721,
+      "grad_norm": 2.848646879196167,
+      "learning_rate": 0.0001871207770240965,
+      "logits/chosen": -0.41031956672668457,
+      "logits/rejected": -0.5503991842269897,
+      "logps/chosen": -6.160996437072754,
+      "logps/rejected": -49.401180267333984,
+      "loss": 0.7412950992584229,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5057318806648254,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11709622293710709,
+      "rewards/margins": 2.803727388381958,
+      "rewards/rejected": -2.686631441116333,
+      "step": 184,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.41283124128312415,
+      "grad_norm": 3.7771029472351074,
+      "learning_rate": 0.00018693895542823476,
+      "logits/chosen": -0.4378977417945862,
+      "logits/rejected": -0.5446797609329224,
+      "logps/chosen": -3.902487277984619,
+      "logps/rejected": -38.302616119384766,
+      "loss": 0.6036807298660278,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3306537866592407,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07923398911952972,
+      "rewards/margins": 2.431417465209961,
+      "rewards/rejected": -2.3521835803985596,
+      "step": 185,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.41506276150627613,
+      "grad_norm": 2.6410114765167236,
+      "learning_rate": 0.00018675594900644394,
+      "logits/chosen": -0.4403599798679352,
+      "logits/rejected": -0.534117579460144,
+      "logps/chosen": -7.202261447906494,
+      "logps/rejected": -40.587371826171875,
+      "loss": 0.6803833842277527,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4558661878108978,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11624391376972198,
+      "rewards/margins": 2.7324059009552,
+      "rewards/rejected": -2.616162061691284,
+      "step": 186,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.4172942817294282,
+      "grad_norm": 27.184694290161133,
+      "learning_rate": 0.0001865717602527816,
+      "logits/chosen": -0.3826313018798828,
+      "logits/rejected": -0.5364388227462769,
+      "logps/chosen": -5.527843952178955,
+      "logps/rejected": -49.32733917236328,
+      "loss": 0.7175433039665222,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5420017242431641,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.13603699207305908,
+      "rewards/margins": 3.4389665126800537,
+      "rewards/rejected": -3.302929401397705,
+      "step": 187,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.4195258019525802,
+      "grad_norm": 2.5546696186065674,
+      "learning_rate": 0.00018638639167741834,
+      "logits/chosen": -0.432650625705719,
+      "logits/rejected": -0.5771442651748657,
+      "logps/chosen": -6.985705375671387,
+      "logps/rejected": -55.28477478027344,
+      "loss": 0.5891926884651184,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.41381514072418213,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2026616781949997,
+      "rewards/margins": 4.167919635772705,
+      "rewards/rejected": -3.9652576446533203,
+      "step": 188,
+      "train_speed(iter/s)": 0.011178
+    },
+    {
+      "epoch": 0.4217573221757322,
+      "grad_norm": 2.758708953857422,
+      "learning_rate": 0.00018619984580660367,
+      "logits/chosen": -0.41937315464019775,
+      "logits/rejected": -0.5321905612945557,
+      "logps/chosen": -6.989339828491211,
+      "logps/rejected": -58.12518310546875,
+      "loss": 0.5879656672477722,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36345142126083374,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.023294318467378616,
+      "rewards/margins": 4.040982246398926,
+      "rewards/rejected": -4.064276695251465,
+      "step": 189,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.42398884239888424,
+      "grad_norm": 11.678010940551758,
+      "learning_rate": 0.00018601212518263156,
+      "logits/chosen": -0.39216288924217224,
+      "logits/rejected": -0.4594748020172119,
+      "logps/chosen": -9.29268741607666,
+      "logps/rejected": -54.65095520019531,
+      "loss": 0.818715512752533,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6221151947975159,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13676013052463531,
+      "rewards/margins": 4.018932819366455,
+      "rewards/rejected": -3.8821723461151123,
+      "step": 190,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.4262203626220363,
+      "grad_norm": 3.9322588443756104,
+      "learning_rate": 0.00018582323236380588,
+      "logits/chosen": -0.4796799421310425,
+      "logits/rejected": -0.5020566582679749,
+      "logps/chosen": -14.528449058532715,
+      "logps/rejected": -43.279300689697266,
+      "loss": 0.9906633496284485,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6794536709785461,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.2910754680633545,
+      "rewards/margins": 2.7544803619384766,
+      "rewards/rejected": -3.045555591583252,
+      "step": 191,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.42845188284518826,
+      "grad_norm": 5.284152507781982,
+      "learning_rate": 0.00018563316992440543,
+      "logits/chosen": -0.4428099989891052,
+      "logits/rejected": -0.5434530973434448,
+      "logps/chosen": -6.242356777191162,
+      "logps/rejected": -46.20015335083008,
+      "loss": 0.8040673732757568,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4949415922164917,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.05515892803668976,
+      "rewards/margins": 2.9070026874542236,
+      "rewards/rejected": -2.9621615409851074,
+      "step": 192,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.4306834030683403,
+      "grad_norm": 11.493171691894531,
+      "learning_rate": 0.00018544194045464886,
+      "logits/chosen": -0.38638293743133545,
+      "logits/rejected": -0.4842715561389923,
+      "logps/chosen": -6.438551425933838,
+      "logps/rejected": -44.880149841308594,
+      "loss": 0.7885379791259766,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5359107255935669,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19068023562431335,
+      "rewards/margins": 2.8829379081726074,
+      "rewards/rejected": -2.692257881164551,
+      "step": 193,
+      "train_speed(iter/s)": 0.011179
+    },
+    {
+      "epoch": 0.43291492329149234,
+      "grad_norm": 2.3137385845184326,
+      "learning_rate": 0.00018524954656065946,
+      "logits/chosen": -0.4063287675380707,
+      "logits/rejected": -0.5103312730789185,
+      "logps/chosen": -7.378753662109375,
+      "logps/rejected": -45.85976791381836,
+      "loss": 0.45938000082969666,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3061022460460663,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.08831208944320679,
+      "rewards/margins": 3.123901605606079,
+      "rewards/rejected": -3.0355896949768066,
+      "step": 194,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.4351464435146444,
+      "grad_norm": 4.795388221740723,
+      "learning_rate": 0.00018505599086442956,
+      "logits/chosen": -0.38035646080970764,
+      "logits/rejected": -0.5444158315658569,
+      "logps/chosen": -2.437512159347534,
+      "logps/rejected": -43.048858642578125,
+      "loss": 0.5650367736816406,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.33980026841163635,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13193824887275696,
+      "rewards/margins": 2.7752439975738525,
+      "rewards/rejected": -2.643305540084839,
+      "step": 195,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.43737796373779636,
+      "grad_norm": 2.4795217514038086,
+      "learning_rate": 0.0001848612760037848,
+      "logits/chosen": -0.3798023462295532,
+      "logits/rejected": -0.5086287260055542,
+      "logps/chosen": -4.153357982635498,
+      "logps/rejected": -47.3394775390625,
+      "loss": 0.4092462360858917,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.22234448790550232,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1915455460548401,
+      "rewards/margins": 2.8723514080047607,
+      "rewards/rejected": -2.6808059215545654,
+      "step": 196,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.4396094839609484,
+      "grad_norm": 4.820924282073975,
+      "learning_rate": 0.0001846654046323482,
+      "logits/chosen": -0.41246578097343445,
+      "logits/rejected": -0.4796575903892517,
+      "logps/chosen": -10.3787841796875,
+      "logps/rejected": -37.14994812011719,
+      "loss": 0.9857741594314575,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.669836163520813,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.05453118681907654,
+      "rewards/margins": 2.0403127670288086,
+      "rewards/rejected": -2.094843864440918,
+      "step": 197,
+      "train_speed(iter/s)": 0.01118
+    },
+    {
+      "epoch": 0.44184100418410044,
+      "grad_norm": 3.00054931640625,
+      "learning_rate": 0.0001844683794195041,
+      "logits/chosen": -0.3768506646156311,
+      "logits/rejected": -0.5438990592956543,
+      "logps/chosen": -6.3500471115112305,
+      "logps/rejected": -49.48595428466797,
+      "loss": 0.6618455648422241,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4003591537475586,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.02325659617781639,
+      "rewards/margins": 3.2074074745178223,
+      "rewards/rejected": -3.1841506958007812,
+      "step": 198,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.4440725244072524,
+      "grad_norm": 5.5015339851379395,
+      "learning_rate": 0.00018427020305036157,
+      "logits/chosen": -0.43915611505508423,
+      "logits/rejected": -0.5123714804649353,
+      "logps/chosen": -5.833986759185791,
+      "logps/rejected": -41.57982635498047,
+      "loss": 0.7684890031814575,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4649631679058075,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04928220808506012,
+      "rewards/margins": 2.619971752166748,
+      "rewards/rejected": -2.6692538261413574,
+      "step": 199,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.44630404463040446,
+      "grad_norm": 4.47312068939209,
+      "learning_rate": 0.00018407087822571794,
+      "logits/chosen": -0.41636520624160767,
+      "logits/rejected": -0.5203005075454712,
+      "logps/chosen": -9.286731719970703,
+      "logps/rejected": -41.12049102783203,
+      "loss": 0.8878820538520813,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6213392615318298,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1641845703125,
+      "rewards/margins": 2.593937397003174,
+      "rewards/rejected": -2.4297525882720947,
+      "step": 200,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.4485355648535565,
+      "grad_norm": 2.523305654525757,
+      "learning_rate": 0.00018387040766202202,
+      "logits/chosen": -0.43077439069747925,
+      "logits/rejected": -0.5606883764266968,
+      "logps/chosen": -9.173616409301758,
+      "logps/rejected": -54.819252014160156,
+      "loss": 0.6095305681228638,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.43501588702201843,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17188499867916107,
+      "rewards/margins": 3.581376791000366,
+      "rewards/rejected": -3.409491539001465,
+      "step": 201,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.4507670850767085,
+      "grad_norm": 10.484657287597656,
+      "learning_rate": 0.00018366879409133702,
+      "logits/chosen": -0.46561458706855774,
+      "logits/rejected": -0.5806446075439453,
+      "logps/chosen": -9.230829238891602,
+      "logps/rejected": -50.296417236328125,
+      "loss": 0.8357149362564087,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5318824052810669,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04186338931322098,
+      "rewards/margins": 3.0313618183135986,
+      "rewards/rejected": -3.0732247829437256,
+      "step": 202,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.4529986052998605,
+      "grad_norm": 3.0593442916870117,
+      "learning_rate": 0.00018346604026130335,
+      "logits/chosen": -0.4838520884513855,
+      "logits/rejected": -0.563227117061615,
+      "logps/chosen": -6.622610092163086,
+      "logps/rejected": -33.531341552734375,
+      "loss": 0.7672325372695923,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.47228333353996277,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16688543558120728,
+      "rewards/margins": 2.1572673320770264,
+      "rewards/rejected": -1.9903819561004639,
+      "step": 203,
+      "train_speed(iter/s)": 0.01117
+    },
+    {
+      "epoch": 0.45523012552301256,
+      "grad_norm": 3.2974071502685547,
+      "learning_rate": 0.00018326214893510113,
+      "logits/chosen": -0.48938241600990295,
+      "logits/rejected": -0.5406135320663452,
+      "logps/chosen": -6.855633735656738,
+      "logps/rejected": -32.516056060791016,
+      "loss": 0.7795042991638184,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.463142991065979,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.058080192655324936,
+      "rewards/margins": 2.187054395675659,
+      "rewards/rejected": -2.128974199295044,
+      "step": 204,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.45746164574616455,
+      "grad_norm": 2.7023813724517822,
+      "learning_rate": 0.00018305712289141263,
+      "logits/chosen": -0.503454864025116,
+      "logits/rejected": -0.5921227931976318,
+      "logps/chosen": -7.617156982421875,
+      "logps/rejected": -42.7328987121582,
+      "loss": 0.6725366711616516,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3977656364440918,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.02913985215127468,
+      "rewards/margins": 2.757948398590088,
+      "rewards/rejected": -2.728808641433716,
+      "step": 205,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.4596931659693166,
+      "grad_norm": 3.1167116165161133,
+      "learning_rate": 0.00018285096492438424,
+      "logits/chosen": -0.5165062546730042,
+      "logits/rejected": -0.6217053532600403,
+      "logps/chosen": -7.3234543800354,
+      "logps/rejected": -50.999603271484375,
+      "loss": 0.6318544745445251,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3780009150505066,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.00554865226149559,
+      "rewards/margins": 3.287566661834717,
+      "rewards/rejected": -3.2820186614990234,
+      "step": 206,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.4619246861924686,
+      "grad_norm": 9.524746894836426,
+      "learning_rate": 0.00018264367784358854,
+      "logits/chosen": -0.4920929968357086,
+      "logits/rejected": -0.5764138102531433,
+      "logps/chosen": -7.860991954803467,
+      "logps/rejected": -42.47827911376953,
+      "loss": 0.7360129356384277,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5072849988937378,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2539522051811218,
+      "rewards/margins": 2.9447669982910156,
+      "rewards/rejected": -2.690814733505249,
+      "step": 207,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.46415620641562066,
+      "grad_norm": 3.1537561416625977,
+      "learning_rate": 0.00018243526447398595,
+      "logits/chosen": -0.49637073278427124,
+      "logits/rejected": -0.5572264194488525,
+      "logps/chosen": -7.864004135131836,
+      "logps/rejected": -36.40027618408203,
+      "loss": 0.701461911201477,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4231345057487488,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.05218251794576645,
+      "rewards/margins": 2.490140438079834,
+      "rewards/rejected": -2.437958002090454,
+      "step": 208,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.46638772663877265,
+      "grad_norm": 5.860493183135986,
+      "learning_rate": 0.00018222572765588625,
+      "logits/chosen": -0.4768957793712616,
+      "logits/rejected": -0.5333793759346008,
+      "logps/chosen": -14.414131164550781,
+      "logps/rejected": -43.239593505859375,
+      "loss": 0.8101565837860107,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.49351921677589417,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.05914236605167389,
+      "rewards/margins": 2.7074344158172607,
+      "rewards/rejected": -2.648291826248169,
+      "step": 209,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.4686192468619247,
+      "grad_norm": 2.79776930809021,
+      "learning_rate": 0.00018201507024490988,
+      "logits/chosen": -0.4704314172267914,
+      "logits/rejected": -0.567833662033081,
+      "logps/chosen": -14.108932495117188,
+      "logps/rejected": -44.836395263671875,
+      "loss": 0.8302231431007385,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6059545278549194,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17180928587913513,
+      "rewards/margins": 2.3872344493865967,
+      "rewards/rejected": -2.2154252529144287,
+      "step": 210,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.4708507670850767,
+      "grad_norm": 1.4490225315093994,
+      "learning_rate": 0.0001818032951119489,
+      "logits/chosen": -0.45868298411369324,
+      "logits/rejected": -0.6028948426246643,
+      "logps/chosen": -3.0846667289733887,
+      "logps/rejected": -51.68486404418945,
+      "loss": 0.36262989044189453,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.25144004821777344,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2839798927307129,
+      "rewards/margins": 3.57785964012146,
+      "rewards/rejected": -3.293879747390747,
+      "step": 211,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4730822873082287,
+      "grad_norm": 2.7278997898101807,
+      "learning_rate": 0.00018159040514312803,
+      "logits/chosen": -0.5122844576835632,
+      "logits/rejected": -0.5693905353546143,
+      "logps/chosen": -8.715449333190918,
+      "logps/rejected": -31.896915435791016,
+      "loss": 0.7132190465927124,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.39171797037124634,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2175295054912567,
+      "rewards/margins": 2.153326988220215,
+      "rewards/rejected": -1.9357972145080566,
+      "step": 212,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.47531380753138075,
+      "grad_norm": 2.76104474067688,
+      "learning_rate": 0.00018137640323976536,
+      "logits/chosen": -0.5042344331741333,
+      "logits/rejected": -0.5739261507987976,
+      "logps/chosen": -12.65793228149414,
+      "logps/rejected": -42.124107360839844,
+      "loss": 0.7560016512870789,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5502416491508484,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03646032512187958,
+      "rewards/margins": 2.767151355743408,
+      "rewards/rejected": -2.7306909561157227,
+      "step": 213,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4775453277545328,
+      "grad_norm": 4.021885871887207,
+      "learning_rate": 0.00018116129231833249,
+      "logits/chosen": -0.466595321893692,
+      "logits/rejected": -0.5636504888534546,
+      "logps/chosen": -8.149341583251953,
+      "logps/rejected": -38.28250503540039,
+      "loss": 0.7789570093154907,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4304181933403015,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0456395149230957,
+      "rewards/margins": 2.337042808532715,
+      "rewards/rejected": -2.2914035320281982,
+      "step": 214,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4797768479776848,
+      "grad_norm": 2.7031266689300537,
+      "learning_rate": 0.00018094507531041517,
+      "logits/chosen": -0.5194039940834045,
+      "logits/rejected": -0.5763382315635681,
+      "logps/chosen": -12.234911918640137,
+      "logps/rejected": -41.63436508178711,
+      "loss": 0.6696392893791199,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.397487074136734,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.020977359265089035,
+      "rewards/margins": 2.5402636528015137,
+      "rewards/rejected": -2.5192863941192627,
+      "step": 215,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4820083682008368,
+      "grad_norm": 4.239037990570068,
+      "learning_rate": 0.00018072775516267306,
+      "logits/chosen": -0.4820006191730499,
+      "logits/rejected": -0.5642452239990234,
+      "logps/chosen": -5.411629676818848,
+      "logps/rejected": -49.11735534667969,
+      "loss": 0.6698524951934814,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4091280698776245,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.007008063141256571,
+      "rewards/margins": 2.734046459197998,
+      "rewards/rejected": -2.7270381450653076,
+      "step": 216,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.48423988842398885,
+      "grad_norm": 2.3891801834106445,
+      "learning_rate": 0.00018050933483679976,
+      "logits/chosen": -0.48481085896492004,
+      "logits/rejected": -0.6319360136985779,
+      "logps/chosen": -6.989506721496582,
+      "logps/rejected": -56.728424072265625,
+      "loss": 0.601786732673645,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.35297003388404846,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13874617218971252,
+      "rewards/margins": 3.526907205581665,
+      "rewards/rejected": -3.3881607055664062,
+      "step": 217,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4864714086471409,
+      "grad_norm": 6.219696044921875,
+      "learning_rate": 0.0001802898173094824,
+      "logits/chosen": -0.5356330871582031,
+      "logits/rejected": -0.6473451852798462,
+      "logps/chosen": -8.472892761230469,
+      "logps/rejected": -51.16962432861328,
+      "loss": 0.783514142036438,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5371319651603699,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.06705744564533234,
+      "rewards/margins": 3.750828266143799,
+      "rewards/rejected": -3.6837711334228516,
+      "step": 218,
+      "train_speed(iter/s)": 0.011172
+    },
+    {
+      "epoch": 0.4887029288702929,
+      "grad_norm": 6.937330722808838,
+      "learning_rate": 0.0001800692055723609,
+      "logits/chosen": -0.6238746047019958,
+      "logits/rejected": -0.6939642429351807,
+      "logps/chosen": -4.523331642150879,
+      "logps/rejected": -37.64277267456055,
+      "loss": 0.6180140376091003,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3569566607475281,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.11646758764982224,
+      "rewards/margins": 2.8116490840911865,
+      "rewards/rejected": -2.695181369781494,
+      "step": 219,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.4909344490934449,
+      "grad_norm": 4.587141036987305,
+      "learning_rate": 0.0001798475026319875,
+      "logits/chosen": -0.5524365901947021,
+      "logits/rejected": -0.7282678484916687,
+      "logps/chosen": -4.9026618003845215,
+      "logps/rejected": -58.83467483520508,
+      "loss": 0.5599265098571777,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3686913251876831,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08274698257446289,
+      "rewards/margins": 4.399111747741699,
+      "rewards/rejected": -4.316364765167236,
+      "step": 220,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.49316596931659695,
+      "grad_norm": 19.358779907226562,
+      "learning_rate": 0.00017962471150978563,
+      "logits/chosen": -0.6010394096374512,
+      "logits/rejected": -0.7070757150650024,
+      "logps/chosen": -7.744332313537598,
+      "logps/rejected": -56.298973083496094,
+      "loss": 1.0826630592346191,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.7130725979804993,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.036551520228385925,
+      "rewards/margins": 4.22437858581543,
+      "rewards/rejected": -4.260930061340332,
+      "step": 221,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.49539748953974894,
+      "grad_norm": 1.9726710319519043,
+      "learning_rate": 0.00017940083524200858,
+      "logits/chosen": -0.5176704525947571,
+      "logits/rejected": -0.6265690326690674,
+      "logps/chosen": -8.897793769836426,
+      "logps/rejected": -51.35803985595703,
+      "loss": 0.5861097574234009,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4163362979888916,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3185967803001404,
+      "rewards/margins": 3.5779831409454346,
+      "rewards/rejected": -3.2593865394592285,
+      "step": 222,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.497629009762901,
+      "grad_norm": 2.865757465362549,
+      "learning_rate": 0.00017917587687969847,
+      "logits/chosen": -0.5248193740844727,
+      "logits/rejected": -0.6183274984359741,
+      "logps/chosen": -11.439458847045898,
+      "logps/rejected": -43.991424560546875,
+      "loss": 0.8551426529884338,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6576440334320068,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24198690056800842,
+      "rewards/margins": 3.191100597381592,
+      "rewards/rejected": -2.9491138458251953,
+      "step": 223,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.499860529986053,
+      "grad_norm": 2.936124324798584,
+      "learning_rate": 0.00017894983948864431,
+      "logits/chosen": -0.5203258991241455,
+      "logits/rejected": -0.6186901926994324,
+      "logps/chosen": -8.132195472717285,
+      "logps/rejected": -41.595703125,
+      "loss": 0.7639778256416321,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4688246548175812,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03628700599074364,
+      "rewards/margins": 2.6348683834075928,
+      "rewards/rejected": -2.598581314086914,
+      "step": 224,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.502092050209205,
+      "grad_norm": 3.2935783863067627,
+      "learning_rate": 0.0001787227261493405,
+      "logits/chosen": -0.49744516611099243,
+      "logits/rejected": -0.6266440749168396,
+      "logps/chosen": -6.2615861892700195,
+      "logps/rejected": -52.76634216308594,
+      "loss": 0.5845623016357422,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36887243390083313,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10732433944940567,
+      "rewards/margins": 3.68102765083313,
+      "rewards/rejected": -3.5737037658691406,
+      "step": 225,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.5043235704323571,
+      "grad_norm": 2.1799988746643066,
+      "learning_rate": 0.0001784945399569447,
+      "logits/chosen": -0.4298093914985657,
+      "logits/rejected": -0.6444836854934692,
+      "logps/chosen": -3.285033941268921,
+      "logps/rejected": -62.29474639892578,
+      "loss": 0.3315008282661438,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.21272185444831848,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2050013542175293,
+      "rewards/margins": 4.134213924407959,
+      "rewards/rejected": -3.9292120933532715,
+      "step": 226,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.5065550906555091,
+      "grad_norm": 1.7241166830062866,
+      "learning_rate": 0.00017826528402123564,
+      "logits/chosen": -0.48651599884033203,
+      "logits/rejected": -0.6437865495681763,
+      "logps/chosen": -2.7627596855163574,
+      "logps/rejected": -52.23710632324219,
+      "loss": 0.3623155653476715,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.25128594040870667,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20487122237682343,
+      "rewards/margins": 3.8199470043182373,
+      "rewards/rejected": -3.6150758266448975,
+      "step": 227,
+      "train_speed(iter/s)": 0.011174
+    },
+    {
+      "epoch": 0.5087866108786611,
+      "grad_norm": 2.1939537525177,
+      "learning_rate": 0.00017803496146657084,
+      "logits/chosen": -0.4696894884109497,
+      "logits/rejected": -0.648972749710083,
+      "logps/chosen": -6.876445770263672,
+      "logps/rejected": -58.56797409057617,
+      "loss": 0.6069039106369019,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.48535749316215515,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30394309759140015,
+      "rewards/margins": 4.106232643127441,
+      "rewards/rejected": -3.8022897243499756,
+      "step": 228,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.5110181311018132,
+      "grad_norm": 2.2029025554656982,
+      "learning_rate": 0.00017780357543184397,
+      "logits/chosen": -0.5041614174842834,
+      "logits/rejected": -0.6466209292411804,
+      "logps/chosen": -6.405285835266113,
+      "logps/rejected": -66.68818664550781,
+      "loss": 0.530963122844696,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.36968475580215454,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1667797863483429,
+      "rewards/margins": 5.374972343444824,
+      "rewards/rejected": -5.208192825317383,
+      "step": 229,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.5132496513249651,
+      "grad_norm": 2.9095795154571533,
+      "learning_rate": 0.000177571129070442,
+      "logits/chosen": -0.5456891655921936,
+      "logits/rejected": -0.5758720636367798,
+      "logps/chosen": -9.691763877868652,
+      "logps/rejected": -49.30378723144531,
+      "loss": 0.6552941203117371,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4235466718673706,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.05060029774904251,
+      "rewards/margins": 3.4885663986206055,
+      "rewards/rejected": -3.4379661083221436,
+      "step": 230,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.5154811715481171,
+      "grad_norm": 2.6140189170837402,
+      "learning_rate": 0.00017733762555020234,
+      "logits/chosen": -0.526019811630249,
+      "logits/rejected": -0.6267409324645996,
+      "logps/chosen": -7.562019348144531,
+      "logps/rejected": -59.53757858276367,
+      "loss": 0.45963698625564575,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3218066692352295,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11903621256351471,
+      "rewards/margins": 4.636847019195557,
+      "rewards/rejected": -4.517810821533203,
+      "step": 231,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.5177126917712692,
+      "grad_norm": 7.271316051483154,
+      "learning_rate": 0.0001771030680533697,
+      "logits/chosen": -0.4977180063724518,
+      "logits/rejected": -0.6347447633743286,
+      "logps/chosen": -6.1880202293396,
+      "logps/rejected": -57.32111358642578,
+      "loss": 0.6974989771842957,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.41052868962287903,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06481020152568817,
+      "rewards/margins": 4.053691864013672,
+      "rewards/rejected": -3.988882064819336,
+      "step": 232,
+      "train_speed(iter/s)": 0.011175
+    },
+    {
+      "epoch": 0.5199442119944212,
+      "grad_norm": 6.688237190246582,
+      "learning_rate": 0.00017686745977655254,
+      "logits/chosen": -0.5404477715492249,
+      "logits/rejected": -0.591978907585144,
+      "logps/chosen": -7.557736873626709,
+      "logps/rejected": -53.3175048828125,
+      "loss": 0.6092088222503662,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40488630533218384,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.009284242987632751,
+      "rewards/margins": 4.1996588706970215,
+      "rewards/rejected": -4.190374851226807,
+      "step": 233,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5221757322175732,
+      "grad_norm": 2.909499168395996,
+      "learning_rate": 0.00017663080393067967,
+      "logits/chosen": -0.41550740599632263,
+      "logits/rejected": -0.6418639421463013,
+      "logps/chosen": -9.175922393798828,
+      "logps/rejected": -79.41323852539062,
+      "loss": 0.7257620096206665,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.47780197858810425,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10455615818500519,
+      "rewards/margins": 5.70869255065918,
+      "rewards/rejected": -5.6041364669799805,
+      "step": 234,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5244072524407253,
+      "grad_norm": 24.34648895263672,
+      "learning_rate": 0.00017639310374095642,
+      "logits/chosen": -0.474945992231369,
+      "logits/rejected": -0.6495036482810974,
+      "logps/chosen": -9.139482498168945,
+      "logps/rejected": -61.76782989501953,
+      "loss": 0.7843772172927856,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6071273684501648,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08099109679460526,
+      "rewards/margins": 4.556737422943115,
+      "rewards/rejected": -4.475746154785156,
+      "step": 235,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5266387726638773,
+      "grad_norm": 3.9490129947662354,
+      "learning_rate": 0.00017615436244682067,
+      "logits/chosen": -0.48849430680274963,
+      "logits/rejected": -0.6626330614089966,
+      "logps/chosen": -6.473770618438721,
+      "logps/rejected": -63.68229675292969,
+      "loss": 0.7651278972625732,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5939050912857056,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05846280977129936,
+      "rewards/margins": 4.720561981201172,
+      "rewards/rejected": -4.662098407745361,
+      "step": 236,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5288702928870292,
+      "grad_norm": 2.3226640224456787,
+      "learning_rate": 0.0001759145833018988,
+      "logits/chosen": -0.5129532814025879,
+      "logits/rejected": -0.6029571294784546,
+      "logps/chosen": -11.16122055053711,
+      "logps/rejected": -65.20454406738281,
+      "loss": 0.6758726239204407,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.44432199001312256,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06612987071275711,
+      "rewards/margins": 4.690124034881592,
+      "rewards/rejected": -4.62399435043335,
+      "step": 237,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5311018131101813,
+      "grad_norm": 4.086884498596191,
+      "learning_rate": 0.00017567376957396125,
+      "logits/chosen": -0.4485701620578766,
+      "logits/rejected": -0.6184876561164856,
+      "logps/chosen": -9.818546295166016,
+      "logps/rejected": -64.69741821289062,
+      "loss": 0.841502845287323,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.6483633518218994,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15229403972625732,
+      "rewards/margins": 4.889117240905762,
+      "rewards/rejected": -4.736823558807373,
+      "step": 238,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 5.736257076263428,
+      "learning_rate": 0.00017543192454487795,
+      "logits/chosen": -0.5332171320915222,
+      "logits/rejected": -0.6654965877532959,
+      "logps/chosen": -8.099287986755371,
+      "logps/rejected": -68.04266357421875,
+      "loss": 0.6135991215705872,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.47898998856544495,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.012953575700521469,
+      "rewards/margins": 5.387064456939697,
+      "rewards/rejected": -5.374111175537109,
+      "step": 239,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5355648535564853,
+      "grad_norm": 2.407405138015747,
+      "learning_rate": 0.0001751890515105738,
+      "logits/chosen": -0.5446051955223083,
+      "logits/rejected": -0.6599375009536743,
+      "logps/chosen": -5.214753150939941,
+      "logps/rejected": -70.1378173828125,
+      "loss": 0.6014682054519653,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4095730781555176,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1154915988445282,
+      "rewards/margins": 5.451560020446777,
+      "rewards/rejected": -5.336068153381348,
+      "step": 240,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5377963737796374,
+      "grad_norm": 5.806000232696533,
+      "learning_rate": 0.00017494515378098347,
+      "logits/chosen": -0.5061202049255371,
+      "logits/rejected": -0.6401721835136414,
+      "logps/chosen": -7.594027996063232,
+      "logps/rejected": -72.66096496582031,
+      "loss": 0.7293487191200256,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.509558379650116,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.20420843362808228,
+      "rewards/margins": 5.691208839416504,
+      "rewards/rejected": -5.895417213439941,
+      "step": 241,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5400278940027894,
+      "grad_norm": 20.491920471191406,
+      "learning_rate": 0.00017470023468000654,
+      "logits/chosen": -0.5332112312316895,
+      "logits/rejected": -0.6652044653892517,
+      "logps/chosen": -4.244685649871826,
+      "logps/rejected": -67.0967788696289,
+      "loss": 0.4344191253185272,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.27993929386138916,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.039192792028188705,
+      "rewards/margins": 5.225668907165527,
+      "rewards/rejected": -5.18647575378418,
+      "step": 242,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5422594142259414,
+      "grad_norm": 1.5459198951721191,
+      "learning_rate": 0.00017445429754546209,
+      "logits/chosen": -0.4623527526855469,
+      "logits/rejected": -0.5880333185195923,
+      "logps/chosen": -6.493972301483154,
+      "logps/rejected": -61.213016510009766,
+      "loss": 0.5002475380897522,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.3536112904548645,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23787753283977509,
+      "rewards/margins": 4.298797607421875,
+      "rewards/rejected": -4.060920715332031,
+      "step": 243,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5444909344490935,
+      "grad_norm": 10.530829429626465,
+      "learning_rate": 0.00017420734572904309,
+      "logits/chosen": -0.4680861234664917,
+      "logits/rejected": -0.6114482879638672,
+      "logps/chosen": -6.2320733070373535,
+      "logps/rejected": -51.97254180908203,
+      "loss": 0.6590880155563354,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.4217822849750519,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1015012115240097,
+      "rewards/margins": 3.5804498195648193,
+      "rewards/rejected": -3.4789485931396484,
+      "step": 244,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5467224546722455,
+      "grad_norm": 4.540682792663574,
+      "learning_rate": 0.00017395938259627102,
+      "logits/chosen": -0.4017217457294464,
+      "logits/rejected": -0.5551414489746094,
+      "logps/chosen": -4.653961658477783,
+      "logps/rejected": -60.12483215332031,
+      "loss": 0.417266309261322,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.26140686869621277,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2787860631942749,
+      "rewards/margins": 3.9614157676696777,
+      "rewards/rejected": -3.682629108428955,
+      "step": 245,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5489539748953974,
+      "grad_norm": 2.821626663208008,
+      "learning_rate": 0.00017371041152644976,
+      "logits/chosen": -0.46805882453918457,
+      "logits/rejected": -0.5824432373046875,
+      "logps/chosen": -7.418338775634766,
+      "logps/rejected": -49.86051940917969,
+      "loss": 0.7901802659034729,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5593172311782837,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2499873787164688,
+      "rewards/margins": 3.6353776454925537,
+      "rewards/rejected": -3.385390043258667,
+      "step": 246,
+      "train_speed(iter/s)": 0.011177
+    },
+    {
+      "epoch": 0.5511854951185495,
+      "grad_norm": 2.110387086868286,
+      "learning_rate": 0.00017346043591261957,
+      "logits/chosen": -0.441771000623703,
+      "logits/rejected": -0.6270588040351868,
+      "logps/chosen": -7.553977012634277,
+      "logps/rejected": -58.14564514160156,
+      "loss": 0.569815993309021,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.40740612149238586,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2955455183982849,
+      "rewards/margins": 3.5217554569244385,
+      "rewards/rejected": -3.226210117340088,
+      "step": 247,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5534170153417015,
+      "grad_norm": 2.2252297401428223,
+      "learning_rate": 0.0001732094591615109,
+      "logits/chosen": -0.3921899199485779,
+      "logits/rejected": -0.5801397562026978,
+      "logps/chosen": -8.181288719177246,
+      "logps/rejected": -63.77415084838867,
+      "loss": 0.5467262268066406,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.37672924995422363,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2877017557621002,
+      "rewards/margins": 4.03120231628418,
+      "rewards/rejected": -3.7435007095336914,
+      "step": 248,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5556485355648536,
+      "grad_norm": 2.368830680847168,
+      "learning_rate": 0.00017295748469349804,
+      "logits/chosen": -0.4548183083534241,
+      "logits/rejected": -0.6428787112236023,
+      "logps/chosen": -5.975998878479004,
+      "logps/rejected": -64.11628723144531,
+      "loss": 0.6057455539703369,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.5057395696640015,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24285957217216492,
+      "rewards/margins": 4.522610664367676,
+      "rewards/rejected": -4.279751300811768,
+      "step": 249,
+      "train_speed(iter/s)": 0.011176
+    },
+    {
+      "epoch": 0.5578800557880056,
+      "grad_norm": 5.085129261016846,
+      "learning_rate": 0.00017270451594255233,
+      "logits/chosen": -0.5124991536140442,
+      "logits/rejected": -0.6001466512680054,
+      "logps/chosen": -5.997179985046387,
+      "logps/rejected": -46.71837615966797,
+      "loss": 0.5066366791725159,
+      "memory(GiB)": 46.71,
+      "nll_loss": 0.31767114996910095,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16393466293811798,
+      "rewards/margins": 3.424452066421509,
+      "rewards/rejected": -3.260517120361328,
+      "step": 250,
+      "train_speed(iter/s)": 0.011176
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 896,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3756117867823104e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}