diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,2860 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 782,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00128,
+      "grad_norm": 17.769851684570312,
+      "learning_rate": 2.5e-08,
+      "logits/chosen": -2.6875,
+      "logits/rejected": -2.66796875,
+      "logps/chosen": -218.5,
+      "logps/rejected": -331.5,
+      "loss": 0.9825439453125,
+      "memory(GiB)": 23.56,
+      "nll_loss": 0.29150390625,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.019634
+    },
+    {
+      "epoch": 0.0064,
+      "grad_norm": 15.24601936340332,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -2.767578125,
+      "logits/rejected": -2.6923828125,
+      "logps/chosen": -203.5625,
+      "logps/rejected": -420.625,
+      "loss": 0.992431640625,
+      "memory(GiB)": 32.57,
+      "nll_loss": 0.2962646484375,
+      "rewards/accuracies": 0.21484375,
+      "rewards/chosen": 0.0005412101745605469,
+      "rewards/margins": 7.82012939453125e-05,
+      "rewards/rejected": 0.00047397613525390625,
+      "step": 5,
+      "train_speed(iter/s)": 0.050573
+    },
+    {
+      "epoch": 0.0128,
+      "grad_norm": 19.849214553833008,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.77734375,
+      "logits/rejected": -2.6734375953674316,
+      "logps/chosen": -197.64999389648438,
+      "logps/rejected": -486.6000061035156,
+      "loss": 0.9816162109375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28559571504592896,
+      "rewards/accuracies": 0.2718749940395355,
+      "rewards/chosen": 0.0008754730224609375,
+      "rewards/margins": -0.0011947632301598787,
+      "rewards/rejected": 0.0020652771927416325,
+      "step": 10,
+      "train_speed(iter/s)": 0.062812
+    },
+    {
+      "epoch": 0.0192,
+      "grad_norm": 15.909340858459473,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -2.7640624046325684,
+      "logits/rejected": -2.668750047683716,
+      "logps/chosen": -230.0500030517578,
+      "logps/rejected": -471.6000061035156,
+      "loss": 1.0011962890625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31132811307907104,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": 0.00835266150534153,
+      "rewards/margins": 0.02167968824505806,
+      "rewards/rejected": -0.013314628973603249,
+      "step": 15,
+      "train_speed(iter/s)": 0.068328
+    },
+    {
+      "epoch": 0.0256,
+      "grad_norm": 15.320168495178223,
+      "learning_rate": 5e-07,
+      "logits/chosen": -2.739062547683716,
+      "logits/rejected": -2.7007813453674316,
+      "logps/chosen": -232.3000030517578,
+      "logps/rejected": -400.6000061035156,
+      "loss": 1.014892578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.32075196504592896,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": 0.01239852886646986,
+      "rewards/margins": 0.0016508102416992188,
+      "rewards/rejected": 0.010778045281767845,
+      "step": 20,
+      "train_speed(iter/s)": 0.071546
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 13.883173942565918,
+      "learning_rate": 6.249999999999999e-07,
+      "logits/chosen": -2.741406202316284,
+      "logits/rejected": -2.6429686546325684,
+      "logps/chosen": -224.10000610351562,
+      "logps/rejected": -419.8999938964844,
+      "loss": 0.999462890625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3099121153354645,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": 0.03140296787023544,
+      "rewards/margins": 0.009646987542510033,
+      "rewards/rejected": 0.021758269518613815,
+      "step": 25,
+      "train_speed(iter/s)": 0.073811
+    },
+    {
+      "epoch": 0.0384,
+      "grad_norm": 17.052732467651367,
+      "learning_rate": 7.5e-07,
+      "logits/chosen": -2.746875047683716,
+      "logits/rejected": -2.6578125953674316,
+      "logps/chosen": -225.64999389648438,
+      "logps/rejected": -418.5,
+      "loss": 0.98515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31513673067092896,
+      "rewards/accuracies": 0.5296875238418579,
+      "rewards/chosen": 0.0872802734375,
+      "rewards/margins": 0.054694365710020065,
+      "rewards/rejected": 0.03258209303021431,
+      "step": 30,
+      "train_speed(iter/s)": 0.074808
+    },
+    {
+      "epoch": 0.0448,
+      "grad_norm": 15.013062477111816,
+      "learning_rate": 8.75e-07,
+      "logits/chosen": -2.772656202316284,
+      "logits/rejected": -2.663281202316284,
+      "logps/chosen": -206.85000610351562,
+      "logps/rejected": -415.1000061035156,
+      "loss": 0.948095703125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2845214903354645,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.14262695610523224,
+      "rewards/margins": 0.07264938205480576,
+      "rewards/rejected": 0.07000770419836044,
+      "step": 35,
+      "train_speed(iter/s)": 0.075872
+    },
+    {
+      "epoch": 0.0512,
+      "grad_norm": 12.79289436340332,
+      "learning_rate": 1e-06,
+      "logits/chosen": -2.746875047683716,
+      "logits/rejected": -2.6851563453674316,
+      "logps/chosen": -225.0,
+      "logps/rejected": -414.1000061035156,
+      "loss": 0.953515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3084472715854645,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.260498046875,
+      "rewards/margins": 0.12696532905101776,
+      "rewards/rejected": 0.13334961235523224,
+      "step": 40,
+      "train_speed(iter/s)": 0.076715
+    },
+    {
+      "epoch": 0.0576,
+      "grad_norm": 14.1936674118042,
+      "learning_rate": 9.998879644803925e-07,
+      "logits/chosen": -2.742968797683716,
+      "logits/rejected": -2.589062452316284,
+      "logps/chosen": -204.9499969482422,
+      "logps/rejected": -444.70001220703125,
+      "loss": 0.90703125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.290283203125,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": 0.40351563692092896,
+      "rewards/margins": 0.21682128310203552,
+      "rewards/rejected": 0.18662109971046448,
+      "step": 45,
+      "train_speed(iter/s)": 0.077534
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 10.757648468017578,
+      "learning_rate": 9.995519081294008e-07,
+      "logits/chosen": -2.73828125,
+      "logits/rejected": -2.64453125,
+      "logps/chosen": -205.1999969482422,
+      "logps/rejected": -384.3500061035156,
+      "loss": 0.8928955078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2987304627895355,
+      "rewards/accuracies": 0.6734374761581421,
+      "rewards/chosen": 0.4620117247104645,
+      "rewards/margins": 0.28678664565086365,
+      "rewards/rejected": 0.17512206733226776,
+      "step": 50,
+      "train_speed(iter/s)": 0.078246
+    },
+    {
+      "epoch": 0.0704,
+      "grad_norm": 11.549966812133789,
+      "learning_rate": 9.989919815480165e-07,
+      "logits/chosen": -2.706249952316284,
+      "logits/rejected": -2.604687452316284,
+      "logps/chosen": -206.6999969482422,
+      "logps/rejected": -426.6000061035156,
+      "loss": 0.8850830078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29228514432907104,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": 0.4903320372104645,
+      "rewards/margins": 0.3165283203125,
+      "rewards/rejected": 0.17380523681640625,
+      "step": 55,
+      "train_speed(iter/s)": 0.078902
+    },
+    {
+      "epoch": 0.0768,
+      "grad_norm": 10.8017578125,
+      "learning_rate": 9.982084356629014e-07,
+      "logits/chosen": -2.686718702316284,
+      "logits/rejected": -2.602343797683716,
+      "logps/chosen": -206.39999389648438,
+      "logps/rejected": -455.3999938964844,
+      "loss": 0.8768798828125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.294677734375,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.535351574420929,
+      "rewards/margins": 0.39509278535842896,
+      "rewards/rejected": 0.14046935737133026,
+      "step": 60,
+      "train_speed(iter/s)": 0.079446
+    },
+    {
+      "epoch": 0.0832,
+      "grad_norm": 11.038475036621094,
+      "learning_rate": 9.97201621613937e-07,
+      "logits/chosen": -2.7007813453674316,
+      "logits/rejected": -2.640625,
+      "logps/chosen": -205.9499969482422,
+      "logps/rejected": -416.5,
+      "loss": 0.8729248046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29584962129592896,
+      "rewards/accuracies": 0.660937488079071,
+      "rewards/chosen": 0.541210949420929,
+      "rewards/margins": 0.4347167909145355,
+      "rewards/rejected": 0.10672302544116974,
+      "step": 65,
+      "train_speed(iter/s)": 0.079945
+    },
+    {
+      "epoch": 0.0896,
+      "grad_norm": 9.712876319885254,
+      "learning_rate": 9.95971990596864e-07,
+      "logits/chosen": -2.7046875953674316,
+      "logits/rejected": -2.667187452316284,
+      "logps/chosen": -211.4499969482422,
+      "logps/rejected": -445.0,
+      "loss": 0.854296875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3075195252895355,
+      "rewards/accuracies": 0.698437511920929,
+      "rewards/chosen": 0.524707019329071,
+      "rewards/margins": 0.51416015625,
+      "rewards/rejected": 0.01068725623190403,
+      "step": 70,
+      "train_speed(iter/s)": 0.080368
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 10.94192123413086,
+      "learning_rate": 9.94520093661082e-07,
+      "logits/chosen": -2.690624952316284,
+      "logits/rejected": -2.6429686546325684,
+      "logps/chosen": -214.1999969482422,
+      "logps/rejected": -399.29998779296875,
+      "loss": 0.875146484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2998046875,
+      "rewards/accuracies": 0.6890624761581421,
+      "rewards/chosen": 0.509570300579071,
+      "rewards/margins": 0.48920899629592896,
+      "rewards/rejected": 0.020348358899354935,
+      "step": 75,
+      "train_speed(iter/s)": 0.080761
+    },
+    {
+      "epoch": 0.1024,
+      "grad_norm": 12.23570442199707,
+      "learning_rate": 9.928465814627014e-07,
+      "logits/chosen": -2.6859374046325684,
+      "logits/rejected": -2.676562547683716,
+      "logps/chosen": -206.5,
+      "logps/rejected": -508.79998779296875,
+      "loss": 0.836328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29765623807907104,
+      "rewards/accuracies": 0.707812488079071,
+      "rewards/chosen": 0.49541014432907104,
+      "rewards/margins": 0.583203136920929,
+      "rewards/rejected": -0.087799072265625,
+      "step": 80,
+      "train_speed(iter/s)": 0.081095
+    },
+    {
+      "epoch": 0.1088,
+      "grad_norm": 10.154147148132324,
+      "learning_rate": 9.90952203972957e-07,
+      "logits/chosen": -2.7249999046325684,
+      "logits/rejected": -2.663281202316284,
+      "logps/chosen": -204.10000610351562,
+      "logps/rejected": -430.5,
+      "loss": 0.8203369140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29389649629592896,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.525195300579071,
+      "rewards/margins": 0.6031249761581421,
+      "rewards/rejected": -0.07814941555261612,
+      "step": 85,
+      "train_speed(iter/s)": 0.081414
+    },
+    {
+      "epoch": 0.1152,
+      "grad_norm": 10.787557601928711,
+      "learning_rate": 9.888378101421147e-07,
+      "logits/chosen": -2.7289061546325684,
+      "logits/rejected": -2.6500000953674316,
+      "logps/chosen": -206.89999389648438,
+      "logps/rejected": -415.1000061035156,
+      "loss": 0.84151611328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.306884765625,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.56005859375,
+      "rewards/margins": 0.615429699420929,
+      "rewards/rejected": -0.05517578125,
+      "step": 90,
+      "train_speed(iter/s)": 0.08169
+    },
+    {
+      "epoch": 0.1216,
+      "grad_norm": 9.400640487670898,
+      "learning_rate": 9.8650434751902e-07,
+      "logits/chosen": -2.734375,
+      "logits/rejected": -2.6953125,
+      "logps/chosen": -208.5,
+      "logps/rejected": -458.20001220703125,
+      "loss": 0.84471435546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30131834745407104,
+      "rewards/accuracies": 0.714062511920929,
+      "rewards/chosen": 0.522656261920929,
+      "rewards/margins": 0.624707043170929,
+      "rewards/rejected": -0.10174636542797089,
+      "step": 95,
+      "train_speed(iter/s)": 0.081934
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 10.11943531036377,
+      "learning_rate": 9.839528618264632e-07,
+      "logits/chosen": -2.742968797683716,
+      "logits/rejected": -2.6953125,
+      "logps/chosen": -219.0,
+      "logps/rejected": -454.29998779296875,
+      "loss": 0.8283203125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2986816465854645,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.556640625,
+      "rewards/margins": 0.659423828125,
+      "rewards/rejected": -0.10334549099206924,
+      "step": 100,
+      "train_speed(iter/s)": 0.082157
+    },
+    {
+      "epoch": 0.1344,
+      "grad_norm": 10.488848686218262,
+      "learning_rate": 9.811844964925453e-07,
+      "logits/chosen": -2.7796874046325684,
+      "logits/rejected": -2.6546874046325684,
+      "logps/chosen": -209.8000030517578,
+      "logps/rejected": -431.5,
+      "loss": 0.85343017578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2945312559604645,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": 0.5352538824081421,
+      "rewards/margins": 0.599658191204071,
+      "rewards/rejected": -0.06458969414234161,
+      "step": 105,
+      "train_speed(iter/s)": 0.081627
+    },
+    {
+      "epoch": 0.1408,
+      "grad_norm": 11.632570266723633,
+      "learning_rate": 9.78200492138261e-07,
+      "logits/chosen": -2.813281297683716,
+      "logits/rejected": -2.7085938453674316,
+      "logps/chosen": -204.8000030517578,
+      "logps/rejected": -438.29998779296875,
+      "loss": 0.80316162109375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27924805879592896,
+      "rewards/accuracies": 0.707812488079071,
+      "rewards/chosen": 0.637988269329071,
+      "rewards/margins": 0.6630859375,
+      "rewards/rejected": -0.02523498609662056,
+      "step": 110,
+      "train_speed(iter/s)": 0.081841
+    },
+    {
+      "epoch": 0.1472,
+      "grad_norm": 9.546564102172852,
+      "learning_rate": 9.75002186021524e-07,
+      "logits/chosen": -2.8031249046325684,
+      "logits/rejected": -2.703906297683716,
+      "logps/chosen": -190.9250030517578,
+      "logps/rejected": -431.6000061035156,
+      "loss": 0.7993896484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28173828125,
+      "rewards/accuracies": 0.7328125238418579,
+      "rewards/chosen": 0.621874988079071,
+      "rewards/margins": 0.6910644769668579,
+      "rewards/rejected": -0.06916351616382599,
+      "step": 115,
+      "train_speed(iter/s)": 0.082038
+    },
+    {
+      "epoch": 0.1536,
+      "grad_norm": 10.464920997619629,
+      "learning_rate": 9.715910114378845e-07,
+      "logits/chosen": -2.780468702316284,
+      "logits/rejected": -2.7125000953674316,
+      "logps/chosen": -205.39999389648438,
+      "logps/rejected": -455.1000061035156,
+      "loss": 0.824658203125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29345703125,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.5582031011581421,
+      "rewards/margins": 0.7310546636581421,
+      "rewards/rejected": -0.17250975966453552,
+      "step": 120,
+      "train_speed(iter/s)": 0.082221
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 10.212596893310547,
+      "learning_rate": 9.679684970782104e-07,
+      "logits/chosen": -2.803906202316284,
+      "logits/rejected": -2.758593797683716,
+      "logps/chosen": -193.8000030517578,
+      "logps/rejected": -401.79998779296875,
+      "loss": 0.79718017578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2833007872104645,
+      "rewards/accuracies": 0.729687511920929,
+      "rewards/chosen": 0.586718738079071,
+      "rewards/margins": 0.743945300579071,
+      "rewards/rejected": -0.15787354111671448,
+      "step": 125,
+      "train_speed(iter/s)": 0.082357
+    },
+    {
+      "epoch": 0.1664,
+      "grad_norm": 8.520217895507812,
+      "learning_rate": 9.641362663436161e-07,
+      "logits/chosen": -2.788281202316284,
+      "logits/rejected": -2.7085938453674316,
+      "logps/chosen": -199.8000030517578,
+      "logps/rejected": -383.70001220703125,
+      "loss": 0.79482421875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2936035096645355,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": 0.615234375,
+      "rewards/margins": 0.7320312261581421,
+      "rewards/rejected": -0.11729736626148224,
+      "step": 130,
+      "train_speed(iter/s)": 0.082521
+    },
+    {
+      "epoch": 0.1728,
+      "grad_norm": 9.796134948730469,
+      "learning_rate": 9.600960366179477e-07,
+      "logits/chosen": -2.848437547683716,
+      "logits/rejected": -2.746875047683716,
+      "logps/chosen": -225.14999389648438,
+      "logps/rejected": -453.8999938964844,
+      "loss": 0.8017578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.308349609375,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.563769519329071,
+      "rewards/margins": 0.8392578363418579,
+      "rewards/rejected": -0.276092529296875,
+      "step": 135,
+      "train_speed(iter/s)": 0.082656
+    },
+    {
+      "epoch": 0.1792,
+      "grad_norm": 9.783967018127441,
+      "learning_rate": 9.558496184981524e-07,
+      "logits/chosen": -2.8046875,
+      "logits/rejected": -2.6953125,
+      "logps/chosen": -197.75,
+      "logps/rejected": -491.8999938964844,
+      "loss": 0.79949951171875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29228514432907104,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": 0.5762695074081421,
+      "rewards/margins": 0.7679687738418579,
+      "rewards/rejected": -0.19173583388328552,
+      "step": 140,
+      "train_speed(iter/s)": 0.082791
+    },
+    {
+      "epoch": 0.1856,
+      "grad_norm": 12.078418731689453,
+      "learning_rate": 9.513989149828717e-07,
+      "logits/chosen": -2.856250047683716,
+      "logits/rejected": -2.72265625,
+      "logps/chosen": -205.39999389648438,
+      "logps/rejected": -463.79998779296875,
+      "loss": 0.79444580078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29316407442092896,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.6298828125,
+      "rewards/margins": 0.8199218511581421,
+      "rewards/rejected": -0.18983764946460724,
+      "step": 145,
+      "train_speed(iter/s)": 0.082913
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 10.10171890258789,
+      "learning_rate": 9.467459206196298e-07,
+      "logits/chosen": -2.80859375,
+      "logits/rejected": -2.723437547683716,
+      "logps/chosen": -208.64999389648438,
+      "logps/rejected": -429.8500061035156,
+      "loss": 0.80247802734375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2978515625,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": 0.565625011920929,
+      "rewards/margins": 0.8257812261581421,
+      "rewards/rejected": -0.26054686307907104,
+      "step": 150,
+      "train_speed(iter/s)": 0.083037
+    },
+    {
+      "epoch": 0.1984,
+      "grad_norm": 11.226574897766113,
+      "learning_rate": 9.418927206109912e-07,
+      "logits/chosen": -2.813281297683716,
+      "logits/rejected": -2.757031202316284,
+      "logps/chosen": -219.3000030517578,
+      "logps/rejected": -431.0,
+      "loss": 0.82308349609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3189453184604645,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.6080077886581421,
+      "rewards/margins": 0.8443359136581421,
+      "rewards/rejected": -0.23585815727710724,
+      "step": 155,
+      "train_speed(iter/s)": 0.083156
+    },
+    {
+      "epoch": 0.2048,
+      "grad_norm": 10.251683235168457,
+      "learning_rate": 9.36841489880095e-07,
+      "logits/chosen": -2.9140625,
+      "logits/rejected": -2.7914061546325684,
+      "logps/chosen": -193.85000610351562,
+      "logps/rejected": -452.3999938964844,
+      "loss": 0.77923583984375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2784667909145355,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.630859375,
+      "rewards/margins": 0.8724609613418579,
+      "rewards/rejected": -0.24244995415210724,
+      "step": 160,
+      "train_speed(iter/s)": 0.083264
+    },
+    {
+      "epoch": 0.2112,
+      "grad_norm": 11.021358489990234,
+      "learning_rate": 9.315944920959798e-07,
+      "logits/chosen": -2.8882813453674316,
+      "logits/rejected": -2.76171875,
+      "logps/chosen": -202.89999389648438,
+      "logps/rejected": -386.79998779296875,
+      "loss": 0.78292236328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2835449278354645,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.632617175579071,
+      "rewards/margins": 0.851367175579071,
+      "rewards/rejected": -0.21873779594898224,
+      "step": 165,
+      "train_speed(iter/s)": 0.083367
+    },
+    {
+      "epoch": 0.2176,
+      "grad_norm": 12.76692008972168,
+      "learning_rate": 9.261540786591376e-07,
+      "logits/chosen": -2.835156202316284,
+      "logits/rejected": -2.7476563453674316,
+      "logps/chosen": -219.14999389648438,
+      "logps/rejected": -466.45001220703125,
+      "loss": 0.80499267578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3189941346645355,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.598437488079071,
+      "rewards/margins": 0.943164050579071,
+      "rewards/rejected": -0.34455567598342896,
+      "step": 170,
+      "train_speed(iter/s)": 0.083464
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 10.589763641357422,
+      "learning_rate": 9.205226876477539e-07,
+      "logits/chosen": -2.8382811546325684,
+      "logits/rejected": -2.7210936546325684,
+      "logps/chosen": -200.1999969482422,
+      "logps/rejected": -411.79998779296875,
+      "loss": 0.81788330078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3031249940395355,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.5677734613418579,
+      "rewards/margins": 0.854199230670929,
+      "rewards/rejected": -0.28624266386032104,
+      "step": 175,
+      "train_speed(iter/s)": 0.083554
+    },
+    {
+      "epoch": 0.2304,
+      "grad_norm": 9.48745346069336,
+      "learning_rate": 9.147028427251009e-07,
+      "logits/chosen": -2.8375000953674316,
+      "logits/rejected": -2.7484374046325684,
+      "logps/chosen": -217.9499969482422,
+      "logps/rejected": -475.5,
+      "loss": 0.82000732421875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.310791015625,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": 0.4823242127895355,
+      "rewards/margins": 0.9468749761581421,
+      "rewards/rejected": -0.4643188416957855,
+      "step": 180,
+      "train_speed(iter/s)": 0.083641
+    },
+    {
+      "epoch": 0.2368,
+      "grad_norm": 10.949711799621582,
+      "learning_rate": 9.086971520085782e-07,
+      "logits/chosen": -2.8765625953674316,
+      "logits/rejected": -2.7953124046325684,
+      "logps/chosen": -207.75,
+      "logps/rejected": -441.70001220703125,
+      "loss": 0.799853515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30029296875,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.569140613079071,
+      "rewards/margins": 0.9800781011581421,
+      "rewards/rejected": -0.4107910096645355,
+      "step": 185,
+      "train_speed(iter/s)": 0.083711
+    },
+    {
+      "epoch": 0.2432,
+      "grad_norm": 9.860713958740234,
+      "learning_rate": 9.025083069009063e-07,
+      "logits/chosen": -2.8765625953674316,
+      "logits/rejected": -2.821093797683716,
+      "logps/chosen": -204.89999389648438,
+      "logps/rejected": -442.1000061035156,
+      "loss": 0.7978759765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28857421875,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.551562488079071,
+      "rewards/margins": 0.950488269329071,
+      "rewards/rejected": -0.39948731660842896,
+      "step": 190,
+      "train_speed(iter/s)": 0.083784
+    },
+    {
+      "epoch": 0.2496,
+      "grad_norm": 10.864822387695312,
+      "learning_rate": 8.961390808839944e-07,
+      "logits/chosen": -2.852343797683716,
+      "logits/rejected": -2.757031202316284,
+      "logps/chosen": -234.6999969482422,
+      "logps/rejected": -510.79998779296875,
+      "loss": 0.8002685546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.32036131620407104,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.5894531011581421,
+      "rewards/margins": 0.9781249761581421,
+      "rewards/rejected": -0.38850098848342896,
+      "step": 195,
+      "train_speed(iter/s)": 0.08385
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 9.288454055786133,
+      "learning_rate": 8.895923282760279e-07,
+      "logits/chosen": -2.86328125,
+      "logits/rejected": -2.7953124046325684,
+      "logps/chosen": -210.39999389648438,
+      "logps/rejected": -457.3999938964844,
+      "loss": 0.7908935546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2958984375,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.543652355670929,
+      "rewards/margins": 0.9458984136581421,
+      "rewards/rejected": -0.4026855528354645,
+      "step": 200,
+      "train_speed(iter/s)": 0.083923
+    },
+    {
+      "epoch": 0.2624,
+      "grad_norm": 9.009343147277832,
+      "learning_rate": 8.828709829523275e-07,
+      "logits/chosen": -2.8726563453674316,
+      "logits/rejected": -2.780468702316284,
+      "logps/chosen": -194.5,
+      "logps/rejected": -479.0,
+      "loss": 0.7591064453125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2847656309604645,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": 0.5716797113418579,
+      "rewards/margins": 0.990429699420929,
+      "rewards/rejected": -0.4192871153354645,
+      "step": 205,
+      "train_speed(iter/s)": 0.08361
+    },
+    {
+      "epoch": 0.2688,
+      "grad_norm": 9.1495361328125,
+      "learning_rate": 8.759780570305564e-07,
+      "logits/chosen": -2.878124952316284,
+      "logits/rejected": -2.796875,
+      "logps/chosen": -205.8000030517578,
+      "logps/rejected": -444.29998779296875,
+      "loss": 0.78328857421875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3006347715854645,
+      "rewards/accuracies": 0.7484375238418579,
+      "rewards/chosen": 0.557324230670929,
+      "rewards/margins": 0.982226550579071,
+      "rewards/rejected": -0.42534178495407104,
+      "step": 210,
+      "train_speed(iter/s)": 0.083681
+    },
+    {
+      "epoch": 0.2752,
+      "grad_norm": 9.955195426940918,
+      "learning_rate": 8.689166395208636e-07,
+      "logits/chosen": -2.890625,
+      "logits/rejected": -2.78515625,
+      "logps/chosen": -229.4499969482422,
+      "logps/rejected": -450.0,
+      "loss": 0.78609619140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3167480528354645,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.525683581829071,
+      "rewards/margins": 1.076757788658142,
+      "rewards/rejected": -0.5508788824081421,
+      "step": 215,
+      "train_speed(iter/s)": 0.083754
+    },
+    {
+      "epoch": 0.2816,
+      "grad_norm": 10.033121109008789,
+      "learning_rate": 8.616898949415686e-07,
+      "logits/chosen": -2.852343797683716,
+      "logits/rejected": -2.789843797683716,
+      "logps/chosen": -204.25,
+      "logps/rejected": -465.20001220703125,
+      "loss": 0.75965576171875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28950196504592896,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.5868164300918579,
+      "rewards/margins": 1.072265625,
+      "rewards/rejected": -0.4854736328125,
+      "step": 220,
+      "train_speed(iter/s)": 0.083819
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 12.535127639770508,
+      "learning_rate": 8.543010619010076e-07,
+      "logits/chosen": -2.8695311546325684,
+      "logits/rejected": -2.792187452316284,
+      "logps/chosen": -205.9499969482422,
+      "logps/rejected": -469.0,
+      "loss": 0.77884521484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30302733182907104,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.48383790254592896,
+      "rewards/margins": 1.1123046875,
+      "rewards/rejected": -0.628222644329071,
+      "step": 225,
+      "train_speed(iter/s)": 0.083877
+    },
+    {
+      "epoch": 0.2944,
+      "grad_norm": 8.69505786895752,
+      "learning_rate": 8.467534516461761e-07,
+      "logits/chosen": -2.8695311546325684,
+      "logits/rejected": -2.811718702316284,
+      "logps/chosen": -218.6999969482422,
+      "logps/rejected": -467.25,
+      "loss": 0.79259033203125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.316162109375,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.5428222417831421,
+      "rewards/margins": 1.0798828601837158,
+      "rewards/rejected": -0.537841796875,
+      "step": 230,
+      "train_speed(iter/s)": 0.083934
+    },
+    {
+      "epoch": 0.3008,
+      "grad_norm": 9.911195755004883,
+      "learning_rate": 8.390504465788215e-07,
+      "logits/chosen": -2.930468797683716,
+      "logits/rejected": -2.8046875,
+      "logps/chosen": -207.14999389648438,
+      "logps/rejected": -392.70001220703125,
+      "loss": 0.7896484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.290283203125,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.529589831829071,
+      "rewards/margins": 1.034765601158142,
+      "rewards/rejected": -0.5060058832168579,
+      "step": 235,
+      "train_speed(iter/s)": 0.083974
+    },
+    {
+      "epoch": 0.3072,
+      "grad_norm": 12.016138076782227,
+      "learning_rate": 8.311954987396444e-07,
+      "logits/chosen": -2.918750047683716,
+      "logits/rejected": -2.8578124046325684,
+      "logps/chosen": -199.0500030517578,
+      "logps/rejected": -379.0,
+      "loss": 0.7730712890625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28022462129592896,
+      "rewards/accuracies": 0.729687511920929,
+      "rewards/chosen": 0.566210925579071,
+      "rewards/margins": 0.9869629144668579,
+      "rewards/rejected": -0.4202514588832855,
+      "step": 240,
+      "train_speed(iter/s)": 0.08402
+    },
+    {
+      "epoch": 0.3136,
+      "grad_norm": 9.696317672729492,
+      "learning_rate": 8.231921282612956e-07,
+      "logits/chosen": -2.899218797683716,
+      "logits/rejected": -2.8148436546325684,
+      "logps/chosen": -202.3000030517578,
+      "logps/rejected": -436.1000061035156,
+      "loss": 0.7557861328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29912108182907104,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.5469726324081421,
+      "rewards/margins": 1.129003882408142,
+      "rewards/rejected": -0.581591784954071,
+      "step": 245,
+      "train_speed(iter/s)": 0.084066
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 9.822099685668945,
+      "learning_rate": 8.150439217908556e-07,
+      "logits/chosen": -2.8828125,
+      "logits/rejected": -2.803906202316284,
+      "logps/chosen": -221.4499969482422,
+      "logps/rejected": -473.70001220703125,
+      "loss": 0.788458251953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31396484375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.5254882574081421,
+      "rewards/margins": 1.1140625476837158,
+      "rewards/rejected": -0.588330090045929,
+      "step": 250,
+      "train_speed(iter/s)": 0.084119
+    },
+    {
+      "epoch": 0.3264,
+      "grad_norm": 8.301116943359375,
+      "learning_rate": 8.067545308825075e-07,
+      "logits/chosen": -2.9437499046325684,
+      "logits/rejected": -2.859375,
+      "logps/chosen": -194.5,
+      "logps/rejected": -435.70001220703125,
+      "loss": 0.71109619140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27783203125,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.5865234136581421,
+      "rewards/margins": 1.225195288658142,
+      "rewards/rejected": -0.639941394329071,
+      "step": 255,
+      "train_speed(iter/s)": 0.084162
+    },
+    {
+      "epoch": 0.3328,
+      "grad_norm": 10.535124778747559,
+      "learning_rate": 7.983276703611221e-07,
+      "logits/chosen": -2.9117188453674316,
+      "logits/rejected": -2.852343797683716,
+      "logps/chosen": -205.60000610351562,
+      "logps/rejected": -421.79998779296875,
+      "loss": 0.78076171875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.298828125,
+      "rewards/accuracies": 0.745312511920929,
+      "rewards/chosen": 0.4937500059604645,
+      "rewards/margins": 1.0968749523162842,
+      "rewards/rejected": -0.603808581829071,
+      "step": 260,
+      "train_speed(iter/s)": 0.084181
+    },
+    {
+      "epoch": 0.3392,
+      "grad_norm": 12.616129875183105,
+      "learning_rate": 7.897671166574877e-07,
+      "logits/chosen": -2.903125047683716,
+      "logits/rejected": -2.8140625953674316,
+      "logps/chosen": -215.3000030517578,
+      "logps/rejected": -429.79998779296875,
+      "loss": 0.760791015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.300048828125,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.48046875,
+      "rewards/margins": 1.2130858898162842,
+      "rewards/rejected": -0.731982409954071,
+      "step": 265,
+      "train_speed(iter/s)": 0.084231
+    },
+    {
+      "epoch": 0.3456,
+      "grad_norm": 12.294151306152344,
+      "learning_rate": 7.810767061159341e-07,
+      "logits/chosen": -2.9281249046325684,
+      "logits/rejected": -2.835156202316284,
+      "logps/chosen": -223.89999389648438,
+      "logps/rejected": -462.79998779296875,
+      "loss": 0.7583251953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30522459745407104,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": 0.5538085699081421,
+      "rewards/margins": 1.266210913658142,
+      "rewards/rejected": -0.71142578125,
+      "step": 270,
+      "train_speed(iter/s)": 0.084265
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 9.941428184509277,
+      "learning_rate": 7.722603332751034e-07,
+      "logits/chosen": -2.9632811546325684,
+      "logits/rejected": -2.844531297683716,
+      "logps/chosen": -215.25,
+      "logps/rejected": -444.3999938964844,
+      "loss": 0.8036865234375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.300048828125,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.505175769329071,
+      "rewards/margins": 1.0949218273162842,
+      "rewards/rejected": -0.5890868902206421,
+      "step": 275,
+      "train_speed(iter/s)": 0.084301
+    },
+    {
+      "epoch": 0.3584,
+      "grad_norm": 10.58115005493164,
+      "learning_rate": 7.633219491226448e-07,
+      "logits/chosen": -2.94921875,
+      "logits/rejected": -2.8695311546325684,
+      "logps/chosen": -184.64999389648438,
+      "logps/rejected": -458.6000061035156,
+      "loss": 0.7162353515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.274658203125,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.586621105670929,
+      "rewards/margins": 1.2373046875,
+      "rewards/rejected": -0.6509765386581421,
+      "step": 280,
+      "train_speed(iter/s)": 0.084343
+    },
+    {
+      "epoch": 0.3648,
+      "grad_norm": 9.514520645141602,
+      "learning_rate": 7.542655593246103e-07,
+      "logits/chosen": -2.9390625953674316,
+      "logits/rejected": -2.83984375,
+      "logps/chosen": -213.0500030517578,
+      "logps/rejected": -395.6000061035156,
+      "loss": 0.77989501953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30024415254592896,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.517138659954071,
+      "rewards/margins": 1.0822265148162842,
+      "rewards/rejected": -0.564160168170929,
+      "step": 285,
+      "train_speed(iter/s)": 0.084388
+    },
+    {
+      "epoch": 0.3712,
+      "grad_norm": 16.591611862182617,
+      "learning_rate": 7.450952224303468e-07,
+      "logits/chosen": -2.918750047683716,
+      "logits/rejected": -2.82421875,
+      "logps/chosen": -226.89999389648438,
+      "logps/rejected": -456.70001220703125,
+      "loss": 0.823974609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3056640625,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": 0.483154296875,
+      "rewards/margins": 1.0153319835662842,
+      "rewards/rejected": -0.532519519329071,
+      "step": 290,
+      "train_speed(iter/s)": 0.084428
+    },
+    {
+      "epoch": 0.3776,
+      "grad_norm": 9.35551643371582,
+      "learning_rate": 7.358150480536903e-07,
+      "logits/chosen": -2.9437499046325684,
+      "logits/rejected": -2.825000047683716,
+      "logps/chosen": -203.25,
+      "logps/rejected": -442.6000061035156,
+      "loss": 0.729248046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2872070372104645,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.5810546875,
+      "rewards/margins": 1.1746094226837158,
+      "rewards/rejected": -0.5936034917831421,
+      "step": 295,
+      "train_speed(iter/s)": 0.084468
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 10.175483703613281,
+      "learning_rate": 7.26429195031274e-07,
+      "logits/chosen": -2.9203124046325684,
+      "logits/rejected": -2.83984375,
+      "logps/chosen": -214.10000610351562,
+      "logps/rejected": -428.0,
+      "loss": 0.75811767578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30561524629592896,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": 0.578320324420929,
+      "rewards/margins": 1.187890648841858,
+      "rewards/rejected": -0.60888671875,
+      "step": 300,
+      "train_speed(iter/s)": 0.084504
+    },
+    {
+      "epoch": 0.3904,
+      "grad_norm": 8.5143404006958,
+      "learning_rate": 7.16941869558779e-07,
+      "logits/chosen": -2.917187452316284,
+      "logits/rejected": -2.807812452316284,
+      "logps/chosen": -210.14999389648438,
+      "logps/rejected": -444.0,
+      "loss": 0.7674072265625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2982421815395355,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.529492199420929,
+      "rewards/margins": 1.193945288658142,
+      "rewards/rejected": -0.6636718511581421,
+      "step": 305,
+      "train_speed(iter/s)": 0.084275
+    },
+    {
+      "epoch": 0.3968,
+      "grad_norm": 10.271326065063477,
+      "learning_rate": 7.073573233059616e-07,
+      "logits/chosen": -2.924999952316284,
+      "logits/rejected": -2.821093797683716,
+      "logps/chosen": -198.6999969482422,
+      "logps/rejected": -459.8999938964844,
+      "loss": 0.7200927734375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2811523377895355,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.4892578125,
+      "rewards/margins": 1.3126952648162842,
+      "rewards/rejected": -0.8232421875,
+      "step": 310,
+      "train_speed(iter/s)": 0.084314
+    },
+    {
+      "epoch": 0.4032,
+      "grad_norm": 8.532340049743652,
+      "learning_rate": 6.976798515113003e-07,
+      "logits/chosen": -2.88671875,
+      "logits/rejected": -2.840625047683716,
+      "logps/chosen": -208.9499969482422,
+      "logps/rejected": -453.5,
+      "loss": 0.74866943359375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2962890565395355,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.4480957090854645,
+      "rewards/margins": 1.264257788658142,
+      "rewards/rejected": -0.8165038824081421,
+      "step": 315,
+      "train_speed(iter/s)": 0.084342
+    },
+    {
+      "epoch": 0.4096,
+      "grad_norm": 9.349964141845703,
+      "learning_rate": 6.87913791057119e-07,
+      "logits/chosen": -2.922656297683716,
+      "logits/rejected": -2.8148436546325684,
+      "logps/chosen": -215.85000610351562,
+      "logps/rejected": -440.79998779296875,
+      "loss": 0.7771484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29814451932907104,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.48857420682907104,
+      "rewards/margins": 1.1326172351837158,
+      "rewards/rejected": -0.6444336175918579,
+      "step": 320,
+      "train_speed(iter/s)": 0.084339
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 9.788792610168457,
+      "learning_rate": 6.780635185260478e-07,
+      "logits/chosen": -2.918750047683716,
+      "logits/rejected": -2.8109374046325684,
+      "logps/chosen": -218.1999969482422,
+      "logps/rejected": -440.0,
+      "loss": 0.76365966796875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30561524629592896,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": 0.5625976324081421,
+      "rewards/margins": 1.238671898841858,
+      "rewards/rejected": -0.67626953125,
+      "step": 325,
+      "train_speed(iter/s)": 0.084318
+    },
+    {
+      "epoch": 0.4224,
+      "grad_norm": 9.145480155944824,
+      "learning_rate": 6.681334482396919e-07,
+      "logits/chosen": -2.9039063453674316,
+      "logits/rejected": -2.859375,
+      "logps/chosen": -239.25,
+      "logps/rejected": -405.29998779296875,
+      "loss": 0.80654296875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.32246094942092896,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.480224609375,
+      "rewards/margins": 1.132226586341858,
+      "rewards/rejected": -0.652783215045929,
+      "step": 330,
+      "train_speed(iter/s)": 0.084337
+    },
+    {
+      "epoch": 0.4288,
+      "grad_norm": 8.718400001525879,
+      "learning_rate": 6.581280302803881e-07,
+      "logits/chosen": -2.88671875,
+      "logits/rejected": -2.8531250953674316,
+      "logps/chosen": -207.9499969482422,
+      "logps/rejected": -432.0,
+      "loss": 0.76123046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.303466796875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.503173828125,
+      "rewards/margins": 1.217382788658142,
+      "rewards/rejected": -0.7139648199081421,
+      "step": 335,
+      "train_speed(iter/s)": 0.084342
+    },
+    {
+      "epoch": 0.4352,
+      "grad_norm": 9.433004379272461,
+      "learning_rate": 6.480517484969365e-07,
+      "logits/chosen": -2.921093702316284,
+      "logits/rejected": -2.8453125953674316,
+      "logps/chosen": -206.1999969482422,
+      "logps/rejected": -498.0,
+      "loss": 0.731048583984375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2965331971645355,
+      "rewards/accuracies": 0.801562488079071,
+      "rewards/chosen": 0.56689453125,
+      "rewards/margins": 1.2658202648162842,
+      "rewards/rejected": -0.6991211175918579,
+      "step": 340,
+      "train_speed(iter/s)": 0.084329
+    },
+    {
+      "epoch": 0.4416,
+      "grad_norm": 8.827898025512695,
+      "learning_rate": 6.379091184951983e-07,
+      "logits/chosen": -2.870312452316284,
+      "logits/rejected": -2.8062500953674316,
+      "logps/chosen": -224.89999389648438,
+      "logps/rejected": -464.6000061035156,
+      "loss": 0.763507080078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3106445372104645,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.4715332090854645,
+      "rewards/margins": 1.28515625,
+      "rewards/rejected": -0.81494140625,
+      "step": 345,
+      "train_speed(iter/s)": 0.08435
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 9.436395645141602,
+      "learning_rate": 6.277046856144632e-07,
+      "logits/chosen": -2.928906202316284,
+      "logits/rejected": -2.8492188453674316,
+      "logps/chosen": -212.39999389648438,
+      "logps/rejected": -486.29998779296875,
+      "loss": 0.734429931640625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29936522245407104,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.5538085699081421,
+      "rewards/margins": 1.330078125,
+      "rewards/rejected": -0.775390625,
+      "step": 350,
+      "train_speed(iter/s)": 0.084365
+    },
+    {
+      "epoch": 0.4544,
+      "grad_norm": 9.7159423828125,
+      "learning_rate": 6.174430228904919e-07,
+      "logits/chosen": -2.956249952316284,
+      "logits/rejected": -2.83984375,
+      "logps/chosen": -202.39999389648438,
+      "logps/rejected": -489.20001220703125,
+      "loss": 0.73070068359375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2900390625,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.494140625,
+      "rewards/margins": 1.331445336341858,
+      "rewards/rejected": -0.836865246295929,
+      "step": 355,
+      "train_speed(iter/s)": 0.084374
+    },
+    {
+      "epoch": 0.4608,
+      "grad_norm": 10.715283393859863,
+      "learning_rate": 6.071287290061451e-07,
+      "logits/chosen": -2.924999952316284,
+      "logits/rejected": -2.8187499046325684,
+      "logps/chosen": -211.89999389648438,
+      "logps/rejected": -451.20001220703125,
+      "loss": 0.7508544921875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2901367247104645,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.4722045958042145,
+      "rewards/margins": 1.2810547351837158,
+      "rewards/rejected": -0.809277355670929,
+      "step": 360,
+      "train_speed(iter/s)": 0.084407
+    },
+    {
+      "epoch": 0.4672,
+      "grad_norm": 8.77979564666748,
+      "learning_rate": 5.96766426230522e-07,
+      "logits/chosen": -2.9156250953674316,
+      "logits/rejected": -2.8296875953674316,
+      "logps/chosen": -201.8000030517578,
+      "logps/rejected": -458.1000061035156,
+      "loss": 0.724267578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.285888671875,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.5166015625,
+      "rewards/margins": 1.334375023841858,
+      "rewards/rejected": -0.818359375,
+      "step": 365,
+      "train_speed(iter/s)": 0.084437
+    },
+    {
+      "epoch": 0.4736,
+      "grad_norm": 8.621392250061035,
+      "learning_rate": 5.863607583475258e-07,
+      "logits/chosen": -2.9429688453674316,
+      "logits/rejected": -2.870312452316284,
+      "logps/chosen": -190.39999389648438,
+      "logps/rejected": -423.70001220703125,
+      "loss": 0.7619873046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28120118379592896,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": 0.46757811307907104,
+      "rewards/margins": 1.193359375,
+      "rewards/rejected": -0.725146472454071,
+      "step": 370,
+      "train_speed(iter/s)": 0.084466
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 10.51291275024414,
+      "learning_rate": 5.759163885747889e-07,
+      "logits/chosen": -2.88671875,
+      "logits/rejected": -2.8140625953674316,
+      "logps/chosen": -225.8000030517578,
+      "logps/rejected": -437.3999938964844,
+      "loss": 0.80009765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3206543028354645,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": 0.44873046875,
+      "rewards/margins": 1.228124976158142,
+      "rewards/rejected": -0.77880859375,
+      "step": 375,
+      "train_speed(iter/s)": 0.084498
+    },
+    {
+      "epoch": 0.4864,
+      "grad_norm": 10.516741752624512,
+      "learning_rate": 5.654379974738892e-07,
+      "logits/chosen": -2.90625,
+      "logits/rejected": -2.811718702316284,
+      "logps/chosen": -208.1999969482422,
+      "logps/rejected": -427.3999938964844,
+      "loss": 0.7852294921875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28955078125,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.46015626192092896,
+      "rewards/margins": 1.220312476158142,
+      "rewards/rejected": -0.760449230670929,
+      "step": 380,
+      "train_speed(iter/s)": 0.084517
+    },
+    {
+      "epoch": 0.4928,
+      "grad_norm": 8.877614974975586,
+      "learning_rate": 5.549302808527934e-07,
+      "logits/chosen": -2.948437452316284,
+      "logits/rejected": -2.835156202316284,
+      "logps/chosen": -191.75,
+      "logps/rejected": -410.8999938964844,
+      "loss": 0.73380126953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2803710997104645,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.513476550579071,
+      "rewards/margins": 1.3224608898162842,
+      "rewards/rejected": -0.8094238042831421,
+      "step": 385,
+      "train_speed(iter/s)": 0.08452
+    },
+    {
+      "epoch": 0.4992,
+      "grad_norm": 10.124411582946777,
+      "learning_rate": 5.443979476614674e-07,
+      "logits/chosen": -2.914843797683716,
+      "logits/rejected": -2.842968702316284,
+      "logps/chosen": -219.64999389648438,
+      "logps/rejected": -402.1499938964844,
+      "loss": 0.781396484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29643553495407104,
+      "rewards/accuracies": 0.7328125238418579,
+      "rewards/chosen": 0.4615722596645355,
+      "rewards/margins": 1.2492187023162842,
+      "rewards/rejected": -0.788378894329071,
+      "step": 390,
+      "train_speed(iter/s)": 0.084524
+    },
+    {
+      "epoch": 0.5056,
+      "grad_norm": 10.239745140075684,
+      "learning_rate": 5.338457178815984e-07,
+      "logits/chosen": -2.907031297683716,
+      "logits/rejected": -2.805468797683716,
+      "logps/chosen": -224.3000030517578,
+      "logps/rejected": -474.70001220703125,
+      "loss": 0.77469482421875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29707032442092896,
+      "rewards/accuracies": 0.760937511920929,
+      "rewards/chosen": 0.4485107362270355,
+      "rewards/margins": 1.267187476158142,
+      "rewards/rejected": -0.8184570074081421,
+      "step": 395,
+      "train_speed(iter/s)": 0.084539
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 10.185434341430664,
+      "learning_rate": 5.232783204113719e-07,
+      "logits/chosen": -2.9078125953674316,
+      "logits/rejected": -2.793750047683716,
+      "logps/chosen": -221.14999389648438,
+      "logps/rejected": -448.79998779296875,
+      "loss": 0.76617431640625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3104003965854645,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.4823242127895355,
+      "rewards/margins": 1.3152344226837158,
+      "rewards/rejected": -0.833300769329071,
+      "step": 400,
+      "train_speed(iter/s)": 0.084559
+    },
+    {
+      "epoch": 0.5184,
+      "grad_norm": 9.6346435546875,
+      "learning_rate": 5.12700490946254e-07,
+      "logits/chosen": -2.9164061546325684,
+      "logits/rejected": -2.797656297683716,
+      "logps/chosen": -209.4499969482422,
+      "logps/rejected": -440.1000061035156,
+      "loss": 0.7625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2940917909145355,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.4845016598701477,
+      "rewards/margins": 1.2638671398162842,
+      "rewards/rejected": -0.779101550579071,
+      "step": 405,
+      "train_speed(iter/s)": 0.084376
+    },
+    {
+      "epoch": 0.5248,
+      "grad_norm": 7.919604778289795,
+      "learning_rate": 5.021169698567264e-07,
+      "logits/chosen": -2.901562452316284,
+      "logits/rejected": -2.7890625,
+      "logps/chosen": -189.10000610351562,
+      "logps/rejected": -421.6000061035156,
+      "loss": 0.715911865234375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2859863340854645,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.5425781011581421,
+      "rewards/margins": 1.278906226158142,
+      "rewards/rejected": -0.7369629144668579,
+      "step": 410,
+      "train_speed(iter/s)": 0.084405
+    },
+    {
+      "epoch": 0.5312,
+      "grad_norm": 9.456597328186035,
+      "learning_rate": 4.915325000639271e-07,
+      "logits/chosen": -2.905468702316284,
+      "logits/rejected": -2.8382811546325684,
+      "logps/chosen": -211.6999969482422,
+      "logps/rejected": -403.04998779296875,
+      "loss": 0.76348876953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29692381620407104,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": 0.46015626192092896,
+      "rewards/margins": 1.2185547351837158,
+      "rewards/rejected": -0.758496105670929,
+      "step": 415,
+      "train_speed(iter/s)": 0.084432
+    },
+    {
+      "epoch": 0.5376,
+      "grad_norm": 8.738895416259766,
+      "learning_rate": 4.809518249141483e-07,
+      "logits/chosen": -2.9124999046325684,
+      "logits/rejected": -2.7874999046325684,
+      "logps/chosen": -212.89999389648438,
+      "logps/rejected": -476.79998779296875,
+      "loss": 0.7605224609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31005859375,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.48095703125,
+      "rewards/margins": 1.341796875,
+      "rewards/rejected": -0.861328125,
+      "step": 420,
+      "train_speed(iter/s)": 0.08446
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 9.516702651977539,
+      "learning_rate": 4.703796860531429e-07,
+      "logits/chosen": -2.914843797683716,
+      "logits/rejected": -2.789843797683716,
+      "logps/chosen": -209.0,
+      "logps/rejected": -450.20001220703125,
+      "loss": 0.7545166015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31000977754592896,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": 0.5224609375,
+      "rewards/margins": 1.3156249523162842,
+      "rewards/rejected": -0.79345703125,
+      "step": 425,
+      "train_speed(iter/s)": 0.084487
+    },
+    {
+      "epoch": 0.5504,
+      "grad_norm": 10.566126823425293,
+      "learning_rate": 4.598208213011934e-07,
+      "logits/chosen": -2.91015625,
+      "logits/rejected": -2.858593702316284,
+      "logps/chosen": -202.75,
+      "logps/rejected": -428.6000061035156,
+      "loss": 0.7319091796875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2850585877895355,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.516552746295929,
+      "rewards/margins": 1.3244140148162842,
+      "rewards/rejected": -0.808398425579071,
+      "step": 430,
+      "train_speed(iter/s)": 0.084512
+    },
+    {
+      "epoch": 0.5568,
+      "grad_norm": 9.938565254211426,
+      "learning_rate": 4.4927996252989527e-07,
+      "logits/chosen": -2.9312500953674316,
+      "logits/rejected": -2.8140625953674316,
+      "logps/chosen": -201.8000030517578,
+      "logps/rejected": -414.29998779296875,
+      "loss": 0.75994873046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28632813692092896,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.47392576932907104,
+      "rewards/margins": 1.161718726158142,
+      "rewards/rejected": -0.687695324420929,
+      "step": 435,
+      "train_speed(iter/s)": 0.084539
+    },
+    {
+      "epoch": 0.5632,
+      "grad_norm": 8.543007850646973,
+      "learning_rate": 4.387618335416073e-07,
+      "logits/chosen": -2.8882813453674316,
+      "logits/rejected": -2.8531250953674316,
+      "logps/chosen": -216.14999389648438,
+      "logps/rejected": -417.1000061035156,
+      "loss": 0.7775634765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3060058653354645,
+      "rewards/accuracies": 0.7484375238418579,
+      "rewards/chosen": 0.515380859375,
+      "rewards/margins": 1.130859375,
+      "rewards/rejected": -0.6161133050918579,
+      "step": 440,
+      "train_speed(iter/s)": 0.084564
+    },
+    {
+      "epoch": 0.5696,
+      "grad_norm": 7.860912799835205,
+      "learning_rate": 4.282711479525151e-07,
+      "logits/chosen": -2.956249952316284,
+      "logits/rejected": -2.8570313453674316,
+      "logps/chosen": -192.5500030517578,
+      "logps/rejected": -391.79998779296875,
+      "loss": 0.7064453125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2862304747104645,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.5794922113418579,
+      "rewards/margins": 1.318945288658142,
+      "rewards/rejected": -0.740429699420929,
+      "step": 445,
+      "train_speed(iter/s)": 0.084569
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 10.7115478515625,
+      "learning_rate": 4.17812607080263e-07,
+      "logits/chosen": -2.936718702316284,
+      "logits/rejected": -2.807812452316284,
+      "logps/chosen": -220.4499969482422,
+      "logps/rejected": -453.6000061035156,
+      "loss": 0.763739013671875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30659180879592896,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.529003918170929,
+      "rewards/margins": 1.267578125,
+      "rewards/rejected": -0.739453136920929,
+      "step": 450,
+      "train_speed(iter/s)": 0.08457
+    },
+    {
+      "epoch": 0.5824,
+      "grad_norm": 10.662744522094727,
+      "learning_rate": 4.073908978370944e-07,
+      "logits/chosen": -2.91796875,
+      "logits/rejected": -2.831249952316284,
+      "logps/chosen": -199.64999389648438,
+      "logps/rejected": -418.70001220703125,
+      "loss": 0.72474365234375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28447264432907104,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.5567871332168579,
+      "rewards/margins": 1.273046851158142,
+      "rewards/rejected": -0.7176758050918579,
+      "step": 455,
+      "train_speed(iter/s)": 0.084565
+    },
+    {
+      "epoch": 0.5888,
+      "grad_norm": 9.964385986328125,
+      "learning_rate": 3.970106906294509e-07,
+      "logits/chosen": -2.9007811546325684,
+      "logits/rejected": -2.805468797683716,
+      "logps/chosen": -186.0500030517578,
+      "logps/rejected": -444.79998779296875,
+      "loss": 0.711279296875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2850585877895355,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.52294921875,
+      "rewards/margins": 1.276757836341858,
+      "rewards/rejected": -0.753125011920929,
+      "step": 460,
+      "train_speed(iter/s)": 0.084576
+    },
+    {
+      "epoch": 0.5952,
+      "grad_norm": 11.822455406188965,
+      "learning_rate": 3.8667663726496445e-07,
+      "logits/chosen": -2.913281202316284,
+      "logits/rejected": -2.8648438453674316,
+      "logps/chosen": -216.1999969482422,
+      "logps/rejected": -430.1000061035156,
+      "loss": 0.76988525390625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3001464903354645,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.47761231660842896,
+      "rewards/margins": 1.2379882335662842,
+      "rewards/rejected": -0.7605956792831421,
+      "step": 465,
+      "train_speed(iter/s)": 0.084591
+    },
+    {
+      "epoch": 0.6016,
+      "grad_norm": 9.923301696777344,
+      "learning_rate": 3.7639336886778895e-07,
+      "logits/chosen": -2.9273438453674316,
+      "logits/rejected": -2.8609375953674316,
+      "logps/chosen": -199.9499969482422,
+      "logps/rejected": -409.0,
+      "loss": 0.71455078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28227537870407104,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.48332518339157104,
+      "rewards/margins": 1.420312523841858,
+      "rewards/rejected": -0.9369140863418579,
+      "step": 470,
+      "train_speed(iter/s)": 0.084608
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 10.85775089263916,
+      "learning_rate": 3.661654938031968e-07,
+      "logits/chosen": -2.907031297683716,
+      "logits/rejected": -2.854687452316284,
+      "logps/chosen": -212.89999389648438,
+      "logps/rejected": -425.5,
+      "loss": 0.7620361328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3114257752895355,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.47172850370407104,
+      "rewards/margins": 1.297460913658142,
+      "rewards/rejected": -0.8252929449081421,
+      "step": 475,
+      "train_speed(iter/s)": 0.084609
+    },
+    {
+      "epoch": 0.6144,
+      "grad_norm": 8.722611427307129,
+      "learning_rate": 3.559975956123772e-07,
+      "logits/chosen": -2.944531202316284,
+      "logits/rejected": -2.856250047683716,
+      "logps/chosen": -203.75,
+      "logps/rejected": -484.20001220703125,
+      "loss": 0.747760009765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2865234315395355,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": 0.49882811307907104,
+      "rewards/margins": 1.337890625,
+      "rewards/rejected": -0.838916003704071,
+      "step": 480,
+      "train_speed(iter/s)": 0.084618
+    },
+    {
+      "epoch": 0.6208,
+      "grad_norm": 10.854005813598633,
+      "learning_rate": 3.458942309583588e-07,
+      "logits/chosen": -2.9507813453674316,
+      "logits/rejected": -2.885937452316284,
+      "logps/chosen": -203.10000610351562,
+      "logps/rejected": -458.5,
+      "loss": 0.77774658203125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28984373807907104,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.4956298768520355,
+      "rewards/margins": 1.2228515148162842,
+      "rewards/rejected": -0.72705078125,
+      "step": 485,
+      "train_speed(iter/s)": 0.084632
+    },
+    {
+      "epoch": 0.6272,
+      "grad_norm": 12.782820701599121,
+      "learning_rate": 3.3585992758397653e-07,
+      "logits/chosen": -2.910937547683716,
+      "logits/rejected": -2.8257813453674316,
+      "logps/chosen": -209.5500030517578,
+      "logps/rejected": -425.6000061035156,
+      "loss": 0.76260986328125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30322265625,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": 0.4853515625,
+      "rewards/margins": 1.196679711341858,
+      "rewards/rejected": -0.710253894329071,
+      "step": 490,
+      "train_speed(iter/s)": 0.084645
+    },
+    {
+      "epoch": 0.6336,
+      "grad_norm": 7.905208110809326,
+      "learning_rate": 3.2589918228280066e-07,
+      "logits/chosen": -2.91015625,
+      "logits/rejected": -2.8570313453674316,
+      "logps/chosen": -198.5500030517578,
+      "logps/rejected": -418.70001220703125,
+      "loss": 0.72608642578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28901368379592896,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.534960925579071,
+      "rewards/margins": 1.2451171875,
+      "rewards/rejected": -0.710156261920929,
+      "step": 495,
+      "train_speed(iter/s)": 0.084665
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 10.61026668548584,
+      "learning_rate": 3.1601645888393273e-07,
+      "logits/chosen": -2.905468702316284,
+      "logits/rejected": -2.811718702316284,
+      "logps/chosen": -213.1999969482422,
+      "logps/rejected": -486.70001220703125,
+      "loss": 0.774609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.30585938692092896,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.514941394329071,
+      "rewards/margins": 1.2021484375,
+      "rewards/rejected": -0.687207043170929,
+      "step": 500,
+      "train_speed(iter/s)": 0.084686
+    },
+    {
+      "epoch": 0.6464,
+      "grad_norm": 10.817185401916504,
+      "learning_rate": 3.0621618625157745e-07,
+      "logits/chosen": -2.8921875953674316,
+      "logits/rejected": -2.778125047683716,
+      "logps/chosen": -202.8000030517578,
+      "logps/rejected": -434.25,
+      "loss": 0.750970458984375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29736328125,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.47929686307907104,
+      "rewards/margins": 1.2705078125,
+      "rewards/rejected": -0.791699230670929,
+      "step": 505,
+      "train_speed(iter/s)": 0.084543
+    },
+    {
+      "epoch": 0.6528,
+      "grad_norm": 9.68419361114502,
+      "learning_rate": 2.965027563002814e-07,
+      "logits/chosen": -2.870312452316284,
+      "logits/rejected": -2.80078125,
+      "logps/chosen": -214.64999389648438,
+      "logps/rejected": -397.5,
+      "loss": 0.76470947265625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3104492127895355,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.4925781190395355,
+      "rewards/margins": 1.2218749523162842,
+      "rewards/rejected": -0.7293945550918579,
+      "step": 510,
+      "train_speed(iter/s)": 0.084563
+    },
+    {
+      "epoch": 0.6592,
+      "grad_norm": 10.539556503295898,
+      "learning_rate": 2.8688052202673186e-07,
+      "logits/chosen": -2.9156250953674316,
+      "logits/rejected": -2.83203125,
+      "logps/chosen": -210.0500030517578,
+      "logps/rejected": -411.0,
+      "loss": 0.75809326171875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29252928495407104,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": 0.4942382872104645,
+      "rewards/margins": 1.212304711341858,
+      "rewards/rejected": -0.717089831829071,
+      "step": 515,
+      "train_speed(iter/s)": 0.084582
+    },
+    {
+      "epoch": 0.6656,
+      "grad_norm": 8.15670394897461,
+      "learning_rate": 2.773537955589951e-07,
+      "logits/chosen": -2.9000000953674316,
+      "logits/rejected": -2.793750047683716,
+      "logps/chosen": -189.60000610351562,
+      "logps/rejected": -438.70001220703125,
+      "loss": 0.7152099609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2918945252895355,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.526904284954071,
+      "rewards/margins": 1.4005858898162842,
+      "rewards/rejected": -0.8738769292831421,
+      "step": 520,
+      "train_speed(iter/s)": 0.084603
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 11.093331336975098,
+      "learning_rate": 2.679268462240708e-07,
+      "logits/chosen": -2.90234375,
+      "logits/rejected": -2.805468797683716,
+      "logps/chosen": -201.5,
+      "logps/rejected": -427.6000061035156,
+      "loss": 0.75528564453125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2938476502895355,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": 0.4699859619140625,
+      "rewards/margins": 1.2257812023162842,
+      "rewards/rejected": -0.755859375,
+      "step": 525,
+      "train_speed(iter/s)": 0.084624
+    },
+    {
+      "epoch": 0.6784,
+      "grad_norm": 7.358150959014893,
+      "learning_rate": 2.5860389863462763e-07,
+      "logits/chosen": -2.9515624046325684,
+      "logits/rejected": -2.807812452316284,
+      "logps/chosen": -205.25,
+      "logps/rejected": -462.6000061035156,
+      "loss": 0.727587890625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28173828125,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": 0.44951170682907104,
+      "rewards/margins": 1.290429711341858,
+      "rewards/rejected": -0.8404296636581421,
+      "step": 530,
+      "train_speed(iter/s)": 0.084642
+    },
+    {
+      "epoch": 0.6848,
+      "grad_norm": 9.283976554870605,
+      "learning_rate": 2.4938913079577497e-07,
+      "logits/chosen": -2.9203124046325684,
+      "logits/rejected": -2.8187499046325684,
+      "logps/chosen": -205.5500030517578,
+      "logps/rejected": -452.79998779296875,
+      "loss": 0.773809814453125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2977539002895355,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": 0.4309448301792145,
+      "rewards/margins": 1.229882836341858,
+      "rewards/rejected": -0.7984863519668579,
+      "step": 535,
+      "train_speed(iter/s)": 0.084663
+    },
+    {
+      "epoch": 0.6912,
+      "grad_norm": 8.79653549194336,
+      "learning_rate": 2.402866722327248e-07,
+      "logits/chosen": -2.9195313453674316,
+      "logits/rejected": -2.8304686546325684,
+      "logps/chosen": -198.3000030517578,
+      "logps/rejected": -469.1499938964844,
+      "loss": 0.743145751953125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28076171875,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.520703136920929,
+      "rewards/margins": 1.208593726158142,
+      "rewards/rejected": -0.688183605670929,
+      "step": 540,
+      "train_speed(iter/s)": 0.084682
+    },
+    {
+      "epoch": 0.6976,
+      "grad_norm": 9.134532928466797,
+      "learning_rate": 2.3130060214017604e-07,
+      "logits/chosen": -2.930468797683716,
+      "logits/rejected": -2.871875047683716,
+      "logps/chosen": -195.9499969482422,
+      "logps/rejected": -404.0,
+      "loss": 0.712310791015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27885740995407104,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.5575195550918579,
+      "rewards/margins": 1.3566405773162842,
+      "rewards/rejected": -0.800097644329071,
+      "step": 545,
+      "train_speed(iter/s)": 0.0847
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 12.105146408081055,
+      "learning_rate": 2.2243494755425713e-07,
+      "logits/chosen": -2.9039063453674316,
+      "logits/rejected": -2.87109375,
+      "logps/chosen": -208.75,
+      "logps/rejected": -398.79998779296875,
+      "loss": 0.74788818359375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29218751192092896,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.46821290254592896,
+      "rewards/margins": 1.271875023841858,
+      "rewards/rejected": -0.8038085699081421,
+      "step": 550,
+      "train_speed(iter/s)": 0.084717
+    },
+    {
+      "epoch": 0.7104,
+      "grad_norm": 8.976114273071289,
+      "learning_rate": 2.1369368154784052e-07,
+      "logits/chosen": -2.965625047683716,
+      "logits/rejected": -2.836718797683716,
+      "logps/chosen": -192.10000610351562,
+      "logps/rejected": -421.70001220703125,
+      "loss": 0.717724609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27412110567092896,
+      "rewards/accuracies": 0.754687488079071,
+      "rewards/chosen": 0.5479491949081421,
+      "rewards/margins": 1.28515625,
+      "rewards/rejected": -0.7374511957168579,
+      "step": 555,
+      "train_speed(iter/s)": 0.084735
+    },
+    {
+      "epoch": 0.7168,
+      "grad_norm": 9.977028846740723,
+      "learning_rate": 2.050807214500428e-07,
+      "logits/chosen": -2.9625000953674316,
+      "logits/rejected": -2.842968702316284,
+      "logps/chosen": -229.85000610351562,
+      "logps/rejected": -434.0,
+      "loss": 0.7812744140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3194335997104645,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.534497082233429,
+      "rewards/margins": 1.298242211341858,
+      "rewards/rejected": -0.76318359375,
+      "step": 560,
+      "train_speed(iter/s)": 0.084751
+    },
+    {
+      "epoch": 0.7232,
+      "grad_norm": 11.700894355773926,
+      "learning_rate": 1.9659992709070344e-07,
+      "logits/chosen": -2.930468797683716,
+      "logits/rejected": -2.7890625,
+      "logps/chosen": -201.89999389648438,
+      "logps/rejected": -481.1000061035156,
+      "loss": 0.71917724609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28779298067092896,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.5340820550918579,
+      "rewards/margins": 1.317773461341858,
+      "rewards/rejected": -0.7837890386581421,
+      "step": 565,
+      "train_speed(iter/s)": 0.084739
+    },
+    {
+      "epoch": 0.7296,
+      "grad_norm": 8.115226745605469,
+      "learning_rate": 1.8825509907063326e-07,
+      "logits/chosen": -2.9375,
+      "logits/rejected": -2.84375,
+      "logps/chosen": -191.9499969482422,
+      "logps/rejected": -460.79998779296875,
+      "loss": 0.701971435546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2848144471645355,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5508788824081421,
+      "rewards/margins": 1.4044921398162842,
+      "rewards/rejected": -0.8525390625,
+      "step": 570,
+      "train_speed(iter/s)": 0.084745
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 10.647868156433105,
+      "learning_rate": 1.8004997705840564e-07,
+      "logits/chosen": -2.921875,
+      "logits/rejected": -2.8109374046325684,
+      "logps/chosen": -213.35000610351562,
+      "logps/rejected": -535.9000244140625,
+      "loss": 0.75244140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2974609434604645,
+      "rewards/accuracies": 0.745312511920929,
+      "rewards/chosen": 0.50830078125,
+      "rewards/margins": 1.355078101158142,
+      "rewards/rejected": -0.846728503704071,
+      "step": 575,
+      "train_speed(iter/s)": 0.084736
+    },
+    {
+      "epoch": 0.7424,
+      "grad_norm": 9.362266540527344,
+      "learning_rate": 1.719882381144525e-07,
+      "logits/chosen": -2.92578125,
+      "logits/rejected": -2.84765625,
+      "logps/chosen": -202.1999969482422,
+      "logps/rejected": -415.0,
+      "loss": 0.72916259765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.289794921875,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.5228027105331421,
+      "rewards/margins": 1.336328148841858,
+      "rewards/rejected": -0.81298828125,
+      "step": 580,
+      "train_speed(iter/s)": 0.084743
+    },
+    {
+      "epoch": 0.7488,
+      "grad_norm": 9.079050064086914,
+      "learning_rate": 1.6407349504322032e-07,
+      "logits/chosen": -2.9320311546325684,
+      "logits/rejected": -2.8570313453674316,
+      "logps/chosen": -201.10000610351562,
+      "logps/rejected": -425.6000061035156,
+      "loss": 0.741131591796875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2859863340854645,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": 0.46961671113967896,
+      "rewards/margins": 1.3302733898162842,
+      "rewards/rejected": -0.861132800579071,
+      "step": 585,
+      "train_speed(iter/s)": 0.084743
+    },
+    {
+      "epoch": 0.7552,
+      "grad_norm": 10.37470531463623,
+      "learning_rate": 1.5630929477411898e-07,
+      "logits/chosen": -2.922656297683716,
+      "logits/rejected": -2.8382811546325684,
+      "logps/chosen": -221.9499969482422,
+      "logps/rejected": -468.29998779296875,
+      "loss": 0.756744384765625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3133789002895355,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.5062500238418579,
+      "rewards/margins": 1.326757788658142,
+      "rewards/rejected": -0.820849597454071,
+      "step": 590,
+      "train_speed(iter/s)": 0.084738
+    },
+    {
+      "epoch": 0.7616,
+      "grad_norm": 10.721070289611816,
+      "learning_rate": 1.4869911677199475e-07,
+      "logits/chosen": -2.8929686546325684,
+      "logits/rejected": -2.809375047683716,
+      "logps/chosen": -217.5,
+      "logps/rejected": -495.29998779296875,
+      "loss": 0.77445068359375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3114257752895355,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.455810546875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -0.94189453125,
+      "step": 595,
+      "train_speed(iter/s)": 0.084725
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 11.324858665466309,
+      "learning_rate": 1.412463714778343e-07,
+      "logits/chosen": -2.921875,
+      "logits/rejected": -2.8421874046325684,
+      "logps/chosen": -233.35000610351562,
+      "logps/rejected": -469.20001220703125,
+      "loss": 0.79466552734375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31977540254592896,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.5077148675918579,
+      "rewards/margins": 1.2169921398162842,
+      "rewards/rejected": -0.7088867425918579,
+      "step": 600,
+      "train_speed(iter/s)": 0.084733
+    },
+    {
+      "epoch": 0.7744,
+      "grad_norm": 11.218781471252441,
+      "learning_rate": 1.3395439878040422e-07,
+      "logits/chosen": -2.9273438453674316,
+      "logits/rejected": -2.8695311546325684,
+      "logps/chosen": -211.85000610351562,
+      "logps/rejected": -406.79998779296875,
+      "loss": 0.764471435546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3001953065395355,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": 0.4898437559604645,
+      "rewards/margins": 1.2726562023162842,
+      "rewards/rejected": -0.781933605670929,
+      "step": 605,
+      "train_speed(iter/s)": 0.084573
+    },
+    {
+      "epoch": 0.7808,
+      "grad_norm": 10.250137329101562,
+      "learning_rate": 1.2682646651950473e-07,
+      "logits/chosen": -2.9585938453674316,
+      "logits/rejected": -2.81640625,
+      "logps/chosen": -202.64999389648438,
+      "logps/rejected": -441.6000061035156,
+      "loss": 0.7609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.282470703125,
+      "rewards/accuracies": 0.754687488079071,
+      "rewards/chosen": 0.512622058391571,
+      "rewards/margins": 1.276757836341858,
+      "rewards/rejected": -0.7650390863418579,
+      "step": 610,
+      "train_speed(iter/s)": 0.084587
+    },
+    {
+      "epoch": 0.7872,
+      "grad_norm": 9.014904975891113,
+      "learning_rate": 1.198657690215142e-07,
+      "logits/chosen": -2.9124999046325684,
+      "logits/rejected": -2.8226561546325684,
+      "logps/chosen": -203.60000610351562,
+      "logps/rejected": -478.79998779296875,
+      "loss": 0.7029541015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29765623807907104,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": 0.5479491949081421,
+      "rewards/margins": 1.4249999523162842,
+      "rewards/rejected": -0.8775390386581421,
+      "step": 615,
+      "train_speed(iter/s)": 0.084601
+    },
+    {
+      "epoch": 0.7936,
+      "grad_norm": 8.6657075881958,
+      "learning_rate": 1.1307542566787659e-07,
+      "logits/chosen": -2.9468750953674316,
+      "logits/rejected": -2.809375047683716,
+      "logps/chosen": -198.0500030517578,
+      "logps/rejected": -434.6000061035156,
+      "loss": 0.73013916015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27314454317092896,
+      "rewards/accuracies": 0.7484375238418579,
+      "rewards/chosen": 0.52490234375,
+      "rewards/margins": 1.2576172351837158,
+      "rewards/rejected": -0.732421875,
+      "step": 620,
+      "train_speed(iter/s)": 0.084604
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 9.32559585571289,
+      "learning_rate": 1.0645847949717568e-07,
+      "logits/chosen": -2.936718702316284,
+      "logits/rejected": -2.84375,
+      "logps/chosen": -190.89999389648438,
+      "logps/rejected": -396.1499938964844,
+      "loss": 0.70264892578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2806640565395355,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.559374988079071,
+      "rewards/margins": 1.329687476158142,
+      "rewards/rejected": -0.770703136920929,
+      "step": 625,
+      "train_speed(iter/s)": 0.084599
+    },
+    {
+      "epoch": 0.8064,
+      "grad_norm": 9.89014720916748,
+      "learning_rate": 1.000178958414214e-07,
+      "logits/chosen": -2.9085936546325684,
+      "logits/rejected": -2.842968702316284,
+      "logps/chosen": -213.25,
+      "logps/rejected": -460.5,
+      "loss": 0.780517578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3036132752895355,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.441162109375,
+      "rewards/margins": 1.2902343273162842,
+      "rewards/rejected": -0.848925769329071,
+      "step": 630,
+      "train_speed(iter/s)": 0.084599
+    },
+    {
+      "epoch": 0.8128,
+      "grad_norm": 10.590904235839844,
+      "learning_rate": 9.375656099715934e-08,
+      "logits/chosen": -2.9164061546325684,
+      "logits/rejected": -2.842968702316284,
+      "logps/chosen": -215.64999389648438,
+      "logps/rejected": -466.0,
+      "loss": 0.7617431640625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2984375059604645,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.45751953125,
+      "rewards/margins": 1.282812476158142,
+      "rewards/rejected": -0.825390636920929,
+      "step": 635,
+      "train_speed(iter/s)": 0.084611
+    },
+    {
+      "epoch": 0.8192,
+      "grad_norm": 7.914394855499268,
+      "learning_rate": 8.76772809320006e-08,
+      "logits/chosen": -2.8984375,
+      "logits/rejected": -2.81640625,
+      "logps/chosen": -203.85000610351562,
+      "logps/rejected": -471.20001220703125,
+      "loss": 0.72257080078125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.3045410215854645,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.536914050579071,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -0.893261730670929,
+      "step": 640,
+      "train_speed(iter/s)": 0.084619
+    },
+    {
+      "epoch": 0.8256,
+      "grad_norm": 12.118059158325195,
+      "learning_rate": 8.178278002714878e-08,
+      "logits/chosen": -2.9234375953674316,
+      "logits/rejected": -2.793750047683716,
+      "logps/chosen": -199.60000610351562,
+      "logps/rejected": -438.8999938964844,
+      "loss": 0.7381103515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29252928495407104,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": 0.5438232421875,
+      "rewards/margins": 1.340234398841858,
+      "rewards/rejected": -0.7959960699081421,
+      "step": 645,
+      "train_speed(iter/s)": 0.084633
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 8.742040634155273,
+      "learning_rate": 7.607569985649064e-08,
+      "logits/chosen": -2.891406297683716,
+      "logits/rejected": -2.879687547683716,
+      "logps/chosen": -207.3000030517578,
+      "logps/rejected": -405.29998779296875,
+      "loss": 0.74339599609375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2965331971645355,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": 0.4720214903354645,
+      "rewards/margins": 1.343164086341858,
+      "rewards/rejected": -0.872753918170929,
+      "step": 650,
+      "train_speed(iter/s)": 0.084648
+    },
+    {
+      "epoch": 0.8384,
+      "grad_norm": 9.125042915344238,
+      "learning_rate": 7.055859800279579e-08,
+      "logits/chosen": -2.96484375,
+      "logits/rejected": -2.8882813453674316,
+      "logps/chosen": -194.89999389648438,
+      "logps/rejected": -447.6000061035156,
+      "loss": 0.72674560546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.27998048067092896,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": 0.517578125,
+      "rewards/margins": 1.278222680091858,
+      "rewards/rejected": -0.7603759765625,
+      "step": 655,
+      "train_speed(iter/s)": 0.084662
+    },
+    {
+      "epoch": 0.8448,
+      "grad_norm": 10.046883583068848,
+      "learning_rate": 6.523394691155565e-08,
+      "logits/chosen": -2.914843797683716,
+      "logits/rejected": -2.796093702316284,
+      "logps/chosen": -203.9499969482422,
+      "logps/rejected": -491.79998779296875,
+      "loss": 0.71722412109375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2900390625,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.49189454317092896,
+      "rewards/margins": 1.380273461341858,
+      "rewards/rejected": -0.8882812261581421,
+      "step": 660,
+      "train_speed(iter/s)": 0.084679
+    },
+    {
+      "epoch": 0.8512,
+      "grad_norm": 12.332063674926758,
+      "learning_rate": 6.010413278297737e-08,
+      "logits/chosen": -2.9398436546325684,
+      "logits/rejected": -2.882031202316284,
+      "logps/chosen": -218.60000610351562,
+      "logps/rejected": -439.6000061035156,
+      "loss": 0.78310546875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.302734375,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": 0.47797852754592896,
+      "rewards/margins": 1.3015625476837158,
+      "rewards/rejected": -0.8248046636581421,
+      "step": 665,
+      "train_speed(iter/s)": 0.084693
+    },
+    {
+      "epoch": 0.8576,
+      "grad_norm": 9.431097984313965,
+      "learning_rate": 5.517145450262639e-08,
+      "logits/chosen": -2.94140625,
+      "logits/rejected": -2.825000047683716,
+      "logps/chosen": -202.85000610351562,
+      "logps/rejected": -451.70001220703125,
+      "loss": 0.71043701171875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2853027284145355,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.529248058795929,
+      "rewards/margins": 1.364843726158142,
+      "rewards/rejected": -0.8353515863418579,
+      "step": 670,
+      "train_speed(iter/s)": 0.084707
+    },
+    {
+      "epoch": 0.864,
+      "grad_norm": 10.46705436706543,
+      "learning_rate": 5.043812261119951e-08,
+      "logits/chosen": -2.921093702316284,
+      "logits/rejected": -2.87109375,
+      "logps/chosen": -210.5,
+      "logps/rejected": -403.1000061035156,
+      "loss": 0.7483154296875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2974609434604645,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.5101073980331421,
+      "rewards/margins": 1.3132812976837158,
+      "rewards/rejected": -0.803662121295929,
+      "step": 675,
+      "train_speed(iter/s)": 0.084697
+    },
+    {
+      "epoch": 0.8704,
+      "grad_norm": 7.990355014801025,
+      "learning_rate": 4.590625831388834e-08,
+      "logits/chosen": -2.9195313453674316,
+      "logits/rejected": -2.793750047683716,
+      "logps/chosen": -192.1999969482422,
+      "logps/rejected": -427.20001220703125,
+      "loss": 0.69468994140625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.286376953125,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": 0.5430663824081421,
+      "rewards/margins": 1.393945336341858,
+      "rewards/rejected": -0.8515625,
+      "step": 680,
+      "train_speed(iter/s)": 0.084703
+    },
+    {
+      "epoch": 0.8768,
+      "grad_norm": 9.987874984741211,
+      "learning_rate": 4.1577892529778393e-08,
+      "logits/chosen": -2.9242186546325684,
+      "logits/rejected": -2.8460936546325684,
+      "logps/chosen": -196.14999389648438,
+      "logps/rejected": -425.8999938964844,
+      "loss": 0.72318115234375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2911621034145355,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.5281738042831421,
+      "rewards/margins": 1.380468726158142,
+      "rewards/rejected": -0.851367175579071,
+      "step": 685,
+      "train_speed(iter/s)": 0.084691
+    },
+    {
+      "epoch": 0.8832,
+      "grad_norm": 9.257203102111816,
+      "learning_rate": 3.74549649817083e-08,
+      "logits/chosen": -2.9625000953674316,
+      "logits/rejected": -2.8687500953674316,
+      "logps/chosen": -198.0500030517578,
+      "logps/rejected": -411.70001220703125,
+      "loss": 0.74117431640625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2782226502895355,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.4981445372104645,
+      "rewards/margins": 1.2863280773162842,
+      "rewards/rejected": -0.787646472454071,
+      "step": 690,
+      "train_speed(iter/s)": 0.084698
+    },
+    {
+      "epoch": 0.8896,
+      "grad_norm": 11.192628860473633,
+      "learning_rate": 3.353932332699866e-08,
+      "logits/chosen": -2.9398436546325684,
+      "logits/rejected": -2.8414063453674316,
+      "logps/chosen": -216.39999389648438,
+      "logps/rejected": -509.70001220703125,
+      "loss": 0.73873291015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.31201171875,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": 0.49609375,
+      "rewards/margins": 1.3982422351837158,
+      "rewards/rejected": -0.90234375,
+      "step": 695,
+      "train_speed(iter/s)": 0.084701
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 10.48532485961914,
+      "learning_rate": 2.983272232943901e-08,
+      "logits/chosen": -2.926562547683716,
+      "logits/rejected": -2.917187452316284,
+      "logps/chosen": -240.60000610351562,
+      "logps/rejected": -455.5,
+      "loss": 0.84154052734375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.33232420682907104,
+      "rewards/accuracies": 0.7359374761581421,
+      "rewards/chosen": 0.4315185546875,
+      "rewards/margins": 1.250585913658142,
+      "rewards/rejected": -0.820117175579071,
+      "step": 700,
+      "train_speed(iter/s)": 0.084705
+    },
+    {
+      "epoch": 0.9024,
+      "grad_norm": 9.226845741271973,
+      "learning_rate": 2.63368230729043e-08,
+      "logits/chosen": -2.9195313453674316,
+      "logits/rejected": -2.79296875,
+      "logps/chosen": -207.85000610351562,
+      "logps/rejected": -411.20001220703125,
+      "loss": 0.74794921875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29374998807907104,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.513427734375,
+      "rewards/margins": 1.251562476158142,
+      "rewards/rejected": -0.7386718988418579,
+      "step": 705,
+      "train_speed(iter/s)": 0.084552
+    },
+    {
+      "epoch": 0.9088,
+      "grad_norm": 13.927331924438477,
+      "learning_rate": 2.3053192216953397e-08,
+      "logits/chosen": -2.9195313453674316,
+      "logits/rejected": -2.832812547683716,
+      "logps/chosen": -200.39999389648438,
+      "logps/rejected": -442.20001220703125,
+      "loss": 0.73919677734375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28959959745407104,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.4505859315395355,
+      "rewards/margins": 1.329687476158142,
+      "rewards/rejected": -0.878857433795929,
+      "step": 710,
+      "train_speed(iter/s)": 0.084559
+    },
+    {
+      "epoch": 0.9152,
+      "grad_norm": 9.653718948364258,
+      "learning_rate": 1.9983301294742737e-08,
+      "logits/chosen": -2.9398436546325684,
+      "logits/rejected": -2.862499952316284,
+      "logps/chosen": -212.75,
+      "logps/rejected": -406.5,
+      "loss": 0.7784423828125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29277342557907104,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": 0.45478516817092896,
+      "rewards/margins": 1.17041015625,
+      "rewards/rejected": -0.715624988079071,
+      "step": 715,
+      "train_speed(iter/s)": 0.084562
+    },
+    {
+      "epoch": 0.9216,
+      "grad_norm": 11.44400405883789,
+      "learning_rate": 1.7128526053570958e-08,
+      "logits/chosen": -2.950000047683716,
+      "logits/rejected": -2.860156297683716,
+      "logps/chosen": -193.14999389648438,
+      "logps/rejected": -470.8999938964844,
+      "loss": 0.6990966796875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28081053495407104,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.5241943597793579,
+      "rewards/margins": 1.434960961341858,
+      "rewards/rejected": -0.9111328125,
+      "step": 720,
+      "train_speed(iter/s)": 0.084562
+    },
+    {
+      "epoch": 0.928,
+      "grad_norm": 8.83023452758789,
+      "learning_rate": 1.4490145838347955e-08,
+      "logits/chosen": -2.975781202316284,
+      "logits/rejected": -2.8531250953674316,
+      "logps/chosen": -192.0,
+      "logps/rejected": -427.3999938964844,
+      "loss": 0.7030517578125,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2828125059604645,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.512744128704071,
+      "rewards/margins": 1.3894531726837158,
+      "rewards/rejected": -0.876171886920929,
+      "step": 725,
+      "train_speed(iter/s)": 0.084556
+    },
+    {
+      "epoch": 0.9344,
+      "grad_norm": 11.11131763458252,
+      "learning_rate": 1.2069343018267063e-08,
+      "logits/chosen": -2.96484375,
+      "logits/rejected": -2.9117188453674316,
+      "logps/chosen": -211.0,
+      "logps/rejected": -433.6000061035156,
+      "loss": 0.75550537109375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.299072265625,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.5169922113418579,
+      "rewards/margins": 1.3732421398162842,
+      "rewards/rejected": -0.8545898199081421,
+      "step": 730,
+      "train_speed(iter/s)": 0.084564
+    },
+    {
+      "epoch": 0.9408,
+      "grad_norm": 9.674853324890137,
+      "learning_rate": 9.867202456935619e-09,
+      "logits/chosen": -2.9359374046325684,
+      "logits/rejected": -2.83203125,
+      "logps/chosen": -211.9499969482422,
+      "logps/rejected": -418.79998779296875,
+      "loss": 0.744586181640625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29594725370407104,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": 0.5079101324081421,
+      "rewards/margins": 1.292382836341858,
+      "rewards/rejected": -0.784472644329071,
+      "step": 735,
+      "train_speed(iter/s)": 0.084575
+    },
+    {
+      "epoch": 0.9472,
+      "grad_norm": 10.080160140991211,
+      "learning_rate": 7.884711026201584e-09,
+      "logits/chosen": -2.953906297683716,
+      "logits/rejected": -2.827343702316284,
+      "logps/chosen": -204.25,
+      "logps/rejected": -458.29998779296875,
+      "loss": 0.7572998046875,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28886717557907104,
+      "rewards/accuracies": 0.739062488079071,
+      "rewards/chosen": 0.45917969942092896,
+      "rewards/margins": 1.2659180164337158,
+      "rewards/rejected": -0.8077148199081421,
+      "step": 740,
+      "train_speed(iter/s)": 0.084572
+    },
+    {
+      "epoch": 0.9536,
+      "grad_norm": 9.772847175598145,
+      "learning_rate": 6.1227571638954316e-09,
+      "logits/chosen": -2.9312500953674316,
+      "logits/rejected": -2.8375000953674316,
+      "logps/chosen": -200.1999969482422,
+      "logps/rejected": -443.5,
+      "loss": 0.721087646484375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.28471678495407104,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.5352538824081421,
+      "rewards/margins": 1.3361327648162842,
+      "rewards/rejected": -0.800000011920929,
+      "step": 745,
+      "train_speed(iter/s)": 0.084571
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 10.025954246520996,
+      "learning_rate": 4.58213047568301e-09,
+      "logits/chosen": -2.87890625,
+      "logits/rejected": -2.866406202316284,
+      "logps/chosen": -208.625,
+      "logps/rejected": -437.0,
+      "loss": 0.746490478515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2996582090854645,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.4754882752895355,
+      "rewards/margins": 1.3244140148162842,
+      "rewards/rejected": -0.848925769329071,
+      "step": 750,
+      "train_speed(iter/s)": 0.084575
+    },
+    {
+      "epoch": 0.9664,
+      "grad_norm": 13.55732536315918,
+      "learning_rate": 3.2635213812104434e-09,
+      "logits/chosen": -2.922656297683716,
+      "logits/rejected": -2.854687452316284,
+      "logps/chosen": -226.4499969482422,
+      "logps/rejected": -433.20001220703125,
+      "loss": 0.73880615234375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.32158201932907104,
+      "rewards/accuracies": 0.7953125238418579,
+      "rewards/chosen": 0.478271484375,
+      "rewards/margins": 1.438085913658142,
+      "rewards/rejected": -0.9603515863418579,
+      "step": 755,
+      "train_speed(iter/s)": 0.084586
+    },
+    {
+      "epoch": 0.9728,
+      "grad_norm": 9.440918922424316,
+      "learning_rate": 2.1675208046980085e-09,
+      "logits/chosen": -2.926562547683716,
+      "logits/rejected": -2.874218702316284,
+      "logps/chosen": -201.3000030517578,
+      "logps/rejected": -469.1000061035156,
+      "loss": 0.72701416015625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2911132872104645,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.5536133050918579,
+      "rewards/margins": 1.4249999523162842,
+      "rewards/rejected": -0.8716796636581421,
+      "step": 760,
+      "train_speed(iter/s)": 0.084592
+    },
+    {
+      "epoch": 0.9792,
+      "grad_norm": 12.171716690063477,
+      "learning_rate": 1.2946199101219746e-09,
+      "logits/chosen": -2.910937547683716,
+      "logits/rejected": -2.844531297683716,
+      "logps/chosen": -207.4499969482422,
+      "logps/rejected": -441.29998779296875,
+      "loss": 0.75218505859375,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.29472655057907104,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.49921876192092896,
+      "rewards/margins": 1.2900390625,
+      "rewards/rejected": -0.790722668170929,
+      "step": 765,
+      "train_speed(iter/s)": 0.084589
+    },
+    {
+      "epoch": 0.9856,
+      "grad_norm": 9.241211891174316,
+      "learning_rate": 6.452098811035034e-10,
+      "logits/chosen": -2.9320311546325684,
+      "logits/rejected": -2.8203125,
+      "logps/chosen": -204.0,
+      "logps/rejected": -492.8999938964844,
+      "loss": 0.75738525390625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.2901855409145355,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.435882568359375,
+      "rewards/margins": 1.2693359851837158,
+      "rewards/rejected": -0.8335937261581421,
+      "step": 770,
+      "train_speed(iter/s)": 0.084591
+    },
+    {
+      "epoch": 0.992,
+      "grad_norm": 9.569814682006836,
+      "learning_rate": 2.1958174560282594e-10,
+      "logits/chosen": -2.91796875,
+      "logits/rejected": -2.8492188453674316,
+      "logps/chosen": -224.6999969482422,
+      "logps/rejected": -418.6000061035156,
+      "loss": 0.7611572265625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.313232421875,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.521923840045929,
+      "rewards/margins": 1.3253905773162842,
+      "rewards/rejected": -0.803417980670929,
+      "step": 775,
+      "train_speed(iter/s)": 0.084601
+    },
+    {
+      "epoch": 0.9984,
+      "grad_norm": 11.742775917053223,
+      "learning_rate": 1.7926245497179583e-11,
+      "logits/chosen": -2.9476561546325684,
+      "logits/rejected": -2.8421874046325684,
+      "logps/chosen": -207.35000610351562,
+      "logps/rejected": -420.0,
+      "loss": 0.74849853515625,
+      "memory(GiB)": 37.1,
+      "nll_loss": 0.289306640625,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.542187511920929,
+      "rewards/margins": 1.268164038658142,
+      "rewards/rejected": -0.725537121295929,
+      "step": 780,
+      "train_speed(iter/s)": 0.084612
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 782,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 326150815350784.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}