diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,1176 +1,10854 @@
 {
-  "best_global_step": null,
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.6648199445983379,
-  "eval_steps": 500,
-  "global_step": 60,
+  "best_global_step": 600,
+  "best_metric": 0.24665305,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/dpo_phase2/v1-20250528-012105/checkpoint-600",
+  "epoch": 1.210047967684928,
+  "eval_steps": 300,
+  "global_step": 600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.6796875,
-      "completions/min_length": 3.0,
-      "epoch": 0.0110803324099723,
-      "grad_norm": 1.3400153486450683,
-      "kl": 0.0,
-      "learning_rate": 1.1111111111111112e-05,
-      "loss": -9.313225746154785e-10,
-      "memory(GiB)": 77.38,
-      "reward": -0.8125,
-      "reward_std": 0.38430261611938477,
-      "rewards/Response_no_think_reward/mean": -0.8125,
-      "rewards/Response_no_think_reward/std": 0.6958821415901184,
+      "epoch": 0.002019691996970462,
+      "grad_norm": 12.761810302734375,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": -0.5775864720344543,
+      "logits/rejected": -0.7030954360961914,
+      "logps/chosen": -6.236894130706787,
+      "logps/rejected": -24.46524429321289,
+      "loss": 1.2380319833755493,
+      "memory(GiB)": 32.64,
+      "nll_loss": 0.5448847413063049,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
       "step": 1,
-      "train_speed(iter/s)": 0.000753
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "epoch": 0.0221606648199446,
-      "grad_norm": 1.3454285871298992,
-      "kl": 0.0,
-      "learning_rate": 2.2222222222222223e-05,
-      "loss": -9.313225746154785e-10,
-      "memory(GiB)": 78.17,
+      "train_speed(iter/s)": 0.0187
+    },
+    {
+      "epoch": 0.004039383993940924,
+      "grad_norm": 8.313699722290039,
+      "learning_rate": 8.000000000000001e-06,
+      "logits/chosen": -0.8238492012023926,
+      "logits/rejected": -0.8350682258605957,
+      "logps/chosen": -5.732274055480957,
+      "logps/rejected": -11.927969932556152,
+      "loss": 1.4877614974975586,
+      "memory(GiB)": 32.64,
+      "nll_loss": 0.7946141958236694,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
       "step": 2,
-      "train_speed(iter/s)": 0.001317
-    },
-    {
-      "clip_ratio/high_max": 0.0009920635493472219,
-      "clip_ratio/high_mean": 0.0009920635493472219,
-      "clip_ratio/low_mean": 0.0012351538171060383,
-      "clip_ratio/low_min": 0.0012351538171060383,
-      "clip_ratio/region_mean": 0.00222721736645326,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.03125,
-      "completions/min_length": 3.0,
-      "epoch": 0.0332409972299169,
-      "grad_norm": 1.1739378406354548,
-      "kl": 0.0013750234156759689,
-      "learning_rate": 3.3333333333333335e-05,
-      "loss": 0.0005532659706659615,
-      "memory(GiB)": 78.17,
-      "reward": -0.0078125,
-      "reward_std": 0.7346676588058472,
-      "rewards/Response_no_think_reward/mean": -0.0078125,
-      "rewards/Response_no_think_reward/std": 1.411405086517334,
+      "train_speed(iter/s)": 0.020044
+    },
+    {
+      "epoch": 0.006059075990911386,
+      "grad_norm": 21.77679443359375,
+      "learning_rate": 1.2e-05,
+      "logits/chosen": -0.6877153515815735,
+      "logits/rejected": -0.7822633981704712,
+      "logps/chosen": -7.046360015869141,
+      "logps/rejected": -14.666805267333984,
+      "loss": 1.5761629343032837,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.8832842707633972,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0033746182452887297,
+      "rewards/margins": 0.0006902526365593076,
+      "rewards/rejected": 0.0026843654923141003,
       "step": 3,
-      "train_speed(iter/s)": 0.001417
-    },
-    {
-      "clip_ratio/high_max": 0.0006313131307251751,
-      "clip_ratio/high_mean": 0.0006313131307251751,
-      "clip_ratio/low_mean": 0.0009640990756452084,
-      "clip_ratio/low_min": 0.0009640990756452084,
-      "clip_ratio/region_mean": 0.0015954122063703835,
-      "epoch": 0.0443213296398892,
-      "grad_norm": 1.1825343839681348,
-      "kl": 0.0013630353023472708,
-      "learning_rate": 4.4444444444444447e-05,
-      "loss": -0.0013326075859367847,
-      "memory(GiB)": 78.17,
+      "train_speed(iter/s)": 0.020481
+    },
+    {
+      "epoch": 0.008078767987881848,
+      "grad_norm": 12.999642372131348,
+      "learning_rate": 1.6000000000000003e-05,
+      "logits/chosen": -0.7259287238121033,
+      "logits/rejected": -0.8152387142181396,
+      "logps/chosen": -4.8744587898254395,
+      "logps/rejected": -14.308876991271973,
+      "loss": 1.4366666078567505,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.7484119534492493,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0021331249736249447,
+      "rewards/margins": 0.009978920221328735,
+      "rewards/rejected": -0.007845795713365078,
       "step": 4,
-      "train_speed(iter/s)": 0.001735
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.005316304974257946,
-      "clip_ratio/low_min": 0.005316304974257946,
-      "clip_ratio/region_mean": 0.005316304974257946,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.3828125,
-      "completions/min_length": 3.0,
-      "epoch": 0.055401662049861494,
-      "grad_norm": 1.3399484117828366,
-      "kl": 0.0029429409933072748,
-      "learning_rate": 5.555555555555556e-05,
-      "loss": 0.0014606926124542952,
-      "memory(GiB)": 78.25,
-      "reward": -0.15625,
-      "reward_std": 0.9831876158714294,
-      "rewards/Response_no_think_reward/mean": -0.15625,
-      "rewards/Response_no_think_reward/std": 1.3541275262832642,
+      "train_speed(iter/s)": 0.020724
+    },
+    {
+      "epoch": 0.01009845998485231,
+      "grad_norm": 12.841984748840332,
+      "learning_rate": 2e-05,
+      "logits/chosen": -0.681520402431488,
+      "logits/rejected": -0.7851653695106506,
+      "logps/chosen": -5.029726982116699,
+      "logps/rejected": -16.70825958251953,
+      "loss": 1.2887729406356812,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.6066412329673767,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.02295457012951374,
+      "rewards/margins": 0.022426389157772064,
+      "rewards/rejected": 0.0005281821941025555,
       "step": 5,
-      "train_speed(iter/s)": 0.001752
-    },
-    {
-      "clip_ratio/high_max": 0.0014204545877873898,
-      "clip_ratio/high_mean": 0.0014204545877873898,
-      "clip_ratio/low_mean": 0.0008680555620230734,
-      "clip_ratio/low_min": 0.0008680555620230734,
-      "clip_ratio/region_mean": 0.002288510149810463,
-      "completions/clipped_ratio": 0.03125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.484375,
-      "completions/min_length": 2.0,
-      "epoch": 0.0664819944598338,
-      "grad_norm": 1.600021993041462,
-      "kl": 0.007914411224192008,
-      "learning_rate": 6.666666666666667e-05,
-      "loss": -0.0006625128444284201,
-      "memory(GiB)": 77.89,
-      "reward": -0.1328125,
-      "reward_std": 0.7212049961090088,
-      "rewards/Response_no_think_reward/mean": -0.1328125,
-      "rewards/Response_no_think_reward/std": 1.3653247356414795,
+      "train_speed(iter/s)": 0.020852
+    },
+    {
+      "epoch": 0.012118151981822771,
+      "grad_norm": 13.818325996398926,
+      "learning_rate": 2.4e-05,
+      "logits/chosen": -0.6980650424957275,
+      "logits/rejected": -0.8158392310142517,
+      "logps/chosen": -4.0964484214782715,
+      "logps/rejected": -15.3903169631958,
+      "loss": 1.3540700674057007,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.6896780133247375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06763897091150284,
+      "rewards/margins": 0.06137145683169365,
+      "rewards/rejected": 0.00626751035451889,
       "step": 6,
-      "train_speed(iter/s)": 0.009313
-    },
-    {
-      "clip_ratio/high_max": 0.020065165997948498,
-      "clip_ratio/high_mean": 0.020065165997948498,
-      "clip_ratio/low_mean": 0.03063795086927712,
-      "clip_ratio/low_min": 0.03063795086927712,
-      "clip_ratio/region_mean": 0.050703116634394974,
-      "epoch": 0.07756232686980609,
-      "grad_norm": 0.9793694784898841,
-      "kl": 0.03891617391491309,
-      "learning_rate": 7.777777777777778e-05,
-      "loss": -0.00832418818026781,
-      "memory(GiB)": 77.89,
+      "train_speed(iter/s)": 0.020943
+    },
+    {
+      "epoch": 0.014137843978793235,
+      "grad_norm": 8.670878410339355,
+      "learning_rate": 2.8000000000000003e-05,
+      "logits/chosen": -0.6072264909744263,
+      "logits/rejected": -0.7461887001991272,
+      "logps/chosen": -6.871906757354736,
+      "logps/rejected": -19.193540573120117,
+      "loss": 1.1981984376907349,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.5265603065490723,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04815364256501198,
+      "rewards/margins": 0.053483281284570694,
+      "rewards/rejected": -0.005329643841832876,
       "step": 7,
-      "train_speed(iter/s)": 0.0084
-    },
-    {
-      "clip_ratio/high_max": 0.0005165289039723575,
-      "clip_ratio/high_mean": 0.0005165289039723575,
-      "clip_ratio/low_mean": 0.0012338250526227057,
-      "clip_ratio/low_min": 0.0012338250526227057,
-      "clip_ratio/region_mean": 0.0017503539565950632,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.734375,
-      "completions/min_length": 3.0,
-      "epoch": 0.0886426592797784,
-      "grad_norm": 1.3164563077884395,
-      "kl": 0.05195357115007937,
-      "learning_rate": 8.888888888888889e-05,
-      "loss": -0.00010046618990600109,
-      "memory(GiB)": 78.03,
-      "reward": -0.2734375,
-      "reward_std": 0.7209804058074951,
-      "rewards/Response_no_think_reward/mean": -0.2734375,
-      "rewards/Response_no_think_reward/std": 1.27183198928833,
+      "train_speed(iter/s)": 0.020996
+    },
+    {
+      "epoch": 0.016157535975763696,
+      "grad_norm": 4.796175003051758,
+      "learning_rate": 3.2000000000000005e-05,
+      "logits/chosen": -0.7197842597961426,
+      "logits/rejected": -0.7792064547538757,
+      "logps/chosen": -7.072351455688477,
+      "logps/rejected": -14.618125915527344,
+      "loss": 1.1894290447235107,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.4693164825439453,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.13033029437065125,
+      "rewards/margins": -0.01030611153692007,
+      "rewards/rejected": 0.1406363844871521,
       "step": 8,
-      "train_speed(iter/s)": 0.004412
+      "train_speed(iter/s)": 0.021046
     },
     {
-      "clip_ratio/high_max": 0.011744739342248067,
-      "clip_ratio/high_mean": 0.011744739342248067,
-      "clip_ratio/low_mean": 0.021301756787579507,
-      "clip_ratio/low_min": 0.021301756787579507,
-      "clip_ratio/region_mean": 0.033046496275346726,
-      "epoch": 0.0997229916897507,
-      "grad_norm": 0.7228491518243964,
-      "kl": 0.14250310882925987,
-      "learning_rate": 0.0001,
-      "loss": -0.009305480867624283,
-      "memory(GiB)": 78.03,
+      "epoch": 0.018177227972734158,
+      "grad_norm": 6.6790852546691895,
+      "learning_rate": 3.6e-05,
+      "logits/chosen": -0.689906656742096,
+      "logits/rejected": -0.8183166980743408,
+      "logps/chosen": -1.6468329429626465,
+      "logps/rejected": -13.756692886352539,
+      "loss": 0.8348377346992493,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.2248959094285965,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1438780128955841,
+      "rewards/margins": 0.22025901079177856,
+      "rewards/rejected": -0.07638098299503326,
       "step": 9,
-      "train_speed(iter/s)": 0.004494
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.140625,
-      "completions/min_length": 3.0,
-      "epoch": 0.11080332409972299,
-      "grad_norm": 0.7948888253152248,
-      "kl": 0.04085417848546058,
-      "learning_rate": 0.00011111111111111112,
-      "loss": 0.00047483143862336874,
-      "memory(GiB)": 78.03,
-      "reward": 0.0546875,
-      "reward_std": 0.6801962852478027,
-      "rewards/Response_no_think_reward/mean": 0.0546875,
-      "rewards/Response_no_think_reward/std": 1.4380027055740356,
+      "train_speed(iter/s)": 0.021091
+    },
+    {
+      "epoch": 0.02019691996970462,
+      "grad_norm": 11.680863380432129,
+      "learning_rate": 4e-05,
+      "logits/chosen": -0.7386271953582764,
+      "logits/rejected": -0.8098344206809998,
+      "logps/chosen": -3.001471996307373,
+      "logps/rejected": -16.38292694091797,
+      "loss": 1.0789214372634888,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.5013705492019653,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18567466735839844,
+      "rewards/margins": 0.40445780754089355,
+      "rewards/rejected": -0.21878314018249512,
       "step": 10,
-      "train_speed(iter/s)": 0.002899
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 17.5078125,
-      "completions/min_length": 4.0,
-      "epoch": 0.12188365650969529,
-      "grad_norm": 0.0004263039062328966,
-      "kl": 0.04189836711157113,
-      "learning_rate": 0.00012222222222222224,
-      "loss": 5.555100869969465e-06,
-      "memory(GiB)": 77.88,
-      "reward": 0.0,
-      "reward_std": 0.0,
-      "rewards/Response_no_think_reward/mean": 0.0,
-      "rewards/Response_no_think_reward/std": 0.0,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.02221661196667508,
+      "grad_norm": 6.016479015350342,
+      "learning_rate": 4.4000000000000006e-05,
+      "logits/chosen": -0.6199511289596558,
+      "logits/rejected": -0.7641665935516357,
+      "logps/chosen": -1.786482810974121,
+      "logps/rejected": -23.814146041870117,
+      "loss": 0.6213081479072571,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.17753979563713074,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1519775092601776,
+      "rewards/margins": 0.6798387169837952,
+      "rewards/rejected": -0.5278611779212952,
       "step": 11,
-      "train_speed(iter/s)": 0.017791
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "epoch": 0.1329639889196676,
-      "grad_norm": 0.001194388957130102,
-      "kl": 0.06704438821179792,
-      "learning_rate": 0.00013333333333333334,
-      "loss": 9.91918750514742e-06,
-      "memory(GiB)": 77.88,
+      "train_speed(iter/s)": 0.021205
+    },
+    {
+      "epoch": 0.024236303963645543,
+      "grad_norm": 19.226726531982422,
+      "learning_rate": 4.8e-05,
+      "logits/chosen": -0.7735041975975037,
+      "logits/rejected": -0.8182462453842163,
+      "logps/chosen": -6.024149417877197,
+      "logps/rejected": -14.866905212402344,
+      "loss": 1.467666506767273,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.8242200016975403,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03484102711081505,
+      "rewards/margins": 0.42743903398513794,
+      "rewards/rejected": -0.4622800350189209,
       "step": 12,
-      "train_speed(iter/s)": 0.014858
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 19.8515625,
-      "completions/min_length": 2.0,
-      "epoch": 0.1440443213296399,
-      "grad_norm": 0.00032184673828801084,
-      "kl": 0.07992906332947314,
-      "learning_rate": 0.00014444444444444444,
-      "loss": 7.416723747155629e-06,
-      "memory(GiB)": 77.92,
-      "reward": 0.0,
-      "reward_std": 0.0,
-      "rewards/Response_no_think_reward/mean": 0.0,
-      "rewards/Response_no_think_reward/std": 0.0,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 0.026255995960616008,
+      "grad_norm": 3.4483253955841064,
+      "learning_rate": 5.2000000000000004e-05,
+      "logits/chosen": -0.754300594329834,
+      "logits/rejected": -0.7772647738456726,
+      "logps/chosen": -3.619762659072876,
+      "logps/rejected": -17.524477005004883,
+      "loss": 0.7115153074264526,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2013324648141861,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10463516414165497,
+      "rewards/margins": 0.7878110408782959,
+      "rewards/rejected": -0.6831759214401245,
       "step": 13,
-      "train_speed(iter/s)": 0.010543
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "epoch": 0.15512465373961218,
-      "grad_norm": 0.0009186451648310945,
-      "kl": 0.13713529019150883,
-      "learning_rate": 0.00015555555555555556,
-      "loss": 1.3553013559430838e-05,
-      "memory(GiB)": 77.92,
+      "train_speed(iter/s)": 0.021242
+    },
+    {
+      "epoch": 0.02827568795758647,
+      "grad_norm": 15.451835632324219,
+      "learning_rate": 5.6000000000000006e-05,
+      "logits/chosen": -0.7475101947784424,
+      "logits/rejected": -0.8498263359069824,
+      "logps/chosen": -3.9441192150115967,
+      "logps/rejected": -21.486186981201172,
+      "loss": 0.9213531613349915,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.5798130035400391,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09564964473247528,
+      "rewards/margins": 1.039682149887085,
+      "rewards/rejected": -0.9440325498580933,
       "step": 14,
-      "train_speed(iter/s)": 0.009842
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 19.9609375,
-      "completions/min_length": 3.0,
-      "epoch": 0.16620498614958448,
-      "grad_norm": 0.0007871986446541219,
-      "kl": 0.10129890241660178,
-      "learning_rate": 0.0001666666666666667,
-      "loss": 1.3785504052066244e-05,
-      "memory(GiB)": 77.92,
-      "reward": 0.0,
-      "reward_std": 0.0,
-      "rewards/Response_no_think_reward/mean": 0.0,
-      "rewards/Response_no_think_reward/std": 0.0,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.03029537995455693,
+      "grad_norm": 3.8979263305664062,
+      "learning_rate": 6e-05,
+      "logits/chosen": -0.6522198915481567,
+      "logits/rejected": -0.7548784017562866,
+      "logps/chosen": -2.765364408493042,
+      "logps/rejected": -21.602596282958984,
+      "loss": 0.6480428576469421,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.23036827147006989,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13951444625854492,
+      "rewards/margins": 0.8476398587226868,
+      "rewards/rejected": -0.7081253528594971,
       "step": 15,
-      "train_speed(iter/s)": 0.008057
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0028207504947204143,
-      "clip_ratio/low_min": 0.0028207504947204143,
-      "clip_ratio/region_mean": 0.0028207504947204143,
-      "completions/clipped_ratio": 0.109375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 21.1796875,
-      "completions/min_length": 3.0,
-      "epoch": 0.1772853185595568,
-      "grad_norm": 1.5221332407610604,
-      "kl": 0.12953427506727166,
-      "learning_rate": 0.00017777777777777779,
-      "loss": 0.0005249045789241791,
-      "memory(GiB)": 77.88,
-      "reward": -0.140625,
-      "reward_std": 0.747445285320282,
-      "rewards/Response_no_think_reward/mean": -0.140625,
-      "rewards/Response_no_think_reward/std": 1.3500328063964844,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.03231507195152739,
+      "grad_norm": 7.998473644256592,
+      "learning_rate": 6.400000000000001e-05,
+      "logits/chosen": -0.7795235514640808,
+      "logits/rejected": -0.8571889400482178,
+      "logps/chosen": -2.1100573539733887,
+      "logps/rejected": -13.082208633422852,
+      "loss": 0.8766142129898071,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.41500476002693176,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11589755117893219,
+      "rewards/margins": 0.6957570910453796,
+      "rewards/rejected": -0.5798596143722534,
       "step": 16,
-      "train_speed(iter/s)": 0.0232
-    },
-    {
-      "clip_ratio/high_max": 0.014893613493768498,
-      "clip_ratio/high_mean": 0.014893613493768498,
-      "clip_ratio/low_mean": 0.08722472144290805,
-      "clip_ratio/low_min": 0.08722472144290805,
-      "clip_ratio/region_mean": 0.10211833566427231,
-      "epoch": 0.1883656509695291,
-      "grad_norm": 1.2273096796368343,
-      "kl": 9.949020208441652,
-      "learning_rate": 0.00018888888888888888,
-      "loss": -0.009794662706553936,
-      "memory(GiB)": 77.88,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.034334763948497854,
+      "grad_norm": 2.908738613128662,
+      "learning_rate": 6.800000000000001e-05,
+      "logits/chosen": -0.7900448441505432,
+      "logits/rejected": -0.8519578576087952,
+      "logps/chosen": -3.0429553985595703,
+      "logps/rejected": -13.749216079711914,
+      "loss": 0.7060399651527405,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.27812597155570984,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40308868885040283,
+      "rewards/margins": 0.7511462569236755,
+      "rewards/rejected": -0.3480575382709503,
       "step": 17,
-      "train_speed(iter/s)": 0.019341
-    },
-    {
-      "clip_ratio/high_max": 0.00042517005931586027,
-      "clip_ratio/high_mean": 0.00042517005931586027,
-      "clip_ratio/low_mean": 0.005602272489340976,
-      "clip_ratio/low_min": 0.005602272489340976,
-      "clip_ratio/region_mean": 0.006027442548656836,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.5234375,
-      "completions/min_length": 3.0,
-      "epoch": 0.1994459833795014,
-      "grad_norm": 1.6723327645413764,
-      "kl": 19.73964432394132,
-      "learning_rate": 0.0002,
-      "loss": 0.0016888766549527645,
-      "memory(GiB)": 77.92,
-      "reward": -0.5078125,
-      "reward_std": 0.46106040477752686,
-      "rewards/Response_no_think_reward/mean": -0.5078125,
-      "rewards/Response_no_think_reward/std": 1.0038986206054688,
+      "train_speed(iter/s)": 0.021313
+    },
+    {
+      "epoch": 0.036354455945468316,
+      "grad_norm": 4.66619348526001,
+      "learning_rate": 7.2e-05,
+      "logits/chosen": -0.7611753344535828,
+      "logits/rejected": -0.8464637398719788,
+      "logps/chosen": -3.652045488357544,
+      "logps/rejected": -19.29679298400879,
+      "loss": 0.9751962423324585,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4190734326839447,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.08509862422943115,
+      "rewards/margins": 0.42228081822395325,
+      "rewards/rejected": -0.3371821641921997,
       "step": 18,
-      "train_speed(iter/s)": 0.012746
-    },
-    {
-      "clip_ratio/high_max": 0.008699395693838596,
-      "clip_ratio/high_mean": 0.008699395693838596,
-      "clip_ratio/low_mean": 0.09401972405612469,
-      "clip_ratio/low_min": 0.09401972405612469,
-      "clip_ratio/region_mean": 0.10271912068128586,
-      "epoch": 0.21052631578947367,
-      "grad_norm": 0.9659705365958429,
-      "kl": 28.792964363470674,
-      "learning_rate": 0.00019999578095183124,
-      "loss": -0.005614398047327995,
-      "memory(GiB)": 77.92,
+      "train_speed(iter/s)": 0.021323
+    },
+    {
+      "epoch": 0.03837414794243878,
+      "grad_norm": 4.54453706741333,
+      "learning_rate": 7.6e-05,
+      "logits/chosen": -0.7284001708030701,
+      "logits/rejected": -0.7670996189117432,
+      "logps/chosen": -6.018934726715088,
+      "logps/rejected": -13.253512382507324,
+      "loss": 1.2025885581970215,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.45651721954345703,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0025453297421336174,
+      "rewards/margins": 0.006808534264564514,
+      "rewards/rejected": -0.004263207316398621,
       "step": 19,
-      "train_speed(iter/s)": 0.011863
-    },
-    {
-      "clip_ratio/high_max": 0.0009057971183210611,
-      "clip_ratio/high_mean": 0.0009057971183210611,
-      "clip_ratio/low_mean": 0.0047033663431648165,
-      "clip_ratio/low_min": 0.0047033663431648165,
-      "clip_ratio/region_mean": 0.005609163461485878,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 17.703125,
-      "completions/min_length": 4.0,
-      "epoch": 0.22160664819944598,
-      "grad_norm": 2.0349350259537244,
-      "kl": 0.8193748028716072,
-      "learning_rate": 0.00019998312416333227,
-      "loss": 0.0007828018860891461,
-      "memory(GiB)": 77.92,
-      "reward": -0.40625,
-      "reward_std": 0.5371267199516296,
-      "rewards/Response_no_think_reward/mean": -0.40625,
-      "rewards/Response_no_think_reward/std": 1.1734429597854614,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.04039383993940924,
+      "grad_norm": 2.965843677520752,
+      "learning_rate": 8e-05,
+      "logits/chosen": -0.674504280090332,
+      "logits/rejected": -0.7847579121589661,
+      "logps/chosen": -3.122269630432129,
+      "logps/rejected": -16.265222549438477,
+      "loss": 0.9356863498687744,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2812672257423401,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12974591553211212,
+      "rewards/margins": 0.18470463156700134,
+      "rewards/rejected": -0.05495870113372803,
       "step": 20,
-      "train_speed(iter/s)": 0.009276
-    },
-    {
-      "clip_ratio/high_max": 0.017557535058585927,
-      "clip_ratio/high_mean": 0.017557535058585927,
-      "clip_ratio/low_mean": 0.0823797988705337,
-      "clip_ratio/low_min": 0.0823797988705337,
-      "clip_ratio/region_mean": 0.09993733279407024,
-      "epoch": 0.23268698060941828,
-      "grad_norm": 5.547205800030314,
-      "kl": 155.78269671928138,
-      "learning_rate": 0.00019996203070249516,
-      "loss": 0.028484804555773735,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021338
+    },
+    {
+      "epoch": 0.0424135319363797,
+      "grad_norm": 3.544822931289673,
+      "learning_rate": 8.4e-05,
+      "logits/chosen": -0.8259390592575073,
+      "logits/rejected": -0.8455062508583069,
+      "logps/chosen": -6.160022258758545,
+      "logps/rejected": -8.875776290893555,
+      "loss": 1.0958666801452637,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4123613238334656,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.17430700361728668,
+      "rewards/margins": 0.11852678656578064,
+      "rewards/rejected": 0.05578019842505455,
       "step": 21,
-      "train_speed(iter/s)": 0.008942
-    },
-    {
-      "clip_ratio/high_max": 0.00173778529278934,
-      "clip_ratio/high_mean": 0.00173778529278934,
-      "clip_ratio/low_mean": 0.004023336310638115,
-      "clip_ratio/low_min": 0.004023336310638115,
-      "clip_ratio/region_mean": 0.005761121603427455,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.359375,
-      "completions/min_length": 4.0,
-      "epoch": 0.24376731301939059,
-      "grad_norm": 1.1706684735088349,
-      "kl": 0.29493264015764,
-      "learning_rate": 0.00019993250234920636,
-      "loss": 0.0013634033966809511,
-      "memory(GiB)": 79.88,
-      "reward": 0.1640625,
-      "reward_std": 0.6870359778404236,
-      "rewards/Response_no_think_reward/mean": 0.1640625,
-      "rewards/Response_no_think_reward/std": 1.4240750074386597,
+      "train_speed(iter/s)": 0.021349
+    },
+    {
+      "epoch": 0.04443322393335016,
+      "grad_norm": 2.8963093757629395,
+      "learning_rate": 8.800000000000001e-05,
+      "logits/chosen": -0.7398238778114319,
+      "logits/rejected": -0.83761066198349,
+      "logps/chosen": -2.3067612648010254,
+      "logps/rejected": -13.408963203430176,
+      "loss": 0.9400858879089355,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.26946064829826355,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1603003889322281,
+      "rewards/margins": 0.15373794734477997,
+      "rewards/rejected": 0.00656244158744812,
       "step": 22,
-      "train_speed(iter/s)": 0.007618
-    },
-    {
-      "clip_ratio/high_max": 0.01892658555880189,
-      "clip_ratio/high_mean": 0.01892658555880189,
-      "clip_ratio/low_mean": 0.06755852687638253,
-      "clip_ratio/low_min": 0.06755852687638253,
-      "clip_ratio/region_mean": 0.0864851123187691,
-      "epoch": 0.2548476454293629,
-      "grad_norm": 0.5069456216354334,
-      "kl": 0.7371144236531109,
-      "learning_rate": 0.0001998945415950969,
-      "loss": -0.015337318181991577,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021358
+    },
+    {
+      "epoch": 0.046452915930320623,
+      "grad_norm": 3.5623438358306885,
+      "learning_rate": 9.200000000000001e-05,
+      "logits/chosen": -0.6750966906547546,
+      "logits/rejected": -0.7796332836151123,
+      "logps/chosen": -3.6001622676849365,
+      "logps/rejected": -14.151020050048828,
+      "loss": 1.0653876066207886,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4920450747013092,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2231697142124176,
+      "rewards/margins": 0.34812790155410767,
+      "rewards/rejected": -0.12495820969343185,
       "step": 23,
-      "train_speed(iter/s)": 0.007472
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.00311830299324356,
-      "clip_ratio/low_min": 0.00311830299324356,
-      "clip_ratio/region_mean": 0.00311830299324356,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.4296875,
-      "completions/min_length": 5.0,
-      "epoch": 0.2659279778393352,
-      "grad_norm": 1.2087135470902275,
-      "kl": 0.9872721234764867,
-      "learning_rate": 0.00019984815164333163,
-      "loss": 0.0027771026361733675,
-      "memory(GiB)": 79.88,
-      "reward": -0.109375,
-      "reward_std": 0.3798917531967163,
-      "rewards/Response_no_think_reward/mean": -0.109375,
-      "rewards/Response_no_think_reward/std": 1.1310532093048096,
+      "train_speed(iter/s)": 0.021362
+    },
+    {
+      "epoch": 0.048472607927291085,
+      "grad_norm": 3.5425710678100586,
+      "learning_rate": 9.6e-05,
+      "logits/chosen": -0.7490979433059692,
+      "logits/rejected": -0.8129041194915771,
+      "logps/chosen": -2.327343463897705,
+      "logps/rejected": -10.883130073547363,
+      "loss": 0.921205461025238,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2920321822166443,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09307915717363358,
+      "rewards/margins": 0.1912679672241211,
+      "rewards/rejected": -0.09818882495164871,
       "step": 24,
-      "train_speed(iter/s)": 0.006636
-    },
-    {
-      "clip_ratio/high_max": 0.0029115322104189545,
-      "clip_ratio/high_mean": 0.0029115322104189545,
-      "clip_ratio/low_mean": 0.07530217664316297,
-      "clip_ratio/low_min": 0.07530217664316297,
-      "clip_ratio/region_mean": 0.07821370894089341,
-      "epoch": 0.2770083102493075,
-      "grad_norm": 0.702071136643931,
-      "kl": 26.457206293552645,
-      "learning_rate": 0.00019979333640833947,
-      "loss": -0.003078086068853736,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021367
+    },
+    {
+      "epoch": 0.05049229992426155,
+      "grad_norm": 4.050752639770508,
+      "learning_rate": 0.0001,
+      "logits/chosen": -0.6667495965957642,
+      "logits/rejected": -0.7579994201660156,
+      "logps/chosen": -6.037406921386719,
+      "logps/rejected": -19.315378189086914,
+      "loss": 0.8225780725479126,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.3186819851398468,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3055117130279541,
+      "rewards/margins": 0.4999930262565613,
+      "rewards/rejected": -0.19448129832744598,
       "step": 25,
-      "train_speed(iter/s)": 0.006569
-    },
-    {
-      "clip_ratio/high_max": 0.0023347785463556647,
-      "clip_ratio/high_mean": 0.0023347785463556647,
-      "clip_ratio/low_mean": 0.01951372090843506,
-      "clip_ratio/low_min": 0.01951372090843506,
-      "clip_ratio/region_mean": 0.021848499542102218,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.6953125,
-      "completions/min_length": 4.0,
-      "epoch": 0.2880886426592798,
-      "grad_norm": 1.6527510231667384,
-      "kl": 7.873713016277179,
-      "learning_rate": 0.00019973010051548275,
-      "loss": 0.0009247222333215177,
-      "memory(GiB)": 79.88,
-      "reward": 0.015625,
-      "reward_std": 0.7059091925621033,
-      "rewards/Response_no_think_reward/mean": 0.015625,
-      "rewards/Response_no_think_reward/std": 1.2610729932785034,
+      "train_speed(iter/s)": 0.021364
+    },
+    {
+      "epoch": 0.052511991921232015,
+      "grad_norm": 3.7214102745056152,
+      "learning_rate": 0.00010400000000000001,
+      "logits/chosen": -0.7550854086875916,
+      "logits/rejected": -0.8332770466804504,
+      "logps/chosen": -3.076296329498291,
+      "logps/rejected": -13.947905540466309,
+      "loss": 1.0035364627838135,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.3730461597442627,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03216666728258133,
+      "rewards/margins": 0.19776780903339386,
+      "rewards/rejected": -0.16560114920139313,
       "step": 26,
-      "train_speed(iter/s)": 0.005996
-    },
-    {
-      "clip_ratio/high_max": 0.01952397992135957,
-      "clip_ratio/high_mean": 0.01952397992135957,
-      "clip_ratio/low_mean": 0.1283843811834231,
-      "clip_ratio/low_min": 0.1283843811834231,
-      "clip_ratio/region_mean": 0.1479083604644984,
-      "epoch": 0.29916897506925205,
-      "grad_norm": 12.590003285629427,
-      "kl": 741.7941563371569,
-      "learning_rate": 0.000199658449300667,
-      "loss": 0.036141443997621536,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.05453168391820248,
+      "grad_norm": 7.799258708953857,
+      "learning_rate": 0.00010800000000000001,
+      "logits/chosen": -0.6102021932601929,
+      "logits/rejected": -0.7568979263305664,
+      "logps/chosen": -5.391008377075195,
+      "logps/rejected": -21.710432052612305,
+      "loss": 1.2860000133514404,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6317353844642639,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.007743582129478455,
+      "rewards/margins": 0.258750855922699,
+      "rewards/rejected": -0.2510072588920593,
       "step": 27,
-      "train_speed(iter/s)": 0.005965
-    },
-    {
-      "clip_ratio/high_max": 0.0036726996186189353,
-      "clip_ratio/high_mean": 0.0036726996186189353,
-      "clip_ratio/low_mean": 0.005622756987577304,
-      "clip_ratio/low_min": 0.005622756987577304,
-      "clip_ratio/region_mean": 0.00929545663530007,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.8203125,
-      "completions/min_length": 4.0,
-      "epoch": 0.31024930747922436,
-      "grad_norm": 2.1964117931286085,
-      "kl": 11.635613721329719,
-      "learning_rate": 0.00019957838880989078,
-      "loss": 0.002877343213185668,
-      "memory(GiB)": 79.88,
-      "reward": 0.390625,
-      "reward_std": 0.6595449447631836,
-      "rewards/Response_no_think_reward/mean": 0.390625,
-      "rewards/Response_no_think_reward/std": 1.1379936933517456,
+      "train_speed(iter/s)": 0.021361
+    },
+    {
+      "epoch": 0.05655137591517294,
+      "grad_norm": 3.4134862422943115,
+      "learning_rate": 0.00011200000000000001,
+      "logits/chosen": -0.681388795375824,
+      "logits/rejected": -0.7808853983879089,
+      "logps/chosen": -5.752387523651123,
+      "logps/rejected": -19.651212692260742,
+      "loss": 1.1130492687225342,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4609695374965668,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.029274294152855873,
+      "rewards/margins": 0.19754451513290405,
+      "rewards/rejected": -0.16827021539211273,
       "step": 28,
-      "train_speed(iter/s)": 0.005568
-    },
-    {
-      "clip_ratio/high_max": 0.017762314528226852,
-      "clip_ratio/high_mean": 0.017762314528226852,
-      "clip_ratio/low_mean": 0.047843097941949964,
-      "clip_ratio/low_min": 0.047843097941949964,
-      "clip_ratio/region_mean": 0.06560541247017682,
-      "epoch": 0.32132963988919666,
-      "grad_norm": 1.3735654410758578,
-      "kl": 80.49331146203622,
-      "learning_rate": 0.00019948992579873538,
-      "loss": -0.005675393156707287,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021358
+    },
+    {
+      "epoch": 0.0585710679121434,
+      "grad_norm": 2.6125388145446777,
+      "learning_rate": 0.000116,
+      "logits/chosen": -0.6846833229064941,
+      "logits/rejected": -0.7732559442520142,
+      "logps/chosen": -2.147914171218872,
+      "logps/rejected": -14.99377727508545,
+      "loss": 0.6967631578445435,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.18463140726089478,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3106439709663391,
+      "rewards/margins": 0.524000346660614,
+      "rewards/rejected": -0.2133564054965973,
       "step": 29,
-      "train_speed(iter/s)": 0.005557
-    },
-    {
-      "clip_ratio/high_max": 0.00618883827701211,
-      "clip_ratio/high_mean": 0.00618883827701211,
-      "clip_ratio/low_mean": 0.005286419298499823,
-      "clip_ratio/low_min": 0.005286419298499823,
-      "clip_ratio/region_mean": 0.011475257691927254,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.484375,
-      "completions/min_length": 3.0,
-      "epoch": 0.33240997229916897,
-      "grad_norm": 1.1914361173581312,
-      "kl": 3.281426345667569,
-      "learning_rate": 0.00019939306773179497,
-      "loss": 0.0003165462985634804,
-      "memory(GiB)": 79.88,
-      "reward": 0.5625,
-      "reward_std": 0.4207349121570587,
-      "rewards/Response_no_think_reward/mean": 0.5625,
-      "rewards/Response_no_think_reward/std": 1.3265905380249023,
+      "train_speed(iter/s)": 0.021362
+    },
+    {
+      "epoch": 0.06059075990911386,
+      "grad_norm": 3.1886954307556152,
+      "learning_rate": 0.00012,
+      "logits/chosen": -0.7671568393707275,
+      "logits/rejected": -0.8747203946113586,
+      "logps/chosen": -1.0704294443130493,
+      "logps/rejected": -15.217928886413574,
+      "loss": 0.6793683171272278,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.11626588553190231,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2649898827075958,
+      "rewards/margins": 0.3756120800971985,
+      "rewards/rejected": -0.11062214523553848,
       "step": 30,
-      "train_speed(iter/s)": 0.005222
-    },
-    {
-      "clip_ratio/high_max": 0.05031776078976691,
-      "clip_ratio/high_mean": 0.05031776078976691,
-      "clip_ratio/low_mean": 0.041757271508686244,
-      "clip_ratio/low_min": 0.041757271508686244,
-      "clip_ratio/region_mean": 0.09207503264769912,
-      "epoch": 0.34349030470914127,
-      "grad_norm": 0.9882009232899025,
-      "kl": 9.828547842378612,
-      "learning_rate": 0.0001992878227820465,
-      "loss": -0.0019915460143238306,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.06261045190608432,
+      "grad_norm": 3.672961711883545,
+      "learning_rate": 0.000124,
+      "logits/chosen": -0.6060948967933655,
+      "logits/rejected": -0.7454618811607361,
+      "logps/chosen": -3.801156997680664,
+      "logps/rejected": -23.42005157470703,
+      "loss": 1.0564124584197998,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4279017746448517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18183766305446625,
+      "rewards/margins": 0.23221838474273682,
+      "rewards/rejected": -0.050380729138851166,
       "step": 31,
-      "train_speed(iter/s)": 0.005221
-    },
-    {
-      "clip_ratio/high_max": 0.0014815493486821651,
-      "clip_ratio/high_mean": 0.0014815493486821651,
-      "clip_ratio/low_mean": 0.00400914296915289,
-      "clip_ratio/low_min": 0.00400914296915289,
-      "clip_ratio/region_mean": 0.005490692317835055,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 19.078125,
-      "completions/min_length": 4.0,
-      "epoch": 0.3545706371191136,
-      "grad_norm": 1.1670914229830547,
-      "kl": 3.5491041047498584,
-      "learning_rate": 0.00019917419983016025,
-      "loss": 0.0021022851578891277,
-      "memory(GiB)": 79.88,
-      "reward": 0.25,
-      "reward_std": 0.7367533445358276,
-      "rewards/Response_no_think_reward/mean": 0.25,
-      "rewards/Response_no_think_reward/std": 1.3631829023361206,
+      "train_speed(iter/s)": 0.021365
+    },
+    {
+      "epoch": 0.06463014390305478,
+      "grad_norm": 5.934133529663086,
+      "learning_rate": 0.00012800000000000002,
+      "logits/chosen": -0.7708523869514465,
+      "logits/rejected": -0.81435227394104,
+      "logps/chosen": -2.663968801498413,
+      "logps/rejected": -11.808816909790039,
+      "loss": 0.9843453168869019,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.37907925248146057,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18125101923942566,
+      "rewards/margins": 0.27721551060676575,
+      "rewards/rejected": -0.0959644690155983,
       "step": 32,
-      "train_speed(iter/s)": 0.004948
-    },
-    {
-      "clip_ratio/high_max": 0.02484438387909904,
-      "clip_ratio/high_mean": 0.02484438387909904,
-      "clip_ratio/low_mean": 0.0674317826051265,
-      "clip_ratio/low_min": 0.0674317826051265,
-      "clip_ratio/region_mean": 0.09227616689167917,
-      "epoch": 0.3656509695290859,
-      "grad_norm": 0.9959477925686483,
-      "kl": 51.25826009634329,
-      "learning_rate": 0.00019905220846375032,
-      "loss": -0.002379298210144043,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.06664983590002524,
+      "grad_norm": 3.0649428367614746,
+      "learning_rate": 0.000132,
+      "logits/chosen": -0.651940107345581,
+      "logits/rejected": -0.7574964761734009,
+      "logps/chosen": -1.5061438083648682,
+      "logps/rejected": -21.272098541259766,
+      "loss": 0.670626163482666,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.23393775522708893,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.312156617641449,
+      "rewards/margins": 0.6968778967857361,
+      "rewards/rejected": -0.38472118973731995,
       "step": 33,
-      "train_speed(iter/s)": 0.004958
-    },
-    {
-      "clip_ratio/high_max": 0.0034505740622989833,
-      "clip_ratio/high_mean": 0.0034505740622989833,
-      "clip_ratio/low_mean": 0.0069533721543848515,
-      "clip_ratio/low_min": 0.0069533721543848515,
-      "clip_ratio/region_mean": 0.010403946158476174,
-      "completions/clipped_ratio": 0.03125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.4375,
-      "completions/min_length": 4.0,
-      "epoch": 0.3767313019390582,
-      "grad_norm": 1.1883165046483204,
-      "kl": 6.391523744328879,
-      "learning_rate": 0.00019892185897656578,
-      "loss": 0.0010706284083425999,
-      "memory(GiB)": 79.88,
-      "reward": 0.078125,
-      "reward_std": 0.3766237795352936,
-      "rewards/Response_no_think_reward/mean": 0.078125,
-      "rewards/Response_no_think_reward/std": 1.1126288175582886,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.06866952789699571,
+      "grad_norm": 3.6389997005462646,
+      "learning_rate": 0.00013600000000000003,
+      "logits/chosen": -0.6423748731613159,
+      "logits/rejected": -0.6685101985931396,
+      "logps/chosen": -6.548682689666748,
+      "logps/rejected": -13.626079559326172,
+      "loss": 1.1170402765274048,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.480682373046875,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2068958729505539,
+      "rewards/margins": 0.31287699937820435,
+      "rewards/rejected": -0.10598116368055344,
       "step": 34,
-      "train_speed(iter/s)": 0.004716
-    },
-    {
-      "clip_ratio/high_max": 0.01320959790609777,
-      "clip_ratio/high_mean": 0.01320959790609777,
-      "clip_ratio/low_mean": 0.059380761347711086,
-      "clip_ratio/low_min": 0.059380761347711086,
-      "clip_ratio/region_mean": 0.07259036041796207,
-      "epoch": 0.3878116343490305,
-      "grad_norm": 0.527136254278729,
-      "kl": 34.938005739822984,
-      "learning_rate": 0.00019878316236762196,
-      "loss": -0.005408844910562038,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021372
+    },
+    {
+      "epoch": 0.07068921989396618,
+      "grad_norm": 2.625232219696045,
+      "learning_rate": 0.00014,
+      "logits/chosen": -0.5362560749053955,
+      "logits/rejected": -0.7028357982635498,
+      "logps/chosen": -3.359612464904785,
+      "logps/rejected": -33.268436431884766,
+      "loss": 0.8328565955162048,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.28516510128974915,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05316687747836113,
+      "rewards/margins": 0.685168981552124,
+      "rewards/rejected": -0.6320021748542786,
       "step": 35,
-      "train_speed(iter/s)": 0.004731
-    },
-    {
-      "clip_ratio/high_max": 0.002702809404581785,
-      "clip_ratio/high_mean": 0.002702809404581785,
-      "clip_ratio/low_mean": 0.0033881079871207476,
-      "clip_ratio/low_min": 0.0033881079871207476,
-      "clip_ratio/region_mean": 0.006090917449910194,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.703125,
-      "completions/min_length": 4.0,
-      "epoch": 0.3988919667590028,
-      "grad_norm": 1.0383854818250358,
-      "kl": 10.416337528149597,
-      "learning_rate": 0.00019863613034027224,
-      "loss": 0.0015072962269186974,
-      "memory(GiB)": 79.88,
-      "reward": 0.5,
-      "reward_std": 0.4501557946205139,
-      "rewards/Response_no_think_reward/mean": 0.5,
-      "rewards/Response_no_think_reward/std": 1.1294203996658325,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.07270891189093663,
+      "grad_norm": 2.7919955253601074,
+      "learning_rate": 0.000144,
+      "logits/chosen": -0.6247209906578064,
+      "logits/rejected": -0.7358378767967224,
+      "logps/chosen": -4.632122039794922,
+      "logps/rejected": -18.708053588867188,
+      "loss": 0.719286322593689,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25582897663116455,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.32600465416908264,
+      "rewards/margins": 0.7712743282318115,
+      "rewards/rejected": -0.44526970386505127,
       "step": 36,
-      "train_speed(iter/s)": 0.004567
-    },
-    {
-      "clip_ratio/high_max": 0.04597766371443868,
-      "clip_ratio/high_mean": 0.04597766371443868,
-      "clip_ratio/low_mean": 0.029037245316430926,
-      "clip_ratio/low_min": 0.029037245316430926,
-      "clip_ratio/region_mean": 0.07501490786671638,
-      "epoch": 0.4099722991689751,
-      "grad_norm": 1.198632656262216,
-      "kl": 1.9380782267544419,
-      "learning_rate": 0.00019848077530122083,
-      "loss": -0.00020142836729064584,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.0747286038879071,
+      "grad_norm": 2.5269482135772705,
+      "learning_rate": 0.000148,
+      "logits/chosen": -0.6388774514198303,
+      "logits/rejected": -0.7476755976676941,
+      "logps/chosen": -1.6513766050338745,
+      "logps/rejected": -20.447532653808594,
+      "loss": 0.652652382850647,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.181876540184021,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2968735098838806,
+      "rewards/margins": 0.7243574857711792,
+      "rewards/rejected": -0.4274839758872986,
       "step": 37,
-      "train_speed(iter/s)": 0.004584
-    },
-    {
-      "clip_ratio/high_max": 0.0011905284482054412,
-      "clip_ratio/high_mean": 0.0011905284482054412,
-      "clip_ratio/low_mean": 0.013364006066694856,
-      "clip_ratio/low_min": 0.013364006066694856,
-      "clip_ratio/region_mean": 0.014554534514900297,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.28125,
-      "completions/min_length": 3.0,
-      "epoch": 0.42105263157894735,
-      "grad_norm": 1.9489606220013782,
-      "kl": 19.19968014501501,
-      "learning_rate": 0.0001983171103594755,
-      "loss": 0.0004887112881988287,
-      "memory(GiB)": 79.88,
-      "reward": 0.0625,
-      "reward_std": 0.47397348284721375,
-      "rewards/Response_no_think_reward/mean": 0.0625,
-      "rewards/Response_no_think_reward/std": 0.9618715047836304,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.07674829588487755,
+      "grad_norm": 2.419584274291992,
+      "learning_rate": 0.000152,
+      "logits/chosen": -0.5964392423629761,
+      "logits/rejected": -0.7218829393386841,
+      "logps/chosen": -1.5039414167404175,
+      "logps/rejected": -27.090576171875,
+      "loss": 0.6621359586715698,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.14563477039337158,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.28821805119514465,
+      "rewards/margins": 0.77447509765625,
+      "rewards/rejected": -0.48625698685646057,
       "step": 38,
-      "train_speed(iter/s)": 0.004438
-    },
-    {
-      "clip_ratio/high_max": 0.0027730573201552033,
-      "clip_ratio/high_mean": 0.0027730573201552033,
-      "clip_ratio/low_mean": 0.11966108158230782,
-      "clip_ratio/low_min": 0.11966108158230782,
-      "clip_ratio/region_mean": 0.12243413866963238,
-      "epoch": 0.43213296398891965,
-      "grad_norm": 1.0018802051645932,
-      "kl": 49.89589491917286,
-      "learning_rate": 0.0001981451493252418,
-      "loss": -0.009153969585895538,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.07876798788184802,
+      "grad_norm": 3.088440179824829,
+      "learning_rate": 0.00015600000000000002,
+      "logits/chosen": -0.7012370824813843,
+      "logits/rejected": -0.8179476261138916,
+      "logps/chosen": -1.9244059324264526,
+      "logps/rejected": -21.519695281982422,
+      "loss": 0.6807724237442017,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.244890034198761,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15054234862327576,
+      "rewards/margins": 0.9000046849250793,
+      "rewards/rejected": -0.749462366104126,
       "step": 39,
-      "train_speed(iter/s)": 0.004457
-    },
-    {
-      "clip_ratio/high_max": 0.0013888889225199819,
-      "clip_ratio/high_mean": 0.0013888889225199819,
-      "clip_ratio/low_mean": 0.01340760855237022,
-      "clip_ratio/low_min": 0.01340760855237022,
-      "clip_ratio/region_mean": 0.014796497474890202,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 8.34375,
-      "completions/min_length": 3.0,
-      "epoch": 0.44321329639889195,
-      "grad_norm": 1.5554099620650559,
-      "kl": 4.425425759982318,
-      "learning_rate": 0.0001979649067087574,
-      "loss": 0.006227841135114431,
-      "memory(GiB)": 79.88,
-      "reward": 0.109375,
-      "reward_std": 0.5266703963279724,
-      "rewards/Response_no_think_reward/mean": 0.109375,
-      "rewards/Response_no_think_reward/std": 1.0057659149169922,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.08078767987881848,
+      "grad_norm": 2.713752508163452,
+      "learning_rate": 0.00016,
+      "logits/chosen": -0.6608636379241943,
+      "logits/rejected": -0.7343001365661621,
+      "logps/chosen": -4.701538562774658,
+      "logps/rejected": -27.00352668762207,
+      "loss": 0.6774569749832153,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2570435106754303,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13876499235630035,
+      "rewards/margins": 1.0153288841247559,
+      "rewards/rejected": -0.8765639066696167,
       "step": 40,
-      "train_speed(iter/s)": 0.004236
-    },
-    {
-      "clip_ratio/high_max": 0.017265826056245714,
-      "clip_ratio/high_mean": 0.017265826056245714,
-      "clip_ratio/low_mean": 0.11041530082002282,
-      "clip_ratio/low_min": 0.11041530082002282,
-      "clip_ratio/region_mean": 0.1276811266434379,
-      "epoch": 0.45429362880886426,
-      "grad_norm": 6.231748769450823,
-      "kl": 522.7659375867806,
-      "learning_rate": 0.00019777639771906795,
-      "loss": -0.0034558051265776157,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.08280737187578895,
+      "grad_norm": 4.948164463043213,
+      "learning_rate": 0.000164,
+      "logits/chosen": -0.6391008496284485,
+      "logits/rejected": -0.6671096682548523,
+      "logps/chosen": -8.970189094543457,
+      "logps/rejected": -14.413579940795898,
+      "loss": 1.0303566455841064,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.41976141929626465,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0029628686606884003,
+      "rewards/margins": 0.573236882686615,
+      "rewards/rejected": -0.5761997699737549,
       "step": 41,
-      "train_speed(iter/s)": 0.004255
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.006427675718441606,
-      "clip_ratio/low_min": 0.006427675718441606,
-      "clip_ratio/region_mean": 0.006427675718441606,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.34375,
-      "completions/min_length": 5.0,
-      "epoch": 0.46537396121883656,
-      "grad_norm": 1.0020303907992343,
-      "kl": 6.527791675332992,
-      "learning_rate": 0.00019757963826274357,
-      "loss": -0.0012322411639615893,
-      "memory(GiB)": 79.88,
-      "reward": 0.1484375,
-      "reward_std": 0.21095064282417297,
-      "rewards/Response_no_think_reward/mean": 0.1484375,
-      "rewards/Response_no_think_reward/std": 0.628582239151001,
+      "train_speed(iter/s)": 0.021375
+    },
+    {
+      "epoch": 0.0848270638727594,
+      "grad_norm": 4.40416955947876,
+      "learning_rate": 0.000168,
+      "logits/chosen": -0.6820927858352661,
+      "logits/rejected": -0.7476735711097717,
+      "logps/chosen": -4.897199630737305,
+      "logps/rejected": -17.570022583007812,
+      "loss": 0.9847887754440308,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.42955508828163147,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.08176113665103912,
+      "rewards/margins": 0.5534911155700684,
+      "rewards/rejected": -0.47172996401786804,
       "step": 42,
-      "train_speed(iter/s)": 0.004165
-    },
-    {
-      "clip_ratio/high_max": 0.01585913705639541,
-      "clip_ratio/high_mean": 0.01585913705639541,
-      "clip_ratio/low_mean": 0.026421017944812775,
-      "clip_ratio/low_min": 0.026421017944812775,
-      "clip_ratio/region_mean": 0.04228015476837754,
-      "epoch": 0.47645429362880887,
-      "grad_norm": 0.4174666284613835,
-      "kl": 30.479038277953805,
-      "learning_rate": 0.0001973746449425368,
-      "loss": -0.009352116845548153,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021376
+    },
+    {
+      "epoch": 0.08684675586972987,
+      "grad_norm": 5.150269508361816,
+      "learning_rate": 0.000172,
+      "logits/chosen": -0.7095463275909424,
+      "logits/rejected": -0.7945126891136169,
+      "logps/chosen": -4.1012396812438965,
+      "logps/rejected": -20.216747283935547,
+      "loss": 0.956037700176239,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5109948515892029,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19462859630584717,
+      "rewards/margins": 0.853682279586792,
+      "rewards/rejected": -0.6590536832809448,
       "step": 43,
-      "train_speed(iter/s)": 0.004185
-    },
-    {
-      "clip_ratio/high_max": 0.0026535717770457268,
-      "clip_ratio/high_mean": 0.0026535717770457268,
-      "clip_ratio/low_mean": 0.008503502060193568,
-      "clip_ratio/low_min": 0.008503502060193568,
-      "clip_ratio/region_mean": 0.011157073837239295,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.953125,
-      "completions/min_length": 5.0,
-      "epoch": 0.48753462603878117,
-      "grad_norm": 2.3666909999114085,
-      "kl": 21.929430570075056,
-      "learning_rate": 0.0001971614350559814,
-      "loss": 0.0009473273530602455,
-      "memory(GiB)": 79.88,
-      "reward": -0.03125,
-      "reward_std": 0.3844763934612274,
-      "rewards/Response_no_think_reward/mean": -0.03125,
-      "rewards/Response_no_think_reward/std": 0.6750619411468506,
+      "train_speed(iter/s)": 0.021378
+    },
+    {
+      "epoch": 0.08886644786670032,
+      "grad_norm": 4.477358341217041,
+      "learning_rate": 0.00017600000000000002,
+      "logits/chosen": -0.7657253742218018,
+      "logits/rejected": -0.8225359320640564,
+      "logps/chosen": -2.71124267578125,
+      "logps/rejected": -13.512589454650879,
+      "loss": 0.8599216341972351,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.27771878242492676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13035942614078522,
+      "rewards/margins": 0.38512665033340454,
+      "rewards/rejected": -0.25476720929145813,
       "step": 44,
-      "train_speed(iter/s)": 0.0041
-    },
-    {
-      "clip_ratio/high_max": 0.003926915815100074,
-      "clip_ratio/high_mean": 0.003926915815100074,
-      "clip_ratio/low_mean": 0.1417178469710052,
-      "clip_ratio/low_min": 0.1417178469710052,
-      "clip_ratio/region_mean": 0.14564476208761334,
-      "epoch": 0.4986149584487535,
-      "grad_norm": 51.49679854517617,
-      "kl": 4715.846689800266,
-      "learning_rate": 0.00019694002659393305,
-      "loss": 0.22465737164020538,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021381
+    },
+    {
+      "epoch": 0.09088613986367079,
+      "grad_norm": 5.4391255378723145,
+      "learning_rate": 0.00018,
+      "logits/chosen": -0.6582808494567871,
+      "logits/rejected": -0.7074710726737976,
+      "logps/chosen": -5.520257472991943,
+      "logps/rejected": -18.50946044921875,
+      "loss": 0.8143137693405151,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2541295886039734,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06929375976324081,
+      "rewards/margins": 0.6602391004562378,
+      "rewards/rejected": -0.590945303440094,
       "step": 45,
-      "train_speed(iter/s)": 0.004121
-    },
-    {
-      "clip_ratio/high_max": 0.0025718168762978166,
-      "clip_ratio/high_mean": 0.0025718168762978166,
-      "clip_ratio/low_mean": 0.0054531682981178164,
-      "clip_ratio/low_min": 0.0054531682981178164,
-      "clip_ratio/region_mean": 0.008024985319934785,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.453125,
-      "completions/min_length": 3.0,
-      "epoch": 0.5096952908587258,
-      "grad_norm": 7.932036097415596,
-      "kl": 134.56803331989795,
-      "learning_rate": 0.0001967104382390511,
-      "loss": 0.00671145785599947,
-      "memory(GiB)": 79.88,
-      "reward": 0.109375,
-      "reward_std": 0.21731878817081451,
-      "rewards/Response_no_think_reward/mean": 0.109375,
-      "rewards/Response_no_think_reward/std": 0.6303901672363281,
+      "train_speed(iter/s)": 0.02138
+    },
+    {
+      "epoch": 0.09290583186064125,
+      "grad_norm": 4.585366249084473,
+      "learning_rate": 0.00018400000000000003,
+      "logits/chosen": -0.722661018371582,
+      "logits/rejected": -0.7770288586616516,
+      "logps/chosen": -3.6351494789123535,
+      "logps/rejected": -10.902822494506836,
+      "loss": 1.1608294248580933,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.47840094566345215,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.09331849962472916,
+      "rewards/margins": 0.1440279632806778,
+      "rewards/rejected": -0.05070946365594864,
       "step": 46,
-      "train_speed(iter/s)": 0.004027
-    },
-    {
-      "clip_ratio/high_max": 0.013239058025646955,
-      "clip_ratio/high_mean": 0.013239058025646955,
-      "clip_ratio/low_mean": 0.03283043531700969,
-      "clip_ratio/low_min": 0.03283043531700969,
-      "clip_ratio/region_mean": 0.04606949305161834,
-      "epoch": 0.5207756232686981,
-      "grad_norm": 2.141404413074471,
-      "kl": 68.84463222045451,
-      "learning_rate": 0.00019647268936422206,
-      "loss": -0.005232019349932671,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021384
+    },
+    {
+      "epoch": 0.09492552385761172,
+      "grad_norm": 4.578927040100098,
+      "learning_rate": 0.000188,
+      "logits/chosen": -0.6195976734161377,
+      "logits/rejected": -0.7779173254966736,
+      "logps/chosen": -0.9245270490646362,
+      "logps/rejected": -30.03981590270996,
+      "loss": 0.5132064819335938,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.10489758849143982,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2798718214035034,
+      "rewards/margins": 0.944153904914856,
+      "rewards/rejected": -0.6642820835113525,
       "step": 47,
-      "train_speed(iter/s)": 0.004048
-    },
-    {
-      "clip_ratio/high_max": 0.0005482456181198359,
-      "clip_ratio/high_mean": 0.0005482456181198359,
-      "clip_ratio/low_mean": 0.005675954627804458,
-      "clip_ratio/low_min": 0.005675954627804458,
-      "clip_ratio/region_mean": 0.006224200245924294,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 6.5078125,
-      "completions/min_length": 3.0,
-      "epoch": 0.5318559556786704,
-      "grad_norm": 2.4236302292247642,
-      "kl": 80.9618124118424,
-      "learning_rate": 0.00019622680003092503,
-      "loss": 0.005468316841870546,
-      "memory(GiB)": 79.88,
-      "reward": 0.0703125,
-      "reward_std": 0.1054728701710701,
-      "rewards/Response_no_think_reward/mean": 0.0703125,
-      "rewards/Response_no_think_reward/std": 0.50483638048172,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.09694521585458217,
+      "grad_norm": 6.127289772033691,
+      "learning_rate": 0.000192,
+      "logits/chosen": -0.6809214949607849,
+      "logits/rejected": -0.7704883813858032,
+      "logps/chosen": -1.4336328506469727,
+      "logps/rejected": -16.669113159179688,
+      "loss": 0.8413453102111816,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.27693358063697815,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1910504847764969,
+      "rewards/margins": 0.4971596598625183,
+      "rewards/rejected": -0.30610913038253784,
       "step": 48,
-      "train_speed(iter/s)": 0.003952
-    },
-    {
-      "clip_ratio/high_max": 0.0366715406998992,
-      "clip_ratio/high_mean": 0.0366715406998992,
-      "clip_ratio/low_mean": 0.0379418209195137,
-      "clip_ratio/low_min": 0.0379418209195137,
-      "clip_ratio/region_mean": 0.07461336255073547,
-      "epoch": 0.5429362880886427,
-      "grad_norm": 27.44200416529675,
-      "kl": 31.79825783398701,
-      "learning_rate": 0.0001959727909875389,
-      "loss": 0.08109890669584274,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.09896490785155264,
+      "grad_norm": 5.920216083526611,
+      "learning_rate": 0.000196,
+      "logits/chosen": -0.744787335395813,
+      "logits/rejected": -0.7133767604827881,
+      "logps/chosen": -8.577417373657227,
+      "logps/rejected": -15.383255958557129,
+      "loss": 1.23928964138031,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5441281199455261,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1016860231757164,
+      "rewards/margins": 0.13402265310287476,
+      "rewards/rejected": -0.23570865392684937,
       "step": 49,
-      "train_speed(iter/s)": 0.003972
-    },
-    {
-      "clip_ratio/high_max": 0.002227982331532985,
-      "clip_ratio/high_mean": 0.002227982331532985,
-      "clip_ratio/low_mean": 0.006371813942678273,
-      "clip_ratio/low_min": 0.006371813942678273,
-      "clip_ratio/region_mean": 0.008599796216003597,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.3125,
-      "completions/min_length": 5.0,
-      "epoch": 0.554016620498615,
-      "grad_norm": 1.8915627714701582,
-      "kl": 32.8292202873854,
-      "learning_rate": 0.00019571068366759143,
-      "loss": 0.004485747776925564,
-      "memory(GiB)": 79.88,
-      "reward": 0.046875,
-      "reward_std": 0.2615792155265808,
-      "rewards/Response_no_think_reward/mean": 0.046875,
-      "rewards/Response_no_think_reward/std": 0.5731440782546997,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.1009845998485231,
+      "grad_norm": 5.736071586608887,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.7741532325744629,
+      "logits/rejected": -0.8851832747459412,
+      "logps/chosen": -4.090970039367676,
+      "logps/rejected": -21.758840560913086,
+      "loss": 1.0278420448303223,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5550558567047119,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15652738511562347,
+      "rewards/margins": 0.8236760497093201,
+      "rewards/rejected": -0.6671487092971802,
       "step": 50,
-      "train_speed(iter/s)": 0.003883
-    },
-    {
-      "clip_ratio/high_max": 0.02533616591244936,
-      "clip_ratio/high_mean": 0.02533616591244936,
-      "clip_ratio/low_mean": 0.05068295169621706,
-      "clip_ratio/low_min": 0.05068295169621706,
-      "clip_ratio/region_mean": 0.07601911667734385,
-      "epoch": 0.5650969529085873,
-      "grad_norm": 3.4383916295396406,
-      "kl": 341.1058924796962,
-      "learning_rate": 0.00019544050018795075,
-      "loss": -2.989325366797857e-06,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021383
+    },
+    {
+      "epoch": 0.10300429184549356,
+      "grad_norm": 4.201787948608398,
+      "learning_rate": 0.0001999994415122672,
+      "logits/chosen": -0.8024938106536865,
+      "logits/rejected": -0.8299172520637512,
+      "logps/chosen": -6.9087419509887695,
+      "logps/rejected": -10.72697639465332,
+      "loss": 1.1162101030349731,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4278947114944458,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.060360271483659744,
+      "rewards/margins": 0.15178537368774414,
+      "rewards/rejected": -0.0914250910282135,
       "step": 51,
-      "train_speed(iter/s)": 0.003903
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0021551724057644606,
-      "clip_ratio/low_min": 0.0021551724057644606,
-      "clip_ratio/region_mean": 0.0021551724057644606,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 24.0,
-      "completions/mean_length": 6.3046875,
-      "completions/min_length": 6.0,
-      "epoch": 0.5761772853185596,
-      "grad_norm": 0.7839236512036062,
-      "kl": 20.716427901759744,
-      "learning_rate": 0.0001951622633469592,
-      "loss": 0.0010195983340963721,
-      "memory(GiB)": 79.88,
-      "reward": 0.0390625,
-      "reward_std": 0.09916213154792786,
-      "rewards/Response_no_think_reward/mean": 0.0390625,
-      "rewards/Response_no_think_reward/std": 0.3847965598106384,
+      "train_speed(iter/s)": 0.021239
+    },
+    {
+      "epoch": 0.10502398384246403,
+      "grad_norm": 3.155092477798462,
+      "learning_rate": 0.0001999977660553069,
+      "logits/chosen": -0.6624072790145874,
+      "logits/rejected": -0.8125584125518799,
+      "logps/chosen": -1.1845072507858276,
+      "logps/rejected": -28.393592834472656,
+      "loss": 0.5598468780517578,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.15286707878112793,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2649957835674286,
+      "rewards/margins": 1.0407716035842896,
+      "rewards/rejected": -0.7757757902145386,
       "step": 52,
-      "train_speed(iter/s)": 0.003843
-    },
-    {
-      "clip_ratio/high_max": 0.0052083334885537624,
-      "clip_ratio/high_mean": 0.0052083334885537624,
-      "clip_ratio/low_mean": 0.031070403289049864,
-      "clip_ratio/low_min": 0.031070403289049864,
-      "clip_ratio/region_mean": 0.036278736777603626,
-      "epoch": 0.5872576177285319,
-      "grad_norm": 0.5191411391520966,
-      "kl": 69.86534339073114,
-      "learning_rate": 0.00019487599662250943,
-      "loss": -0.0016538399504497647,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.02124
+    },
+    {
+      "epoch": 0.10704367583943449,
+      "grad_norm": 3.9601948261260986,
+      "learning_rate": 0.0001999949736478336,
+      "logits/chosen": -0.8051952719688416,
+      "logits/rejected": -0.8654646873474121,
+      "logps/chosen": -3.793642997741699,
+      "logps/rejected": -17.214599609375,
+      "loss": 0.738736629486084,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.23666247725486755,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17774169147014618,
+      "rewards/margins": 0.6443970799446106,
+      "rewards/rejected": -0.4666553735733032,
       "step": 53,
-      "train_speed(iter/s)": 0.003863
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0052784152794629335,
-      "clip_ratio/low_min": 0.0052784152794629335,
-      "clip_ratio/region_mean": 0.0052784152794629335,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.3046875,
-      "completions/min_length": 3.0,
-      "epoch": 0.5983379501385041,
-      "grad_norm": 2.311740957637672,
-      "kl": 2.841107105177798,
-      "learning_rate": 0.00019458172417006347,
-      "loss": 0.004196230787783861,
-      "memory(GiB)": 79.88,
-      "reward": 0.3984375,
-      "reward_std": 0.26962828636169434,
-      "rewards/Response_no_think_reward/mean": 0.3984375,
-      "rewards/Response_no_think_reward/std": 1.0221164226531982,
+      "train_speed(iter/s)": 0.021244
+    },
+    {
+      "epoch": 0.10906336783640495,
+      "grad_norm": 2.747492790222168,
+      "learning_rate": 0.0001999910643210378,
+      "logits/chosen": -0.7448392510414124,
+      "logits/rejected": -0.9058634042739868,
+      "logps/chosen": -1.6127703189849854,
+      "logps/rejected": -30.806520462036133,
+      "loss": 0.5075910091400146,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.17231889069080353,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.4633626639842987,
+      "rewards/margins": 1.3208293914794922,
+      "rewards/rejected": -0.8574665784835815,
       "step": 54,
-      "train_speed(iter/s)": 0.003692
-    },
-    {
-      "clip_ratio/high_max": 0.017684109043329954,
-      "clip_ratio/high_mean": 0.017684109043329954,
-      "clip_ratio/low_mean": 0.09186162473633885,
-      "clip_ratio/low_min": 0.09186162473633885,
-      "clip_ratio/region_mean": 0.10954573331400752,
-      "epoch": 0.6094182825484764,
-      "grad_norm": 2.0764162059344025,
-      "kl": 263.63854900409933,
-      "learning_rate": 0.00019427947082061432,
-      "loss": 0.006979572586715221,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021246
+    },
+    {
+      "epoch": 0.11108305983337541,
+      "grad_norm": 3.485903739929199,
+      "learning_rate": 0.00019998603811858571,
+      "logits/chosen": -0.6839566230773926,
+      "logits/rejected": -0.8555458784103394,
+      "logps/chosen": -2.3723244667053223,
+      "logps/rejected": -34.97600173950195,
+      "loss": 0.7429671883583069,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3656916618347168,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.24485386908054352,
+      "rewards/margins": 1.4793577194213867,
+      "rewards/rejected": -1.2345038652420044,
       "step": 55,
-      "train_speed(iter/s)": 0.003712
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.018976572435349226,
-      "clip_ratio/low_min": 0.018976572435349226,
-      "clip_ratio/region_mean": 0.018976572435349226,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 8.0,
-      "completions/min_length": 3.0,
-      "epoch": 0.6204986149584487,
-      "grad_norm": 2.3049148508787534,
-      "kl": 227.90118128014728,
-      "learning_rate": 0.00019396926207859084,
-      "loss": 0.00933245662599802,
-      "memory(GiB)": 79.88,
-      "reward": -0.1171875,
-      "reward_std": 0.33797404170036316,
-      "rewards/Response_no_think_reward/mean": -0.1171875,
-      "rewards/Response_no_think_reward/std": 0.7594143748283386,
+      "train_speed(iter/s)": 0.021247
+    },
+    {
+      "epoch": 0.11310275183034588,
+      "grad_norm": 4.815752029418945,
+      "learning_rate": 0.0001999798950966188,
+      "logits/chosen": -0.7993583679199219,
+      "logits/rejected": -0.8806532025337219,
+      "logps/chosen": -5.939935207366943,
+      "logps/rejected": -18.04892349243164,
+      "loss": 1.1477632522583008,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5634173154830933,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.026777414605021477,
+      "rewards/margins": 0.612076461315155,
+      "rewards/rejected": -0.5852990746498108,
       "step": 56,
-      "train_speed(iter/s)": 0.003618
-    },
-    {
-      "clip_ratio/high_max": 0.024031779495999217,
-      "clip_ratio/high_mean": 0.024031779495999217,
-      "clip_ratio/low_mean": 0.10989384504500777,
-      "clip_ratio/low_min": 0.10989384504500777,
-      "clip_ratio/region_mean": 0.13392562558874488,
-      "epoch": 0.631578947368421,
-      "grad_norm": 408.32691013228697,
-      "kl": 23.09741603245493,
-      "learning_rate": 0.0001936511241197055,
-      "loss": 1.690829873085022,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021251
+    },
+    {
+      "epoch": 0.11512244382731633,
+      "grad_norm": 2.9355173110961914,
+      "learning_rate": 0.00019997263532375303,
+      "logits/chosen": -0.7389899492263794,
+      "logits/rejected": -0.8667410612106323,
+      "logps/chosen": -2.2958154678344727,
+      "logps/rejected": -32.464534759521484,
+      "loss": 0.6630210876464844,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3285917043685913,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1826460361480713,
+      "rewards/margins": 1.4345746040344238,
+      "rewards/rejected": -1.251928448677063,
       "step": 57,
-      "train_speed(iter/s)": 0.003638
-    },
-    {
-      "clip_ratio/high_max": 0.00028280544211156666,
-      "clip_ratio/high_mean": 0.00028280544211156666,
-      "clip_ratio/low_mean": 0.023381341248750687,
-      "clip_ratio/low_min": 0.023381341248750687,
-      "clip_ratio/region_mean": 0.023664146661758423,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.25,
-      "completions/min_length": 3.0,
-      "epoch": 0.6426592797783933,
-      "grad_norm": 2.545625270383155,
-      "kl": 130.25380403241434,
-      "learning_rate": 0.0001933250837887457,
-      "loss": 0.014269158244132996,
-      "memory(GiB)": 79.88,
-      "reward": 0.203125,
-      "reward_std": 0.3562659025192261,
-      "rewards/Response_no_think_reward/mean": 0.203125,
-      "rewards/Response_no_think_reward/std": 1.0600800514221191,
+      "train_speed(iter/s)": 0.021253
+    },
+    {
+      "epoch": 0.1171421358242868,
+      "grad_norm": 4.644790172576904,
+      "learning_rate": 0.0001999642588810784,
+      "logits/chosen": -0.8502639532089233,
+      "logits/rejected": -0.9041623473167419,
+      "logps/chosen": -4.175015449523926,
+      "logps/rejected": -20.228912353515625,
+      "loss": 1.007567048072815,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4676174223423004,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.053391821682453156,
+      "rewards/margins": 0.8722233176231384,
+      "rewards/rejected": -0.925615131855011,
       "step": 58,
-      "train_speed(iter/s)": 0.003488
-    },
-    {
-      "clip_ratio/high_max": 0.0029069767333567142,
-      "clip_ratio/high_mean": 0.0029069767333567142,
-      "clip_ratio/low_mean": 0.15388594008982182,
-      "clip_ratio/low_min": 0.15388594008982182,
-      "clip_ratio/region_mean": 0.15679291728883982,
-      "epoch": 0.6537396121883656,
-      "grad_norm": 16.25636826382254,
-      "kl": 3078.323552307884,
-      "learning_rate": 0.0001929911685973088,
-      "loss": 0.1047079786658287,
-      "memory(GiB)": 79.88,
+      "train_speed(iter/s)": 0.021258
+    },
+    {
+      "epoch": 0.11916182782125725,
+      "grad_norm": 4.834179878234863,
+      "learning_rate": 0.00019995476586215762,
+      "logits/chosen": -0.8684061169624329,
+      "logits/rejected": -0.9059044718742371,
+      "logps/chosen": -9.152887344360352,
+      "logps/rejected": -15.517383575439453,
+      "loss": 1.118729591369629,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6267417073249817,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0668405294418335,
+      "rewards/margins": 0.7597751021385193,
+      "rewards/rejected": -0.6929346323013306,
       "step": 59,
-      "train_speed(iter/s)": 0.003508
-    },
-    {
-      "clip_ratio/high_max": 0.0012499999720603228,
-      "clip_ratio/high_mean": 0.0012499999720603228,
-      "clip_ratio/low_mean": 0.032458390574902296,
-      "clip_ratio/low_min": 0.032458390574902296,
-      "clip_ratio/region_mean": 0.03370839054696262,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.96875,
-      "completions/min_length": 4.0,
-      "epoch": 0.6648199445983379,
-      "grad_norm": 7.944638763107582,
-      "kl": 1613.5422123023309,
-      "learning_rate": 0.00019264940672148018,
-      "loss": 0.06612621992826462,
-      "memory(GiB)": 79.88,
-      "reward": -0.2109375,
-      "reward_std": 0.21648234128952026,
-      "rewards/Response_no_think_reward/mean": -0.2109375,
-      "rewards/Response_no_think_reward/std": 0.4636782705783844,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.12118151981822772,
+      "grad_norm": 3.052858591079712,
+      "learning_rate": 0.00019994415637302547,
+      "logits/chosen": -0.7785470485687256,
+      "logits/rejected": -0.8464182019233704,
+      "logps/chosen": -2.411245107650757,
+      "logps/rejected": -17.925155639648438,
+      "loss": 0.7822959423065186,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25340673327445984,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.19467662274837494,
+      "rewards/margins": 0.8461788892745972,
+      "rewards/rejected": -0.6515023112297058,
       "step": 60,
-      "train_speed(iter/s)": 0.003389
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.12320121181519818,
+      "grad_norm": 3.3108043670654297,
+      "learning_rate": 0.0001999324305321873,
+      "logits/chosen": -0.762412428855896,
+      "logits/rejected": -0.8276241421699524,
+      "logps/chosen": -2.0741934776306152,
+      "logps/rejected": -14.743797302246094,
+      "loss": 0.8085264563560486,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25528597831726074,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13805074989795685,
+      "rewards/margins": 0.4351329803466797,
+      "rewards/rejected": -0.29708221554756165,
+      "step": 61,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.12522090381216863,
+      "grad_norm": 3.6591594219207764,
+      "learning_rate": 0.00019991958847061784,
+      "logits/chosen": -0.6577974557876587,
+      "logits/rejected": -0.7462488412857056,
+      "logps/chosen": -5.370046138763428,
+      "logps/rejected": -15.76892375946045,
+      "loss": 0.9229806661605835,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3679361343383789,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14663854241371155,
+      "rewards/margins": 0.4843655228614807,
+      "rewards/rejected": -0.33772704005241394,
+      "step": 62,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.1272405958091391,
+      "grad_norm": 3.069516658782959,
+      "learning_rate": 0.00019990563033175983,
+      "logits/chosen": -0.6640661954879761,
+      "logits/rejected": -0.7672103643417358,
+      "logps/chosen": -2.8094382286071777,
+      "logps/rejected": -18.480703353881836,
+      "loss": 0.7683196663856506,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2581484913825989,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2657431662082672,
+      "rewards/margins": 0.5364934802055359,
+      "rewards/rejected": -0.27075034379959106,
+      "step": 63,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.12926028780610957,
+      "grad_norm": 4.207716464996338,
+      "learning_rate": 0.0001998905562715222,
+      "logits/chosen": -0.7866019010543823,
+      "logits/rejected": -0.8437267541885376,
+      "logps/chosen": -6.750940799713135,
+      "logps/rejected": -15.358600616455078,
+      "loss": 1.2175216674804688,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6082065105438232,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.004789266735315323,
+      "rewards/margins": 0.4517655074596405,
+      "rewards/rejected": -0.44697627425193787,
+      "step": 64,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.13127997980308004,
+      "grad_norm": 2.928544282913208,
+      "learning_rate": 0.0001998743664582786,
+      "logits/chosen": -0.7847710251808167,
+      "logits/rejected": -0.8567973971366882,
+      "logps/chosen": -2.9689300060272217,
+      "logps/rejected": -19.40102767944336,
+      "loss": 0.9175847768783569,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4591127634048462,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08043977618217468,
+      "rewards/margins": 0.8204008340835571,
+      "rewards/rejected": -0.7399609088897705,
+      "step": 65,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.13329967180005048,
+      "grad_norm": 3.2160611152648926,
+      "learning_rate": 0.00019985706107286514,
+      "logits/chosen": -0.737562358379364,
+      "logits/rejected": -0.8171918988227844,
+      "logps/chosen": -1.0520405769348145,
+      "logps/rejected": -13.78677749633789,
+      "loss": 0.6010702252388,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.1751595437526703,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.34129399061203003,
+      "rewards/margins": 0.7596877813339233,
+      "rewards/rejected": -0.41839370131492615,
+      "step": 66,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.13531936379702095,
+      "grad_norm": 3.3036932945251465,
+      "learning_rate": 0.00019983864030857882,
+      "logits/chosen": -0.8425466418266296,
+      "logits/rejected": -0.8621857762336731,
+      "logps/chosen": -3.0595273971557617,
+      "logps/rejected": -10.996197700500488,
+      "loss": 0.8735880851745605,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.35458827018737793,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18186795711517334,
+      "rewards/margins": 0.6314299702644348,
+      "rewards/rejected": -0.4495620131492615,
+      "step": 67,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.13733905579399142,
+      "grad_norm": 4.266996383666992,
+      "learning_rate": 0.000199819104371175,
+      "logits/chosen": -0.780097246170044,
+      "logits/rejected": -0.8485308289527893,
+      "logps/chosen": -2.523211717605591,
+      "logps/rejected": -16.111953735351562,
+      "loss": 0.8541173934936523,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3557310998439789,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15383680164813995,
+      "rewards/margins": 0.7679665684700012,
+      "rewards/rejected": -0.6141297817230225,
+      "step": 68,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.13935874779096188,
+      "grad_norm": 4.107142925262451,
+      "learning_rate": 0.0001997984534788654,
+      "logits/chosen": -0.8730877041816711,
+      "logits/rejected": -0.9365532398223877,
+      "logps/chosen": -3.6425538063049316,
+      "logps/rejected": -14.239816665649414,
+      "loss": 0.9654451608657837,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5073843002319336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17483213543891907,
+      "rewards/margins": 0.7276958227157593,
+      "rewards/rejected": -0.5528636574745178,
+      "step": 69,
+      "train_speed(iter/s)": 0.021301
+    },
+    {
+      "epoch": 0.14137843978793235,
+      "grad_norm": 6.40019416809082,
+      "learning_rate": 0.00019977668786231534,
+      "logits/chosen": -0.8017210960388184,
+      "logits/rejected": -0.8579236268997192,
+      "logps/chosen": -3.737541437149048,
+      "logps/rejected": -21.258663177490234,
+      "loss": 0.9284271001815796,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.45301198959350586,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02088707685470581,
+      "rewards/margins": 1.0906426906585693,
+      "rewards/rejected": -1.0697555541992188,
+      "step": 70,
+      "train_speed(iter/s)": 0.021303
+    },
+    {
+      "epoch": 0.1433981317849028,
+      "grad_norm": 2.583921432495117,
+      "learning_rate": 0.0001997538077646414,
+      "logits/chosen": -0.7329834699630737,
+      "logits/rejected": -0.8454375863075256,
+      "logps/chosen": -1.1047859191894531,
+      "logps/rejected": -17.00843620300293,
+      "loss": 0.5504162907600403,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.13712885975837708,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.37011057138442993,
+      "rewards/margins": 0.8445523381233215,
+      "rewards/rejected": -0.4744417369365692,
+      "step": 71,
+      "train_speed(iter/s)": 0.021305
+    },
+    {
+      "epoch": 0.14541782378187326,
+      "grad_norm": 2.9413509368896484,
+      "learning_rate": 0.00019972981344140874,
+      "logits/chosen": -0.7188448309898376,
+      "logits/rejected": -0.8286901116371155,
+      "logps/chosen": -2.631627321243286,
+      "logps/rejected": -22.459745407104492,
+      "loss": 0.7896307706832886,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3527805805206299,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12620781362056732,
+      "rewards/margins": 1.1612414121627808,
+      "rewards/rejected": -1.0350335836410522,
+      "step": 72,
+      "train_speed(iter/s)": 0.021307
+    },
+    {
+      "epoch": 0.14743751577884373,
+      "grad_norm": 3.0875403881073,
+      "learning_rate": 0.000199704705160628,
+      "logits/chosen": -0.7547197341918945,
+      "logits/rejected": -0.8351807594299316,
+      "logps/chosen": -5.00991153717041,
+      "logps/rejected": -30.629638671875,
+      "loss": 0.8512266874313354,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.39438357949256897,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1695525050163269,
+      "rewards/margins": 1.1737316846847534,
+      "rewards/rejected": -1.0041792392730713,
+      "step": 73,
+      "train_speed(iter/s)": 0.021308
+    },
+    {
+      "epoch": 0.1494572077758142,
+      "grad_norm": 3.0828444957733154,
+      "learning_rate": 0.0001996784832027525,
+      "logits/chosen": -0.7492446899414062,
+      "logits/rejected": -0.8275682926177979,
+      "logps/chosen": -3.108285903930664,
+      "logps/rejected": -18.224658966064453,
+      "loss": 0.7956384420394897,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2778869569301605,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1457613706588745,
+      "rewards/margins": 0.7900225520133972,
+      "rewards/rejected": -0.6442610621452332,
+      "step": 74,
+      "train_speed(iter/s)": 0.021311
+    },
+    {
+      "epoch": 0.15147689977278464,
+      "grad_norm": 2.5818111896514893,
+      "learning_rate": 0.00019965114786067516,
+      "logits/chosen": -0.7824280858039856,
+      "logits/rejected": -0.8512482643127441,
+      "logps/chosen": -0.3878008723258972,
+      "logps/rejected": -19.119585037231445,
+      "loss": 0.3888098895549774,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.06809913367033005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3290511965751648,
+      "rewards/margins": 1.1828463077545166,
+      "rewards/rejected": -0.853795051574707,
+      "step": 75,
+      "train_speed(iter/s)": 0.021315
+    },
+    {
+      "epoch": 0.1534965917697551,
+      "grad_norm": 3.1177899837493896,
+      "learning_rate": 0.000199622699439725,
+      "logits/chosen": -0.7851688861846924,
+      "logits/rejected": -0.8108853101730347,
+      "logps/chosen": -1.9916081428527832,
+      "logps/rejected": -18.17394256591797,
+      "loss": 0.6110791563987732,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25529026985168457,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.44155535101890564,
+      "rewards/margins": 1.2954609394073486,
+      "rewards/rejected": -0.8539055585861206,
+      "step": 76,
+      "train_speed(iter/s)": 0.021318
+    },
+    {
+      "epoch": 0.15551628376672558,
+      "grad_norm": 4.25754976272583,
+      "learning_rate": 0.00019959313825766386,
+      "logits/chosen": -0.6911004185676575,
+      "logits/rejected": -0.7695882320404053,
+      "logps/chosen": -2.4477198123931885,
+      "logps/rejected": -31.02164077758789,
+      "loss": 0.7074019908905029,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.29494914412498474,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19254879653453827,
+      "rewards/margins": 1.7098207473754883,
+      "rewards/rejected": -1.5172719955444336,
+      "step": 77,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.15753597576369605,
+      "grad_norm": 7.972916603088379,
+      "learning_rate": 0.00019956246464468294,
+      "logits/chosen": -0.6392947435379028,
+      "logits/rejected": -0.7352097630500793,
+      "logps/chosen": -4.085236549377441,
+      "logps/rejected": -42.829315185546875,
+      "loss": 0.8604206442832947,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.34664490818977356,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.04581055790185928,
+      "rewards/margins": 1.8653144836425781,
+      "rewards/rejected": -1.9111250638961792,
+      "step": 78,
+      "train_speed(iter/s)": 0.021318
+    },
+    {
+      "epoch": 0.1595556677606665,
+      "grad_norm": 9.261566162109375,
+      "learning_rate": 0.00019953067894339896,
+      "logits/chosen": -0.7612121105194092,
+      "logits/rejected": -0.7584162950515747,
+      "logps/chosen": -22.329971313476562,
+      "logps/rejected": -21.844179153442383,
+      "loss": 1.6152374744415283,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.7144608497619629,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5578466653823853,
+      "rewards/margins": 0.5714378952980042,
+      "rewards/rejected": -1.1292846202850342,
+      "step": 79,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16157535975763695,
+      "grad_norm": 2.517524242401123,
+      "learning_rate": 0.00019949778150885042,
+      "logits/chosen": -0.7058742046356201,
+      "logits/rejected": -0.8091444373130798,
+      "logps/chosen": -2.800210952758789,
+      "logps/rejected": -20.613197326660156,
+      "loss": 0.5419849157333374,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.18958702683448792,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1262124478816986,
+      "rewards/margins": 1.2596018314361572,
+      "rewards/rejected": -1.1333893537521362,
+      "step": 80,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16359505175460742,
+      "grad_norm": 2.3371551036834717,
+      "learning_rate": 0.00019946377270849356,
+      "logits/chosen": -0.6530874967575073,
+      "logits/rejected": -0.7903028130531311,
+      "logps/chosen": -1.8574573993682861,
+      "logps/rejected": -34.57234573364258,
+      "loss": 0.47842535376548767,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.20502685010433197,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21239632368087769,
+      "rewards/margins": 1.7234400510787964,
+      "rewards/rejected": -1.5110437870025635,
+      "step": 81,
+      "train_speed(iter/s)": 0.021321
+    },
+    {
+      "epoch": 0.1656147437515779,
+      "grad_norm": 2.623552083969116,
+      "learning_rate": 0.00019942865292219838,
+      "logits/chosen": -0.5469449162483215,
+      "logits/rejected": -0.7048972249031067,
+      "logps/chosen": -4.390412330627441,
+      "logps/rejected": -31.797592163085938,
+      "loss": 0.6188596487045288,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.28328126668930054,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1721397191286087,
+      "rewards/margins": 1.405639410018921,
+      "rewards/rejected": -1.2334996461868286,
+      "step": 82,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16763443574854833,
+      "grad_norm": 3.757638931274414,
+      "learning_rate": 0.00019939242254224423,
+      "logits/chosen": -0.7363824248313904,
+      "logits/rejected": -0.8039526343345642,
+      "logps/chosen": -4.921553134918213,
+      "logps/rejected": -18.160133361816406,
+      "loss": 1.017354965209961,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4084692597389221,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.042609840631484985,
+      "rewards/margins": 0.6757681965827942,
+      "rewards/rejected": -0.7183780074119568,
+      "step": 83,
+      "train_speed(iter/s)": 0.021321
+    },
+    {
+      "epoch": 0.1696541277455188,
+      "grad_norm": 2.6959855556488037,
+      "learning_rate": 0.00019935508197331555,
+      "logits/chosen": -0.601901650428772,
+      "logits/rejected": -0.7518936395645142,
+      "logps/chosen": -2.1843678951263428,
+      "logps/rejected": -32.7392578125,
+      "loss": 0.5553274154663086,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.21038973331451416,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22294606268405914,
+      "rewards/margins": 1.4514588117599487,
+      "rewards/rejected": -1.2285128831863403,
+      "step": 84,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.17167381974248927,
+      "grad_norm": 3.8080639839172363,
+      "learning_rate": 0.00019931663163249742,
+      "logits/chosen": -0.8070834875106812,
+      "logits/rejected": -0.8214148283004761,
+      "logps/chosen": -4.387559413909912,
+      "logps/rejected": -13.731513977050781,
+      "loss": 0.8594411611557007,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2712303698062897,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08239315450191498,
+      "rewards/margins": 0.5998476147651672,
+      "rewards/rejected": -0.5174545049667358,
+      "step": 85,
+      "train_speed(iter/s)": 0.021322
+    },
+    {
+      "epoch": 0.17369351173945974,
+      "grad_norm": 4.74073600769043,
+      "learning_rate": 0.00019927707194927066,
+      "logits/chosen": -0.770719051361084,
+      "logits/rejected": -0.802936315536499,
+      "logps/chosen": -3.8553550243377686,
+      "logps/rejected": -22.488487243652344,
+      "loss": 0.7815805077552795,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.432535856962204,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1827356368303299,
+      "rewards/margins": 1.2188729047775269,
+      "rewards/rejected": -1.0361372232437134,
+      "step": 86,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.1757132037364302,
+      "grad_norm": 6.638978481292725,
+      "learning_rate": 0.0001992364033655072,
+      "logits/chosen": -0.6007230281829834,
+      "logits/rejected": -0.7821131944656372,
+      "logps/chosen": -0.698825478553772,
+      "logps/rejected": -31.5311279296875,
+      "loss": 0.4713582396507263,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.09712451696395874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.23644955456256866,
+      "rewards/margins": 1.2481129169464111,
+      "rewards/rejected": -1.0116633176803589,
+      "step": 87,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.17773289573340065,
+      "grad_norm": 5.0127739906311035,
+      "learning_rate": 0.00019919462633546519,
+      "logits/chosen": -0.7654692530632019,
+      "logits/rejected": -0.8514058589935303,
+      "logps/chosen": -4.281129837036133,
+      "logps/rejected": -17.92319107055664,
+      "loss": 1.0033936500549316,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5217462182044983,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14046959578990936,
+      "rewards/margins": 0.7611122131347656,
+      "rewards/rejected": -0.6206426620483398,
+      "step": 88,
+      "train_speed(iter/s)": 0.021326
+    },
+    {
+      "epoch": 0.17975258773037112,
+      "grad_norm": 5.285451889038086,
+      "learning_rate": 0.00019915174132578378,
+      "logits/chosen": -0.6645584106445312,
+      "logits/rejected": -0.789848804473877,
+      "logps/chosen": -5.77445650100708,
+      "logps/rejected": -37.78670883178711,
+      "loss": 0.6340041160583496,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.26252368092536926,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07033467292785645,
+      "rewards/margins": 1.5086185932159424,
+      "rewards/rejected": -1.4382836818695068,
+      "step": 89,
+      "train_speed(iter/s)": 0.021326
+    },
+    {
+      "epoch": 0.18177227972734158,
+      "grad_norm": 6.486772060394287,
+      "learning_rate": 0.000199107748815478,
+      "logits/chosen": -0.5895614624023438,
+      "logits/rejected": -0.7669257521629333,
+      "logps/chosen": -2.622659206390381,
+      "logps/rejected": -42.33251953125,
+      "loss": 0.7003304958343506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30153194069862366,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2086510956287384,
+      "rewards/margins": 1.978740930557251,
+      "rewards/rejected": -1.770089864730835,
+      "step": 90,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.18379197172431205,
+      "grad_norm": 4.038703918457031,
+      "learning_rate": 0.00019906264929593347,
+      "logits/chosen": -0.6498143076896667,
+      "logits/rejected": -0.7417197823524475,
+      "logps/chosen": -6.643020153045654,
+      "logps/rejected": -26.659536361694336,
+      "loss": 0.5661642551422119,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23169369995594025,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.25588488578796387,
+      "rewards/margins": 1.4719629287719727,
+      "rewards/rejected": -1.2160780429840088,
+      "step": 91,
+      "train_speed(iter/s)": 0.021323
+    },
+    {
+      "epoch": 0.1858116637212825,
+      "grad_norm": 2.8225672245025635,
+      "learning_rate": 0.00019901644327090064,
+      "logits/chosen": -0.7718486189842224,
+      "logits/rejected": -0.8817813992500305,
+      "logps/chosen": -2.4756038188934326,
+      "logps/rejected": -29.404544830322266,
+      "loss": 0.4890742599964142,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24528944492340088,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2983011305332184,
+      "rewards/margins": 1.8501620292663574,
+      "rewards/rejected": -1.5518608093261719,
+      "step": 92,
+      "train_speed(iter/s)": 0.021325
+    },
+    {
+      "epoch": 0.18783135571825296,
+      "grad_norm": 7.307145595550537,
+      "learning_rate": 0.00019896913125648955,
+      "logits/chosen": -0.8246339559555054,
+      "logits/rejected": -0.8683527708053589,
+      "logps/chosen": -6.234779357910156,
+      "logps/rejected": -18.237667083740234,
+      "loss": 1.4174689054489136,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7513330578804016,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1854678839445114,
+      "rewards/margins": 0.6419495940208435,
+      "rewards/rejected": -0.8274174332618713,
+      "step": 93,
+      "train_speed(iter/s)": 0.021328
+    },
+    {
+      "epoch": 0.18985104771522343,
+      "grad_norm": 5.936891078948975,
+      "learning_rate": 0.00019892071378116376,
+      "logits/chosen": -0.7434009313583374,
+      "logits/rejected": -0.8614495396614075,
+      "logps/chosen": -4.436880111694336,
+      "logps/rejected": -27.014469146728516,
+      "loss": 0.9648631811141968,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44619184732437134,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08920584619045258,
+      "rewards/margins": 1.2521498203277588,
+      "rewards/rejected": -1.341355562210083,
+      "step": 94,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.1918707397121939,
+      "grad_norm": 11.764037132263184,
+      "learning_rate": 0.0001988711913857346,
+      "logits/chosen": -0.5709604024887085,
+      "logits/rejected": -0.7604628801345825,
+      "logps/chosen": -5.328193187713623,
+      "logps/rejected": -44.60304260253906,
+      "loss": 0.8271011114120483,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5297277569770813,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0564165934920311,
+      "rewards/margins": 2.2518301010131836,
+      "rewards/rejected": -2.3082468509674072,
+      "step": 95,
+      "train_speed(iter/s)": 0.021329
+    },
+    {
+      "epoch": 0.19389043170916434,
+      "grad_norm": 4.440858840942383,
+      "learning_rate": 0.00019882056462335512,
+      "logits/chosen": -0.716915488243103,
+      "logits/rejected": -0.8559072017669678,
+      "logps/chosen": -4.923259258270264,
+      "logps/rejected": -22.751483917236328,
+      "loss": 1.0631147623062134,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5847352743148804,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04070784151554108,
+      "rewards/margins": 0.9810720086097717,
+      "rewards/rejected": -0.9403641223907471,
+      "step": 96,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.1959101237061348,
+      "grad_norm": 4.534213542938232,
+      "learning_rate": 0.00019876883405951377,
+      "logits/chosen": -0.7735016345977783,
+      "logits/rejected": -0.8521608114242554,
+      "logps/chosen": -2.242138147354126,
+      "logps/rejected": -16.696674346923828,
+      "loss": 0.8221019506454468,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30803507566452026,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.23840653896331787,
+      "rewards/margins": 0.7663496732711792,
+      "rewards/rejected": -0.5279431939125061,
+      "step": 97,
+      "train_speed(iter/s)": 0.021332
+    },
+    {
+      "epoch": 0.19792981570310528,
+      "grad_norm": 6.823293685913086,
+      "learning_rate": 0.0001987160002720283,
+      "logits/chosen": -0.7845535278320312,
+      "logits/rejected": -0.8421120643615723,
+      "logps/chosen": -8.860337257385254,
+      "logps/rejected": -18.79197883605957,
+      "loss": 0.9576016068458557,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.476947546005249,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1753305345773697,
+      "rewards/margins": 1.0159931182861328,
+      "rewards/rejected": -0.8406625986099243,
+      "step": 98,
+      "train_speed(iter/s)": 0.021334
+    },
+    {
+      "epoch": 0.19994950770007575,
+      "grad_norm": 3.7400314807891846,
+      "learning_rate": 0.00019866206385103915,
+      "logits/chosen": -0.708592414855957,
+      "logits/rejected": -0.7664063572883606,
+      "logps/chosen": -5.081974983215332,
+      "logps/rejected": -16.826196670532227,
+      "loss": 1.022936224937439,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4172855615615845,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.049844808876514435,
+      "rewards/margins": 0.2858249545097351,
+      "rewards/rejected": -0.23598016798496246,
+      "step": 99,
+      "train_speed(iter/s)": 0.021335
+    },
+    {
+      "epoch": 0.2019691996970462,
+      "grad_norm": 4.397147178649902,
+      "learning_rate": 0.00019860702539900287,
+      "logits/chosen": -0.5574980974197388,
+      "logits/rejected": -0.7396320700645447,
+      "logps/chosen": -2.61395001411438,
+      "logps/rejected": -33.31013107299805,
+      "loss": 0.8842970132827759,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3539983034133911,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.06448046863079071,
+      "rewards/margins": 0.9230259656906128,
+      "rewards/rejected": -0.8585454225540161,
+      "step": 100,
+      "train_speed(iter/s)": 0.021334
+    },
+    {
+      "epoch": 0.20398889169401666,
+      "grad_norm": 3.6620795726776123,
+      "learning_rate": 0.0001985508855306855,
+      "logits/chosen": -0.7978264093399048,
+      "logits/rejected": -0.8318206667900085,
+      "logps/chosen": -3.0011239051818848,
+      "logps/rejected": -13.4129638671875,
+      "loss": 0.869354784488678,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28029945492744446,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17673368752002716,
+      "rewards/margins": 0.44924402236938477,
+      "rewards/rejected": -0.2725103795528412,
+      "step": 101,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.20600858369098712,
+      "grad_norm": 2.9251015186309814,
+      "learning_rate": 0.00019849364487315558,
+      "logits/chosen": -0.6999006271362305,
+      "logits/rejected": -0.7932964563369751,
+      "logps/chosen": -4.888404369354248,
+      "logps/rejected": -15.926616668701172,
+      "loss": 0.9600812792778015,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4099733531475067,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16860802471637726,
+      "rewards/margins": 0.5318610668182373,
+      "rewards/rejected": -0.36325308680534363,
+      "step": 102,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.2080282756879576,
+      "grad_norm": 3.172440767288208,
+      "learning_rate": 0.00019843530406577723,
+      "logits/chosen": -0.7246884107589722,
+      "logits/rejected": -0.7787467837333679,
+      "logps/chosen": -3.1632981300354004,
+      "logps/rejected": -21.545318603515625,
+      "loss": 0.7317727208137512,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3409346342086792,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1461772322654724,
+      "rewards/margins": 1.200048565864563,
+      "rewards/rejected": -1.0538712739944458,
+      "step": 103,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.21004796768492806,
+      "grad_norm": 3.71928334236145,
+      "learning_rate": 0.00019837586376020294,
+      "logits/chosen": -0.6723949313163757,
+      "logits/rejected": -0.7337789535522461,
+      "logps/chosen": -9.011453628540039,
+      "logps/rejected": -24.695667266845703,
+      "loss": 0.8767037987709045,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40366095304489136,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.017453588545322418,
+      "rewards/margins": 1.022116780281067,
+      "rewards/rejected": -1.0395703315734863,
+      "step": 104,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.2120676596818985,
+      "grad_norm": 2.393298625946045,
+      "learning_rate": 0.00019831532462036636,
+      "logits/chosen": -0.5787035226821899,
+      "logits/rejected": -0.7247171998023987,
+      "logps/chosen": -3.2338626384735107,
+      "logps/rejected": -30.85213851928711,
+      "loss": 0.5790133476257324,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23591873049736023,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.273136168718338,
+      "rewards/margins": 1.4806981086730957,
+      "rewards/rejected": -1.2075618505477905,
+      "step": 105,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.21408735167886897,
+      "grad_norm": 3.9807705879211426,
+      "learning_rate": 0.0001982536873224748,
+      "logits/chosen": -0.6726570129394531,
+      "logits/rejected": -0.7844520807266235,
+      "logps/chosen": -11.104076385498047,
+      "logps/rejected": -29.71591567993164,
+      "loss": 0.5819936990737915,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16406820714473724,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07211651653051376,
+      "rewards/margins": 1.3368288278579712,
+      "rewards/rejected": -1.2647120952606201,
+      "step": 106,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.21610704367583944,
+      "grad_norm": 14.505188941955566,
+      "learning_rate": 0.00019819095255500178,
+      "logits/chosen": -0.6649254560470581,
+      "logits/rejected": -0.7490218877792358,
+      "logps/chosen": -6.695674896240234,
+      "logps/rejected": -36.64976501464844,
+      "loss": 0.9944093227386475,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5400701761245728,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.20841172337532043,
+      "rewards/margins": 1.6346428394317627,
+      "rewards/rejected": -1.4262311458587646,
+      "step": 107,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.2181267356728099,
+      "grad_norm": 5.73347282409668,
+      "learning_rate": 0.00019812712101867922,
+      "logits/chosen": -0.7590409517288208,
+      "logits/rejected": -0.8766330480575562,
+      "logps/chosen": -1.7668696641921997,
+      "logps/rejected": -23.773542404174805,
+      "loss": 0.6851338744163513,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25430163741111755,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15534955263137817,
+      "rewards/margins": 1.3926160335540771,
+      "rewards/rejected": -1.2372663021087646,
+      "step": 108,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.22014642766978035,
+      "grad_norm": 2.351529121398926,
+      "learning_rate": 0.00019806219342648977,
+      "logits/chosen": -0.7287034392356873,
+      "logits/rejected": -0.8862197995185852,
+      "logps/chosen": -0.5065869688987732,
+      "logps/rejected": -29.2410945892334,
+      "loss": 0.35556384921073914,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10009891539812088,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.37776732444763184,
+      "rewards/margins": 1.7389698028564453,
+      "rewards/rejected": -1.361202597618103,
+      "step": 109,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.22216611966675082,
+      "grad_norm": 4.292262077331543,
+      "learning_rate": 0.0001979961705036587,
+      "logits/chosen": -0.7477302551269531,
+      "logits/rejected": -0.8113406300544739,
+      "logps/chosen": -4.461059093475342,
+      "logps/rejected": -25.79521942138672,
+      "loss": 0.9549956321716309,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4869251251220703,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08557192981243134,
+      "rewards/margins": 1.1444835662841797,
+      "rewards/rejected": -1.2300554513931274,
+      "step": 110,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.22418581166372128,
+      "grad_norm": 3.231773853302002,
+      "learning_rate": 0.0001979290529876458,
+      "logits/chosen": -0.6231730580329895,
+      "logits/rejected": -0.8174745440483093,
+      "logps/chosen": -0.6967498064041138,
+      "logps/rejected": -34.312564849853516,
+      "loss": 0.4151938259601593,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14568573236465454,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3004146218299866,
+      "rewards/margins": 1.8394732475280762,
+      "rewards/rejected": -1.5390586853027344,
+      "step": 111,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.22620550366069175,
+      "grad_norm": 17.476547241210938,
+      "learning_rate": 0.00019786084162813733,
+      "logits/chosen": -0.7515086531639099,
+      "logits/rejected": -0.8278781771659851,
+      "logps/chosen": -8.48164176940918,
+      "logps/rejected": -27.406028747558594,
+      "loss": 1.484164834022522,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8838226795196533,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3509487807750702,
+      "rewards/margins": 1.2049919366836548,
+      "rewards/rejected": -1.5559407472610474,
+      "step": 112,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.2282251956576622,
+      "grad_norm": 4.023671627044678,
+      "learning_rate": 0.00019779153718703745,
+      "logits/chosen": -0.6617270708084106,
+      "logits/rejected": -0.8029566407203674,
+      "logps/chosen": -3.8847668170928955,
+      "logps/rejected": -29.473770141601562,
+      "loss": 0.8206814527511597,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.33473947644233704,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1446269452571869,
+      "rewards/margins": 1.4900355339050293,
+      "rewards/rejected": -1.3454086780548096,
+      "step": 113,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.23024488765463266,
+      "grad_norm": 1.5781970024108887,
+      "learning_rate": 0.00019772114043845965,
+      "logits/chosen": -0.7092657089233398,
+      "logits/rejected": -0.8382660150527954,
+      "logps/chosen": -0.47141486406326294,
+      "logps/rejected": -40.95867919921875,
+      "loss": 0.3395133912563324,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10217627137899399,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24633201956748962,
+      "rewards/margins": 2.3744595050811768,
+      "rewards/rejected": -2.1281275749206543,
+      "step": 114,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.23226457965160313,
+      "grad_norm": 3.4185638427734375,
+      "learning_rate": 0.00019764965216871846,
+      "logits/chosen": -0.6823985576629639,
+      "logits/rejected": -0.8108518123626709,
+      "logps/chosen": -3.519054412841797,
+      "logps/rejected": -34.14986038208008,
+      "loss": 0.6295965909957886,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34174439311027527,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1087992712855339,
+      "rewards/margins": 1.8799338340759277,
+      "rewards/rejected": -1.7711347341537476,
+      "step": 115,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.2342842716485736,
+      "grad_norm": 5.202569961547852,
+      "learning_rate": 0.00019757707317632028,
+      "logits/chosen": -0.7702198624610901,
+      "logits/rejected": -0.7985497117042542,
+      "logps/chosen": -8.110039710998535,
+      "logps/rejected": -23.981826782226562,
+      "loss": 0.9200264811515808,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40714696049690247,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.011667863465845585,
+      "rewards/margins": 1.2929402589797974,
+      "rewards/rejected": -1.2812724113464355,
+      "step": 116,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.23630396364554407,
+      "grad_norm": 3.570646286010742,
+      "learning_rate": 0.0001975034042719546,
+      "logits/chosen": -0.6833267211914062,
+      "logits/rejected": -0.832187294960022,
+      "logps/chosen": -5.206616401672363,
+      "logps/rejected": -43.44145965576172,
+      "loss": 0.7497320175170898,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4067853093147278,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1555595099925995,
+      "rewards/margins": 2.0963921546936035,
+      "rewards/rejected": -1.940832495689392,
+      "step": 117,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.2383236556425145,
+      "grad_norm": 2.559943914413452,
+      "learning_rate": 0.0001974286462784851,
+      "logits/chosen": -0.7291467785835266,
+      "logits/rejected": -0.8261964321136475,
+      "logps/chosen": -1.4934219121932983,
+      "logps/rejected": -20.029233932495117,
+      "loss": 0.575518786907196,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24626033008098602,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.35854265093803406,
+      "rewards/margins": 1.5817933082580566,
+      "rewards/rejected": -1.2232506275177002,
+      "step": 118,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.24034334763948498,
+      "grad_norm": 8.765168190002441,
+      "learning_rate": 0.00019735280003094015,
+      "logits/chosen": -0.7664045095443726,
+      "logits/rejected": -0.8301156759262085,
+      "logps/chosen": -11.605274200439453,
+      "logps/rejected": -33.71162414550781,
+      "loss": 1.43560791015625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.0074396133422852,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5399270057678223,
+      "rewards/margins": 1.4373902082443237,
+      "rewards/rejected": -1.9773170948028564,
+      "step": 119,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24236303963645545,
+      "grad_norm": 4.5908522605896,
+      "learning_rate": 0.00019727586637650373,
+      "logits/chosen": -0.6932565569877625,
+      "logits/rejected": -0.7234666347503662,
+      "logps/chosen": -17.431007385253906,
+      "logps/rejected": -34.0074462890625,
+      "loss": 1.0241637229919434,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6038533449172974,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17532162368297577,
+      "rewards/margins": 1.6745084524154663,
+      "rewards/rejected": -1.849830150604248,
+      "step": 120,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24438273163342591,
+      "grad_norm": 4.291138648986816,
+      "learning_rate": 0.00019719784617450593,
+      "logits/chosen": -0.655267596244812,
+      "logits/rejected": -0.7929190993309021,
+      "logps/chosen": -6.5861992835998535,
+      "logps/rejected": -43.83182907104492,
+      "loss": 0.696444571018219,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4287887215614319,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.015995100140571594,
+      "rewards/margins": 2.3263349533081055,
+      "rewards/rejected": -2.342329978942871,
+      "step": 121,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24640242363039636,
+      "grad_norm": 3.7837073802948,
+      "learning_rate": 0.0001971187402964132,
+      "logits/chosen": -0.6419543027877808,
+      "logits/rejected": -0.7936818599700928,
+      "logps/chosen": -5.325220108032227,
+      "logps/rejected": -35.03059768676758,
+      "loss": 0.760798990726471,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4128337502479553,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.004157397896051407,
+      "rewards/margins": 1.6647117137908936,
+      "rewards/rejected": -1.6688693761825562,
+      "step": 122,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24842211562736682,
+      "grad_norm": 3.332064390182495,
+      "learning_rate": 0.00019703854962581884,
+      "logits/chosen": -0.767221987247467,
+      "logits/rejected": -0.8504694700241089,
+      "logps/chosen": -1.6606477499008179,
+      "logps/rejected": -30.287626266479492,
+      "loss": 0.5448625087738037,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1951584815979004,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18560823798179626,
+      "rewards/margins": 1.3272532224655151,
+      "rewards/rejected": -1.1416449546813965,
+      "step": 123,
+      "train_speed(iter/s)": 0.021289
+    },
+    {
+      "epoch": 0.25044180762433726,
+      "grad_norm": 2.538031816482544,
+      "learning_rate": 0.00019695727505843297,
+      "logits/chosen": -0.8040828704833984,
+      "logits/rejected": -0.8591914176940918,
+      "logps/chosen": -2.396242618560791,
+      "logps/rejected": -17.942195892333984,
+      "loss": 0.6009309887886047,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1990574449300766,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.31234806776046753,
+      "rewards/margins": 0.9391263723373413,
+      "rewards/rejected": -0.6267784237861633,
+      "step": 124,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.25246149962130776,
+      "grad_norm": 2.4364805221557617,
+      "learning_rate": 0.00019687491750207254,
+      "logits/chosen": -0.8258935213088989,
+      "logits/rejected": -0.8962618112564087,
+      "logps/chosen": -1.0966453552246094,
+      "logps/rejected": -16.684877395629883,
+      "loss": 0.6394660472869873,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15617281198501587,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2913311719894409,
+      "rewards/margins": 0.7776257395744324,
+      "rewards/rejected": -0.48629459738731384,
+      "step": 125,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.2544811916182782,
+      "grad_norm": 4.309374809265137,
+      "learning_rate": 0.00019679147787665126,
+      "logits/chosen": -0.7291491627693176,
+      "logits/rejected": -0.8647027611732483,
+      "logps/chosen": -2.546858549118042,
+      "logps/rejected": -25.51129150390625,
+      "loss": 0.7438904047012329,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29447755217552185,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24194566905498505,
+      "rewards/margins": 0.7109313607215881,
+      "rewards/rejected": -0.4689857065677643,
+      "step": 126,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.2565008836152487,
+      "grad_norm": 3.316589117050171,
+      "learning_rate": 0.00019670695711416928,
+      "logits/chosen": -0.7973051071166992,
+      "logits/rejected": -0.9037990570068359,
+      "logps/chosen": -1.6116214990615845,
+      "logps/rejected": -24.904531478881836,
+      "loss": 0.4576399624347687,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17375069856643677,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22464421391487122,
+      "rewards/margins": 1.4565354585647583,
+      "rewards/rejected": -1.2318912744522095,
+      "step": 127,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.25852057561221914,
+      "grad_norm": 4.841390132904053,
+      "learning_rate": 0.00019662135615870275,
+      "logits/chosen": -0.7546025514602661,
+      "logits/rejected": -0.9116957187652588,
+      "logps/chosen": -2.7441368103027344,
+      "logps/rejected": -25.96759796142578,
+      "loss": 0.8130377531051636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3844357430934906,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1914481222629547,
+      "rewards/margins": 1.4247140884399414,
+      "rewards/rejected": -1.2332658767700195,
+      "step": 128,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.2605402676091896,
+      "grad_norm": 6.839901447296143,
+      "learning_rate": 0.0001965346759663934,
+      "logits/chosen": -0.6810730695724487,
+      "logits/rejected": -0.8468779921531677,
+      "logps/chosen": -5.32491397857666,
+      "logps/rejected": -42.42945861816406,
+      "loss": 1.0299216508865356,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6279267072677612,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.009131789207458496,
+      "rewards/margins": 1.819998025894165,
+      "rewards/rejected": -1.829129934310913,
+      "step": 129,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.2625599596061601,
+      "grad_norm": 2.0061724185943604,
+      "learning_rate": 0.00019644691750543767,
+      "logits/chosen": -0.8441457152366638,
+      "logits/rejected": -0.9423956274986267,
+      "logps/chosen": -0.5530110597610474,
+      "logps/rejected": -32.842193603515625,
+      "loss": 0.25913411378860474,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05609491094946861,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27188676595687866,
+      "rewards/margins": 2.330637216567993,
+      "rewards/rejected": -2.0587503910064697,
+      "step": 130,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.2645796516031305,
+      "grad_norm": 4.784903526306152,
+      "learning_rate": 0.00019635808175607605,
+      "logits/chosen": -0.8365444540977478,
+      "logits/rejected": -0.9138449430465698,
+      "logps/chosen": -3.4112894535064697,
+      "logps/rejected": -23.938806533813477,
+      "loss": 0.7469592094421387,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31692469120025635,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09105291962623596,
+      "rewards/margins": 1.37967050075531,
+      "rewards/rejected": -1.288617730140686,
+      "step": 131,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.26659934360010096,
+      "grad_norm": 9.284527778625488,
+      "learning_rate": 0.00019626816971058205,
+      "logits/chosen": -0.8304141759872437,
+      "logits/rejected": -0.9100163578987122,
+      "logps/chosen": -3.7839527130126953,
+      "logps/rejected": -18.869930267333984,
+      "loss": 1.1523946523666382,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6580032706260681,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.240356907248497,
+      "rewards/margins": 1.1313059329986572,
+      "rewards/rejected": -0.890949010848999,
+      "step": 132,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.26861903559707145,
+      "grad_norm": 4.299342632293701,
+      "learning_rate": 0.00019617718237325115,
+      "logits/chosen": -0.6347696781158447,
+      "logits/rejected": -0.7560770511627197,
+      "logps/chosen": -3.5856218338012695,
+      "logps/rejected": -30.4008846282959,
+      "loss": 0.739059567451477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38467657566070557,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13314950466156006,
+      "rewards/margins": 1.3657082319259644,
+      "rewards/rejected": -1.2325587272644043,
+      "step": 133,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2706387275940419,
+      "grad_norm": 4.246809482574463,
+      "learning_rate": 0.00019608512076038962,
+      "logits/chosen": -0.7022749781608582,
+      "logits/rejected": -0.7915350794792175,
+      "logps/chosen": -4.822379112243652,
+      "logps/rejected": -24.07721710205078,
+      "loss": 0.8055979609489441,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3307011127471924,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.11167091131210327,
+      "rewards/margins": 1.3118420839309692,
+      "rewards/rejected": -1.2001712322235107,
+      "step": 134,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2726584195910124,
+      "grad_norm": 2.9997761249542236,
+      "learning_rate": 0.0001959919859003031,
+      "logits/chosen": -0.6153374910354614,
+      "logits/rejected": -0.7747887969017029,
+      "logps/chosen": -0.7465253472328186,
+      "logps/rejected": -28.333953857421875,
+      "loss": 0.5413554906845093,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14097237586975098,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33766934275627136,
+      "rewards/margins": 0.9080523252487183,
+      "rewards/rejected": -0.5703830718994141,
+      "step": 135,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.27467811158798283,
+      "grad_norm": 3.5653076171875,
+      "learning_rate": 0.00019589777883328505,
+      "logits/chosen": -0.7418694496154785,
+      "logits/rejected": -0.8762567043304443,
+      "logps/chosen": -1.4205451011657715,
+      "logps/rejected": -21.64703941345215,
+      "loss": 0.6491340398788452,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16818463802337646,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2140020728111267,
+      "rewards/margins": 0.9494506120681763,
+      "rewards/rejected": -0.7354484796524048,
+      "step": 136,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2766978035849533,
+      "grad_norm": 2.6075761318206787,
+      "learning_rate": 0.00019580250061160539,
+      "logits/chosen": -0.5225172638893127,
+      "logits/rejected": -0.7319562435150146,
+      "logps/chosen": -2.6468346118927,
+      "logps/rejected": -41.52344512939453,
+      "loss": 0.5949529409408569,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19340890645980835,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2633662819862366,
+      "rewards/margins": 1.1732457876205444,
+      "rewards/rejected": -0.9098795056343079,
+      "step": 137,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.27871749558192377,
+      "grad_norm": 3.9614176750183105,
+      "learning_rate": 0.00019570615229949842,
+      "logits/chosen": -0.636820375919342,
+      "logits/rejected": -0.7707249522209167,
+      "logps/chosen": -3.3777596950531006,
+      "logps/rejected": -26.16946029663086,
+      "loss": 0.758621335029602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3399200439453125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15843479335308075,
+      "rewards/margins": 0.947920024394989,
+      "rewards/rejected": -0.7894852161407471,
+      "step": 138,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.2807371875788942,
+      "grad_norm": 2.576108455657959,
+      "learning_rate": 0.00019560873497315116,
+      "logits/chosen": -0.6899356842041016,
+      "logits/rejected": -0.7887363433837891,
+      "logps/chosen": -1.9815049171447754,
+      "logps/rejected": -15.912079811096191,
+      "loss": 0.7393437623977661,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24023714661598206,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.33446478843688965,
+      "rewards/margins": 0.6615203619003296,
+      "rewards/rejected": -0.32705551385879517,
+      "step": 139,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2827568795758647,
+      "grad_norm": 3.3134891986846924,
+      "learning_rate": 0.00019551024972069126,
+      "logits/chosen": -0.8353362679481506,
+      "logits/rejected": -0.8483090996742249,
+      "logps/chosen": -1.7245616912841797,
+      "logps/rejected": -15.235477447509766,
+      "loss": 0.5436122417449951,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1701793223619461,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3035600185394287,
+      "rewards/margins": 1.1392890214920044,
+      "rewards/rejected": -0.8357290029525757,
+      "step": 140,
+      "train_speed(iter/s)": 0.021301
+    },
+    {
+      "epoch": 0.28477657157283515,
+      "grad_norm": 4.701112270355225,
+      "learning_rate": 0.0001954106976421748,
+      "logits/chosen": -0.7467123866081238,
+      "logits/rejected": -0.742473840713501,
+      "logps/chosen": -7.344741344451904,
+      "logps/rejected": -13.826343536376953,
+      "loss": 1.0974572896957397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.45263901352882385,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08538863807916641,
+      "rewards/margins": 0.4929981231689453,
+      "rewards/rejected": -0.5783867835998535,
+      "step": 141,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2867962635698056,
+      "grad_norm": 2.704679012298584,
+      "learning_rate": 0.00019531007984957408,
+      "logits/chosen": -0.5686406493186951,
+      "logits/rejected": -0.7427109479904175,
+      "logps/chosen": -2.140307664871216,
+      "logps/rejected": -37.99428939819336,
+      "loss": 0.5559962391853333,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15430670976638794,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3728140592575073,
+      "rewards/margins": 2.170938730239868,
+      "rewards/rejected": -1.7981246709823608,
+      "step": 142,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2888159555667761,
+      "grad_norm": 6.6994123458862305,
+      "learning_rate": 0.0001952083974667652,
+      "logits/chosen": -0.6588355898857117,
+      "logits/rejected": -0.7316503524780273,
+      "logps/chosen": -7.539848804473877,
+      "logps/rejected": -26.308818817138672,
+      "loss": 1.176750659942627,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5222424268722534,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.17204269766807556,
+      "rewards/margins": 0.8352524042129517,
+      "rewards/rejected": -1.0072951316833496,
+      "step": 143,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2908356475637465,
+      "grad_norm": 5.0174384117126465,
+      "learning_rate": 0.00019510565162951537,
+      "logits/chosen": -0.720901608467102,
+      "logits/rejected": -0.7480478882789612,
+      "logps/chosen": -6.40681266784668,
+      "logps/rejected": -27.90705680847168,
+      "loss": 0.6913059949874878,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3720777928829193,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31874871253967285,
+      "rewards/margins": 2.0500223636627197,
+      "rewards/rejected": -1.7312736511230469,
+      "step": 144,
+      "train_speed(iter/s)": 0.021303
+    },
+    {
+      "epoch": 0.29285533956071697,
+      "grad_norm": 6.825158596038818,
+      "learning_rate": 0.00019500184348547042,
+      "logits/chosen": -0.7821186780929565,
+      "logits/rejected": -0.8765726685523987,
+      "logps/chosen": -3.37898588180542,
+      "logps/rejected": -19.76290512084961,
+      "loss": 0.8391829133033752,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.339937299489975,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13619036972522736,
+      "rewards/margins": 1.1103405952453613,
+      "rewards/rejected": -0.9741502404212952,
+      "step": 145,
+      "train_speed(iter/s)": 0.021304
+    },
+    {
+      "epoch": 0.29487503155768746,
+      "grad_norm": 2.3948497772216797,
+      "learning_rate": 0.00019489697419414182,
+      "logits/chosen": -0.8977736830711365,
+      "logits/rejected": -0.954646110534668,
+      "logps/chosen": -1.5781641006469727,
+      "logps/rejected": -27.0587100982666,
+      "loss": 0.4295532703399658,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1162162497639656,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24644073843955994,
+      "rewards/margins": 1.8821758031845093,
+      "rewards/rejected": -1.6357351541519165,
+      "step": 146,
+      "train_speed(iter/s)": 0.021306
+    },
+    {
+      "epoch": 0.2968947235546579,
+      "grad_norm": 6.068948268890381,
+      "learning_rate": 0.00019479104492689384,
+      "logits/chosen": -0.7972388863563538,
+      "logits/rejected": -0.9000754356384277,
+      "logps/chosen": -8.90436840057373,
+      "logps/rejected": -43.9642448425293,
+      "loss": 1.004248023033142,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7405007481575012,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2410261034965515,
+      "rewards/margins": 2.683554172515869,
+      "rewards/rejected": -2.9245800971984863,
+      "step": 147,
+      "train_speed(iter/s)": 0.021307
+    },
+    {
+      "epoch": 0.2989144155516284,
+      "grad_norm": 6.354671478271484,
+      "learning_rate": 0.00019468405686693044,
+      "logits/chosen": -0.8239361643791199,
+      "logits/rejected": -0.8870259523391724,
+      "logps/chosen": -3.0636796951293945,
+      "logps/rejected": -32.0543327331543,
+      "loss": 0.6938296556472778,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32726508378982544,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2392302304506302,
+      "rewards/margins": 2.2771260738372803,
+      "rewards/rejected": -2.037895917892456,
+      "step": 148,
+      "train_speed(iter/s)": 0.021309
+    },
+    {
+      "epoch": 0.30093410754859884,
+      "grad_norm": 2.746241331100464,
+      "learning_rate": 0.00019457601120928194,
+      "logits/chosen": -0.7442300915718079,
+      "logits/rejected": -0.8740307092666626,
+      "logps/chosen": -0.39770960807800293,
+      "logps/rejected": -24.435047149658203,
+      "loss": 0.323159396648407,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05953114107251167,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28615039587020874,
+      "rewards/margins": 1.8557114601135254,
+      "rewards/rejected": -1.5695610046386719,
+      "step": 149,
+      "train_speed(iter/s)": 0.02131
+    },
+    {
+      "epoch": 0.3029537995455693,
+      "grad_norm": 7.339154243469238,
+      "learning_rate": 0.0001944669091607919,
+      "logits/chosen": -0.7669613361358643,
+      "logits/rejected": -0.8887062072753906,
+      "logps/chosen": -6.112335205078125,
+      "logps/rejected": -46.467247009277344,
+      "loss": 0.7917571663856506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44829899072647095,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07194096595048904,
+      "rewards/margins": 2.9925551414489746,
+      "rewards/rejected": -3.0644962787628174,
+      "step": 150,
+      "train_speed(iter/s)": 0.02131
+    },
+    {
+      "epoch": 0.3049734915425398,
+      "grad_norm": 13.051281929016113,
+      "learning_rate": 0.00019435675194010336,
+      "logits/chosen": -0.8439881801605225,
+      "logits/rejected": -0.93268883228302,
+      "logps/chosen": -4.412451267242432,
+      "logps/rejected": -46.751976013183594,
+      "loss": 0.8531545400619507,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44560468196868896,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06836838275194168,
+      "rewards/margins": 3.4782376289367676,
+      "rewards/rejected": -3.5466065406799316,
+      "step": 151,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.3069931835395102,
+      "grad_norm": 3.5028750896453857,
+      "learning_rate": 0.00019424554077764546,
+      "logits/chosen": -0.7861919403076172,
+      "logits/rejected": -0.915184497833252,
+      "logps/chosen": -2.617560386657715,
+      "logps/rejected": -56.570960998535156,
+      "loss": 0.4112832546234131,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2496473491191864,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.11519647389650345,
+      "rewards/margins": 3.7134275436401367,
+      "rewards/rejected": -3.598231315612793,
+      "step": 152,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.3090128755364807,
+      "grad_norm": 6.6451520919799805,
+      "learning_rate": 0.00019413327691561967,
+      "logits/chosen": -0.7669755816459656,
+      "logits/rejected": -0.8034001588821411,
+      "logps/chosen": -9.02087688446045,
+      "logps/rejected": -36.971290588378906,
+      "loss": 0.8952721357345581,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42217180132865906,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07903412729501724,
+      "rewards/margins": 2.4241135120391846,
+      "rewards/rejected": -2.503147602081299,
+      "step": 153,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.31103256753345115,
+      "grad_norm": 6.9827470779418945,
+      "learning_rate": 0.00019401996160798573,
+      "logits/chosen": -0.7891846895217896,
+      "logits/rejected": -0.8431411981582642,
+      "logps/chosen": -12.274852752685547,
+      "logps/rejected": -40.9384765625,
+      "loss": 1.0033327341079712,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5167030692100525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11504307389259338,
+      "rewards/margins": 2.439614772796631,
+      "rewards/rejected": -2.5546579360961914,
+      "step": 154,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.3130522595304216,
+      "grad_norm": 5.614060878753662,
+      "learning_rate": 0.0001939055961204478,
+      "logits/chosen": -0.8796770572662354,
+      "logits/rejected": -0.9147617816925049,
+      "logps/chosen": -7.191051959991455,
+      "logps/rejected": -29.8262882232666,
+      "loss": 0.8995785117149353,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.49978020787239075,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.055766861885786057,
+      "rewards/margins": 1.4225391149520874,
+      "rewards/rejected": -1.4783059358596802,
+      "step": 155,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.3150719515273921,
+      "grad_norm": 5.3426737785339355,
+      "learning_rate": 0.00019379018173044037,
+      "logits/chosen": -0.8801127076148987,
+      "logits/rejected": -0.9478501081466675,
+      "logps/chosen": -5.170099258422852,
+      "logps/rejected": -29.119112014770508,
+      "loss": 0.7686877846717834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36578065156936646,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10118754208087921,
+      "rewards/margins": 1.4974370002746582,
+      "rewards/rejected": -1.3962492942810059,
+      "step": 156,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.31709164352436253,
+      "grad_norm": 6.185426235198975,
+      "learning_rate": 0.00019367371972711385,
+      "logits/chosen": -0.883526623249054,
+      "logits/rejected": -0.9433069229125977,
+      "logps/chosen": -2.170651435852051,
+      "logps/rejected": -28.505441665649414,
+      "loss": 0.7070222496986389,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.333578884601593,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17996083199977875,
+      "rewards/margins": 1.5758326053619385,
+      "rewards/rejected": -1.395871639251709,
+      "step": 157,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.319111335521333,
+      "grad_norm": 4.604299068450928,
+      "learning_rate": 0.0001935562114113202,
+      "logits/chosen": -0.874811053276062,
+      "logits/rejected": -0.9069212079048157,
+      "logps/chosen": -6.411377906799316,
+      "logps/rejected": -28.813356399536133,
+      "loss": 0.8389774560928345,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4572408199310303,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1476302295923233,
+      "rewards/margins": 1.0184513330459595,
+      "rewards/rejected": -0.8708210587501526,
+      "step": 158,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.32113102751830347,
+      "grad_norm": 2.668325424194336,
+      "learning_rate": 0.00019343765809559852,
+      "logits/chosen": -0.9191151261329651,
+      "logits/rejected": -0.9647189378738403,
+      "logps/chosen": -2.182382106781006,
+      "logps/rejected": -27.029888153076172,
+      "loss": 0.6324246525764465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3407551646232605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22473560273647308,
+      "rewards/margins": 1.382723093032837,
+      "rewards/rejected": -1.1579875946044922,
+      "step": 159,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.3231507195152739,
+      "grad_norm": 3.9123880863189697,
+      "learning_rate": 0.00019331806110416027,
+      "logits/chosen": -0.9419438242912292,
+      "logits/rejected": -0.9589441418647766,
+      "logps/chosen": -2.685596466064453,
+      "logps/rejected": -16.038766860961914,
+      "loss": 0.8650633692741394,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3469831943511963,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12647470831871033,
+      "rewards/margins": 0.6830577254295349,
+      "rewards/rejected": -0.5565829873085022,
+      "step": 160,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.3251704115122444,
+      "grad_norm": 3.562811851501465,
+      "learning_rate": 0.00019319742177287448,
+      "logits/chosen": -0.895343542098999,
+      "logits/rejected": -0.9661368727684021,
+      "logps/chosen": -3.5327184200286865,
+      "logps/rejected": -20.103837966918945,
+      "loss": 0.8163770437240601,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42449474334716797,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2701226472854614,
+      "rewards/margins": 1.0340347290039062,
+      "rewards/rejected": -0.7639120221138,
+      "step": 161,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.32719010350921485,
+      "grad_norm": 6.563948631286621,
+      "learning_rate": 0.00019307574144925287,
+      "logits/chosen": -0.9778874516487122,
+      "logits/rejected": -1.0073670148849487,
+      "logps/chosen": -2.233975887298584,
+      "logps/rejected": -14.128082275390625,
+      "loss": 0.8808709979057312,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3910011947154999,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16543619334697723,
+      "rewards/margins": 0.6482046246528625,
+      "rewards/rejected": -0.48276838660240173,
+      "step": 162,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.3292097955061853,
+      "grad_norm": 2.812866449356079,
+      "learning_rate": 0.0001929530214924348,
+      "logits/chosen": -0.8719493746757507,
+      "logits/rejected": -0.9090234041213989,
+      "logps/chosen": -2.135589361190796,
+      "logps/rejected": -34.29349136352539,
+      "loss": 0.6473661661148071,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23261404037475586,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12609758973121643,
+      "rewards/margins": 1.2335658073425293,
+      "rewards/rejected": -1.1074683666229248,
+      "step": 163,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.3312294875031558,
+      "grad_norm": 6.056075096130371,
+      "learning_rate": 0.0001928292632731721,
+      "logits/chosen": -0.927330732345581,
+      "logits/rejected": -0.9583979845046997,
+      "logps/chosen": -2.9721577167510986,
+      "logps/rejected": -37.22073745727539,
+      "loss": 0.5903728604316711,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22814247012138367,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24338555335998535,
+      "rewards/margins": 1.3612616062164307,
+      "rewards/rejected": -1.1178761720657349,
+      "step": 164,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.3332491795001262,
+      "grad_norm": 3.5847744941711426,
+      "learning_rate": 0.00019270446817381377,
+      "logits/chosen": -0.9705857038497925,
+      "logits/rejected": -0.9790812134742737,
+      "logps/chosen": -2.228187084197998,
+      "logps/rejected": -22.54074478149414,
+      "loss": 0.582669198513031,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3219188451766968,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31063607335090637,
+      "rewards/margins": 1.6680660247802734,
+      "rewards/rejected": -1.3574302196502686,
+      "step": 165,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.33526887149709667,
+      "grad_norm": 3.5586040019989014,
+      "learning_rate": 0.00019257863758829035,
+      "logits/chosen": -0.9236764311790466,
+      "logits/rejected": -0.9713916182518005,
+      "logps/chosen": -2.841557264328003,
+      "logps/rejected": -41.60185241699219,
+      "loss": 0.667453408241272,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.45156461000442505,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1021651178598404,
+      "rewards/margins": 2.591975450515747,
+      "rewards/rejected": -2.4898104667663574,
+      "step": 166,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.33728856349406716,
+      "grad_norm": 11.787897109985352,
+      "learning_rate": 0.00019245177292209867,
+      "logits/chosen": -0.9395866394042969,
+      "logits/rejected": -0.9611519575119019,
+      "logps/chosen": -5.94741153717041,
+      "logps/rejected": -48.858680725097656,
+      "loss": 1.042246699333191,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6328973174095154,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07254321873188019,
+      "rewards/margins": 2.8825278282165527,
+      "rewards/rejected": -2.955070734024048,
+      "step": 167,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.3393082554910376,
+      "grad_norm": 2.1068975925445557,
+      "learning_rate": 0.00019232387559228587,
+      "logits/chosen": -0.9090099930763245,
+      "logits/rejected": -0.9743850827217102,
+      "logps/chosen": -3.3460116386413574,
+      "logps/rejected": -49.71737289428711,
+      "loss": 0.3520747423171997,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20601019263267517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.022673189640045166,
+      "rewards/margins": 3.1425955295562744,
+      "rewards/rejected": -3.165269136428833,
+      "step": 168,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.3413279474880081,
+      "grad_norm": 13.362913131713867,
+      "learning_rate": 0.0001921949470274338,
+      "logits/chosen": -0.9718740582466125,
+      "logits/rejected": -0.9933693408966064,
+      "logps/chosen": -16.02916717529297,
+      "logps/rejected": -40.69719696044922,
+      "loss": 1.3985366821289062,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.129913330078125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9396551847457886,
+      "rewards/margins": 1.5676053762435913,
+      "rewards/rejected": -2.507260799407959,
+      "step": 169,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.34334763948497854,
+      "grad_norm": 12.149360656738281,
+      "learning_rate": 0.00019206498866764288,
+      "logits/chosen": -0.9391348361968994,
+      "logits/rejected": -0.9958175420761108,
+      "logps/chosen": -4.999204635620117,
+      "logps/rejected": -34.91378402709961,
+      "loss": 1.1959863901138306,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.683914303779602,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1860208809375763,
+      "rewards/margins": 2.0848278999328613,
+      "rewards/rejected": -2.2708487510681152,
+      "step": 170,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.345367331481949,
+      "grad_norm": 4.593865871429443,
+      "learning_rate": 0.0001919340019645161,
+      "logits/chosen": -0.8601446151733398,
+      "logits/rejected": -0.9445533752441406,
+      "logps/chosen": -6.576286792755127,
+      "logps/rejected": -35.74972152709961,
+      "loss": 0.8745996952056885,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.48976749181747437,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1266278177499771,
+      "rewards/margins": 1.351570963859558,
+      "rewards/rejected": -1.478198766708374,
+      "step": 171,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.3473870234789195,
+      "grad_norm": 6.634700775146484,
+      "learning_rate": 0.00019180198838114282,
+      "logits/chosen": -0.9056757688522339,
+      "logits/rejected": -0.9392012357711792,
+      "logps/chosen": -1.7248777151107788,
+      "logps/rejected": -23.6937255859375,
+      "loss": 0.8178808093070984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37585797905921936,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18669246137142181,
+      "rewards/margins": 1.2759226560592651,
+      "rewards/rejected": -1.0892301797866821,
+      "step": 172,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.3494067154758899,
+      "grad_norm": 6.2399725914001465,
+      "learning_rate": 0.00019166894939208237,
+      "logits/chosen": -0.8763759732246399,
+      "logits/rejected": -0.8875811100006104,
+      "logps/chosen": -4.987262725830078,
+      "logps/rejected": -23.384920120239258,
+      "loss": 0.953944742679596,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5001811385154724,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11620373278856277,
+      "rewards/margins": 1.522448182106018,
+      "rewards/rejected": -1.6386518478393555,
+      "step": 173,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.3514264074728604,
+      "grad_norm": 2.378427743911743,
+      "learning_rate": 0.0001915348864833476,
+      "logits/chosen": -0.8616642951965332,
+      "logits/rejected": -0.9031955003738403,
+      "logps/chosen": -2.75161075592041,
+      "logps/rejected": -17.232093811035156,
+      "loss": 0.7261631488800049,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3087785542011261,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24855753779411316,
+      "rewards/margins": 0.8990098237991333,
+      "rewards/rejected": -0.650452196598053,
+      "step": 174,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.35344609946983085,
+      "grad_norm": 2.4263904094696045,
+      "learning_rate": 0.00019139980115238827,
+      "logits/chosen": -0.8389776945114136,
+      "logits/rejected": -0.8697970509529114,
+      "logps/chosen": -4.028109073638916,
+      "logps/rejected": -18.706117630004883,
+      "loss": 0.780457615852356,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2751180827617645,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.006659353151917458,
+      "rewards/margins": 0.7851797342300415,
+      "rewards/rejected": -0.791839063167572,
+      "step": 175,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.3554657914668013,
+      "grad_norm": 2.074300765991211,
+      "learning_rate": 0.00019126369490807447,
+      "logits/chosen": -0.8048940896987915,
+      "logits/rejected": -0.884000301361084,
+      "logps/chosen": -1.4055310487747192,
+      "logps/rejected": -25.309282302856445,
+      "loss": 0.4804037809371948,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14255806803703308,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40084120631217957,
+      "rewards/margins": 1.3346383571624756,
+      "rewards/rejected": -0.9337972402572632,
+      "step": 176,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.3574854834637718,
+      "grad_norm": 3.744903802871704,
+      "learning_rate": 0.00019112656927067955,
+      "logits/chosen": -0.8155094981193542,
+      "logits/rejected": -0.8395469784736633,
+      "logps/chosen": -11.99010944366455,
+      "logps/rejected": -26.437166213989258,
+      "loss": 0.8095741271972656,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34381529688835144,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11786942183971405,
+      "rewards/margins": 1.111412763595581,
+      "rewards/rejected": -1.2292821407318115,
+      "step": 177,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.35950517546074223,
+      "grad_norm": 2.4447503089904785,
+      "learning_rate": 0.00019098842577186314,
+      "logits/chosen": -0.8492040634155273,
+      "logits/rejected": -0.9052773714065552,
+      "logps/chosen": -2.7093420028686523,
+      "logps/rejected": -21.28760528564453,
+      "loss": 0.6568049192428589,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21920911967754364,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19283369183540344,
+      "rewards/margins": 1.1686737537384033,
+      "rewards/rejected": -0.9758400321006775,
+      "step": 178,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.3615248674577127,
+      "grad_norm": 2.275618314743042,
+      "learning_rate": 0.0001908492659546543,
+      "logits/chosen": -0.7995213866233826,
+      "logits/rejected": -0.9124236106872559,
+      "logps/chosen": -2.1147406101226807,
+      "logps/rejected": -30.72578239440918,
+      "loss": 0.5593333840370178,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1950627565383911,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16707345843315125,
+      "rewards/margins": 1.2975778579711914,
+      "rewards/rejected": -1.1305043697357178,
+      "step": 179,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.36354455945468317,
+      "grad_norm": 3.495725631713867,
+      "learning_rate": 0.00019070909137343408,
+      "logits/chosen": -0.9321005940437317,
+      "logits/rejected": -0.9968794584274292,
+      "logps/chosen": -3.873347759246826,
+      "logps/rejected": -28.580020904541016,
+      "loss": 0.6998752951622009,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.46146360039711,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.157605841755867,
+      "rewards/margins": 1.7749366760253906,
+      "rewards/rejected": -1.6173310279846191,
+      "step": 180,
+      "train_speed(iter/s)": 0.021286
+    },
+    {
+      "epoch": 0.3655642514516536,
+      "grad_norm": 2.6631152629852295,
+      "learning_rate": 0.0001905679035939181,
+      "logits/chosen": -0.9386301636695862,
+      "logits/rejected": -1.0215297937393188,
+      "logps/chosen": -0.5281402468681335,
+      "logps/rejected": -32.908546447753906,
+      "loss": 0.3208782374858856,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11616937071084976,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27659526467323303,
+      "rewards/margins": 2.29314923286438,
+      "rewards/rejected": -2.016554117202759,
+      "step": 181,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.3675839434486241,
+      "grad_norm": 10.830909729003906,
+      "learning_rate": 0.00019042570419313925,
+      "logits/chosen": -0.970279335975647,
+      "logits/rejected": -1.0291110277175903,
+      "logps/chosen": -8.81902027130127,
+      "logps/rejected": -42.53478240966797,
+      "loss": 1.1662126779556274,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6375398635864258,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.26809704303741455,
+      "rewards/margins": 2.575526237487793,
+      "rewards/rejected": -2.843623161315918,
+      "step": 182,
+      "train_speed(iter/s)": 0.021289
+    },
+    {
+      "epoch": 0.36960363544559455,
+      "grad_norm": 9.357040405273438,
+      "learning_rate": 0.0001902824947594299,
+      "logits/chosen": -0.9984275102615356,
+      "logits/rejected": -1.0901621580123901,
+      "logps/chosen": -4.183664321899414,
+      "logps/rejected": -42.764862060546875,
+      "loss": 0.8461188077926636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.662725031375885,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08130670338869095,
+      "rewards/margins": 3.4639174938201904,
+      "rewards/rejected": -3.3826112747192383,
+      "step": 183,
+      "train_speed(iter/s)": 0.02129
+    },
+    {
+      "epoch": 0.371623327442565,
+      "grad_norm": 22.538454055786133,
+      "learning_rate": 0.00019013827689240436,
+      "logits/chosen": -0.9899927973747253,
+      "logits/rejected": -1.0915980339050293,
+      "logps/chosen": -5.971269130706787,
+      "logps/rejected": -41.92043685913086,
+      "loss": 1.3363139629364014,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7483702301979065,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.24273402988910675,
+      "rewards/margins": 2.6009511947631836,
+      "rewards/rejected": -2.8436849117279053,
+      "step": 184,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.3736430194395355,
+      "grad_norm": 5.44999361038208,
+      "learning_rate": 0.0001899930522029408,
+      "logits/chosen": -0.9289810061454773,
+      "logits/rejected": -1.0461653470993042,
+      "logps/chosen": -13.561850547790527,
+      "logps/rejected": -50.344093322753906,
+      "loss": 1.021875262260437,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7987765669822693,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.4458293318748474,
+      "rewards/margins": 2.7672858238220215,
+      "rewards/rejected": -3.2131152153015137,
+      "step": 185,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.3756627114365059,
+      "grad_norm": 9.218350410461426,
+      "learning_rate": 0.00018984682231316333,
+      "logits/chosen": -0.9281139373779297,
+      "logits/rejected": -1.0329409837722778,
+      "logps/chosen": -3.859086036682129,
+      "logps/rejected": -30.334747314453125,
+      "loss": 1.411182165145874,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8451413512229919,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08102743327617645,
+      "rewards/margins": 1.733138084411621,
+      "rewards/rejected": -1.6521105766296387,
+      "step": 186,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.3776824034334764,
+      "grad_norm": 9.405385971069336,
+      "learning_rate": 0.00018969958885642398,
+      "logits/chosen": -0.9063903093338013,
+      "logits/rejected": -1.0287251472473145,
+      "logps/chosen": -5.265182971954346,
+      "logps/rejected": -53.523624420166016,
+      "loss": 0.5908830165863037,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38780495524406433,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0466960184276104,
+      "rewards/margins": 3.7127225399017334,
+      "rewards/rejected": -3.6660263538360596,
+      "step": 187,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.37970209543044686,
+      "grad_norm": 2.527911901473999,
+      "learning_rate": 0.00018955135347728432,
+      "logits/chosen": -0.749373733997345,
+      "logits/rejected": -0.9382051229476929,
+      "logps/chosen": -1.4075103998184204,
+      "logps/rejected": -48.75277328491211,
+      "loss": 0.32720643281936646,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15030686557292938,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2724253535270691,
+      "rewards/margins": 2.877960205078125,
+      "rewards/rejected": -2.605534791946411,
+      "step": 188,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.3817217874274173,
+      "grad_norm": 3.514798164367676,
+      "learning_rate": 0.00018940211783149722,
+      "logits/chosen": -0.9332194924354553,
+      "logits/rejected": -0.9634183049201965,
+      "logps/chosen": -3.9122090339660645,
+      "logps/rejected": -23.81725311279297,
+      "loss": 0.5536522269248962,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2619181275367737,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2673307955265045,
+      "rewards/margins": 1.8769687414169312,
+      "rewards/rejected": -1.609637975692749,
+      "step": 189,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3837414794243878,
+      "grad_norm": 3.0047051906585693,
+      "learning_rate": 0.00018925188358598813,
+      "logits/chosen": -0.874540388584137,
+      "logits/rejected": -0.9682676792144775,
+      "logps/chosen": -1.5275218486785889,
+      "logps/rejected": -36.20308303833008,
+      "loss": 0.476752907037735,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12829114496707916,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1873767375946045,
+      "rewards/margins": 2.56374454498291,
+      "rewards/rejected": -2.3763678073883057,
+      "step": 190,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.38576117142135824,
+      "grad_norm": 5.604814052581787,
+      "learning_rate": 0.0001891006524188368,
+      "logits/chosen": -0.8748272657394409,
+      "logits/rejected": -0.8527596592903137,
+      "logps/chosen": -22.366464614868164,
+      "logps/rejected": -31.544967651367188,
+      "loss": 0.844088077545166,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34909212589263916,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.017623694613575935,
+      "rewards/margins": 1.6229170560836792,
+      "rewards/rejected": -1.6052933931350708,
+      "step": 191,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3877808634183287,
+      "grad_norm": 3.902900457382202,
+      "learning_rate": 0.0001889484260192582,
+      "logits/chosen": -0.8291831612586975,
+      "logits/rejected": -0.9175705909729004,
+      "logps/chosen": -3.4213364124298096,
+      "logps/rejected": -28.334596633911133,
+      "loss": 0.6280531883239746,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2871948778629303,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.20435035228729248,
+      "rewards/margins": 1.6512333154678345,
+      "rewards/rejected": -1.446882963180542,
+      "step": 192,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3898005554152992,
+      "grad_norm": 3.3589909076690674,
+      "learning_rate": 0.00018879520608758394,
+      "logits/chosen": -0.7861786484718323,
+      "logits/rejected": -0.9745673537254333,
+      "logps/chosen": -2.2813668251037598,
+      "logps/rejected": -30.406993865966797,
+      "loss": 0.46809083223342896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24172116816043854,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2660973072052002,
+      "rewards/margins": 1.8777257204055786,
+      "rewards/rejected": -1.6116284132003784,
+      "step": 193,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.3918202474122696,
+      "grad_norm": 3.7471225261688232,
+      "learning_rate": 0.000188640994335243,
+      "logits/chosen": -0.8293952941894531,
+      "logits/rejected": -0.9208282232284546,
+      "logps/chosen": -2.756782293319702,
+      "logps/rejected": -28.273818969726562,
+      "loss": 0.7544887065887451,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3577630817890167,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12236475944519043,
+      "rewards/margins": 1.8170613050460815,
+      "rewards/rejected": -1.6946964263916016,
+      "step": 194,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.3938399394092401,
+      "grad_norm": 3.194539785385132,
+      "learning_rate": 0.00018848579248474288,
+      "logits/chosen": -0.7125469446182251,
+      "logits/rejected": -0.888473391532898,
+      "logps/chosen": -4.232546329498291,
+      "logps/rejected": -34.417152404785156,
+      "loss": 0.634448230266571,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23650826513767242,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3275882303714752,
+      "rewards/margins": 1.5964200496673584,
+      "rewards/rejected": -1.2688318490982056,
+      "step": 195,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.39585963140621055,
+      "grad_norm": 7.319854736328125,
+      "learning_rate": 0.00018832960226965008,
+      "logits/chosen": -0.8826249837875366,
+      "logits/rejected": -0.9558060169219971,
+      "logps/chosen": -2.2309622764587402,
+      "logps/rejected": -23.310834884643555,
+      "loss": 0.7857663035392761,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31681591272354126,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.3184216618537903,
+      "rewards/margins": 1.3100265264511108,
+      "rewards/rejected": -0.9916048049926758,
+      "step": 196,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.397879323403181,
+      "grad_norm": 4.467423439025879,
+      "learning_rate": 0.00018817242543457108,
+      "logits/chosen": -0.7410279512405396,
+      "logits/rejected": -0.8783193826675415,
+      "logps/chosen": -6.2165632247924805,
+      "logps/rejected": -40.68191909790039,
+      "loss": 0.8568143844604492,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4732620418071747,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06536143273115158,
+      "rewards/margins": 1.7412124872207642,
+      "rewards/rejected": -1.6758511066436768,
+      "step": 197,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.3998990154001515,
+      "grad_norm": 3.227818250656128,
+      "learning_rate": 0.0001880142637351325,
+      "logits/chosen": -0.8976326584815979,
+      "logits/rejected": -0.9664545059204102,
+      "logps/chosen": -2.204209804534912,
+      "logps/rejected": -19.951866149902344,
+      "loss": 0.5582164525985718,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18982845544815063,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3193460702896118,
+      "rewards/margins": 1.213867425918579,
+      "rewards/rejected": -0.8945214748382568,
+      "step": 198,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.40191870739712193,
+      "grad_norm": 5.091889381408691,
+      "learning_rate": 0.00018785511893796176,
+      "logits/chosen": -0.813483476638794,
+      "logits/rejected": -0.9725134968757629,
+      "logps/chosen": -1.0746464729309082,
+      "logps/rejected": -31.801054000854492,
+      "loss": 0.38803631067276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18849264085292816,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29135608673095703,
+      "rewards/margins": 1.9811937808990479,
+      "rewards/rejected": -1.6898375749588013,
+      "step": 199,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.4039383993940924,
+      "grad_norm": 3.733981132507324,
+      "learning_rate": 0.00018769499282066717,
+      "logits/chosen": -0.8599748015403748,
+      "logits/rejected": -0.9630089998245239,
+      "logps/chosen": -2.635427236557007,
+      "logps/rejected": -24.271150588989258,
+      "loss": 0.6127274036407471,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2767713665962219,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2570668160915375,
+      "rewards/margins": 1.4302881956100464,
+      "rewards/rejected": -1.1732213497161865,
+      "step": 200,
+      "train_speed(iter/s)": 0.021299
+    },
+    {
+      "epoch": 0.40595809139106287,
+      "grad_norm": 4.356893062591553,
+      "learning_rate": 0.0001875338871718182,
+      "logits/chosen": -0.8239525556564331,
+      "logits/rejected": -0.9406594634056091,
+      "logps/chosen": -3.5920815467834473,
+      "logps/rejected": -33.701141357421875,
+      "loss": 0.7486278414726257,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34808939695358276,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06076638400554657,
+      "rewards/margins": 1.9439024925231934,
+      "rewards/rejected": -1.8831360340118408,
+      "step": 201,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.4079777833880333,
+      "grad_norm": 3.3695449829101562,
+      "learning_rate": 0.00018737180379092537,
+      "logits/chosen": -0.7839801907539368,
+      "logits/rejected": -0.8861851096153259,
+      "logps/chosen": -8.617507934570312,
+      "logps/rejected": -47.99607849121094,
+      "loss": 0.5823696851730347,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3239917457103729,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02249305695295334,
+      "rewards/margins": 3.1341774463653564,
+      "rewards/rejected": -3.1116843223571777,
+      "step": 202,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.4099974753850038,
+      "grad_norm": 2.5396463871002197,
+      "learning_rate": 0.00018720874448842034,
+      "logits/chosen": -0.8206271529197693,
+      "logits/rejected": -1.0252444744110107,
+      "logps/chosen": -1.0570971965789795,
+      "logps/rejected": -47.1510124206543,
+      "loss": 0.324438214302063,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11744873970746994,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.36073753237724304,
+      "rewards/margins": 3.558206558227539,
+      "rewards/rejected": -3.1974687576293945,
+      "step": 203,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.41201716738197425,
+      "grad_norm": 3.693103551864624,
+      "learning_rate": 0.00018704471108563548,
+      "logits/chosen": -0.8473316431045532,
+      "logits/rejected": -0.9990904927253723,
+      "logps/chosen": -4.564356803894043,
+      "logps/rejected": -40.657447814941406,
+      "loss": 0.4926657974720001,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2881479859352112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39431607723236084,
+      "rewards/margins": 2.6521053314208984,
+      "rewards/rejected": -2.257789373397827,
+      "step": 204,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.4140368593789447,
+      "grad_norm": 5.1197896003723145,
+      "learning_rate": 0.00018687970541478364,
+      "logits/chosen": -0.8696497082710266,
+      "logits/rejected": -1.0545570850372314,
+      "logps/chosen": -1.4695444107055664,
+      "logps/rejected": -46.127044677734375,
+      "loss": 0.34414029121398926,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16013705730438232,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33041495084762573,
+      "rewards/margins": 3.691737174987793,
+      "rewards/rejected": -3.3613224029541016,
+      "step": 205,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.4160565513759152,
+      "grad_norm": 11.712461471557617,
+      "learning_rate": 0.00018671372931893773,
+      "logits/chosen": -0.9976873993873596,
+      "logits/rejected": -1.0082725286483765,
+      "logps/chosen": -6.205466270446777,
+      "logps/rejected": -21.121868133544922,
+      "loss": 1.509117841720581,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.9143091440200806,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13785380125045776,
+      "rewards/margins": 1.1608891487121582,
+      "rewards/rejected": -1.2987430095672607,
+      "step": 206,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.4180762433728856,
+      "grad_norm": 4.418537616729736,
+      "learning_rate": 0.00018654678465201,
+      "logits/chosen": -0.9154266119003296,
+      "logits/rejected": -0.981490969657898,
+      "logps/chosen": -9.175884246826172,
+      "logps/rejected": -32.25775146484375,
+      "loss": 0.7383416891098022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3417145013809204,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03129662945866585,
+      "rewards/margins": 2.172858476638794,
+      "rewards/rejected": -2.14156174659729,
+      "step": 207,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.4200959353698561,
+      "grad_norm": 4.682573318481445,
+      "learning_rate": 0.0001863788732787314,
+      "logits/chosen": -0.845721423625946,
+      "logits/rejected": -1.0325840711593628,
+      "logps/chosen": -3.112144708633423,
+      "logps/rejected": -48.1558952331543,
+      "loss": 0.4932224452495575,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3019799590110779,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16389404237270355,
+      "rewards/margins": 3.949674367904663,
+      "rewards/rejected": -3.785780429840088,
+      "step": 208,
+      "train_speed(iter/s)": 0.021267
+    },
+    {
+      "epoch": 0.42211562736682656,
+      "grad_norm": 6.930049419403076,
+      "learning_rate": 0.0001862099970746308,
+      "logits/chosen": -0.8911486268043518,
+      "logits/rejected": -1.048769474029541,
+      "logps/chosen": -3.7173314094543457,
+      "logps/rejected": -37.73887252807617,
+      "loss": 0.6971174478530884,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3776175379753113,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14115534722805023,
+      "rewards/margins": 2.630204200744629,
+      "rewards/rejected": -2.489048719406128,
+      "step": 209,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.424135319363797,
+      "grad_norm": 3.0014336109161377,
+      "learning_rate": 0.00018604015792601396,
+      "logits/chosen": -0.7730808854103088,
+      "logits/rejected": -0.9816968441009521,
+      "logps/chosen": -0.3805132210254669,
+      "logps/rejected": -53.63943099975586,
+      "loss": 0.21242156624794006,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05388123169541359,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3490655720233917,
+      "rewards/margins": 3.7549197673797607,
+      "rewards/rejected": -3.4058542251586914,
+      "step": 210,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.4261550113607675,
+      "grad_norm": 6.788722991943359,
+      "learning_rate": 0.00018586935772994245,
+      "logits/chosen": -0.7963772416114807,
+      "logits/rejected": -1.0568972826004028,
+      "logps/chosen": -1.2283453941345215,
+      "logps/rejected": -59.22148132324219,
+      "loss": 0.18988274037837982,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12394578754901886,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4420156478881836,
+      "rewards/margins": 4.728057384490967,
+      "rewards/rejected": -4.286042213439941,
+      "step": 211,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.42817470335773794,
+      "grad_norm": 2.8118438720703125,
+      "learning_rate": 0.00018569759839421265,
+      "logits/chosen": -0.819808840751648,
+      "logits/rejected": -0.9025678038597107,
+      "logps/chosen": -2.9665253162384033,
+      "logps/rejected": -40.64404296875,
+      "loss": 0.33937186002731323,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11274304240942001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4729611575603485,
+      "rewards/margins": 3.319221019744873,
+      "rewards/rejected": -2.8462600708007812,
+      "step": 212,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.4301943953547084,
+      "grad_norm": 4.191134452819824,
+      "learning_rate": 0.00018552488183733412,
+      "logits/chosen": -0.8641024231910706,
+      "logits/rejected": -0.9831883907318115,
+      "logps/chosen": -7.590506553649902,
+      "logps/rejected": -40.67595672607422,
+      "loss": 0.6653822064399719,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3946245610713959,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.004725143313407898,
+      "rewards/margins": 2.8683340549468994,
+      "rewards/rejected": -2.8730592727661133,
+      "step": 213,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.4322140873516789,
+      "grad_norm": 7.408899307250977,
+      "learning_rate": 0.00018535120998850848,
+      "logits/chosen": -0.9360211491584778,
+      "logits/rejected": -1.0136916637420654,
+      "logps/chosen": -5.061653137207031,
+      "logps/rejected": -31.533058166503906,
+      "loss": 0.8134801983833313,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38820868730545044,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2650642693042755,
+      "rewards/margins": 2.2353947162628174,
+      "rewards/rejected": -1.9703304767608643,
+      "step": 214,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.4342337793486493,
+      "grad_norm": 6.260317802429199,
+      "learning_rate": 0.0001851765847876076,
+      "logits/chosen": -0.9736087322235107,
+      "logits/rejected": -1.0754610300064087,
+      "logps/chosen": -6.311574935913086,
+      "logps/rejected": -41.57005310058594,
+      "loss": 1.0864337682724,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6948180794715881,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11255162954330444,
+      "rewards/margins": 2.995018482208252,
+      "rewards/rejected": -3.107570171356201,
+      "step": 215,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.4362534713456198,
+      "grad_norm": 13.104352951049805,
+      "learning_rate": 0.00018500100818515222,
+      "logits/chosen": -0.6429583430290222,
+      "logits/rejected": -0.9696972966194153,
+      "logps/chosen": -2.051022529602051,
+      "logps/rejected": -83.3814697265625,
+      "loss": 0.6120116710662842,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.312576562166214,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.121861532330513,
+      "rewards/margins": 5.671273708343506,
+      "rewards/rejected": -5.549411773681641,
+      "step": 216,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.43827316334259026,
+      "grad_norm": 7.3358564376831055,
+      "learning_rate": 0.0001848244821422899,
+      "logits/chosen": -1.0172042846679688,
+      "logits/rejected": -1.0574982166290283,
+      "logps/chosen": -3.74454927444458,
+      "logps/rejected": -25.982912063598633,
+      "loss": 0.8323017358779907,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30521970987319946,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1191268116235733,
+      "rewards/margins": 1.8004627227783203,
+      "rewards/rejected": -1.6813361644744873,
+      "step": 217,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.4402928553395607,
+      "grad_norm": 2.6182119846343994,
+      "learning_rate": 0.00018464700863077312,
+      "logits/chosen": -0.995885968208313,
+      "logits/rejected": -1.102242350578308,
+      "logps/chosen": -1.5725170373916626,
+      "logps/rejected": -46.24748229980469,
+      "loss": 0.3959670662879944,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1685146987438202,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18964146077632904,
+      "rewards/margins": 3.7616095542907715,
+      "rewards/rejected": -3.5719680786132812,
+      "step": 218,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.4423125473365312,
+      "grad_norm": 5.691047191619873,
+      "learning_rate": 0.00018446858963293763,
+      "logits/chosen": -0.9241669178009033,
+      "logits/rejected": -1.0488460063934326,
+      "logps/chosen": -2.7383437156677246,
+      "logps/rejected": -53.106632232666016,
+      "loss": 0.46598395705223083,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28670936822891235,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08014581352472305,
+      "rewards/margins": 3.8416402339935303,
+      "rewards/rejected": -3.7614941596984863,
+      "step": 219,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.44433223933350163,
+      "grad_norm": 6.792243957519531,
+      "learning_rate": 0.0001842892271416797,
+      "logits/chosen": -0.867421567440033,
+      "logits/rejected": -1.0047706365585327,
+      "logps/chosen": -3.9948360919952393,
+      "logps/rejected": -40.925697326660156,
+      "loss": 0.559948205947876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2816776633262634,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27128446102142334,
+      "rewards/margins": 2.5824544429779053,
+      "rewards/rejected": -2.3111701011657715,
+      "step": 220,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.44635193133047213,
+      "grad_norm": 2.8233304023742676,
+      "learning_rate": 0.00018410892316043448,
+      "logits/chosen": -0.7455387711524963,
+      "logits/rejected": -1.0043134689331055,
+      "logps/chosen": -2.28206205368042,
+      "logps/rejected": -50.90422821044922,
+      "loss": 0.4489433765411377,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17719773948192596,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17336970567703247,
+      "rewards/margins": 3.006608009338379,
+      "rewards/rejected": -2.833238363265991,
+      "step": 221,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.44837162332744257,
+      "grad_norm": 3.7978217601776123,
+      "learning_rate": 0.00018392767970315313,
+      "logits/chosen": -0.9136677980422974,
+      "logits/rejected": -1.06996488571167,
+      "logps/chosen": -1.5614748001098633,
+      "logps/rejected": -34.510162353515625,
+      "loss": 0.3262411057949066,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1898116022348404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3149145543575287,
+      "rewards/margins": 2.7444238662719727,
+      "rewards/rejected": -2.429509162902832,
+      "step": 222,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.450391315324413,
+      "grad_norm": 6.886748313903809,
+      "learning_rate": 0.00018374549879428062,
+      "logits/chosen": -0.9338952302932739,
+      "logits/rejected": -1.0804352760314941,
+      "logps/chosen": -1.0283788442611694,
+      "logps/rejected": -38.30894088745117,
+      "loss": 0.3192234933376312,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11882957816123962,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24301038682460785,
+      "rewards/margins": 2.798785924911499,
+      "rewards/rejected": -2.5557756423950195,
+      "step": 223,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.4524110073213835,
+      "grad_norm": 6.402444362640381,
+      "learning_rate": 0.000183562382468733,
+      "logits/chosen": -0.9219646453857422,
+      "logits/rejected": -1.0332969427108765,
+      "logps/chosen": -4.296561241149902,
+      "logps/rejected": -32.83451843261719,
+      "loss": 1.1228792667388916,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8876572251319885,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.020390525460243225,
+      "rewards/margins": 2.142768383026123,
+      "rewards/rejected": -2.163159132003784,
+      "step": 224,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.45443069931835395,
+      "grad_norm": 9.758877754211426,
+      "learning_rate": 0.00018337833277187472,
+      "logits/chosen": -0.8922550082206726,
+      "logits/rejected": -0.9859142303466797,
+      "logps/chosen": -7.753718376159668,
+      "logps/rejected": -28.561172485351562,
+      "loss": 1.4200571775436401,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8696328997612,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07261030375957489,
+      "rewards/margins": 1.1861965656280518,
+      "rewards/rejected": -1.2588069438934326,
+      "step": 225,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.4564503913153244,
+      "grad_norm": 7.866222858428955,
+      "learning_rate": 0.0001831933517594957,
+      "logits/chosen": -0.7848281860351562,
+      "logits/rejected": -1.0049529075622559,
+      "logps/chosen": -2.8492534160614014,
+      "logps/rejected": -53.778289794921875,
+      "loss": 0.520083487033844,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3156394362449646,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19083034992218018,
+      "rewards/margins": 3.20526123046875,
+      "rewards/rejected": -3.0144309997558594,
+      "step": 226,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.4584700833122949,
+      "grad_norm": 4.694428443908691,
+      "learning_rate": 0.00018300744149778853,
+      "logits/chosen": -0.9140058755874634,
+      "logits/rejected": -1.035470724105835,
+      "logps/chosen": -0.7746766805648804,
+      "logps/rejected": -31.849538803100586,
+      "loss": 0.41979673504829407,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1669030785560608,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34211465716362,
+      "rewards/margins": 2.157820701599121,
+      "rewards/rejected": -1.8157060146331787,
+      "step": 227,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.4604897753092653,
+      "grad_norm": 4.092323303222656,
+      "learning_rate": 0.00018282060406332512,
+      "logits/chosen": -0.8949604034423828,
+      "logits/rejected": -1.0626131296157837,
+      "logps/chosen": -2.425996780395508,
+      "logps/rejected": -28.149372100830078,
+      "loss": 0.4806259274482727,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22964058816432953,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2916340231895447,
+      "rewards/margins": 1.6805919408798218,
+      "rewards/rejected": -1.3889580965042114,
+      "step": 228,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.4625094673062358,
+      "grad_norm": 4.152270317077637,
+      "learning_rate": 0.0001826328415430339,
+      "logits/chosen": -1.0330709218978882,
+      "logits/rejected": -1.1299700736999512,
+      "logps/chosen": -2.0911142826080322,
+      "logps/rejected": -25.8432674407959,
+      "loss": 0.47937411069869995,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24223287403583527,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22816987335681915,
+      "rewards/margins": 1.9592314958572388,
+      "rewards/rejected": -1.7310616970062256,
+      "step": 229,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.46452915930320626,
+      "grad_norm": 3.1440939903259277,
+      "learning_rate": 0.00018244415603417603,
+      "logits/chosen": -0.9888077974319458,
+      "logits/rejected": -1.087904930114746,
+      "logps/chosen": -1.7838736772537231,
+      "logps/rejected": -31.170108795166016,
+      "loss": 0.5677547454833984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2625994384288788,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.36328813433647156,
+      "rewards/margins": 2.194064140319824,
+      "rewards/rejected": -1.8307762145996094,
+      "step": 230,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4665488513001767,
+      "grad_norm": 3.9995431900024414,
+      "learning_rate": 0.00018225454964432248,
+      "logits/chosen": -0.8009510040283203,
+      "logits/rejected": -0.9875888824462891,
+      "logps/chosen": -4.417139530181885,
+      "logps/rejected": -39.54981231689453,
+      "loss": 0.5815356969833374,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4281938374042511,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21771493554115295,
+      "rewards/margins": 2.3999886512756348,
+      "rewards/rejected": -2.1822738647460938,
+      "step": 231,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4685685432971472,
+      "grad_norm": 5.97980260848999,
+      "learning_rate": 0.00018206402449132995,
+      "logits/chosen": -0.8073972463607788,
+      "logits/rejected": -1.005049705505371,
+      "logps/chosen": -3.3927805423736572,
+      "logps/rejected": -40.57596206665039,
+      "loss": 0.6254691481590271,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36383697390556335,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1796584278345108,
+      "rewards/margins": 1.9802054166793823,
+      "rewards/rejected": -1.8005468845367432,
+      "step": 232,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.47058823529411764,
+      "grad_norm": 3.2046210765838623,
+      "learning_rate": 0.00018187258270331783,
+      "logits/chosen": -0.843147873878479,
+      "logits/rejected": -0.9698415398597717,
+      "logps/chosen": -6.980717182159424,
+      "logps/rejected": -47.44036865234375,
+      "loss": 0.42867493629455566,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22475841641426086,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42296847701072693,
+      "rewards/margins": 3.3137428760528564,
+      "rewards/rejected": -2.8907744884490967,
+      "step": 233,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.47260792729108814,
+      "grad_norm": 5.650546550750732,
+      "learning_rate": 0.00018168022641864377,
+      "logits/chosen": -0.9458031058311462,
+      "logits/rejected": -1.022731900215149,
+      "logps/chosen": -5.699041366577148,
+      "logps/rejected": -27.91295051574707,
+      "loss": 0.6052364706993103,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25157076120376587,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17266854643821716,
+      "rewards/margins": 2.01995587348938,
+      "rewards/rejected": -1.8472874164581299,
+      "step": 234,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.4746276192880586,
+      "grad_norm": 7.562004089355469,
+      "learning_rate": 0.00018148695778588033,
+      "logits/chosen": -0.8491840362548828,
+      "logits/rejected": -0.9722462296485901,
+      "logps/chosen": -8.168458938598633,
+      "logps/rejected": -39.646427154541016,
+      "loss": 1.076995849609375,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6254736185073853,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23536130785942078,
+      "rewards/margins": 2.3698079586029053,
+      "rewards/rejected": -2.134446859359741,
+      "step": 235,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.476647311285029,
+      "grad_norm": 2.8358967304229736,
+      "learning_rate": 0.00018129277896379077,
+      "logits/chosen": -0.8474506139755249,
+      "logits/rejected": -0.9761014580726624,
+      "logps/chosen": -1.0130198001861572,
+      "logps/rejected": -34.24652099609375,
+      "loss": 0.3748179078102112,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11387713998556137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.42450040578842163,
+      "rewards/margins": 2.2273645401000977,
+      "rewards/rejected": -1.8028637170791626,
+      "step": 236,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.4786670032819995,
+      "grad_norm": 24.549625396728516,
+      "learning_rate": 0.00018109769212130487,
+      "logits/chosen": -0.7827628254890442,
+      "logits/rejected": -0.899127721786499,
+      "logps/chosen": -12.455778121948242,
+      "logps/rejected": -39.55864715576172,
+      "loss": 1.0212831497192383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6375862956047058,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05945993959903717,
+      "rewards/margins": 2.1959457397460938,
+      "rewards/rejected": -2.2554056644439697,
+      "step": 237,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.48068669527896996,
+      "grad_norm": 5.986259460449219,
+      "learning_rate": 0.00018090169943749476,
+      "logits/chosen": -0.9809411764144897,
+      "logits/rejected": -1.0568145513534546,
+      "logps/chosen": -2.8511006832122803,
+      "logps/rejected": -30.153661727905273,
+      "loss": 0.6427941918373108,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31609585881233215,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.20257410407066345,
+      "rewards/margins": 2.3083198070526123,
+      "rewards/rejected": -2.105745553970337,
+      "step": 238,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.4827063872759404,
+      "grad_norm": 6.145867824554443,
+      "learning_rate": 0.00018070480310155066,
+      "logits/chosen": -0.8342568874359131,
+      "logits/rejected": -0.9062321782112122,
+      "logps/chosen": -7.222947120666504,
+      "logps/rejected": -37.760414123535156,
+      "loss": 0.8602092862129211,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.47236353158950806,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14982852339744568,
+      "rewards/margins": 2.0299620628356934,
+      "rewards/rejected": -1.8801336288452148,
+      "step": 239,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.4847260792729109,
+      "grad_norm": 5.842182159423828,
+      "learning_rate": 0.0001805070053127563,
+      "logits/chosen": -0.8298416137695312,
+      "logits/rejected": -0.9391087889671326,
+      "logps/chosen": -4.0047993659973145,
+      "logps/rejected": -34.49217224121094,
+      "loss": 0.6390224695205688,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.391327440738678,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.296629935503006,
+      "rewards/margins": 2.591277837753296,
+      "rewards/rejected": -2.294647693634033,
+      "step": 240,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.48674577126988133,
+      "grad_norm": 3.0892484188079834,
+      "learning_rate": 0.0001803083082804645,
+      "logits/chosen": -0.6407896876335144,
+      "logits/rejected": -0.8386159539222717,
+      "logps/chosen": -2.6044552326202393,
+      "logps/rejected": -43.223602294921875,
+      "loss": 0.5524685978889465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23806366324424744,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17818674445152283,
+      "rewards/margins": 2.3168139457702637,
+      "rewards/rejected": -2.138627052307129,
+      "step": 241,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.48876546326685183,
+      "grad_norm": 2.5642099380493164,
+      "learning_rate": 0.00018010871422407236,
+      "logits/chosen": -0.5903664231300354,
+      "logits/rejected": -0.817259669303894,
+      "logps/chosen": -1.1348421573638916,
+      "logps/rejected": -54.072288513183594,
+      "loss": 0.390775203704834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18238955736160278,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31289827823638916,
+      "rewards/margins": 3.0836544036865234,
+      "rewards/rejected": -2.7707560062408447,
+      "step": 242,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.49078515526382227,
+      "grad_norm": 4.531945705413818,
+      "learning_rate": 0.00017990822537299647,
+      "logits/chosen": -0.7170388698577881,
+      "logits/rejected": -0.8259367942810059,
+      "logps/chosen": -6.029257774353027,
+      "logps/rejected": -34.971519470214844,
+      "loss": 0.7846373915672302,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31968259811401367,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.002697870135307312,
+      "rewards/margins": 1.6093158721923828,
+      "rewards/rejected": -1.612013816833496,
+      "step": 243,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4928048472607927,
+      "grad_norm": 3.57047176361084,
+      "learning_rate": 0.00017970684396664813,
+      "logits/chosen": -0.7720434069633484,
+      "logits/rejected": -0.9465057253837585,
+      "logps/chosen": -3.873821258544922,
+      "logps/rejected": -37.18175506591797,
+      "loss": 0.8110936284065247,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44118532538414,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19488100707530975,
+      "rewards/margins": 1.9362576007843018,
+      "rewards/rejected": -1.741376519203186,
+      "step": 244,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4948245392577632,
+      "grad_norm": 5.007617473602295,
+      "learning_rate": 0.0001795045722544083,
+      "logits/chosen": -0.7931674122810364,
+      "logits/rejected": -0.8741809129714966,
+      "logps/chosen": -1.8064591884613037,
+      "logps/rejected": -22.11163330078125,
+      "loss": 0.6888466477394104,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31985488533973694,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28564998507499695,
+      "rewards/margins": 1.2898226976394653,
+      "rewards/rejected": -1.0041728019714355,
+      "step": 245,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.49684423125473365,
+      "grad_norm": 6.91418981552124,
+      "learning_rate": 0.00017930141249560233,
+      "logits/chosen": -0.8298520445823669,
+      "logits/rejected": -0.9262869358062744,
+      "logps/chosen": -1.8928561210632324,
+      "logps/rejected": -22.42644691467285,
+      "loss": 0.5054484605789185,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17920702695846558,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19144940376281738,
+      "rewards/margins": 1.3705973625183105,
+      "rewards/rejected": -1.1791479587554932,
+      "step": 246,
+      "train_speed(iter/s)": 0.021286
+    },
+    {
+      "epoch": 0.4988639232517041,
+      "grad_norm": 5.468963146209717,
+      "learning_rate": 0.00017909736695947485,
+      "logits/chosen": -0.8126583695411682,
+      "logits/rejected": -0.9218447804450989,
+      "logps/chosen": -5.573441982269287,
+      "logps/rejected": -31.411596298217773,
+      "loss": 0.640342652797699,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2689912021160126,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.249068945646286,
+      "rewards/margins": 1.9487626552581787,
+      "rewards/rejected": -1.6996936798095703,
+      "step": 247,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.5008836152486745,
+      "grad_norm": 5.13163423538208,
+      "learning_rate": 0.0001788924379251645,
+      "logits/chosen": -0.6676142811775208,
+      "logits/rejected": -0.7854112982749939,
+      "logps/chosen": -5.0913262367248535,
+      "logps/rejected": -33.67715835571289,
+      "loss": 0.6214959621429443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30665862560272217,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09342917054891586,
+      "rewards/margins": 2.036255121231079,
+      "rewards/rejected": -1.9428261518478394,
+      "step": 248,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.502903307245645,
+      "grad_norm": 5.3085246086120605,
+      "learning_rate": 0.00017868662768167828,
+      "logits/chosen": -0.8283058404922485,
+      "logits/rejected": -0.8981807827949524,
+      "logps/chosen": -3.5498528480529785,
+      "logps/rejected": -25.92713737487793,
+      "loss": 0.8023664355278015,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3884091079235077,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14035727083683014,
+      "rewards/margins": 1.476125717163086,
+      "rewards/rejected": -1.335768461227417,
+      "step": 249,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.5049229992426155,
+      "grad_norm": 3.970473051071167,
+      "learning_rate": 0.0001784799385278661,
+      "logits/chosen": -0.7480765581130981,
+      "logits/rejected": -0.850541889667511,
+      "logps/chosen": -2.1585793495178223,
+      "logps/rejected": -25.883459091186523,
+      "loss": 0.5557476878166199,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23389576375484467,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27245235443115234,
+      "rewards/margins": 1.5237624645233154,
+      "rewards/rejected": -1.2513102293014526,
+      "step": 250,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.506942691239586,
+      "grad_norm": 2.7458653450012207,
+      "learning_rate": 0.00017827237277239514,
+      "logits/chosen": -0.7085766792297363,
+      "logits/rejected": -0.8612480163574219,
+      "logps/chosen": -1.5254360437393188,
+      "logps/rejected": -41.573387145996094,
+      "loss": 0.4261833429336548,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20671844482421875,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23911702632904053,
+      "rewards/margins": 2.4618849754333496,
+      "rewards/rejected": -2.2227678298950195,
+      "step": 251,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5089623832365564,
+      "grad_norm": 4.110019207000732,
+      "learning_rate": 0.00017806393273372395,
+      "logits/chosen": -0.6960774064064026,
+      "logits/rejected": -0.773073673248291,
+      "logps/chosen": -5.731773376464844,
+      "logps/rejected": -40.48484420776367,
+      "loss": 0.6885837912559509,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38311967253685,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13538384437561035,
+      "rewards/margins": 2.754824638366699,
+      "rewards/rejected": -2.619440793991089,
+      "step": 252,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5109820752335269,
+      "grad_norm": 2.490933656692505,
+      "learning_rate": 0.0001778546207400766,
+      "logits/chosen": -0.7139323353767395,
+      "logits/rejected": -0.8478928208351135,
+      "logps/chosen": -2.643089771270752,
+      "logps/rejected": -49.735321044921875,
+      "loss": 0.4857577383518219,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20252788066864014,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2547016441822052,
+      "rewards/margins": 3.3733713626861572,
+      "rewards/rejected": -3.1186697483062744,
+      "step": 253,
+      "train_speed(iter/s)": 0.021258
+    },
+    {
+      "epoch": 0.5130017672304974,
+      "grad_norm": 1.8276517391204834,
+      "learning_rate": 0.00017764443912941672,
+      "logits/chosen": -0.8284695148468018,
+      "logits/rejected": -0.9083284735679626,
+      "logps/chosen": -0.5445702075958252,
+      "logps/rejected": -36.77948760986328,
+      "loss": 0.3219444751739502,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10567262023687363,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28718090057373047,
+      "rewards/margins": 2.473656415939331,
+      "rewards/rejected": -2.1864752769470215,
+      "step": 254,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5150214592274678,
+      "grad_norm": 1.7103385925292969,
+      "learning_rate": 0.00017743339024942135,
+      "logits/chosen": -0.8364546298980713,
+      "logits/rejected": -0.9355967044830322,
+      "logps/chosen": -1.8402678966522217,
+      "logps/rejected": -41.97530746459961,
+      "loss": 0.3561093807220459,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15152454376220703,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3776871860027313,
+      "rewards/margins": 3.4742887020111084,
+      "rewards/rejected": -3.096601963043213,
+      "step": 255,
+      "train_speed(iter/s)": 0.02126
+    },
+    {
+      "epoch": 0.5170411512244383,
+      "grad_norm": 5.862149238586426,
+      "learning_rate": 0.00017722147645745468,
+      "logits/chosen": -0.8007981777191162,
+      "logits/rejected": -0.9000130891799927,
+      "logps/chosen": -4.51971435546875,
+      "logps/rejected": -49.85738754272461,
+      "loss": 0.6844031810760498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23998194932937622,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11539852619171143,
+      "rewards/margins": 3.546253204345703,
+      "rewards/rejected": -3.661651849746704,
+      "step": 256,
+      "train_speed(iter/s)": 0.021261
+    },
+    {
+      "epoch": 0.5190608432214088,
+      "grad_norm": 23.92522621154785,
+      "learning_rate": 0.0001770087001205418,
+      "logits/chosen": -0.7442535161972046,
+      "logits/rejected": -0.8944082260131836,
+      "logps/chosen": -5.961298942565918,
+      "logps/rejected": -71.44552612304688,
+      "loss": 1.19035804271698,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7366040945053101,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2813420295715332,
+      "rewards/margins": 4.797372817993164,
+      "rewards/rejected": -5.078714847564697,
+      "step": 257,
+      "train_speed(iter/s)": 0.021261
+    },
+    {
+      "epoch": 0.5210805352183792,
+      "grad_norm": 9.221707344055176,
+      "learning_rate": 0.00017679506361534215,
+      "logits/chosen": -0.8309729099273682,
+      "logits/rejected": -0.9244940280914307,
+      "logps/chosen": -3.6046130657196045,
+      "logps/rejected": -38.451324462890625,
+      "loss": 1.0726348161697388,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5696147084236145,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.056192442774772644,
+      "rewards/margins": 2.7530393600463867,
+      "rewards/rejected": -2.6968469619750977,
+      "step": 258,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.5231002272153497,
+      "grad_norm": 3.6904711723327637,
+      "learning_rate": 0.0001765805693281231,
+      "logits/chosen": -0.7574508786201477,
+      "logits/rejected": -0.8763286471366882,
+      "logps/chosen": -1.6263139247894287,
+      "logps/rejected": -46.19388198852539,
+      "loss": 0.5760003328323364,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32455456256866455,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.25203612446784973,
+      "rewards/margins": 3.3660078048706055,
+      "rewards/rejected": -3.1139719486236572,
+      "step": 259,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.5251199192123202,
+      "grad_norm": 6.571902751922607,
+      "learning_rate": 0.00017636521965473323,
+      "logits/chosen": -0.7622162103652954,
+      "logits/rejected": -0.8849962949752808,
+      "logps/chosen": -3.6163456439971924,
+      "logps/rejected": -57.187339782714844,
+      "loss": 0.513578474521637,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28239667415618896,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26628056168556213,
+      "rewards/margins": 4.149145603179932,
+      "rewards/rejected": -3.8828649520874023,
+      "step": 260,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.5271396112092905,
+      "grad_norm": 3.009442090988159,
+      "learning_rate": 0.00017614901700057552,
+      "logits/chosen": -0.7535040974617004,
+      "logits/rejected": -0.8197404146194458,
+      "logps/chosen": -1.611877679824829,
+      "logps/rejected": -36.75214767456055,
+      "loss": 0.4263669550418854,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1808186173439026,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.355543851852417,
+      "rewards/margins": 2.128063678741455,
+      "rewards/rejected": -1.7725193500518799,
+      "step": 261,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.529159303206261,
+      "grad_norm": 10.684004783630371,
+      "learning_rate": 0.0001759319637805806,
+      "logits/chosen": -0.7372271418571472,
+      "logits/rejected": -0.8281789422035217,
+      "logps/chosen": -1.6989682912826538,
+      "logps/rejected": -33.11771011352539,
+      "loss": 0.5836552381515503,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24641528725624084,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42360562086105347,
+      "rewards/margins": 2.2778561115264893,
+      "rewards/rejected": -1.8542505502700806,
+      "step": 262,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5311789952032315,
+      "grad_norm": 4.7525248527526855,
+      "learning_rate": 0.00017571406241917968,
+      "logits/chosen": -0.7032569646835327,
+      "logits/rejected": -0.8048897385597229,
+      "logps/chosen": -3.0269763469696045,
+      "logps/rejected": -29.489608764648438,
+      "loss": 0.5058411359786987,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19841410219669342,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27225935459136963,
+      "rewards/margins": 2.0580902099609375,
+      "rewards/rejected": -1.7858312129974365,
+      "step": 263,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5331986872002019,
+      "grad_norm": 2.464600086212158,
+      "learning_rate": 0.0001754953153502775,
+      "logits/chosen": -0.6921654939651489,
+      "logits/rejected": -0.8245174288749695,
+      "logps/chosen": -1.2894872426986694,
+      "logps/rejected": -41.8453369140625,
+      "loss": 0.36821237206459045,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10888063907623291,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26967915892601013,
+      "rewards/margins": 2.722353458404541,
+      "rewards/rejected": -2.452674150466919,
+      "step": 264,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5352183791971724,
+      "grad_norm": 3.666675329208374,
+      "learning_rate": 0.00017527572501722512,
+      "logits/chosen": -0.6526811718940735,
+      "logits/rejected": -0.8120133876800537,
+      "logps/chosen": -5.671820640563965,
+      "logps/rejected": -57.84872055053711,
+      "loss": 0.6365722417831421,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44796761870384216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04257254675030708,
+      "rewards/margins": 3.3321990966796875,
+      "rewards/rejected": -3.289626121520996,
+      "step": 265,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5372380711941429,
+      "grad_norm": 5.50975227355957,
+      "learning_rate": 0.00017505529387279277,
+      "logits/chosen": -0.7191404104232788,
+      "logits/rejected": -0.7549193501472473,
+      "logps/chosen": -11.20957088470459,
+      "logps/rejected": -24.620988845825195,
+      "loss": 0.8978201746940613,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3969779908657074,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3098965585231781,
+      "rewards/margins": 1.0103328227996826,
+      "rewards/rejected": -1.320229411125183,
+      "step": 266,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5392577631911134,
+      "grad_norm": 9.796467781066895,
+      "learning_rate": 0.0001748340243791422,
+      "logits/chosen": -0.6914748549461365,
+      "logits/rejected": -0.8334357738494873,
+      "logps/chosen": -5.802427291870117,
+      "logps/rejected": -41.96017074584961,
+      "loss": 0.6360026597976685,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3448495864868164,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2543778419494629,
+      "rewards/margins": 2.576064348220825,
+      "rewards/rejected": -2.321686267852783,
+      "step": 267,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5412774551880838,
+      "grad_norm": 4.284933567047119,
+      "learning_rate": 0.00017461191900779936,
+      "logits/chosen": -0.713287353515625,
+      "logits/rejected": -0.7692936658859253,
+      "logps/chosen": -4.46176290512085,
+      "logps/rejected": -30.945892333984375,
+      "loss": 0.5797103047370911,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3343329429626465,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1580563336610794,
+      "rewards/margins": 2.0286142826080322,
+      "rewards/rejected": -1.8705579042434692,
+      "step": 268,
+      "train_speed(iter/s)": 0.021267
+    },
+    {
+      "epoch": 0.5432971471850543,
+      "grad_norm": 3.172943353652954,
+      "learning_rate": 0.00017438898023962679,
+      "logits/chosen": -0.7907764911651611,
+      "logits/rejected": -0.8711207509040833,
+      "logps/chosen": -1.7791467905044556,
+      "logps/rejected": -29.88024139404297,
+      "loss": 0.33940574526786804,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14780378341674805,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2693939208984375,
+      "rewards/margins": 2.3542535305023193,
+      "rewards/rejected": -2.0848593711853027,
+      "step": 269,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.5453168391820248,
+      "grad_norm": 5.463645935058594,
+      "learning_rate": 0.00017416521056479577,
+      "logits/chosen": -0.7378922700881958,
+      "logits/rejected": -0.8519225120544434,
+      "logps/chosen": -5.347389221191406,
+      "logps/rejected": -35.428524017333984,
+      "loss": 0.6419250965118408,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.49513840675354004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09582029283046722,
+      "rewards/margins": 2.3689239025115967,
+      "rewards/rejected": -2.2731034755706787,
+      "step": 270,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5473365311789952,
+      "grad_norm": 2.791165828704834,
+      "learning_rate": 0.00017394061248275872,
+      "logits/chosen": -0.7984417080879211,
+      "logits/rejected": -0.8422114849090576,
+      "logps/chosen": -6.475491046905518,
+      "logps/rejected": -26.260494232177734,
+      "loss": 0.47203221917152405,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23484748601913452,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4524388909339905,
+      "rewards/margins": 1.7292096614837646,
+      "rewards/rejected": -1.2767709493637085,
+      "step": 271,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5493562231759657,
+      "grad_norm": 2.7601311206817627,
+      "learning_rate": 0.00017371518850222112,
+      "logits/chosen": -0.7166031002998352,
+      "logits/rejected": -0.8156049251556396,
+      "logps/chosen": -1.3833684921264648,
+      "logps/rejected": -45.072750091552734,
+      "loss": 0.4653846323490143,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.185410737991333,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1616918444633484,
+      "rewards/margins": 2.9714224338531494,
+      "rewards/rejected": -2.8097305297851562,
+      "step": 272,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5513759151729362,
+      "grad_norm": 4.9853434562683105,
+      "learning_rate": 0.00017348894114111344,
+      "logits/chosen": -0.7632589340209961,
+      "logits/rejected": -0.7834702730178833,
+      "logps/chosen": -5.120803356170654,
+      "logps/rejected": -20.066801071166992,
+      "loss": 0.7621564865112305,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.39412054419517517,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09666721522808075,
+      "rewards/margins": 1.2335360050201416,
+      "rewards/rejected": -1.1368687152862549,
+      "step": 273,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.5533956071699065,
+      "grad_norm": 6.144747734069824,
+      "learning_rate": 0.00017326187292656333,
+      "logits/chosen": -0.7423402070999146,
+      "logits/rejected": -0.8353608846664429,
+      "logps/chosen": -3.019026756286621,
+      "logps/rejected": -38.725494384765625,
+      "loss": 0.4919193685054779,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29635170102119446,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20840522646903992,
+      "rewards/margins": 2.5846292972564697,
+      "rewards/rejected": -2.3762240409851074,
+      "step": 274,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.555415299166877,
+      "grad_norm": 5.072936058044434,
+      "learning_rate": 0.00017303398639486695,
+      "logits/chosen": -0.8050971627235413,
+      "logits/rejected": -0.8979619741439819,
+      "logps/chosen": -3.597996473312378,
+      "logps/rejected": -38.68907165527344,
+      "loss": 0.6185693144798279,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36338648200035095,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04303528368473053,
+      "rewards/margins": 2.610353946685791,
+      "rewards/rejected": -2.567318916320801,
+      "step": 275,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.5574349911638475,
+      "grad_norm": 4.154740810394287,
+      "learning_rate": 0.00017280528409146094,
+      "logits/chosen": -0.8580950498580933,
+      "logits/rejected": -0.9338577389717102,
+      "logps/chosen": -1.9045888185501099,
+      "logps/rejected": -29.787948608398438,
+      "loss": 0.6412729024887085,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2939704358577728,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12071260064840317,
+      "rewards/margins": 2.376711130142212,
+      "rewards/rejected": -2.255998373031616,
+      "step": 276,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.5594546831608179,
+      "grad_norm": 4.429795265197754,
+      "learning_rate": 0.00017257576857089397,
+      "logits/chosen": -0.7823255658149719,
+      "logits/rejected": -0.8295686841011047,
+      "logps/chosen": -6.783672332763672,
+      "logps/rejected": -42.67618942260742,
+      "loss": 0.5918665528297424,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3899191915988922,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.04123719036579132,
+      "rewards/margins": 2.868664503097534,
+      "rewards/rejected": -2.827427387237549,
+      "step": 277,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.5614743751577884,
+      "grad_norm": 3.5302765369415283,
+      "learning_rate": 0.00017234544239679806,
+      "logits/chosen": -0.7818064093589783,
+      "logits/rejected": -0.8903212547302246,
+      "logps/chosen": -1.2885193824768066,
+      "logps/rejected": -25.56497573852539,
+      "loss": 0.5648685693740845,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20888982713222504,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.252044677734375,
+      "rewards/margins": 1.763919472694397,
+      "rewards/rejected": -1.5118746757507324,
+      "step": 278,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5634940671547589,
+      "grad_norm": 1.8682705163955688,
+      "learning_rate": 0.0001721143081418601,
+      "logits/chosen": -0.6173131465911865,
+      "logits/rejected": -0.7885771989822388,
+      "logps/chosen": -0.348890095949173,
+      "logps/rejected": -61.137290954589844,
+      "loss": 0.2171340137720108,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0689445212483406,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4386233389377594,
+      "rewards/margins": 4.034773826599121,
+      "rewards/rejected": -3.5961508750915527,
+      "step": 279,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5655137591517294,
+      "grad_norm": 1.9534425735473633,
+      "learning_rate": 0.00017188236838779295,
+      "logits/chosen": -0.6642014384269714,
+      "logits/rejected": -0.8017529249191284,
+      "logps/chosen": -1.7374509572982788,
+      "logps/rejected": -56.20750427246094,
+      "loss": 0.22880004346370697,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09635397791862488,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3913995027542114,
+      "rewards/margins": 3.391270160675049,
+      "rewards/rejected": -2.999870777130127,
+      "step": 280,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.5675334511486998,
+      "grad_norm": 6.31340217590332,
+      "learning_rate": 0.0001716496257253068,
+      "logits/chosen": -0.7913269996643066,
+      "logits/rejected": -0.9220376014709473,
+      "logps/chosen": -3.3653981685638428,
+      "logps/rejected": -47.17043685913086,
+      "loss": 0.7093809843063354,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40024468302726746,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22013141214847565,
+      "rewards/margins": 3.3141214847564697,
+      "rewards/rejected": -3.0939903259277344,
+      "step": 281,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5695531431456703,
+      "grad_norm": 2.4428505897521973,
+      "learning_rate": 0.00017141608275408006,
+      "logits/chosen": -0.8131429553031921,
+      "logits/rejected": -0.9399922490119934,
+      "logps/chosen": -1.98673415184021,
+      "logps/rejected": -42.24494552612305,
+      "loss": 0.4994111657142639,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2407137006521225,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18601495027542114,
+      "rewards/margins": 2.6695024967193604,
+      "rewards/rejected": -2.483487367630005,
+      "step": 282,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5715728351426408,
+      "grad_norm": 1.5563158988952637,
+      "learning_rate": 0.0001711817420827305,
+      "logits/chosen": -0.6783189177513123,
+      "logits/rejected": -0.9088720083236694,
+      "logps/chosen": -0.37805676460266113,
+      "logps/rejected": -64.65815734863281,
+      "loss": 0.18766579031944275,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08086632192134857,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3040025234222412,
+      "rewards/margins": 4.409641265869141,
+      "rewards/rejected": -4.10563850402832,
+      "step": 283,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5735925271396112,
+      "grad_norm": 7.082139015197754,
+      "learning_rate": 0.00017094660632878582,
+      "logits/chosen": -0.7040260434150696,
+      "logits/rejected": -0.7696897387504578,
+      "logps/chosen": -8.167627334594727,
+      "logps/rejected": -38.17253112792969,
+      "loss": 0.9717364311218262,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5700907111167908,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.012362787500023842,
+      "rewards/margins": 2.3398733139038086,
+      "rewards/rejected": -2.327510356903076,
+      "step": 284,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5756122191365817,
+      "grad_norm": 5.067822456359863,
+      "learning_rate": 0.00017071067811865476,
+      "logits/chosen": -0.802982747554779,
+      "logits/rejected": -0.8972690105438232,
+      "logps/chosen": -2.8758363723754883,
+      "logps/rejected": -46.713172912597656,
+      "loss": 0.5463993549346924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.284506618976593,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16443192958831787,
+      "rewards/margins": 3.5550990104675293,
+      "rewards/rejected": -3.3906667232513428,
+      "step": 285,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5776319111335522,
+      "grad_norm": 4.077809810638428,
+      "learning_rate": 0.00017047396008759754,
+      "logits/chosen": -0.8403116464614868,
+      "logits/rejected": -0.8808307647705078,
+      "logps/chosen": -6.7314581871032715,
+      "logps/rejected": -30.9017333984375,
+      "loss": 0.6438797116279602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3068145215511322,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03903551399707794,
+      "rewards/margins": 1.8479769229888916,
+      "rewards/rejected": -1.887012243270874,
+      "step": 286,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.5796516031305226,
+      "grad_norm": 2.9939074516296387,
+      "learning_rate": 0.00017023645487969645,
+      "logits/chosen": -0.7674691081047058,
+      "logits/rejected": -0.8917283415794373,
+      "logps/chosen": -3.4815213680267334,
+      "logps/rejected": -56.125099182128906,
+      "loss": 0.29649537801742554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15909157693386078,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16410662233829498,
+      "rewards/margins": 4.206541061401367,
+      "rewards/rejected": -4.042433738708496,
+      "step": 287,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.581671295127493,
+      "grad_norm": 12.105512619018555,
+      "learning_rate": 0.00016999816514782647,
+      "logits/chosen": -0.7554559707641602,
+      "logits/rejected": -0.8941792845726013,
+      "logps/chosen": -6.372040271759033,
+      "logps/rejected": -39.95315170288086,
+      "loss": 1.1656122207641602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6600521206855774,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05390087515115738,
+      "rewards/margins": 2.4357922077178955,
+      "rewards/rejected": -2.4896934032440186,
+      "step": 288,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.5836909871244635,
+      "grad_norm": 5.215820789337158,
+      "learning_rate": 0.0001697590935536254,
+      "logits/chosen": -0.8481535911560059,
+      "logits/rejected": -0.9250099062919617,
+      "logps/chosen": -2.248663902282715,
+      "logps/rejected": -45.463008880615234,
+      "loss": 0.4629695415496826,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22682401537895203,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0752679854631424,
+      "rewards/margins": 3.593411922454834,
+      "rewards/rejected": -3.5181431770324707,
+      "step": 289,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.5857106791214339,
+      "grad_norm": 7.402690887451172,
+      "learning_rate": 0.00016951924276746425,
+      "logits/chosen": -0.5944700241088867,
+      "logits/rejected": -0.7739753723144531,
+      "logps/chosen": -8.305325508117676,
+      "logps/rejected": -56.55354690551758,
+      "loss": 1.023478627204895,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5631112456321716,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.024727948009967804,
+      "rewards/margins": 3.671443462371826,
+      "rewards/rejected": -3.696171522140503,
+      "step": 290,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.5877303711184044,
+      "grad_norm": 9.028773307800293,
+      "learning_rate": 0.00016927861546841747,
+      "logits/chosen": -0.827506959438324,
+      "logits/rejected": -0.8843482136726379,
+      "logps/chosen": -3.57808256149292,
+      "logps/rejected": -27.147445678710938,
+      "loss": 0.9430906772613525,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36645135283470154,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07185473293066025,
+      "rewards/margins": 1.882130742073059,
+      "rewards/rejected": -1.810275912284851,
+      "step": 291,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.5897500631153749,
+      "grad_norm": 3.0350229740142822,
+      "learning_rate": 0.00016903721434423306,
+      "logits/chosen": -0.9094471335411072,
+      "logits/rejected": -0.9361112713813782,
+      "logps/chosen": -2.4875197410583496,
+      "logps/rejected": -24.18358612060547,
+      "loss": 0.4315643012523651,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24477659165859222,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3049478530883789,
+      "rewards/margins": 2.156770706176758,
+      "rewards/rejected": -1.8518224954605103,
+      "step": 292,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.5917697551123454,
+      "grad_norm": 4.291833877563477,
+      "learning_rate": 0.0001687950420913022,
+      "logits/chosen": -0.7475178241729736,
+      "logits/rejected": -0.8380208015441895,
+      "logps/chosen": -3.275129795074463,
+      "logps/rejected": -32.63922882080078,
+      "loss": 0.6617104411125183,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4119285047054291,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08241842687129974,
+      "rewards/margins": 2.206611394882202,
+      "rewards/rejected": -2.124192953109741,
+      "step": 293,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.5937894471093158,
+      "grad_norm": 4.568887710571289,
+      "learning_rate": 0.00016855210141462963,
+      "logits/chosen": -0.7708112001419067,
+      "logits/rejected": -0.8631462454795837,
+      "logps/chosen": -2.731691837310791,
+      "logps/rejected": -34.596675872802734,
+      "loss": 0.5602784156799316,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3268437683582306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27219781279563904,
+      "rewards/margins": 2.2604424953460693,
+      "rewards/rejected": -1.9882445335388184,
+      "step": 294,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.5958091391062863,
+      "grad_norm": 4.01273250579834,
+      "learning_rate": 0.0001683083950278031,
+      "logits/chosen": -0.8505802750587463,
+      "logits/rejected": -0.9058212637901306,
+      "logps/chosen": -1.9776384830474854,
+      "logps/rejected": -29.04343605041504,
+      "loss": 0.5075977444648743,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26625630259513855,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16404712200164795,
+      "rewards/margins": 2.069391965866089,
+      "rewards/rejected": -1.905344843864441,
+      "step": 295,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.5978288311032568,
+      "grad_norm": 2.308389186859131,
+      "learning_rate": 0.00016806392565296311,
+      "logits/chosen": -0.7509005665779114,
+      "logits/rejected": -0.8670397996902466,
+      "logps/chosen": -2.6714932918548584,
+      "logps/rejected": -31.49539566040039,
+      "loss": 0.4104999899864197,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14024530351161957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3330576419830322,
+      "rewards/margins": 1.9010692834854126,
+      "rewards/rejected": -1.5680116415023804,
+      "step": 296,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.5998485231002272,
+      "grad_norm": 7.128021717071533,
+      "learning_rate": 0.00016781869602077264,
+      "logits/chosen": -0.7379200458526611,
+      "logits/rejected": -0.8878840804100037,
+      "logps/chosen": -3.651395320892334,
+      "logps/rejected": -39.45635986328125,
+      "loss": 0.8120747208595276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.46895888447761536,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07825376093387604,
+      "rewards/margins": 1.777091145515442,
+      "rewards/rejected": -1.698837399482727,
+      "step": 297,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6018682150971977,
+      "grad_norm": 3.926884651184082,
+      "learning_rate": 0.00016757270887038654,
+      "logits/chosen": -0.7688292264938354,
+      "logits/rejected": -0.9089186191558838,
+      "logps/chosen": -1.3693759441375732,
+      "logps/rejected": -37.80522918701172,
+      "loss": 0.4208237826824188,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17390652000904083,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1809903383255005,
+      "rewards/margins": 2.1318840980529785,
+      "rewards/rejected": -1.9508938789367676,
+      "step": 298,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6038879070941682,
+      "grad_norm": 3.3669636249542236,
+      "learning_rate": 0.00016732596694942083,
+      "logits/chosen": -0.74614417552948,
+      "logits/rejected": -0.8711810111999512,
+      "logps/chosen": -4.000098705291748,
+      "logps/rejected": -40.62786102294922,
+      "loss": 0.5480436682701111,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23701626062393188,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11138585954904556,
+      "rewards/margins": 1.9405393600463867,
+      "rewards/rejected": -1.8291537761688232,
+      "step": 299,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6059075990911386,
+      "grad_norm": 2.457587480545044,
+      "learning_rate": 0.00016707847301392236,
+      "logits/chosen": -0.8123277425765991,
+      "logits/rejected": -0.9144627451896667,
+      "logps/chosen": -4.013862133026123,
+      "logps/rejected": -33.994991302490234,
+      "loss": 0.5886133909225464,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31008380651474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23923330008983612,
+      "rewards/margins": 2.1191067695617676,
+      "rewards/rejected": -1.879873514175415,
+      "step": 300,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6059075990911386,
+      "eval_logits/chosen": -0.8244539499282837,
+      "eval_logits/rejected": -0.9301112294197083,
+      "eval_logps/chosen": -2.4614338874816895,
+      "eval_logps/rejected": -37.376708984375,
+      "eval_loss": 0.5653835535049438,
+      "eval_nll_loss": 0.26967209577560425,
+      "eval_rewards/accuracies": 0.862500011920929,
+      "eval_rewards/chosen": 0.19960978627204895,
+      "eval_rewards/margins": 2.2363221645355225,
+      "eval_rewards/rejected": -2.036712169647217,
+      "eval_runtime": 92.5823,
+      "eval_samples_per_second": 0.864,
+      "eval_steps_per_second": 0.432,
+      "step": 300
+    },
+    {
+      "epoch": 0.6079272910881091,
+      "grad_norm": 1.4425960779190063,
+      "learning_rate": 0.00016683022982833757,
+      "logits/chosen": -0.9167453050613403,
+      "logits/rejected": -0.9786884784698486,
+      "logps/chosen": -0.2554780840873718,
+      "logps/rejected": -30.395259857177734,
+      "loss": 0.28913912177085876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.056439101696014404,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2730258107185364,
+      "rewards/margins": 2.289043664932251,
+      "rewards/rejected": -2.0160179138183594,
+      "step": 301,
+      "train_speed(iter/s)": 0.021119
+    },
+    {
+      "epoch": 0.6099469830850796,
+      "grad_norm": 2.9968175888061523,
+      "learning_rate": 0.00016658124016548197,
+      "logits/chosen": -0.9765443801879883,
+      "logits/rejected": -1.0245364904403687,
+      "logps/chosen": -1.0650570392608643,
+      "logps/rejected": -35.4031867980957,
+      "loss": 0.43358710408210754,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18546591699123383,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2088906615972519,
+      "rewards/margins": 2.796689748764038,
+      "rewards/rejected": -2.587799310684204,
+      "step": 302,
+      "train_speed(iter/s)": 0.021121
+    },
+    {
+      "epoch": 0.6119666750820499,
+      "grad_norm": 4.542461395263672,
+      "learning_rate": 0.000166331506806509,
+      "logits/chosen": -0.9329476356506348,
+      "logits/rejected": -1.0012348890304565,
+      "logps/chosen": -2.2945432662963867,
+      "logps/rejected": -31.613216400146484,
+      "loss": 0.5403211116790771,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2446451485157013,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2515539526939392,
+      "rewards/margins": 2.3975443840026855,
+      "rewards/rejected": -2.1459906101226807,
+      "step": 303,
+      "train_speed(iter/s)": 0.021123
+    },
+    {
+      "epoch": 0.6139863670790204,
+      "grad_norm": 3.4531707763671875,
+      "learning_rate": 0.00016608103254087906,
+      "logits/chosen": -0.7963448762893677,
+      "logits/rejected": -0.9296629428863525,
+      "logps/chosen": -3.21559476852417,
+      "logps/rejected": -49.27157211303711,
+      "loss": 0.5262452363967896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2854577302932739,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08577384054660797,
+      "rewards/margins": 3.5433404445648193,
+      "rewards/rejected": -3.457566261291504,
+      "step": 304,
+      "train_speed(iter/s)": 0.021123
+    },
+    {
+      "epoch": 0.6160060590759909,
+      "grad_norm": 5.113759994506836,
+      "learning_rate": 0.00016582982016632818,
+      "logits/chosen": -0.7916707396507263,
+      "logits/rejected": -0.934760570526123,
+      "logps/chosen": -2.8785698413848877,
+      "logps/rejected": -56.01650619506836,
+      "loss": 0.6735590100288391,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42455944418907166,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24037745594978333,
+      "rewards/margins": 3.957566261291504,
+      "rewards/rejected": -3.717188835144043,
+      "step": 305,
+      "train_speed(iter/s)": 0.021124
+    },
+    {
+      "epoch": 0.6180257510729614,
+      "grad_norm": 1.8782538175582886,
+      "learning_rate": 0.00016557787248883696,
+      "logits/chosen": -0.8707424402236938,
+      "logits/rejected": -0.9912842512130737,
+      "logps/chosen": -1.2852058410644531,
+      "logps/rejected": -55.1501579284668,
+      "loss": 0.26773229241371155,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11378525197505951,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.32057979702949524,
+      "rewards/margins": 4.313370227813721,
+      "rewards/rejected": -3.992790460586548,
+      "step": 306,
+      "train_speed(iter/s)": 0.021124
+    },
+    {
+      "epoch": 0.6200454430699318,
+      "grad_norm": 12.83481216430664,
+      "learning_rate": 0.00016532519232259916,
+      "logits/chosen": -0.8531877398490906,
+      "logits/rejected": -0.9290578365325928,
+      "logps/chosen": -3.8104918003082275,
+      "logps/rejected": -58.618621826171875,
+      "loss": 0.3679554760456085,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2755557596683502,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2785298526287079,
+      "rewards/margins": 4.633764743804932,
+      "rewards/rejected": -4.355234622955322,
+      "step": 307,
+      "train_speed(iter/s)": 0.021125
+    },
+    {
+      "epoch": 0.6220651350669023,
+      "grad_norm": 3.644946336746216,
+      "learning_rate": 0.00016507178248999024,
+      "logits/chosen": -0.8718377947807312,
+      "logits/rejected": -1.0232672691345215,
+      "logps/chosen": -1.1764187812805176,
+      "logps/rejected": -66.40010070800781,
+      "loss": 0.4798681139945984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2249489575624466,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23639193177223206,
+      "rewards/margins": 4.824877738952637,
+      "rewards/rejected": -4.5884857177734375,
+      "step": 308,
+      "train_speed(iter/s)": 0.021126
+    },
+    {
+      "epoch": 0.6240848270638728,
+      "grad_norm": 3.6663687229156494,
+      "learning_rate": 0.00016481764582153586,
+      "logits/chosen": -0.8408634066581726,
+      "logits/rejected": -0.9267845153808594,
+      "logps/chosen": -4.433281898498535,
+      "logps/rejected": -32.701011657714844,
+      "loss": 0.47992053627967834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21022652089595795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5157691240310669,
+      "rewards/margins": 2.2275538444519043,
+      "rewards/rejected": -1.7117843627929688,
+      "step": 309,
+      "train_speed(iter/s)": 0.021127
+    },
+    {
+      "epoch": 0.6261045190608432,
+      "grad_norm": 8.538496017456055,
+      "learning_rate": 0.00016456278515588024,
+      "logits/chosen": -0.8827572464942932,
+      "logits/rejected": -0.9595881700515747,
+      "logps/chosen": -5.372483253479004,
+      "logps/rejected": -43.774436950683594,
+      "loss": 0.74099200963974,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4238053560256958,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08053889870643616,
+      "rewards/margins": 2.704007148742676,
+      "rewards/rejected": -2.6234686374664307,
+      "step": 310,
+      "train_speed(iter/s)": 0.021128
+    },
+    {
+      "epoch": 0.6281242110578137,
+      "grad_norm": 3.513566493988037,
+      "learning_rate": 0.00016430720333975452,
+      "logits/chosen": -0.8878616094589233,
+      "logits/rejected": -0.9629725217819214,
+      "logps/chosen": -2.4065513610839844,
+      "logps/rejected": -43.71923065185547,
+      "loss": 0.4641585052013397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2343730330467224,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1930648535490036,
+      "rewards/margins": 3.418928623199463,
+      "rewards/rejected": -3.2258636951446533,
+      "step": 311,
+      "train_speed(iter/s)": 0.021129
+    },
+    {
+      "epoch": 0.6301439030547842,
+      "grad_norm": 6.264689922332764,
+      "learning_rate": 0.00016405090322794483,
+      "logits/chosen": -0.9043357968330383,
+      "logits/rejected": -0.9985758066177368,
+      "logps/chosen": -2.7129530906677246,
+      "logps/rejected": -41.665489196777344,
+      "loss": 0.6599389910697937,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3321758210659027,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17053501307964325,
+      "rewards/margins": 2.564326047897339,
+      "rewards/rejected": -2.3937911987304688,
+      "step": 312,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6321635950517546,
+      "grad_norm": 22.02393913269043,
+      "learning_rate": 0.00016379388768326063,
+      "logits/chosen": -0.8338424563407898,
+      "logits/rejected": -0.9313662052154541,
+      "logps/chosen": -4.164543151855469,
+      "logps/rejected": -50.46995544433594,
+      "loss": 1.0124552249908447,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6529191732406616,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.016213715076446533,
+      "rewards/margins": 3.471205949783325,
+      "rewards/rejected": -3.4549922943115234,
+      "step": 313,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6341832870487251,
+      "grad_norm": 10.904000282287598,
+      "learning_rate": 0.00016353615957650236,
+      "logits/chosen": -0.7540192008018494,
+      "logits/rejected": -0.8433898687362671,
+      "logps/chosen": -10.264131546020508,
+      "logps/rejected": -41.65949630737305,
+      "loss": 0.5326629877090454,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.239348366856575,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.253826379776001,
+      "rewards/margins": 2.7431883811950684,
+      "rewards/rejected": -2.4893617630004883,
+      "step": 314,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6362029790456956,
+      "grad_norm": 3.1705944538116455,
+      "learning_rate": 0.00016327772178642986,
+      "logits/chosen": -0.9517133831977844,
+      "logits/rejected": -1.0139639377593994,
+      "logps/chosen": -0.973305344581604,
+      "logps/rejected": -38.540000915527344,
+      "loss": 0.3936949670314789,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1697213351726532,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20970085263252258,
+      "rewards/margins": 3.3330564498901367,
+      "rewards/rejected": -3.1233556270599365,
+      "step": 315,
+      "train_speed(iter/s)": 0.021133
+    },
+    {
+      "epoch": 0.638222671042666,
+      "grad_norm": 5.624232769012451,
+      "learning_rate": 0.00016301857719972976,
+      "logits/chosen": -0.8001008629798889,
+      "logits/rejected": -0.8618478775024414,
+      "logps/chosen": -1.4323068857192993,
+      "logps/rejected": -35.56591033935547,
+      "loss": 0.4376400411128998,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14159713685512543,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20484629273414612,
+      "rewards/margins": 2.43072247505188,
+      "rewards/rejected": -2.2258763313293457,
+      "step": 316,
+      "train_speed(iter/s)": 0.021134
+    },
+    {
+      "epoch": 0.6402423630396364,
+      "grad_norm": 7.583630561828613,
+      "learning_rate": 0.0001627587287109836,
+      "logits/chosen": -0.8299413919448853,
+      "logits/rejected": -0.9769677519798279,
+      "logps/chosen": -3.7215254306793213,
+      "logps/rejected": -60.84526062011719,
+      "loss": 0.5378049612045288,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32689476013183594,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.025372333824634552,
+      "rewards/margins": 4.722233295440674,
+      "rewards/rejected": -4.6968607902526855,
+      "step": 317,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.6422620550366069,
+      "grad_norm": 8.251932144165039,
+      "learning_rate": 0.00016249817922263517,
+      "logits/chosen": -0.8568125367164612,
+      "logits/rejected": -0.931611955165863,
+      "logps/chosen": -5.391039848327637,
+      "logps/rejected": -45.42852020263672,
+      "loss": 0.5377641916275024,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29841428995132446,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.31022340059280396,
+      "rewards/margins": 3.579132080078125,
+      "rewards/rejected": -3.268908977508545,
+      "step": 318,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.6442817470335774,
+      "grad_norm": 4.026428699493408,
+      "learning_rate": 0.00016223693164495835,
+      "logits/chosen": -0.77679044008255,
+      "logits/rejected": -0.8818118572235107,
+      "logps/chosen": -5.223626136779785,
+      "logps/rejected": -46.39476013183594,
+      "loss": 0.5932655930519104,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30503058433532715,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.009123122319579124,
+      "rewards/margins": 3.3854122161865234,
+      "rewards/rejected": -3.3945353031158447,
+      "step": 319,
+      "train_speed(iter/s)": 0.021136
+    },
+    {
+      "epoch": 0.6463014390305478,
+      "grad_norm": 3.859316110610962,
+      "learning_rate": 0.00016197498889602448,
+      "logits/chosen": -0.9226962924003601,
+      "logits/rejected": -0.984259843826294,
+      "logps/chosen": -0.3310558497905731,
+      "logps/rejected": -39.15312576293945,
+      "loss": 0.21047677099704742,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.053509388118982315,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37339526414871216,
+      "rewards/margins": 3.054718494415283,
+      "rewards/rejected": -2.681323289871216,
+      "step": 320,
+      "train_speed(iter/s)": 0.021137
+    },
+    {
+      "epoch": 0.6483211310275183,
+      "grad_norm": 4.3604888916015625,
+      "learning_rate": 0.00016171235390166985,
+      "logits/chosen": -0.9235320687294006,
+      "logits/rejected": -0.9902734160423279,
+      "logps/chosen": -2.538111448287964,
+      "logps/rejected": -48.80242156982422,
+      "loss": 0.5195378661155701,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23220032453536987,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17571590840816498,
+      "rewards/margins": 3.9404807090759277,
+      "rewards/rejected": -3.7647647857666016,
+      "step": 321,
+      "train_speed(iter/s)": 0.021138
+    },
+    {
+      "epoch": 0.6503408230244888,
+      "grad_norm": 9.97143268585205,
+      "learning_rate": 0.00016144902959546286,
+      "logits/chosen": -0.9121385216712952,
+      "logits/rejected": -0.9658838510513306,
+      "logps/chosen": -5.540125846862793,
+      "logps/rejected": -45.613990783691406,
+      "loss": 0.8853093385696411,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.35688427090644836,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05821647495031357,
+      "rewards/margins": 3.1649882793426514,
+      "rewards/rejected": -3.2232048511505127,
+      "step": 322,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.6523605150214592,
+      "grad_norm": 4.8883819580078125,
+      "learning_rate": 0.0001611850189186714,
+      "logits/chosen": -0.9210085868835449,
+      "logits/rejected": -0.9733687043190002,
+      "logps/chosen": -4.037269592285156,
+      "logps/rejected": -41.21742630004883,
+      "loss": 0.5509124398231506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32746627926826477,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21709240972995758,
+      "rewards/margins": 3.416368246078491,
+      "rewards/rejected": -3.199275493621826,
+      "step": 323,
+      "train_speed(iter/s)": 0.02114
+    },
+    {
+      "epoch": 0.6543802070184297,
+      "grad_norm": 2.4553871154785156,
+      "learning_rate": 0.00016092032482023,
+      "logits/chosen": -0.9412962794303894,
+      "logits/rejected": -1.0209475755691528,
+      "logps/chosen": -0.632290244102478,
+      "logps/rejected": -33.6468505859375,
+      "loss": 0.23645614087581635,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07834430038928986,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43455106019973755,
+      "rewards/margins": 3.022073745727539,
+      "rewards/rejected": -2.587522506713867,
+      "step": 324,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.6563998990154002,
+      "grad_norm": 2.446058511734009,
+      "learning_rate": 0.00016065495025670675,
+      "logits/chosen": -0.8765738606452942,
+      "logits/rejected": -0.9390391111373901,
+      "logps/chosen": -5.492425918579102,
+      "logps/rejected": -34.9351692199707,
+      "loss": 0.5046104788780212,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34279099106788635,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22330977022647858,
+      "rewards/margins": 2.7527146339416504,
+      "rewards/rejected": -2.529404878616333,
+      "step": 325,
+      "train_speed(iter/s)": 0.021143
+    },
+    {
+      "epoch": 0.6584195910123706,
+      "grad_norm": 18.5092716217041,
+      "learning_rate": 0.00016038889819227045,
+      "logits/chosen": -0.8468933701515198,
+      "logits/rejected": -0.9487016201019287,
+      "logps/chosen": -1.3551874160766602,
+      "logps/rejected": -38.7204475402832,
+      "loss": 0.4489996135234833,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2548639476299286,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3463600277900696,
+      "rewards/margins": 2.8273019790649414,
+      "rewards/rejected": -2.4809420108795166,
+      "step": 326,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.6604392830093411,
+      "grad_norm": 6.140425682067871,
+      "learning_rate": 0.00016012217159865739,
+      "logits/chosen": -0.8302001357078552,
+      "logits/rejected": -0.924152135848999,
+      "logps/chosen": -3.6062324047088623,
+      "logps/rejected": -56.65538787841797,
+      "loss": 0.784902811050415,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3868030309677124,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.006980091333389282,
+      "rewards/margins": 3.1850905418395996,
+      "rewards/rejected": -3.178110122680664,
+      "step": 327,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.6624589750063116,
+      "grad_norm": 2.915184259414673,
+      "learning_rate": 0.00015985477345513817,
+      "logits/chosen": -0.7988994121551514,
+      "logits/rejected": -0.9357851147651672,
+      "logps/chosen": -5.732859134674072,
+      "logps/rejected": -39.42655944824219,
+      "loss": 0.347720742225647,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16449037194252014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23062925040721893,
+      "rewards/margins": 2.8876638412475586,
+      "rewards/rejected": -2.6570348739624023,
+      "step": 328,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.664478667003282,
+      "grad_norm": 7.8771162033081055,
+      "learning_rate": 0.00015958670674848442,
+      "logits/chosen": -0.857587456703186,
+      "logits/rejected": -0.9888629913330078,
+      "logps/chosen": -6.550410270690918,
+      "logps/rejected": -41.466896057128906,
+      "loss": 0.709037184715271,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40757060050964355,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.43693459033966064,
+      "rewards/margins": 2.189579486846924,
+      "rewards/rejected": -1.7526450157165527,
+      "step": 329,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.6664983590002524,
+      "grad_norm": 19.00602149963379,
+      "learning_rate": 0.00015931797447293552,
+      "logits/chosen": -0.8882402181625366,
+      "logits/rejected": -0.9668765068054199,
+      "logps/chosen": -3.752054214477539,
+      "logps/rejected": -34.47447204589844,
+      "loss": 0.6838876605033875,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4321058392524719,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22359441220760345,
+      "rewards/margins": 2.5547034740448,
+      "rewards/rejected": -2.331109046936035,
+      "step": 330,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.668518050997223,
+      "grad_norm": 1.7641230821609497,
+      "learning_rate": 0.00015904857963016506,
+      "logits/chosen": -0.7825491428375244,
+      "logits/rejected": -0.9280312061309814,
+      "logps/chosen": -0.6239898204803467,
+      "logps/rejected": -43.15408706665039,
+      "loss": 0.24098820984363556,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09237208962440491,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2947159707546234,
+      "rewards/margins": 3.279745578765869,
+      "rewards/rejected": -2.985029697418213,
+      "step": 331,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.6705377429941933,
+      "grad_norm": 2.2421019077301025,
+      "learning_rate": 0.00015877852522924732,
+      "logits/chosen": -0.8130238652229309,
+      "logits/rejected": -0.9327349662780762,
+      "logps/chosen": -0.8687487244606018,
+      "logps/rejected": -47.515663146972656,
+      "loss": 0.2235592156648636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09912680834531784,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2986675500869751,
+      "rewards/margins": 3.4509923458099365,
+      "rewards/rejected": -3.152324914932251,
+      "step": 332,
+      "train_speed(iter/s)": 0.021147
+    },
+    {
+      "epoch": 0.6725574349911638,
+      "grad_norm": 8.463959693908691,
+      "learning_rate": 0.0001585078142866237,
+      "logits/chosen": -0.8485694527626038,
+      "logits/rejected": -0.9816790819168091,
+      "logps/chosen": -4.081976890563965,
+      "logps/rejected": -34.49391174316406,
+      "loss": 0.8111600279808044,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5827361941337585,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.010378856211900711,
+      "rewards/margins": 2.1897881031036377,
+      "rewards/rejected": -2.1794090270996094,
+      "step": 333,
+      "train_speed(iter/s)": 0.021149
+    },
+    {
+      "epoch": 0.6745771269881343,
+      "grad_norm": 3.4926505088806152,
+      "learning_rate": 0.00015823644982606905,
+      "logits/chosen": -0.9417222142219543,
+      "logits/rejected": -1.0309451818466187,
+      "logps/chosen": -0.7485335469245911,
+      "logps/rejected": -33.3504524230957,
+      "loss": 0.307941198348999,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09651970863342285,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.388710618019104,
+      "rewards/margins": 2.98738956451416,
+      "rewards/rejected": -2.5986788272857666,
+      "step": 334,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.6765968189851048,
+      "grad_norm": 10.243365287780762,
+      "learning_rate": 0.00015796443487865776,
+      "logits/chosen": -0.909482479095459,
+      "logits/rejected": -0.9898862838745117,
+      "logps/chosen": -6.335696220397949,
+      "logps/rejected": -46.39136505126953,
+      "loss": 0.6261657476425171,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.43729183077812195,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27775925397872925,
+      "rewards/margins": 3.944711446762085,
+      "rewards/rejected": -3.666951894760132,
+      "step": 335,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.6786165109820752,
+      "grad_norm": 7.904644012451172,
+      "learning_rate": 0.00015769177248273008,
+      "logits/chosen": -0.9360239505767822,
+      "logits/rejected": -1.049852967262268,
+      "logps/chosen": -7.70269250869751,
+      "logps/rejected": -51.02676773071289,
+      "loss": 0.8781910538673401,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5348291397094727,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.07586444914340973,
+      "rewards/margins": 3.814580202102661,
+      "rewards/rejected": -3.738715648651123,
+      "step": 336,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6806362029790457,
+      "grad_norm": 16.91658592224121,
+      "learning_rate": 0.00015741846568385808,
+      "logits/chosen": -0.8461523652076721,
+      "logits/rejected": -1.0379836559295654,
+      "logps/chosen": -8.801255226135254,
+      "logps/rejected": -61.471771240234375,
+      "loss": 1.1453731060028076,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7435779571533203,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.463644415140152,
+      "rewards/margins": 4.270198345184326,
+      "rewards/rejected": -4.733842849731445,
+      "step": 337,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6826558949760162,
+      "grad_norm": 12.933732986450195,
+      "learning_rate": 0.00015714451753481168,
+      "logits/chosen": -0.6461415886878967,
+      "logits/rejected": -0.9560658931732178,
+      "logps/chosen": -6.069940567016602,
+      "logps/rejected": -53.62686538696289,
+      "loss": 1.2495336532592773,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8185646533966064,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3106077015399933,
+      "rewards/margins": 3.0313639640808105,
+      "rewards/rejected": -3.3419713973999023,
+      "step": 338,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6846755869729866,
+      "grad_norm": 6.386441230773926,
+      "learning_rate": 0.00015686993109552443,
+      "logits/chosen": -0.9001967906951904,
+      "logits/rejected": -1.0288763046264648,
+      "logps/chosen": -4.8095598220825195,
+      "logps/rejected": -63.7664909362793,
+      "loss": 0.6220850348472595,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42656922340393066,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07018661499023438,
+      "rewards/margins": 4.841952323913574,
+      "rewards/rejected": -4.77176570892334,
+      "step": 339,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.6866952789699571,
+      "grad_norm": 5.94568395614624,
+      "learning_rate": 0.00015659470943305955,
+      "logits/chosen": -0.8113903999328613,
+      "logits/rejected": -1.0205042362213135,
+      "logps/chosen": -3.7077159881591797,
+      "logps/rejected": -56.63233947753906,
+      "loss": 0.6539210081100464,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4288255274295807,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.059340257197618484,
+      "rewards/margins": 4.157015800476074,
+      "rewards/rejected": -4.097675323486328,
+      "step": 340,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.6887149709669276,
+      "grad_norm": 3.9790964126586914,
+      "learning_rate": 0.00015631885562157543,
+      "logits/chosen": -0.7372142672538757,
+      "logits/rejected": -0.9902328848838806,
+      "logps/chosen": -4.715322017669678,
+      "logps/rejected": -48.823387145996094,
+      "loss": 0.6435255408287048,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3544943630695343,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05755168944597244,
+      "rewards/margins": 2.823580741882324,
+      "rewards/rejected": -2.766029119491577,
+      "step": 341,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.690734662963898,
+      "grad_norm": 3.6011016368865967,
+      "learning_rate": 0.00015604237274229147,
+      "logits/chosen": -0.9505451321601868,
+      "logits/rejected": -1.0710150003433228,
+      "logps/chosen": -1.6771836280822754,
+      "logps/rejected": -42.753135681152344,
+      "loss": 0.3884910047054291,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2611246109008789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41318997740745544,
+      "rewards/margins": 3.458676338195801,
+      "rewards/rejected": -3.0454862117767334,
+      "step": 342,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.6927543549608685,
+      "grad_norm": 10.275211334228516,
+      "learning_rate": 0.00015576526388345367,
+      "logits/chosen": -0.9035798907279968,
+      "logits/rejected": -1.0091338157653809,
+      "logps/chosen": -4.990538597106934,
+      "logps/rejected": -41.765995025634766,
+      "loss": 0.998170018196106,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7295805215835571,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13218021392822266,
+      "rewards/margins": 3.011774778366089,
+      "rewards/rejected": -2.879594326019287,
+      "step": 343,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.694774046957839,
+      "grad_norm": 8.156365394592285,
+      "learning_rate": 0.0001554875321402999,
+      "logits/chosen": -0.8031080961227417,
+      "logits/rejected": -0.9648129940032959,
+      "logps/chosen": -2.28568696975708,
+      "logps/rejected": -35.34866714477539,
+      "loss": 0.5425023436546326,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24677146971225739,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23254749178886414,
+      "rewards/margins": 2.27335262298584,
+      "rewards/rejected": -2.0408051013946533,
+      "step": 344,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.6967937389548093,
+      "grad_norm": 4.11509370803833,
+      "learning_rate": 0.00015520918061502569,
+      "logits/chosen": -0.8795959949493408,
+      "logits/rejected": -1.0246540307998657,
+      "logps/chosen": -1.0050594806671143,
+      "logps/rejected": -38.232444763183594,
+      "loss": 0.32429325580596924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1644802689552307,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3108518719673157,
+      "rewards/margins": 2.897658109664917,
+      "rewards/rejected": -2.586806297302246,
+      "step": 345,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.6988134309517798,
+      "grad_norm": 3.702918529510498,
+      "learning_rate": 0.00015493021241674918,
+      "logits/chosen": -0.8280074000358582,
+      "logits/rejected": -0.9851573705673218,
+      "logps/chosen": -1.3124903440475464,
+      "logps/rejected": -35.455692291259766,
+      "loss": 0.46161431074142456,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20427490770816803,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2279079556465149,
+      "rewards/margins": 2.2844018936157227,
+      "rewards/rejected": -2.0564939975738525,
+      "step": 346,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7008331229487503,
+      "grad_norm": 7.40757417678833,
+      "learning_rate": 0.0001546506306614768,
+      "logits/chosen": -0.8412176966667175,
+      "logits/rejected": -0.9579489231109619,
+      "logps/chosen": -6.378708839416504,
+      "logps/rejected": -44.914512634277344,
+      "loss": 0.6551076173782349,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3504900634288788,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.00600462406873703,
+      "rewards/margins": 3.1142947673797607,
+      "rewards/rejected": -3.1202995777130127,
+      "step": 347,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.7028528149457208,
+      "grad_norm": 2.9458255767822266,
+      "learning_rate": 0.0001543704384720681,
+      "logits/chosen": -0.7899777889251709,
+      "logits/rejected": -0.9945221543312073,
+      "logps/chosen": -1.4169785976409912,
+      "logps/rejected": -50.57292556762695,
+      "loss": 0.3628111779689789,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15231113135814667,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.226759135723114,
+      "rewards/margins": 3.434785842895508,
+      "rewards/rejected": -3.208026647567749,
+      "step": 348,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.7048725069426912,
+      "grad_norm": 4.447574615478516,
+      "learning_rate": 0.00015408963897820113,
+      "logits/chosen": -0.8848673105239868,
+      "logits/rejected": -0.8949815034866333,
+      "logps/chosen": -9.218939781188965,
+      "logps/rejected": -32.96113967895508,
+      "loss": 0.6976296901702881,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31593263149261475,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1799672544002533,
+      "rewards/margins": 2.5064430236816406,
+      "rewards/rejected": -2.3264756202697754,
+      "step": 349,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.7068921989396617,
+      "grad_norm": 3.5722556114196777,
+      "learning_rate": 0.00015380823531633729,
+      "logits/chosen": -0.9537544250488281,
+      "logits/rejected": -1.0240473747253418,
+      "logps/chosen": -3.9818813800811768,
+      "logps/rejected": -31.99053382873535,
+      "loss": 0.6191223859786987,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34284406900405884,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09681667387485504,
+      "rewards/margins": 2.406569480895996,
+      "rewards/rejected": -2.309752941131592,
+      "step": 350,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.7089118909366322,
+      "grad_norm": 3.7325596809387207,
+      "learning_rate": 0.00015352623062968648,
+      "logits/chosen": -0.922427773475647,
+      "logits/rejected": -0.9690065383911133,
+      "logps/chosen": -4.305528163909912,
+      "logps/rejected": -23.71919059753418,
+      "loss": 0.7693547010421753,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.33499932289123535,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06479213386774063,
+      "rewards/margins": 1.370113730430603,
+      "rewards/rejected": -1.3053216934204102,
+      "step": 351,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.7109315829336026,
+      "grad_norm": 16.7460880279541,
+      "learning_rate": 0.00015324362806817186,
+      "logits/chosen": -0.7807352542877197,
+      "logits/rejected": -0.9729360342025757,
+      "logps/chosen": -7.726840972900391,
+      "logps/rejected": -53.6373176574707,
+      "loss": 1.1061019897460938,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6384282112121582,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.23256784677505493,
+      "rewards/margins": 2.8845200538635254,
+      "rewards/rejected": -3.1170878410339355,
+      "step": 352,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.7129512749305731,
+      "grad_norm": 2.80759596824646,
+      "learning_rate": 0.00015296043078839473,
+      "logits/chosen": -0.7530017495155334,
+      "logits/rejected": -0.9337269067764282,
+      "logps/chosen": -1.4654000997543335,
+      "logps/rejected": -55.76970672607422,
+      "loss": 0.254292368888855,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13821597397327423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28305867314338684,
+      "rewards/margins": 4.615407943725586,
+      "rewards/rejected": -4.3323493003845215,
+      "step": 353,
+      "train_speed(iter/s)": 0.02114
+    },
+    {
+      "epoch": 0.7149709669275436,
+      "grad_norm": 4.502577781677246,
+      "learning_rate": 0.00015267664195359917,
+      "logits/chosen": -0.9706122279167175,
+      "logits/rejected": -1.0545164346694946,
+      "logps/chosen": -1.6742087602615356,
+      "logps/rejected": -41.54838180541992,
+      "loss": 0.518170177936554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2445138841867447,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10661309212446213,
+      "rewards/margins": 3.0441462993621826,
+      "rewards/rejected": -2.937532901763916,
+      "step": 354,
+      "train_speed(iter/s)": 0.021141
+    },
+    {
+      "epoch": 0.716990658924514,
+      "grad_norm": 5.956605434417725,
+      "learning_rate": 0.00015239226473363687,
+      "logits/chosen": -0.9984610080718994,
+      "logits/rejected": -1.0658961534500122,
+      "logps/chosen": -2.667590379714966,
+      "logps/rejected": -30.01835823059082,
+      "loss": 0.6159235239028931,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25425097346305847,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13858197629451752,
+      "rewards/margins": 2.178638219833374,
+      "rewards/rejected": -2.0400562286376953,
+      "step": 355,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.7190103509214845,
+      "grad_norm": 2.670485734939575,
+      "learning_rate": 0.00015210730230493162,
+      "logits/chosen": -0.8936488628387451,
+      "logits/rejected": -1.0093450546264648,
+      "logps/chosen": -3.5686304569244385,
+      "logps/rejected": -55.824180603027344,
+      "loss": 0.3414130210876465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20404773950576782,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14025713503360748,
+      "rewards/margins": 3.9673044681549072,
+      "rewards/rejected": -3.827047109603882,
+      "step": 356,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.721030042918455,
+      "grad_norm": 4.567800521850586,
+      "learning_rate": 0.00015182175785044387,
+      "logits/chosen": -0.981977105140686,
+      "logits/rejected": -0.96744304895401,
+      "logps/chosen": -7.694738388061523,
+      "logps/rejected": -33.95967483520508,
+      "loss": 0.5043447613716125,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24890656769275665,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2846420407295227,
+      "rewards/margins": 2.429332733154297,
+      "rewards/rejected": -2.144690752029419,
+      "step": 357,
+      "train_speed(iter/s)": 0.021143
+    },
+    {
+      "epoch": 0.7230497349154253,
+      "grad_norm": 8.642942428588867,
+      "learning_rate": 0.00015153563455963499,
+      "logits/chosen": -0.9537326097488403,
+      "logits/rejected": -1.1215823888778687,
+      "logps/chosen": -2.9258816242218018,
+      "logps/rejected": -48.52583312988281,
+      "loss": 0.7397590279579163,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4563119411468506,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18413014709949493,
+      "rewards/margins": 4.132811069488525,
+      "rewards/rejected": -3.9486806392669678,
+      "step": 358,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.7250694269123958,
+      "grad_norm": 24.88861083984375,
+      "learning_rate": 0.00015124893562843208,
+      "logits/chosen": -0.9031403660774231,
+      "logits/rejected": -0.9935811758041382,
+      "logps/chosen": -8.023815155029297,
+      "logps/rejected": -40.717247009277344,
+      "loss": 0.8925535082817078,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5370101928710938,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3610239028930664,
+      "rewards/margins": 3.118933916091919,
+      "rewards/rejected": -2.7579100131988525,
+      "step": 359,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.7270891189093663,
+      "grad_norm": 10.635350227355957,
+      "learning_rate": 0.00015096166425919175,
+      "logits/chosen": -0.9903375506401062,
+      "logits/rejected": -1.0812005996704102,
+      "logps/chosen": -3.357611656188965,
+      "logps/rejected": -47.11662292480469,
+      "loss": 0.7105602025985718,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3658413290977478,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13873916864395142,
+      "rewards/margins": 3.40531063079834,
+      "rewards/rejected": -3.266571283340454,
+      "step": 360,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.7291088109063368,
+      "grad_norm": 18.06879234313965,
+      "learning_rate": 0.0001506738236606648,
+      "logits/chosen": -1.099637508392334,
+      "logits/rejected": -1.1945291757583618,
+      "logps/chosen": -8.104093551635742,
+      "logps/rejected": -45.32279968261719,
+      "loss": 1.623255729675293,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.006649374961853,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.31839191913604736,
+      "rewards/margins": 3.014329433441162,
+      "rewards/rejected": -3.332721471786499,
+      "step": 361,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.7311285029033072,
+      "grad_norm": 15.128271102905273,
+      "learning_rate": 0.00015038541704796003,
+      "logits/chosen": -1.1882942914962769,
+      "logits/rejected": -1.2618598937988281,
+      "logps/chosen": -1.8502236604690552,
+      "logps/rejected": -69.4446792602539,
+      "loss": 0.38569730520248413,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24810141324996948,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1845402717590332,
+      "rewards/margins": 6.129173278808594,
+      "rewards/rejected": -5.944633483886719,
+      "step": 362,
+      "train_speed(iter/s)": 0.021148
+    },
+    {
+      "epoch": 0.7331481949002777,
+      "grad_norm": 5.277956485748291,
+      "learning_rate": 0.00015009644764250863,
+      "logits/chosen": -1.163029432296753,
+      "logits/rejected": -1.2435169219970703,
+      "logps/chosen": -5.300413608551025,
+      "logps/rejected": -73.11473846435547,
+      "loss": 0.8010823130607605,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5916219353675842,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.007427714765071869,
+      "rewards/margins": 5.836453914642334,
+      "rewards/rejected": -5.829026222229004,
+      "step": 363,
+      "train_speed(iter/s)": 0.021148
+    },
+    {
+      "epoch": 0.7351678868972482,
+      "grad_norm": 75.7375717163086,
+      "learning_rate": 0.0001498069186720279,
+      "logits/chosen": -1.1564539670944214,
+      "logits/rejected": -1.1993024349212646,
+      "logps/chosen": -2.832427978515625,
+      "logps/rejected": -73.21650695800781,
+      "loss": 0.8798462748527527,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4809403121471405,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17453742027282715,
+      "rewards/margins": 5.794814109802246,
+      "rewards/rejected": -5.620276927947998,
+      "step": 364,
+      "train_speed(iter/s)": 0.021149
+    },
+    {
+      "epoch": 0.7371875788942186,
+      "grad_norm": 2.991504669189453,
+      "learning_rate": 0.00014951683337048537,
+      "logits/chosen": -1.1292376518249512,
+      "logits/rejected": -1.1874810457229614,
+      "logps/chosen": -6.187487602233887,
+      "logps/rejected": -55.482765197753906,
+      "loss": 0.45134469866752625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22427882254123688,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3069310486316681,
+      "rewards/margins": 4.149868488311768,
+      "rewards/rejected": -3.842937707901001,
+      "step": 365,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.7392072708911891,
+      "grad_norm": 10.73784065246582,
+      "learning_rate": 0.00014922619497806277,
+      "logits/chosen": -1.167018175125122,
+      "logits/rejected": -1.2153444290161133,
+      "logps/chosen": -1.954573631286621,
+      "logps/rejected": -61.2001953125,
+      "loss": 0.5314586162567139,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2992059588432312,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14029161632061005,
+      "rewards/margins": 4.6798810958862305,
+      "rewards/rejected": -4.539588928222656,
+      "step": 366,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.7412269628881596,
+      "grad_norm": 5.413460731506348,
+      "learning_rate": 0.0001489350067411196,
+      "logits/chosen": -1.1709797382354736,
+      "logits/rejected": -1.1748571395874023,
+      "logps/chosen": -7.91160774230957,
+      "logps/rejected": -46.791404724121094,
+      "loss": 0.5436979532241821,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30416107177734375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3512665033340454,
+      "rewards/margins": 3.6528432369232178,
+      "rewards/rejected": -3.301577091217041,
+      "step": 367,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.74324665488513,
+      "grad_norm": 7.643467903137207,
+      "learning_rate": 0.00014864327191215702,
+      "logits/chosen": -1.1821328401565552,
+      "logits/rejected": -1.2253855466842651,
+      "logps/chosen": -2.3498167991638184,
+      "logps/rejected": -73.46287536621094,
+      "loss": 0.664942741394043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.454881489276886,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1444016695022583,
+      "rewards/margins": 5.324404716491699,
+      "rewards/rejected": -5.1800031661987305,
+      "step": 368,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.7452663468821005,
+      "grad_norm": 3.926517963409424,
+      "learning_rate": 0.00014835099374978147,
+      "logits/chosen": -1.1555640697479248,
+      "logits/rejected": -1.2078311443328857,
+      "logps/chosen": -0.9639624953269958,
+      "logps/rejected": -67.16943359375,
+      "loss": 0.2394624501466751,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1322186291217804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2654881775379181,
+      "rewards/margins": 6.110224723815918,
+      "rewards/rejected": -5.844736099243164,
+      "step": 369,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.747286038879071,
+      "grad_norm": 4.563858985900879,
+      "learning_rate": 0.00014805817551866838,
+      "logits/chosen": -1.183912992477417,
+      "logits/rejected": -1.2156834602355957,
+      "logps/chosen": -1.4219800233840942,
+      "logps/rejected": -39.744171142578125,
+      "loss": 0.3572657108306885,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15966495871543884,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4396134316921234,
+      "rewards/margins": 3.298137903213501,
+      "rewards/rejected": -2.8585243225097656,
+      "step": 370,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.7493057308760414,
+      "grad_norm": 4.50530481338501,
+      "learning_rate": 0.00014776482048952551,
+      "logits/chosen": -1.1194803714752197,
+      "logits/rejected": -1.1540480852127075,
+      "logps/chosen": -1.7459050416946411,
+      "logps/rejected": -64.54890441894531,
+      "loss": 0.2934742867946625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15494152903556824,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2886925935745239,
+      "rewards/margins": 5.029353141784668,
+      "rewards/rejected": -4.740660190582275,
+      "step": 371,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.7513254228730119,
+      "grad_norm": 7.041140556335449,
+      "learning_rate": 0.00014747093193905657,
+      "logits/chosen": -1.1590553522109985,
+      "logits/rejected": -1.1508252620697021,
+      "logps/chosen": -3.047581434249878,
+      "logps/rejected": -61.70812225341797,
+      "loss": 0.6074828505516052,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3978445827960968,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13817714154720306,
+      "rewards/margins": 4.407755374908447,
+      "rewards/rejected": -4.269577980041504,
+      "step": 372,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.7533451148699823,
+      "grad_norm": 4.780473709106445,
+      "learning_rate": 0.00014717651314992455,
+      "logits/chosen": -1.1245406866073608,
+      "logits/rejected": -1.1541649103164673,
+      "logps/chosen": -2.052983283996582,
+      "logps/rejected": -59.029022216796875,
+      "loss": 0.3612958490848541,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17177711427211761,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1790371835231781,
+      "rewards/margins": 4.619558811187744,
+      "rewards/rejected": -4.440521717071533,
+      "step": 373,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.7553648068669528,
+      "grad_norm": 2.6618669033050537,
+      "learning_rate": 0.00014688156741071514,
+      "logits/chosen": -1.16391921043396,
+      "logits/rejected": -1.2101376056671143,
+      "logps/chosen": -2.5643341541290283,
+      "logps/rejected": -50.99317932128906,
+      "loss": 0.359174907207489,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14501263201236725,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2054893523454666,
+      "rewards/margins": 4.245234489440918,
+      "rewards/rejected": -4.039744853973389,
+      "step": 374,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.7573844988639232,
+      "grad_norm": 2.183286666870117,
+      "learning_rate": 0.00014658609801589982,
+      "logits/chosen": -1.197989583015442,
+      "logits/rejected": -1.232681155204773,
+      "logps/chosen": -1.0838813781738281,
+      "logps/rejected": -53.68153762817383,
+      "loss": 0.27284619212150574,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11144410073757172,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3119613528251648,
+      "rewards/margins": 4.7982072830200195,
+      "rewards/rejected": -4.486246109008789,
+      "step": 375,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.7594041908608937,
+      "grad_norm": 6.413065433502197,
+      "learning_rate": 0.00014629010826579928,
+      "logits/chosen": -1.217897891998291,
+      "logits/rejected": -1.279034972190857,
+      "logps/chosen": -2.029020071029663,
+      "logps/rejected": -67.19940948486328,
+      "loss": 0.3730330765247345,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2322002500295639,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2920472323894501,
+      "rewards/margins": 5.175812244415283,
+      "rewards/rejected": -4.883764266967773,
+      "step": 376,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.7614238828578642,
+      "grad_norm": 3.451570987701416,
+      "learning_rate": 0.0001459936014665464,
+      "logits/chosen": -1.17360258102417,
+      "logits/rejected": -1.2139333486557007,
+      "logps/chosen": -4.350042819976807,
+      "logps/rejected": -56.49632263183594,
+      "loss": 0.5775706171989441,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.39697498083114624,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23329809308052063,
+      "rewards/margins": 4.431912422180176,
+      "rewards/rejected": -4.198614597320557,
+      "step": 377,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7634435748548346,
+      "grad_norm": 5.291248798370361,
+      "learning_rate": 0.00014569658093004935,
+      "logits/chosen": -1.1762566566467285,
+      "logits/rejected": -1.2233586311340332,
+      "logps/chosen": -3.534919261932373,
+      "logps/rejected": -80.47354888916016,
+      "loss": 0.30777013301849365,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2154829502105713,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2291412353515625,
+      "rewards/margins": 6.180717468261719,
+      "rewards/rejected": -5.951576232910156,
+      "step": 378,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7654632668518051,
+      "grad_norm": 10.057852745056152,
+      "learning_rate": 0.00014539904997395468,
+      "logits/chosen": -1.1968324184417725,
+      "logits/rejected": -1.2274653911590576,
+      "logps/chosen": -3.546328544616699,
+      "logps/rejected": -69.02865600585938,
+      "loss": 0.6399840712547302,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4272352159023285,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11763279885053635,
+      "rewards/margins": 5.601714611053467,
+      "rewards/rejected": -5.484081745147705,
+      "step": 379,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7674829588487756,
+      "grad_norm": 4.334756374359131,
+      "learning_rate": 0.00014510101192161018,
+      "logits/chosen": -1.2146495580673218,
+      "logits/rejected": -1.2545320987701416,
+      "logps/chosen": -6.085969924926758,
+      "logps/rejected": -41.06128692626953,
+      "loss": 0.776848316192627,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5693851709365845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19634687900543213,
+      "rewards/margins": 3.313415050506592,
+      "rewards/rejected": -3.1170685291290283,
+      "step": 380,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.769502650845746,
+      "grad_norm": 6.6673479080200195,
+      "learning_rate": 0.00014480247010202775,
+      "logits/chosen": -1.2620035409927368,
+      "logits/rejected": -1.2927439212799072,
+      "logps/chosen": -0.9638640880584717,
+      "logps/rejected": -58.5118293762207,
+      "loss": 0.3140888810157776,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13222388923168182,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43909886479377747,
+      "rewards/margins": 5.030383110046387,
+      "rewards/rejected": -4.591284275054932,
+      "step": 381,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.7715223428427165,
+      "grad_norm": 1.8640215396881104,
+      "learning_rate": 0.00014450342784984633,
+      "logits/chosen": -1.2142149209976196,
+      "logits/rejected": -1.2449159622192383,
+      "logps/chosen": -3.2471883296966553,
+      "logps/rejected": -69.44312286376953,
+      "loss": 0.2647269666194916,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17502562701702118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32493484020233154,
+      "rewards/margins": 5.946244239807129,
+      "rewards/rejected": -5.621310234069824,
+      "step": 382,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.773542034839687,
+      "grad_norm": 4.235260963439941,
+      "learning_rate": 0.00014420388850529446,
+      "logits/chosen": -1.2407033443450928,
+      "logits/rejected": -1.280212163925171,
+      "logps/chosen": -1.4121766090393066,
+      "logps/rejected": -73.05975341796875,
+      "loss": 0.4667883813381195,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23148246109485626,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42525410652160645,
+      "rewards/margins": 5.927689552307129,
+      "rewards/rejected": -5.502435207366943,
+      "step": 383,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.7755617268366574,
+      "grad_norm": 11.74973201751709,
+      "learning_rate": 0.00014390385541415308,
+      "logits/chosen": -1.2475506067276,
+      "logits/rejected": -1.2707443237304688,
+      "logps/chosen": -2.454108953475952,
+      "logps/rejected": -42.124366760253906,
+      "loss": 0.6018331050872803,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3722446858882904,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23790600895881653,
+      "rewards/margins": 3.314408779144287,
+      "rewards/rejected": -3.076502561569214,
+      "step": 384,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7775814188336279,
+      "grad_norm": 2.7534830570220947,
+      "learning_rate": 0.0001436033319277183,
+      "logits/chosen": -1.2355518341064453,
+      "logits/rejected": -1.2741280794143677,
+      "logps/chosen": -0.7196122407913208,
+      "logps/rejected": -107.31944274902344,
+      "loss": 0.1968921422958374,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1137460246682167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2591235637664795,
+      "rewards/margins": 8.63725757598877,
+      "rewards/rejected": -8.378134727478027,
+      "step": 385,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7796011108305984,
+      "grad_norm": 5.864130020141602,
+      "learning_rate": 0.00014330232140276366,
+      "logits/chosen": -1.1965060234069824,
+      "logits/rejected": -1.2489956617355347,
+      "logps/chosen": -3.9303858280181885,
+      "logps/rejected": -66.87359619140625,
+      "loss": 0.6629573106765747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5041288137435913,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20213821530342102,
+      "rewards/margins": 5.429197311401367,
+      "rewards/rejected": -5.2270588874816895,
+      "step": 386,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7816208028275689,
+      "grad_norm": 0.9686940312385559,
+      "learning_rate": 0.0001430008272015029,
+      "logits/chosen": -1.168556571006775,
+      "logits/rejected": -1.2702208757400513,
+      "logps/chosen": -0.1927298754453659,
+      "logps/rejected": -86.75350189208984,
+      "loss": 0.07656482607126236,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.02479482628405094,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3893800973892212,
+      "rewards/margins": 6.8833112716674805,
+      "rewards/rejected": -6.493931770324707,
+      "step": 387,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7836404948245392,
+      "grad_norm": 6.69142484664917,
+      "learning_rate": 0.0001426988526915523,
+      "logits/chosen": -1.1132441759109497,
+      "logits/rejected": -1.2206928730010986,
+      "logps/chosen": -2.1822941303253174,
+      "logps/rejected": -58.27075958251953,
+      "loss": 0.4767617881298065,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2635282874107361,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18697461485862732,
+      "rewards/margins": 4.062845706939697,
+      "rewards/rejected": -3.875871419906616,
+      "step": 388,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.7856601868215097,
+      "grad_norm": 22.11477279663086,
+      "learning_rate": 0.00014239640124589302,
+      "logits/chosen": -1.1612333059310913,
+      "logits/rejected": -1.2209270000457764,
+      "logps/chosen": -5.149811267852783,
+      "logps/rejected": -56.67973327636719,
+      "loss": 0.9844444394111633,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.464616984128952,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.07719806581735611,
+      "rewards/margins": 3.927542209625244,
+      "rewards/rejected": -4.004740238189697,
+      "step": 389,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.7876798788184802,
+      "grad_norm": 4.598104476928711,
+      "learning_rate": 0.0001420934762428335,
+      "logits/chosen": -1.191237211227417,
+      "logits/rejected": -1.2505682706832886,
+      "logps/chosen": -1.6766197681427002,
+      "logps/rejected": -68.58138275146484,
+      "loss": 0.37751591205596924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1912289708852768,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2715514898300171,
+      "rewards/margins": 6.051302909851074,
+      "rewards/rejected": -5.779752731323242,
+      "step": 390,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.7896995708154506,
+      "grad_norm": 5.1656084060668945,
+      "learning_rate": 0.00014179008106597173,
+      "logits/chosen": -1.1447128057479858,
+      "logits/rejected": -1.2311046123504639,
+      "logps/chosen": -1.3589807748794556,
+      "logps/rejected": -69.0888900756836,
+      "loss": 0.37555360794067383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20962855219841003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2835935950279236,
+      "rewards/margins": 6.014133453369141,
+      "rewards/rejected": -5.730539798736572,
+      "step": 391,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 0.7917192628124211,
+      "grad_norm": 4.515217304229736,
+      "learning_rate": 0.0001414862191041574,
+      "logits/chosen": -1.1313319206237793,
+      "logits/rejected": -1.1735104322433472,
+      "logps/chosen": -1.4379080533981323,
+      "logps/rejected": -41.75006103515625,
+      "loss": 0.42747819423675537,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13288989663124084,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.21147757768630981,
+      "rewards/margins": 3.0256876945495605,
+      "rewards/rejected": -2.8142104148864746,
+      "step": 392,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.7937389548093916,
+      "grad_norm": 4.281467437744141,
+      "learning_rate": 0.00014118189375145402,
+      "logits/chosen": -1.129544734954834,
+      "logits/rejected": -1.2433701753616333,
+      "logps/chosen": -1.0194271802902222,
+      "logps/rejected": -68.8100357055664,
+      "loss": 0.4009433388710022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2110380381345749,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42351850867271423,
+      "rewards/margins": 5.658205032348633,
+      "rewards/rejected": -5.234686374664307,
+      "step": 393,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.795758646806362,
+      "grad_norm": 0.9730565547943115,
+      "learning_rate": 0.0001408771084071012,
+      "logits/chosen": -1.098219394683838,
+      "logits/rejected": -1.2555054426193237,
+      "logps/chosen": -0.3178718388080597,
+      "logps/rejected": -82.52906799316406,
+      "loss": 0.08471984416246414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.025890445336699486,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41851806640625,
+      "rewards/margins": 7.259293556213379,
+      "rewards/rejected": -6.840775489807129,
+      "step": 394,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.7977783388033325,
+      "grad_norm": 6.422388076782227,
+      "learning_rate": 0.0001405718664754764,
+      "logits/chosen": -1.0757089853286743,
+      "logits/rejected": -1.2194029092788696,
+      "logps/chosen": -4.098117351531982,
+      "logps/rejected": -79.46504211425781,
+      "loss": 0.9334844946861267,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.670303463935852,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.09435828030109406,
+      "rewards/margins": 6.080230236053467,
+      "rewards/rejected": -6.174588680267334,
+      "step": 395,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.799798030800303,
+      "grad_norm": 3.1609609127044678,
+      "learning_rate": 0.0001402661713660571,
+      "logits/chosen": -1.1343252658843994,
+      "logits/rejected": -1.2600665092468262,
+      "logps/chosen": -1.6706289052963257,
+      "logps/rejected": -69.32755279541016,
+      "loss": 0.22293509542942047,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17708872258663177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18604204058647156,
+      "rewards/margins": 6.022045612335205,
+      "rewards/rejected": -5.836003303527832,
+      "step": 396,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8018177227972734,
+      "grad_norm": 3.5824060440063477,
+      "learning_rate": 0.0001399600264933827,
+      "logits/chosen": -1.1588817834854126,
+      "logits/rejected": -1.2686594724655151,
+      "logps/chosen": -2.480526924133301,
+      "logps/rejected": -77.53683471679688,
+      "loss": 0.400388240814209,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2272554636001587,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2708975374698639,
+      "rewards/margins": 6.85975980758667,
+      "rewards/rejected": -6.58886194229126,
+      "step": 397,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8038374147942439,
+      "grad_norm": 8.915884017944336,
+      "learning_rate": 0.00013965343527701628,
+      "logits/chosen": -1.0931580066680908,
+      "logits/rejected": -1.2515380382537842,
+      "logps/chosen": -6.809703826904297,
+      "logps/rejected": -92.00086975097656,
+      "loss": 1.2307904958724976,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.950491189956665,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.23660066723823547,
+      "rewards/margins": 7.736156463623047,
+      "rewards/rejected": -7.9727559089660645,
+      "step": 398,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8058571067912144,
+      "grad_norm": 16.942861557006836,
+      "learning_rate": 0.00013934640114150656,
+      "logits/chosen": -1.1165294647216797,
+      "logits/rejected": -1.2258877754211426,
+      "logps/chosen": -5.151846885681152,
+      "logps/rejected": -50.20682144165039,
+      "loss": 1.4095838069915771,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.0055750608444214,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08993971347808838,
+      "rewards/margins": 3.8719429969787598,
+      "rewards/rejected": -3.9618825912475586,
+      "step": 399,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.8078767987881847,
+      "grad_norm": 12.716894149780273,
+      "learning_rate": 0.00013903892751634947,
+      "logits/chosen": -1.1147105693817139,
+      "logits/rejected": -1.2179588079452515,
+      "logps/chosen": -2.3593387603759766,
+      "logps/rejected": -60.22994613647461,
+      "loss": 0.3005334734916687,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17930257320404053,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2562834620475769,
+      "rewards/margins": 5.470202445983887,
+      "rewards/rejected": -5.213918685913086,
+      "step": 400,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.8098964907851552,
+      "grad_norm": 3.6407527923583984,
+      "learning_rate": 0.00013873101783594999,
+      "logits/chosen": -0.9179055690765381,
+      "logits/rejected": -1.0948255062103271,
+      "logps/chosen": -7.396864414215088,
+      "logps/rejected": -76.39356994628906,
+      "loss": 0.7279776334762573,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5141282081604004,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07542237639427185,
+      "rewards/margins": 5.8221588134765625,
+      "rewards/rejected": -5.746736526489258,
+      "step": 401,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.8119161827821257,
+      "grad_norm": 4.730077743530273,
+      "learning_rate": 0.00013842267553958371,
+      "logits/chosen": -1.0015758275985718,
+      "logits/rejected": -1.173001766204834,
+      "logps/chosen": -1.2833012342453003,
+      "logps/rejected": -65.50496673583984,
+      "loss": 0.31842923164367676,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15187285840511322,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3435157239437103,
+      "rewards/margins": 5.360050201416016,
+      "rewards/rejected": -5.016533851623535,
+      "step": 402,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.8139358747790962,
+      "grad_norm": 6.681663513183594,
+      "learning_rate": 0.00013811390407135837,
+      "logits/chosen": -1.0345579385757446,
+      "logits/rejected": -1.1669750213623047,
+      "logps/chosen": -2.711009979248047,
+      "logps/rejected": -48.982208251953125,
+      "loss": 0.49727287888526917,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25284403562545776,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15987528860569,
+      "rewards/margins": 4.172056674957275,
+      "rewards/rejected": -4.012181282043457,
+      "step": 403,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.8159555667760666,
+      "grad_norm": 3.184199571609497,
+      "learning_rate": 0.00013780470688017562,
+      "logits/chosen": -1.0638835430145264,
+      "logits/rejected": -1.1304998397827148,
+      "logps/chosen": -4.9558868408203125,
+      "logps/rejected": -44.02895736694336,
+      "loss": 0.6054225564002991,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3829798102378845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39171290397644043,
+      "rewards/margins": 3.7432100772857666,
+      "rewards/rejected": -3.3514974117279053,
+      "step": 404,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.8179752587730371,
+      "grad_norm": 16.700468063354492,
+      "learning_rate": 0.00013749508741969213,
+      "logits/chosen": -0.9126327633857727,
+      "logits/rejected": -0.9981098771095276,
+      "logps/chosen": -8.324628829956055,
+      "logps/rejected": -54.67494201660156,
+      "loss": 1.088732361793518,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7484469413757324,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14237897098064423,
+      "rewards/margins": 3.1626644134521484,
+      "rewards/rejected": -3.3050432205200195,
+      "step": 405,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.8199949507700076,
+      "grad_norm": 4.572319507598877,
+      "learning_rate": 0.00013718504914828135,
+      "logits/chosen": -0.9639456272125244,
+      "logits/rejected": -1.0909277200698853,
+      "logps/chosen": -2.374218225479126,
+      "logps/rejected": -28.60363006591797,
+      "loss": 0.6309629082679749,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3386070728302002,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.38088110089302063,
+      "rewards/margins": 2.043933868408203,
+      "rewards/rejected": -1.6630527973175049,
+      "step": 406,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.822014642766978,
+      "grad_norm": 7.175300598144531,
+      "learning_rate": 0.00013687459552899465,
+      "logits/chosen": -1.011370062828064,
+      "logits/rejected": -1.1552919149398804,
+      "logps/chosen": -0.751252293586731,
+      "logps/rejected": -58.353179931640625,
+      "loss": 0.3189142942428589,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1253315508365631,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2173149585723877,
+      "rewards/margins": 4.045991897583008,
+      "rewards/rejected": -3.828676700592041,
+      "step": 407,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.8240343347639485,
+      "grad_norm": 3.638871908187866,
+      "learning_rate": 0.0001365637300295229,
+      "logits/chosen": -0.9884570240974426,
+      "logits/rejected": -1.1229445934295654,
+      "logps/chosen": -2.4416940212249756,
+      "logps/rejected": -49.12045669555664,
+      "loss": 0.5083186030387878,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30594146251678467,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28965339064598083,
+      "rewards/margins": 3.306330919265747,
+      "rewards/rejected": -3.0166776180267334,
+      "step": 408,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.826054026760919,
+      "grad_norm": 1.9313740730285645,
+      "learning_rate": 0.0001362524561221574,
+      "logits/chosen": -1.0589873790740967,
+      "logits/rejected": -1.155687928199768,
+      "logps/chosen": -0.7494394183158875,
+      "logps/rejected": -61.899314880371094,
+      "loss": 0.16012489795684814,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06837154924869537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.422152578830719,
+      "rewards/margins": 5.1323323249816895,
+      "rewards/rejected": -4.710179805755615,
+      "step": 409,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.8280737187578894,
+      "grad_norm": 2.8826475143432617,
+      "learning_rate": 0.00013594077728375128,
+      "logits/chosen": -1.046625018119812,
+      "logits/rejected": -1.1403248310089111,
+      "logps/chosen": -2.6342790126800537,
+      "logps/rejected": -47.86298370361328,
+      "loss": 0.3936201333999634,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19051307439804077,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13923531770706177,
+      "rewards/margins": 3.3216652870178223,
+      "rewards/rejected": -3.182429790496826,
+      "step": 410,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8300934107548599,
+      "grad_norm": 7.247873306274414,
+      "learning_rate": 0.00013562869699568076,
+      "logits/chosen": -1.1259405612945557,
+      "logits/rejected": -1.1454825401306152,
+      "logps/chosen": -7.434700012207031,
+      "logps/rejected": -49.61088562011719,
+      "loss": 0.960975170135498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7216768860816956,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03636153042316437,
+      "rewards/margins": 3.5744729042053223,
+      "rewards/rejected": -3.6108345985412598,
+      "step": 411,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8321131027518304,
+      "grad_norm": 2.7764785289764404,
+      "learning_rate": 0.00013531621874380613,
+      "logits/chosen": -0.9636606574058533,
+      "logits/rejected": -1.1189731359481812,
+      "logps/chosen": -3.8168373107910156,
+      "logps/rejected": -65.32124328613281,
+      "loss": 0.31505468487739563,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2553892135620117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2123510092496872,
+      "rewards/margins": 5.209415912628174,
+      "rewards/rejected": -4.997064590454102,
+      "step": 412,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8341327947488008,
+      "grad_norm": 4.009552001953125,
+      "learning_rate": 0.0001350033460184327,
+      "logits/chosen": -0.8576051592826843,
+      "logits/rejected": -1.1664515733718872,
+      "logps/chosen": -1.3953887224197388,
+      "logps/rejected": -94.75813293457031,
+      "loss": 0.1995779424905777,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09238594025373459,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33278360962867737,
+      "rewards/margins": 6.894500255584717,
+      "rewards/rejected": -6.5617170333862305,
+      "step": 413,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8361524867457713,
+      "grad_norm": 14.014391899108887,
+      "learning_rate": 0.00013469008231427207,
+      "logits/chosen": -1.1141268014907837,
+      "logits/rejected": -1.1793636083602905,
+      "logps/chosen": -2.8426718711853027,
+      "logps/rejected": -64.9253158569336,
+      "loss": 0.6592072248458862,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44861316680908203,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20249105989933014,
+      "rewards/margins": 5.202639579772949,
+      "rewards/rejected": -5.000148296356201,
+      "step": 414,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8381721787427417,
+      "grad_norm": 6.667239189147949,
+      "learning_rate": 0.00013437643113040301,
+      "logits/chosen": -1.1405432224273682,
+      "logits/rejected": -1.205533504486084,
+      "logps/chosen": -4.049654960632324,
+      "logps/rejected": -65.5449447631836,
+      "loss": 0.955814003944397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7577894926071167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.011894673109054565,
+      "rewards/margins": 5.261639595031738,
+      "rewards/rejected": -5.273534297943115,
+      "step": 415,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.8401918707397122,
+      "grad_norm": 5.3052802085876465,
+      "learning_rate": 0.00013406239597023225,
+      "logits/chosen": -1.1984199285507202,
+      "logits/rejected": -1.2300504446029663,
+      "logps/chosen": -2.0026376247406006,
+      "logps/rejected": -44.80060958862305,
+      "loss": 0.32053616642951965,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1967550367116928,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29741793870925903,
+      "rewards/margins": 3.937462329864502,
+      "rewards/rejected": -3.6400444507598877,
+      "step": 416,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.8422115627366826,
+      "grad_norm": 4.6360578536987305,
+      "learning_rate": 0.0001337479803414555,
+      "logits/chosen": -1.1135053634643555,
+      "logits/rejected": -1.1990042924880981,
+      "logps/chosen": -1.2526822090148926,
+      "logps/rejected": -64.39313507080078,
+      "loss": 0.3324093520641327,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19946074485778809,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22650685906410217,
+      "rewards/margins": 5.059194087982178,
+      "rewards/rejected": -4.8326873779296875,
+      "step": 417,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.8442312547336531,
+      "grad_norm": 5.64458703994751,
+      "learning_rate": 0.0001334331877560182,
+      "logits/chosen": -1.1316285133361816,
+      "logits/rejected": -1.1844420433044434,
+      "logps/chosen": -2.270009994506836,
+      "logps/rejected": -73.60192108154297,
+      "loss": 0.37290775775909424,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2037116289138794,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22195757925510406,
+      "rewards/margins": 6.114259719848633,
+      "rewards/rejected": -5.892301559448242,
+      "step": 418,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.8462509467306236,
+      "grad_norm": 1.5520920753479004,
+      "learning_rate": 0.00013311802173007626,
+      "logits/chosen": -1.1409199237823486,
+      "logits/rejected": -1.203282356262207,
+      "logps/chosen": -2.087766408920288,
+      "logps/rejected": -75.88261413574219,
+      "loss": 0.19689969718456268,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11977825313806534,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1527182012796402,
+      "rewards/margins": 6.459143161773682,
+      "rewards/rejected": -6.306424617767334,
+      "step": 419,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.848270638727594,
+      "grad_norm": 16.449495315551758,
+      "learning_rate": 0.0001328024857839569,
+      "logits/chosen": -1.189044713973999,
+      "logits/rejected": -1.2236502170562744,
+      "logps/chosen": -4.713018894195557,
+      "logps/rejected": -54.21986389160156,
+      "loss": 0.7003884315490723,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4415695369243622,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.018294498324394226,
+      "rewards/margins": 4.57138204574585,
+      "rewards/rejected": -4.589676856994629,
+      "step": 420,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.8502903307245645,
+      "grad_norm": 28.160564422607422,
+      "learning_rate": 0.00013248658344211926,
+      "logits/chosen": -1.0967679023742676,
+      "logits/rejected": -1.1361645460128784,
+      "logps/chosen": -6.815154552459717,
+      "logps/rejected": -83.51364135742188,
+      "loss": 1.2551347017288208,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8729581832885742,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03463226556777954,
+      "rewards/margins": 6.367086410522461,
+      "rewards/rejected": -6.401719093322754,
+      "step": 421,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.852310022721535,
+      "grad_norm": 31.47573471069336,
+      "learning_rate": 0.00013217031823311488,
+      "logits/chosen": -1.1553871631622314,
+      "logits/rejected": -1.2064706087112427,
+      "logps/chosen": -5.438415050506592,
+      "logps/rejected": -65.18042755126953,
+      "loss": 0.8344112038612366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5628060698509216,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07962656021118164,
+      "rewards/margins": 5.513628959655762,
+      "rewards/rejected": -5.434001922607422,
+      "step": 422,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 0.8543297147185054,
+      "grad_norm": 8.784391403198242,
+      "learning_rate": 0.0001318536936895487,
+      "logits/chosen": -1.123747706413269,
+      "logits/rejected": -1.1673023700714111,
+      "logps/chosen": -3.4688830375671387,
+      "logps/rejected": -54.17964172363281,
+      "loss": 0.8599854707717896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5710347890853882,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.038751691579818726,
+      "rewards/margins": 4.241340637207031,
+      "rewards/rejected": -4.2025885581970215,
+      "step": 423,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.8563494067154759,
+      "grad_norm": 22.89176368713379,
+      "learning_rate": 0.00013153671334803905,
+      "logits/chosen": -1.1089229583740234,
+      "logits/rejected": -1.1420047283172607,
+      "logps/chosen": -4.063858985900879,
+      "logps/rejected": -40.476722717285156,
+      "loss": 0.9364627599716187,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6597320437431335,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02382955513894558,
+      "rewards/margins": 3.0120341777801514,
+      "rewards/rejected": -2.9882047176361084,
+      "step": 424,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.8583690987124464,
+      "grad_norm": 4.699946880340576,
+      "learning_rate": 0.00013121938074917865,
+      "logits/chosen": -1.0667518377304077,
+      "logits/rejected": -1.1017130613327026,
+      "logps/chosen": -4.643237113952637,
+      "logps/rejected": -60.97563934326172,
+      "loss": 0.39269566535949707,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3167289197444916,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22700132429599762,
+      "rewards/margins": 4.699875354766846,
+      "rewards/rejected": -4.472874641418457,
+      "step": 425,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.8603887907094168,
+      "grad_norm": 2.7371695041656494,
+      "learning_rate": 0.00013090169943749476,
+      "logits/chosen": -1.076597809791565,
+      "logits/rejected": -1.0993525981903076,
+      "logps/chosen": -2.2984302043914795,
+      "logps/rejected": -54.213531494140625,
+      "loss": 0.3011862337589264,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17973017692565918,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19591867923736572,
+      "rewards/margins": 3.4975361824035645,
+      "rewards/rejected": -3.3016176223754883,
+      "step": 426,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.8624084827063873,
+      "grad_norm": 17.75054168701172,
+      "learning_rate": 0.00013058367296140967,
+      "logits/chosen": -1.046842098236084,
+      "logits/rejected": -1.0758651494979858,
+      "logps/chosen": -4.21718692779541,
+      "logps/rejected": -39.91408920288086,
+      "loss": 0.9825822114944458,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6494057774543762,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.017109857872128487,
+      "rewards/margins": 2.6019086837768555,
+      "rewards/rejected": -2.584798812866211,
+      "step": 427,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8644281747033578,
+      "grad_norm": 2.5992624759674072,
+      "learning_rate": 0.00013026530487320113,
+      "logits/chosen": -1.0623323917388916,
+      "logits/rejected": -1.1041500568389893,
+      "logps/chosen": -1.7210079431533813,
+      "logps/rejected": -52.09022521972656,
+      "loss": 0.3279334008693695,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19231894612312317,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27669844031333923,
+      "rewards/margins": 4.07616662979126,
+      "rewards/rejected": -3.7994682788848877,
+      "step": 428,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8664478667003283,
+      "grad_norm": 3.9208383560180664,
+      "learning_rate": 0.00012994659872896257,
+      "logits/chosen": -1.0508674383163452,
+      "logits/rejected": -1.1102675199508667,
+      "logps/chosen": -3.953282594680786,
+      "logps/rejected": -63.04370880126953,
+      "loss": 0.7971250414848328,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5409887433052063,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06726647913455963,
+      "rewards/margins": 4.291454792022705,
+      "rewards/rejected": -4.224188327789307,
+      "step": 429,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8684675586972986,
+      "grad_norm": 2.1997742652893066,
+      "learning_rate": 0.00012962755808856342,
+      "logits/chosen": -1.0617396831512451,
+      "logits/rejected": -1.0978208780288696,
+      "logps/chosen": -1.448944330215454,
+      "logps/rejected": -47.621673583984375,
+      "loss": 0.17363907396793365,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10171680897474289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2539137005805969,
+      "rewards/margins": 3.7236616611480713,
+      "rewards/rejected": -3.4697484970092773,
+      "step": 430,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8704872506942691,
+      "grad_norm": 5.373311519622803,
+      "learning_rate": 0.00012930818651560934,
+      "logits/chosen": -1.015924096107483,
+      "logits/rejected": -1.0387259721755981,
+      "logps/chosen": -3.2455596923828125,
+      "logps/rejected": -39.7397575378418,
+      "loss": 0.8503936529159546,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.522122859954834,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04042099043726921,
+      "rewards/margins": 2.7263174057006836,
+      "rewards/rejected": -2.7667384147644043,
+      "step": 431,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.8725069426912396,
+      "grad_norm": 10.711043357849121,
+      "learning_rate": 0.00012898848757740246,
+      "logits/chosen": -0.9647800922393799,
+      "logits/rejected": -0.9854017496109009,
+      "logps/chosen": -9.754167556762695,
+      "logps/rejected": -35.71137619018555,
+      "loss": 1.1322778463363647,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6833858489990234,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09812775254249573,
+      "rewards/margins": 1.88960862159729,
+      "rewards/rejected": -1.7914810180664062,
+      "step": 432,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.87452663468821,
+      "grad_norm": 6.029493808746338,
+      "learning_rate": 0.00012866846484490147,
+      "logits/chosen": -1.05926513671875,
+      "logits/rejected": -1.0726540088653564,
+      "logps/chosen": -1.8859708309173584,
+      "logps/rejected": -38.563228607177734,
+      "loss": 0.3466527760028839,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2102343887090683,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.403474360704422,
+      "rewards/margins": 3.249852180480957,
+      "rewards/rejected": -2.8463780879974365,
+      "step": 433,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.8765463266851805,
+      "grad_norm": 9.18895149230957,
+      "learning_rate": 0.0001283481218926818,
+      "logits/chosen": -1.060423731803894,
+      "logits/rejected": -1.0363166332244873,
+      "logps/chosen": -4.131833553314209,
+      "logps/rejected": -75.67689514160156,
+      "loss": 0.7851709723472595,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5702386498451233,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1319551020860672,
+      "rewards/margins": 5.650537490844727,
+      "rewards/rejected": -5.518582820892334,
+      "step": 434,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.878566018682151,
+      "grad_norm": 6.691397666931152,
+      "learning_rate": 0.00012802746229889563,
+      "logits/chosen": -1.0681829452514648,
+      "logits/rejected": -1.0891364812850952,
+      "logps/chosen": -1.9405295848846436,
+      "logps/rejected": -50.287044525146484,
+      "loss": 0.4823431670665741,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2815513014793396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21010005474090576,
+      "rewards/margins": 3.8635170459747314,
+      "rewards/rejected": -3.653416633605957,
+      "step": 435,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 0.8805857106791214,
+      "grad_norm": 4.134374141693115,
+      "learning_rate": 0.00012770648964523194,
+      "logits/chosen": -1.0445590019226074,
+      "logits/rejected": -1.0800678730010986,
+      "logps/chosen": -4.192701816558838,
+      "logps/rejected": -41.08876037597656,
+      "loss": 0.764896035194397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6386605501174927,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05937701463699341,
+      "rewards/margins": 3.2779440879821777,
+      "rewards/rejected": -3.21856689453125,
+      "step": 436,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 0.8826054026760919,
+      "grad_norm": 5.5645976066589355,
+      "learning_rate": 0.0001273852075168765,
+      "logits/chosen": -1.052649736404419,
+      "logits/rejected": -1.0750157833099365,
+      "logps/chosen": -3.2202606201171875,
+      "logps/rejected": -39.1766471862793,
+      "loss": 0.7316163778305054,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.35654330253601074,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04998503625392914,
+      "rewards/margins": 2.966275453567505,
+      "rewards/rejected": -2.916290521621704,
+      "step": 437,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.8846250946730624,
+      "grad_norm": 3.430967330932617,
+      "learning_rate": 0.0001270636195024719,
+      "logits/chosen": -1.0656282901763916,
+      "logits/rejected": -1.0755031108856201,
+      "logps/chosen": -0.5838297605514526,
+      "logps/rejected": -53.800167083740234,
+      "loss": 0.23027637600898743,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11958169937133789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35133183002471924,
+      "rewards/margins": 4.607048034667969,
+      "rewards/rejected": -4.255716323852539,
+      "step": 438,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 0.8866447866700328,
+      "grad_norm": 4.035180568695068,
+      "learning_rate": 0.00012674172919407734,
+      "logits/chosen": -1.035705327987671,
+      "logits/rejected": -1.046338677406311,
+      "logps/chosen": -1.762102484703064,
+      "logps/rejected": -41.274600982666016,
+      "loss": 0.44759538769721985,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17419928312301636,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2473682463169098,
+      "rewards/margins": 3.196018695831299,
+      "rewards/rejected": -2.948650360107422,
+      "step": 439,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.8886644786670033,
+      "grad_norm": 5.988144397735596,
+      "learning_rate": 0.00012641954018712863,
+      "logits/chosen": -1.0401451587677002,
+      "logits/rejected": -1.0481927394866943,
+      "logps/chosen": -1.6200381517410278,
+      "logps/rejected": -40.854026794433594,
+      "loss": 0.4076470732688904,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28579434752464294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2611207365989685,
+      "rewards/margins": 3.813446044921875,
+      "rewards/rejected": -3.5523252487182617,
+      "step": 440,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8906841706639738,
+      "grad_norm": 4.463588237762451,
+      "learning_rate": 0.00012609705608039782,
+      "logits/chosen": -1.0440635681152344,
+      "logits/rejected": -1.038417935371399,
+      "logps/chosen": -1.1576311588287354,
+      "logps/rejected": -78.23765563964844,
+      "loss": 0.2654612958431244,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14011946320533752,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31270933151245117,
+      "rewards/margins": 5.999755859375,
+      "rewards/rejected": -5.687046051025391,
+      "step": 441,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8927038626609443,
+      "grad_norm": 3.0731234550476074,
+      "learning_rate": 0.00012577428047595344,
+      "logits/chosen": -1.016157865524292,
+      "logits/rejected": -1.0280359983444214,
+      "logps/chosen": -3.0516724586486816,
+      "logps/rejected": -60.31180191040039,
+      "loss": 0.4081045091152191,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23720964789390564,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2602730989456177,
+      "rewards/margins": 4.627702236175537,
+      "rewards/rejected": -4.367428779602051,
+      "step": 442,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8947235546579146,
+      "grad_norm": 2.895439624786377,
+      "learning_rate": 0.00012545121697911962,
+      "logits/chosen": -1.0191925764083862,
+      "logits/rejected": -1.0200212001800537,
+      "logps/chosen": -2.8940157890319824,
+      "logps/rejected": -57.44503402709961,
+      "loss": 0.36436915397644043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1371701955795288,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2842387855052948,
+      "rewards/margins": 4.40120792388916,
+      "rewards/rejected": -4.116969108581543,
+      "step": 443,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8967432466548851,
+      "grad_norm": 9.891191482543945,
+      "learning_rate": 0.00012512786919843648,
+      "logits/chosen": -1.0200165510177612,
+      "logits/rejected": -1.032355785369873,
+      "logps/chosen": -2.6891350746154785,
+      "logps/rejected": -47.72188949584961,
+      "loss": 0.7450097799301147,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5138753652572632,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2414613664150238,
+      "rewards/margins": 4.151015281677246,
+      "rewards/rejected": -3.9095540046691895,
+      "step": 444,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.8987629386518556,
+      "grad_norm": 3.1112067699432373,
+      "learning_rate": 0.00012480424074561933,
+      "logits/chosen": -1.025837779045105,
+      "logits/rejected": -1.0225645303726196,
+      "logps/chosen": -1.7361624240875244,
+      "logps/rejected": -58.649635314941406,
+      "loss": 0.2522731125354767,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09138274937868118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2967418134212494,
+      "rewards/margins": 4.663671970367432,
+      "rewards/rejected": -4.3669304847717285,
+      "step": 445,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.900782630648826,
+      "grad_norm": 14.950603485107422,
+      "learning_rate": 0.00012448033523551865,
+      "logits/chosen": -1.0079376697540283,
+      "logits/rejected": -1.0010699033737183,
+      "logps/chosen": -7.840879440307617,
+      "logps/rejected": -26.104421615600586,
+      "loss": 1.1924169063568115,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4921947121620178,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1946742981672287,
+      "rewards/margins": 1.5733067989349365,
+      "rewards/rejected": -1.7679810523986816,
+      "step": 446,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9028023226457965,
+      "grad_norm": 3.8511011600494385,
+      "learning_rate": 0.00012415615628607945,
+      "logits/chosen": -1.015897274017334,
+      "logits/rejected": -1.014409065246582,
+      "logps/chosen": -2.20813250541687,
+      "logps/rejected": -45.50457763671875,
+      "loss": 0.5370233058929443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34335654973983765,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12667731940746307,
+      "rewards/margins": 3.747370958328247,
+      "rewards/rejected": -3.6206939220428467,
+      "step": 447,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.904822014642767,
+      "grad_norm": 4.594604969024658,
+      "learning_rate": 0.0001238317075183011,
+      "logits/chosen": -1.0298758745193481,
+      "logits/rejected": -1.0120558738708496,
+      "logps/chosen": -0.9791274070739746,
+      "logps/rejected": -79.58103942871094,
+      "loss": 0.3356885612010956,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19914807379245758,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22014901041984558,
+      "rewards/margins": 5.962551116943359,
+      "rewards/rejected": -5.742402076721191,
+      "step": 448,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.9068417066397374,
+      "grad_norm": 3.488224983215332,
+      "learning_rate": 0.00012350699255619677,
+      "logits/chosen": -1.0189076662063599,
+      "logits/rejected": -1.0452524423599243,
+      "logps/chosen": -1.572611689567566,
+      "logps/rejected": -41.083641052246094,
+      "loss": 0.36099478602409363,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19940336048603058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20073141157627106,
+      "rewards/margins": 3.2094714641571045,
+      "rewards/rejected": -3.008740186691284,
+      "step": 449,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9088613986367079,
+      "grad_norm": 1.7710275650024414,
+      "learning_rate": 0.00012318201502675285,
+      "logits/chosen": -0.9990462064743042,
+      "logits/rejected": -1.00516939163208,
+      "logps/chosen": -1.445307731628418,
+      "logps/rejected": -54.3629035949707,
+      "loss": 0.2929742634296417,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14184969663619995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3061780631542206,
+      "rewards/margins": 4.353916645050049,
+      "rewards/rejected": -4.047738552093506,
+      "step": 450,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9108810906336784,
+      "grad_norm": 5.618566989898682,
+      "learning_rate": 0.00012285677855988864,
+      "logits/chosen": -1.0002800226211548,
+      "logits/rejected": -1.0010613203048706,
+      "logps/chosen": -3.4435813426971436,
+      "logps/rejected": -65.43953704833984,
+      "loss": 0.5797629952430725,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2722090482711792,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13838529586791992,
+      "rewards/margins": 4.899312973022461,
+      "rewards/rejected": -4.760928153991699,
+      "step": 451,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.9129007826306488,
+      "grad_norm": 22.944427490234375,
+      "learning_rate": 0.00012253128678841568,
+      "logits/chosen": -1.011128544807434,
+      "logits/rejected": -1.0143555402755737,
+      "logps/chosen": -15.104331970214844,
+      "logps/rejected": -42.862266540527344,
+      "loss": 1.11042058467865,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3811737298965454,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3371947109699249,
+      "rewards/margins": 2.6465537548065186,
+      "rewards/rejected": -2.983748435974121,
+      "step": 452,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.9149204746276193,
+      "grad_norm": 4.528737545013428,
+      "learning_rate": 0.0001222055433479972,
+      "logits/chosen": -1.0228255987167358,
+      "logits/rejected": -1.0586241483688354,
+      "logps/chosen": -1.2600595951080322,
+      "logps/rejected": -73.49124145507812,
+      "loss": 0.28426051139831543,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14906807243824005,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2577366530895233,
+      "rewards/margins": 5.7026872634887695,
+      "rewards/rejected": -5.444951057434082,
+      "step": 453,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.9169401666245898,
+      "grad_norm": 4.978267669677734,
+      "learning_rate": 0.0001218795518771075,
+      "logits/chosen": -1.0025545358657837,
+      "logits/rejected": -1.0025382041931152,
+      "logps/chosen": -0.9946606755256653,
+      "logps/rejected": -37.86664581298828,
+      "loss": 0.34124690294265747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09514570236206055,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.231367290019989,
+      "rewards/margins": 2.811678171157837,
+      "rewards/rejected": -2.580310821533203,
+      "step": 454,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.9189598586215603,
+      "grad_norm": 5.106620788574219,
+      "learning_rate": 0.00012155331601699136,
+      "logits/chosen": -1.0310418605804443,
+      "logits/rejected": -1.0054916143417358,
+      "logps/chosen": -1.0205014944076538,
+      "logps/rejected": -85.00090026855469,
+      "loss": 0.18072457611560822,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1114395335316658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41535112261772156,
+      "rewards/margins": 6.101562023162842,
+      "rewards/rejected": -5.686211109161377,
+      "step": 455,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.9209795506185307,
+      "grad_norm": 1.438368797302246,
+      "learning_rate": 0.0001212268394116233,
+      "logits/chosen": -1.0807647705078125,
+      "logits/rejected": -1.057398796081543,
+      "logps/chosen": -0.6302530765533447,
+      "logps/rejected": -55.822792053222656,
+      "loss": 0.25450506806373596,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09770296514034271,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2143504023551941,
+      "rewards/margins": 4.336926460266113,
+      "rewards/rejected": -4.1225762367248535,
+      "step": 456,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.9229992426155011,
+      "grad_norm": 2.4940450191497803,
+      "learning_rate": 0.00012090012570766685,
+      "logits/chosen": -1.047285556793213,
+      "logits/rejected": -1.0609232187271118,
+      "logps/chosen": -3.7559874057769775,
+      "logps/rejected": -58.776737213134766,
+      "loss": 0.2771581709384918,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1527220904827118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3420707583427429,
+      "rewards/margins": 5.054807662963867,
+      "rewards/rejected": -4.712737083435059,
+      "step": 457,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.9250189346124716,
+      "grad_norm": 16.446552276611328,
+      "learning_rate": 0.00012057317855443395,
+      "logits/chosen": -1.1021456718444824,
+      "logits/rejected": -1.1048663854599,
+      "logps/chosen": -5.987412452697754,
+      "logps/rejected": -61.24203872680664,
+      "loss": 0.6597545742988586,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40991318225860596,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.06934911757707596,
+      "rewards/margins": 4.689109802246094,
+      "rewards/rejected": -4.758459091186523,
+      "step": 458,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.927038626609442,
+      "grad_norm": 2.537680149078369,
+      "learning_rate": 0.00012024600160384416,
+      "logits/chosen": -1.1070061922073364,
+      "logits/rejected": -1.1183254718780518,
+      "logps/chosen": -2.1882822513580322,
+      "logps/rejected": -94.47828674316406,
+      "loss": 0.47271794080734253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3363499641418457,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23398056626319885,
+      "rewards/margins": 7.600938320159912,
+      "rewards/rejected": -7.366957187652588,
+      "step": 459,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.9290583186064125,
+      "grad_norm": 3.8238890171051025,
+      "learning_rate": 0.0001199185985103836,
+      "logits/chosen": -1.079871416091919,
+      "logits/rejected": -1.1050368547439575,
+      "logps/chosen": -0.6882074475288391,
+      "logps/rejected": -112.36473846435547,
+      "loss": 0.2024538367986679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1239171102643013,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41684651374816895,
+      "rewards/margins": 8.971007347106934,
+      "rewards/rejected": -8.554160118103027,
+      "step": 460,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.931078010603383,
+      "grad_norm": 7.065299987792969,
+      "learning_rate": 0.00011959097293106465,
+      "logits/chosen": -1.1408017873764038,
+      "logits/rejected": -1.1611324548721313,
+      "logps/chosen": -0.8949813842773438,
+      "logps/rejected": -44.59485626220703,
+      "loss": 0.26141998171806335,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1390543431043625,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3610275685787201,
+      "rewards/margins": 3.8335914611816406,
+      "rewards/rejected": -3.4725637435913086,
+      "step": 461,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.9330977026003534,
+      "grad_norm": 4.476932048797607,
+      "learning_rate": 0.00011926312852538455,
+      "logits/chosen": -1.128575086593628,
+      "logits/rejected": -1.157038688659668,
+      "logps/chosen": -3.6390225887298584,
+      "logps/rejected": -63.7474365234375,
+      "loss": 0.5282421708106995,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2585342824459076,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11798002570867538,
+      "rewards/margins": 5.297179222106934,
+      "rewards/rejected": -5.179199695587158,
+      "step": 462,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.9351173945973239,
+      "grad_norm": 11.82602310180664,
+      "learning_rate": 0.0001189350689552849,
+      "logits/chosen": -1.1367065906524658,
+      "logits/rejected": -1.156144618988037,
+      "logps/chosen": -1.9807525873184204,
+      "logps/rejected": -49.89271926879883,
+      "loss": 0.6038030385971069,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34540605545043945,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.37543025612831116,
+      "rewards/margins": 4.289130687713623,
+      "rewards/rejected": -3.9137003421783447,
+      "step": 463,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.9371370865942944,
+      "grad_norm": 12.497689247131348,
+      "learning_rate": 0.00011860679788511064,
+      "logits/chosen": -1.1170810461044312,
+      "logits/rejected": -1.1292904615402222,
+      "logps/chosen": -4.893659591674805,
+      "logps/rejected": -60.843013763427734,
+      "loss": 0.6962206363677979,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4090201258659363,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10479210317134857,
+      "rewards/margins": 4.898015022277832,
+      "rewards/rejected": -4.793222427368164,
+      "step": 464,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9391567785912648,
+      "grad_norm": 2.3905272483825684,
+      "learning_rate": 0.00011827831898156905,
+      "logits/chosen": -1.119379997253418,
+      "logits/rejected": -1.0900958776474,
+      "logps/chosen": -1.0332272052764893,
+      "logps/rejected": -82.55847930908203,
+      "loss": 0.21928882598876953,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14558973908424377,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21076346933841705,
+      "rewards/margins": 6.53924560546875,
+      "rewards/rejected": -6.328482151031494,
+      "step": 465,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9411764705882353,
+      "grad_norm": 2.804384708404541,
+      "learning_rate": 0.00011794963591368893,
+      "logits/chosen": -1.1550871133804321,
+      "logits/rejected": -1.1876434087753296,
+      "logps/chosen": -1.962457299232483,
+      "logps/rejected": -91.43489837646484,
+      "loss": 0.26453521847724915,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18894466757774353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2565460503101349,
+      "rewards/margins": 7.109828472137451,
+      "rewards/rejected": -6.853282451629639,
+      "step": 466,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9431961625852058,
+      "grad_norm": 6.29252290725708,
+      "learning_rate": 0.00011762075235277943,
+      "logits/chosen": -1.1255277395248413,
+      "logits/rejected": -1.148550271987915,
+      "logps/chosen": -7.0823774337768555,
+      "logps/rejected": -84.76576232910156,
+      "loss": 0.7452417016029358,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6746703386306763,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21994975209236145,
+      "rewards/margins": 6.5102715492248535,
+      "rewards/rejected": -6.730221271514893,
+      "step": 467,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9452158545821763,
+      "grad_norm": 3.2159340381622314,
+      "learning_rate": 0.00011729167197238935,
+      "logits/chosen": -1.0966346263885498,
+      "logits/rejected": -1.103726863861084,
+      "logps/chosen": -4.399909496307373,
+      "logps/rejected": -42.98252487182617,
+      "loss": 0.7106723785400391,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3980051279067993,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04321293905377388,
+      "rewards/margins": 3.2986812591552734,
+      "rewards/rejected": -3.2554681301116943,
+      "step": 468,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 0.9472355465791467,
+      "grad_norm": 3.3617236614227295,
+      "learning_rate": 0.00011696239844826571,
+      "logits/chosen": -1.1088882684707642,
+      "logits/rejected": -1.1284962892532349,
+      "logps/chosen": -1.639062762260437,
+      "logps/rejected": -72.10785675048828,
+      "loss": 0.24586933851242065,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12198629230260849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3906645178794861,
+      "rewards/margins": 6.221713066101074,
+      "rewards/rejected": -5.831048011779785,
+      "step": 469,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 0.9492552385761172,
+      "grad_norm": 2.8374667167663574,
+      "learning_rate": 0.00011663293545831302,
+      "logits/chosen": -1.1557269096374512,
+      "logits/rejected": -1.175348162651062,
+      "logps/chosen": -2.1615452766418457,
+      "logps/rejected": -36.723304748535156,
+      "loss": 0.38634181022644043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22775641083717346,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3754770755767822,
+      "rewards/margins": 3.0530035495758057,
+      "rewards/rejected": -2.6775267124176025,
+      "step": 470,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.9512749305730877,
+      "grad_norm": 11.482617378234863,
+      "learning_rate": 0.00011630328668255206,
+      "logits/chosen": -1.1401901245117188,
+      "logits/rejected": -1.1286134719848633,
+      "logps/chosen": -6.495123386383057,
+      "logps/rejected": -35.29318618774414,
+      "loss": 0.8619951009750366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5049220323562622,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13930773735046387,
+      "rewards/margins": 2.6000585556030273,
+      "rewards/rejected": -2.4607508182525635,
+      "step": 471,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.953294622570058,
+      "grad_norm": 3.4752252101898193,
+      "learning_rate": 0.00011597345580307875,
+      "logits/chosen": -1.162688970565796,
+      "logits/rejected": -1.192884922027588,
+      "logps/chosen": -2.404099225997925,
+      "logps/rejected": -56.021663665771484,
+      "loss": 0.3634865880012512,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23837605118751526,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12834902107715607,
+      "rewards/margins": 4.692193508148193,
+      "rewards/rejected": -4.563844203948975,
+      "step": 472,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 0.9553143145670285,
+      "grad_norm": 2.5039899349212646,
+      "learning_rate": 0.0001156434465040231,
+      "logits/chosen": -1.1668778657913208,
+      "logits/rejected": -1.1930913925170898,
+      "logps/chosen": -3.766876697540283,
+      "logps/rejected": -63.9132194519043,
+      "loss": 0.3028702735900879,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20875141024589539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3659513592720032,
+      "rewards/margins": 5.140607833862305,
+      "rewards/rejected": -4.774656295776367,
+      "step": 473,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.957334006563999,
+      "grad_norm": 2.068408250808716,
+      "learning_rate": 0.00011531326247150803,
+      "logits/chosen": -1.1649010181427002,
+      "logits/rejected": -1.1960549354553223,
+      "logps/chosen": -1.7281254529953003,
+      "logps/rejected": -72.08183288574219,
+      "loss": 0.3556201756000519,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1918855905532837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12103867530822754,
+      "rewards/margins": 5.5760884284973145,
+      "rewards/rejected": -5.45504903793335,
+      "step": 474,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.9593536985609694,
+      "grad_norm": 2.6573944091796875,
+      "learning_rate": 0.00011498290739360815,
+      "logits/chosen": -1.1912912130355835,
+      "logits/rejected": -1.1985927820205688,
+      "logps/chosen": -1.0083485841751099,
+      "logps/rejected": -56.71221923828125,
+      "loss": 0.2199336588382721,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09299471974372864,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2724168002605438,
+      "rewards/margins": 4.650609493255615,
+      "rewards/rejected": -4.37819242477417,
+      "step": 475,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.9613733905579399,
+      "grad_norm": 5.599519729614258,
+      "learning_rate": 0.00011465238496030868,
+      "logits/chosen": -1.186398983001709,
+      "logits/rejected": -1.2222926616668701,
+      "logps/chosen": -2.25126576423645,
+      "logps/rejected": -65.31526184082031,
+      "loss": 0.6140868067741394,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.350625216960907,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11485815793275833,
+      "rewards/margins": 5.405522346496582,
+      "rewards/rejected": -5.2906646728515625,
+      "step": 476,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.9633930825549104,
+      "grad_norm": 1.3937158584594727,
+      "learning_rate": 0.00011432169886346404,
+      "logits/chosen": -1.1549639701843262,
+      "logits/rejected": -1.1829733848571777,
+      "logps/chosen": -2.5441954135894775,
+      "logps/rejected": -70.29151916503906,
+      "loss": 0.24065297842025757,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1281718909740448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2520023286342621,
+      "rewards/margins": 5.15536642074585,
+      "rewards/rejected": -4.903364181518555,
+      "step": 477,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.9654127745518808,
+      "grad_norm": 3.0468287467956543,
+      "learning_rate": 0.00011399085279675687,
+      "logits/chosen": -1.1789178848266602,
+      "logits/rejected": -1.2163946628570557,
+      "logps/chosen": -4.268495082855225,
+      "logps/rejected": -53.77469253540039,
+      "loss": 0.40022939443588257,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2632007300853729,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3904189169406891,
+      "rewards/margins": 4.054966926574707,
+      "rewards/rejected": -3.66454815864563,
+      "step": 478,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.9674324665488513,
+      "grad_norm": 6.615331172943115,
+      "learning_rate": 0.00011365985045565648,
+      "logits/chosen": -1.1960713863372803,
+      "logits/rejected": -1.2022978067398071,
+      "logps/chosen": -4.764765739440918,
+      "logps/rejected": -46.24211120605469,
+      "loss": 0.8375652432441711,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.47868138551712036,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2317066639661789,
+      "rewards/margins": 3.9557816982269287,
+      "rewards/rejected": -3.7240753173828125,
+      "step": 479,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9694521585458218,
+      "grad_norm": 3.0518853664398193,
+      "learning_rate": 0.0001133286955373779,
+      "logits/chosen": -1.1568349599838257,
+      "logits/rejected": -1.1736950874328613,
+      "logps/chosen": -1.8058488368988037,
+      "logps/rejected": -63.62800979614258,
+      "loss": 0.3420032560825348,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1940608024597168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4025638699531555,
+      "rewards/margins": 5.4827446937561035,
+      "rewards/rejected": -5.080180644989014,
+      "step": 480,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9714718505427922,
+      "grad_norm": 6.214394569396973,
+      "learning_rate": 0.00011299739174084025,
+      "logits/chosen": -1.1839512586593628,
+      "logits/rejected": -1.2068456411361694,
+      "logps/chosen": -2.6526894569396973,
+      "logps/rejected": -55.63904571533203,
+      "loss": 0.44367533922195435,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2684517502784729,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1775396466255188,
+      "rewards/margins": 4.404716968536377,
+      "rewards/rejected": -4.227177619934082,
+      "step": 481,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.9734915425397627,
+      "grad_norm": 5.678071975708008,
+      "learning_rate": 0.0001126659427666257,
+      "logits/chosen": -1.1944388151168823,
+      "logits/rejected": -1.2148536443710327,
+      "logps/chosen": -2.0405924320220947,
+      "logps/rejected": -61.48400115966797,
+      "loss": 0.40944910049438477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23419791460037231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34613850712776184,
+      "rewards/margins": 5.159732818603516,
+      "rewards/rejected": -4.813594341278076,
+      "step": 482,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9755112345367332,
+      "grad_norm": 26.189855575561523,
+      "learning_rate": 0.00011233435231693794,
+      "logits/chosen": -1.1763508319854736,
+      "logits/rejected": -1.195464015007019,
+      "logps/chosen": -9.439873695373535,
+      "logps/rejected": -40.02541732788086,
+      "loss": 1.6160178184509277,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8637927174568176,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2953401505947113,
+      "rewards/margins": 2.8487725257873535,
+      "rewards/rejected": -3.1441128253936768,
+      "step": 483,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9775309265337037,
+      "grad_norm": 3.0877511501312256,
+      "learning_rate": 0.00011200262409556097,
+      "logits/chosen": -1.1713593006134033,
+      "logits/rejected": -1.2405827045440674,
+      "logps/chosen": -1.0278502702713013,
+      "logps/rejected": -76.41952514648438,
+      "loss": 0.28457099199295044,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20625659823417664,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32308924198150635,
+      "rewards/margins": 6.389144420623779,
+      "rewards/rejected": -6.066054821014404,
+      "step": 484,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.979550618530674,
+      "grad_norm": 9.65100383758545,
+      "learning_rate": 0.00011167076180781764,
+      "logits/chosen": -1.146728277206421,
+      "logits/rejected": -1.2022031545639038,
+      "logps/chosen": -5.121095657348633,
+      "logps/rejected": -74.87187957763672,
+      "loss": 0.5169023871421814,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4096004366874695,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2158682644367218,
+      "rewards/margins": 5.7417778968811035,
+      "rewards/rejected": -5.525909900665283,
+      "step": 485,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9815703105276445,
+      "grad_norm": 3.232163190841675,
+      "learning_rate": 0.00011133876916052821,
+      "logits/chosen": -1.1618849039077759,
+      "logits/rejected": -1.19745934009552,
+      "logps/chosen": -4.945662498474121,
+      "logps/rejected": -65.05085754394531,
+      "loss": 0.3440502882003784,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2677401304244995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08972585201263428,
+      "rewards/margins": 5.094820976257324,
+      "rewards/rejected": -5.0050950050354,
+      "step": 486,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.983590002524615,
+      "grad_norm": 1.1338063478469849,
+      "learning_rate": 0.0001110066498619692,
+      "logits/chosen": -1.1489181518554688,
+      "logits/rejected": -1.1883840560913086,
+      "logps/chosen": -0.9724054336547852,
+      "logps/rejected": -69.8716812133789,
+      "loss": 0.16405296325683594,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1183013990521431,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37503695487976074,
+      "rewards/margins": 6.176860809326172,
+      "rewards/rejected": -5.801823616027832,
+      "step": 487,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9856096945215854,
+      "grad_norm": 4.550635814666748,
+      "learning_rate": 0.00011067440762183164,
+      "logits/chosen": -1.127516508102417,
+      "logits/rejected": -1.1615864038467407,
+      "logps/chosen": -1.5935579538345337,
+      "logps/rejected": -70.34913635253906,
+      "loss": 0.32236579060554504,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21638807654380798,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30907154083251953,
+      "rewards/margins": 4.958349704742432,
+      "rewards/rejected": -4.649278163909912,
+      "step": 488,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9876293865185559,
+      "grad_norm": 4.5942206382751465,
+      "learning_rate": 0.00011034204615117982,
+      "logits/chosen": -1.1207928657531738,
+      "logits/rejected": -1.1431444883346558,
+      "logps/chosen": -2.02649188041687,
+      "logps/rejected": -69.4359359741211,
+      "loss": 0.45521846413612366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2501518428325653,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21566975116729736,
+      "rewards/margins": 4.797036647796631,
+      "rewards/rejected": -4.581367492675781,
+      "step": 489,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9896490785155264,
+      "grad_norm": 3.483578681945801,
+      "learning_rate": 0.00011000956916240985,
+      "logits/chosen": -1.1226303577423096,
+      "logits/rejected": -1.1501247882843018,
+      "logps/chosen": -5.958000659942627,
+      "logps/rejected": -32.57322311401367,
+      "loss": 0.6992753148078918,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5034775137901306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.47746312618255615,
+      "rewards/margins": 2.7217843532562256,
+      "rewards/rejected": -2.24432110786438,
+      "step": 490,
+      "train_speed(iter/s)": 0.021184
+    },
+    {
+      "epoch": 0.9916687705124968,
+      "grad_norm": 4.195751667022705,
+      "learning_rate": 0.00010967698036920812,
+      "logits/chosen": -1.10448157787323,
+      "logits/rejected": -1.1570674180984497,
+      "logps/chosen": -4.190779685974121,
+      "logps/rejected": -60.6494255065918,
+      "loss": 0.5290452837944031,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3574896454811096,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.35892462730407715,
+      "rewards/margins": 4.584457874298096,
+      "rewards/rejected": -4.225533485412598,
+      "step": 491,
+      "train_speed(iter/s)": 0.021184
+    },
+    {
+      "epoch": 0.9936884625094673,
+      "grad_norm": 1.556215524673462,
+      "learning_rate": 0.00010934428348650986,
+      "logits/chosen": -1.1513900756835938,
+      "logits/rejected": -1.1773016452789307,
+      "logps/chosen": -0.4516194462776184,
+      "logps/rejected": -54.02649688720703,
+      "loss": 0.15559712052345276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06051451712846756,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3142217993736267,
+      "rewards/margins": 4.602743148803711,
+      "rewards/rejected": -4.2885212898254395,
+      "step": 492,
+      "train_speed(iter/s)": 0.021185
+    },
+    {
+      "epoch": 0.9957081545064378,
+      "grad_norm": 1.837632656097412,
+      "learning_rate": 0.00010901148223045761,
+      "logits/chosen": -1.122188925743103,
+      "logits/rejected": -1.1630114316940308,
+      "logps/chosen": -1.2533413171768188,
+      "logps/rejected": -65.71454620361328,
+      "loss": 0.23370426893234253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1294473558664322,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34939849376678467,
+      "rewards/margins": 4.393534183502197,
+      "rewards/rejected": -4.044135093688965,
+      "step": 493,
+      "train_speed(iter/s)": 0.021185
+    },
+    {
+      "epoch": 0.9977278465034082,
+      "grad_norm": 5.143455982208252,
+      "learning_rate": 0.00010867858031835975,
+      "logits/chosen": -1.160941481590271,
+      "logits/rejected": -1.1693061590194702,
+      "logps/chosen": -2.2333078384399414,
+      "logps/rejected": -62.20633316040039,
+      "loss": 0.33971500396728516,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23756271600723267,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17624695599079132,
+      "rewards/margins": 5.301555633544922,
+      "rewards/rejected": -5.125308513641357,
+      "step": 494,
+      "train_speed(iter/s)": 0.021186
+    },
+    {
+      "epoch": 0.9997475385003787,
+      "grad_norm": 1.6146841049194336,
+      "learning_rate": 0.000108345581468649,
+      "logits/chosen": -1.156958818435669,
+      "logits/rejected": -1.2084702253341675,
+      "logps/chosen": -1.2683087587356567,
+      "logps/rejected": -73.02035522460938,
+      "loss": 0.2508525550365448,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12798351049423218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23698541522026062,
+      "rewards/margins": 5.670597553253174,
+      "rewards/rejected": -5.433611869812012,
+      "step": 495,
+      "train_speed(iter/s)": 0.021186
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.6146841049194336,
+      "learning_rate": 0.00010801248940084074,
+      "logits/chosen": -1.271807074546814,
+      "logits/rejected": -1.3348701000213623,
+      "logps/chosen": -4.720621291198768e-05,
+      "logps/rejected": -91.26646423339844,
+      "loss": 1.2297836292418651e-05,
+      "memory(GiB)": 45.64,
+      "nll_loss": 7.867701242503244e-06,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5421666502952576,
+      "rewards/margins": 9.30994987487793,
+      "rewards/rejected": -8.767783164978027,
+      "step": 496,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0020196919969704,
+      "grad_norm": 2.052020311355591,
+      "learning_rate": 0.00010767930783549171,
+      "logits/chosen": -1.1468470096588135,
+      "logits/rejected": -1.1875897645950317,
+      "logps/chosen": -1.5805768966674805,
+      "logps/rejected": -60.07487869262695,
+      "loss": 0.35193732380867004,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22589635848999023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43169844150543213,
+      "rewards/margins": 4.954350471496582,
+      "rewards/rejected": -4.522652626037598,
+      "step": 497,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.004039383993941,
+      "grad_norm": 3.080597400665283,
+      "learning_rate": 0.00010734604049415822,
+      "logits/chosen": -1.1931610107421875,
+      "logits/rejected": -1.2040530443191528,
+      "logps/chosen": -0.3329848051071167,
+      "logps/rejected": -89.8782730102539,
+      "loss": 0.08142131567001343,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.028553346171975136,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3950589895248413,
+      "rewards/margins": 7.771610260009766,
+      "rewards/rejected": -7.376551628112793,
+      "step": 498,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0060590759909114,
+      "grad_norm": 1.200395941734314,
+      "learning_rate": 0.00010701269109935474,
+      "logits/chosen": -1.1844416856765747,
+      "logits/rejected": -1.216762900352478,
+      "logps/chosen": -2.2689549922943115,
+      "logps/rejected": -74.18507385253906,
+      "loss": 0.18864388763904572,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13327570259571075,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5378884673118591,
+      "rewards/margins": 6.781070709228516,
+      "rewards/rejected": -6.2431817054748535,
+      "step": 499,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0080787679878818,
+      "grad_norm": 1.3004249334335327,
+      "learning_rate": 0.00010667926337451217,
+      "logits/chosen": -1.2056668996810913,
+      "logits/rejected": -1.2564384937286377,
+      "logps/chosen": -2.1434969902038574,
+      "logps/rejected": -107.59019470214844,
+      "loss": 0.14688310027122498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1056690588593483,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18269282579421997,
+      "rewards/margins": 9.098722457885742,
+      "rewards/rejected": -8.916029930114746,
+      "step": 500,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0100984599848524,
+      "grad_norm": 10.15276050567627,
+      "learning_rate": 0.00010634576104393643,
+      "logits/chosen": -1.2333784103393555,
+      "logits/rejected": -1.2786855697631836,
+      "logps/chosen": -1.4780651330947876,
+      "logps/rejected": -113.30062866210938,
+      "loss": 0.33663061261177063,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18906863033771515,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19989997148513794,
+      "rewards/margins": 9.928545951843262,
+      "rewards/rejected": -9.728645324707031,
+      "step": 501,
+      "train_speed(iter/s)": 0.021209
+    },
+    {
+      "epoch": 1.0121181519818228,
+      "grad_norm": 1.6605217456817627,
+      "learning_rate": 0.00010601218783276672,
+      "logits/chosen": -1.224414348602295,
+      "logits/rejected": -1.286801815032959,
+      "logps/chosen": -0.5362107753753662,
+      "logps/rejected": -101.3456802368164,
+      "loss": 0.1514112651348114,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07159439474344254,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21423012018203735,
+      "rewards/margins": 8.556539535522461,
+      "rewards/rejected": -8.342309951782227,
+      "step": 502,
+      "train_speed(iter/s)": 0.021209
+    },
+    {
+      "epoch": 1.0141378439787931,
+      "grad_norm": 12.038124084472656,
+      "learning_rate": 0.00010567854746693396,
+      "logits/chosen": -1.2550753355026245,
+      "logits/rejected": -1.3067824840545654,
+      "logps/chosen": -4.968113899230957,
+      "logps/rejected": -76.51419067382812,
+      "loss": 0.6970563530921936,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5403850674629211,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.015964236110448837,
+      "rewards/margins": 6.770257472991943,
+      "rewards/rejected": -6.7542924880981445,
+      "step": 503,
+      "train_speed(iter/s)": 0.02121
+    },
+    {
+      "epoch": 1.0161575359757637,
+      "grad_norm": 5.976532936096191,
+      "learning_rate": 0.00010534484367311923,
+      "logits/chosen": -1.2590959072113037,
+      "logits/rejected": -1.3068499565124512,
+      "logps/chosen": -2.158994436264038,
+      "logps/rejected": -74.8353271484375,
+      "loss": 0.34383249282836914,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22420844435691833,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4135916233062744,
+      "rewards/margins": 6.9369611740112305,
+      "rewards/rejected": -6.523369789123535,
+      "step": 504,
+      "train_speed(iter/s)": 0.02121
+    },
+    {
+      "epoch": 1.0181772279727341,
+      "grad_norm": 8.177599906921387,
+      "learning_rate": 0.00010501108017871192,
+      "logits/chosen": -1.2204450368881226,
+      "logits/rejected": -1.2789673805236816,
+      "logps/chosen": -1.5879818201065063,
+      "logps/rejected": -66.489990234375,
+      "loss": 0.35112184286117554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18649819493293762,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41614973545074463,
+      "rewards/margins": 5.6356306076049805,
+      "rewards/rejected": -5.219481468200684,
+      "step": 505,
+      "train_speed(iter/s)": 0.021211
+    },
+    {
+      "epoch": 1.0201969199697045,
+      "grad_norm": 1.072225570678711,
+      "learning_rate": 0.00010467726071176853,
+      "logits/chosen": -1.2156474590301514,
+      "logits/rejected": -1.2541887760162354,
+      "logps/chosen": -7.711733341217041,
+      "logps/rejected": -65.3000717163086,
+      "loss": 0.1724158525466919,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12215537577867508,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6666063070297241,
+      "rewards/margins": 6.159120559692383,
+      "rewards/rejected": -5.492514133453369,
+      "step": 506,
+      "train_speed(iter/s)": 0.021211
+    },
+    {
+      "epoch": 1.0222166119666751,
+      "grad_norm": 2.4010398387908936,
+      "learning_rate": 0.00010434338900097049,
+      "logits/chosen": -1.2528554201126099,
+      "logits/rejected": -1.3154045343399048,
+      "logps/chosen": -0.8416248559951782,
+      "logps/rejected": -69.87122344970703,
+      "loss": 0.12864626944065094,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07988111674785614,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31406697630882263,
+      "rewards/margins": 6.217660903930664,
+      "rewards/rejected": -5.903594493865967,
+      "step": 507,
+      "train_speed(iter/s)": 0.021212
+    },
+    {
+      "epoch": 1.0242363039636455,
+      "grad_norm": 1.6103118658065796,
+      "learning_rate": 0.00010400946877558293,
+      "logits/chosen": -1.171745777130127,
+      "logits/rejected": -1.205277919769287,
+      "logps/chosen": -1.2725749015808105,
+      "logps/rejected": -75.42990112304688,
+      "loss": 0.17115575075149536,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11690245568752289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42929667234420776,
+      "rewards/margins": 5.8164963722229,
+      "rewards/rejected": -5.387199878692627,
+      "step": 508,
+      "train_speed(iter/s)": 0.021212
+    },
+    {
+      "epoch": 1.0262559959606161,
+      "grad_norm": 7.029423713684082,
+      "learning_rate": 0.00010367550376541283,
+      "logits/chosen": -1.205200433731079,
+      "logits/rejected": -1.2511245012283325,
+      "logps/chosen": -2.5946669578552246,
+      "logps/rejected": -84.32099914550781,
+      "loss": 0.30733728408813477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2792370319366455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25035470724105835,
+      "rewards/margins": 6.80730676651001,
+      "rewards/rejected": -6.556951999664307,
+      "step": 509,
+      "train_speed(iter/s)": 0.021213
+    },
+    {
+      "epoch": 1.0282756879575865,
+      "grad_norm": 0.7719494700431824,
+      "learning_rate": 0.00010334149770076747,
+      "logits/chosen": -1.2033907175064087,
+      "logits/rejected": -1.2671985626220703,
+      "logps/chosen": -0.1477760523557663,
+      "logps/rejected": -67.04902648925781,
+      "loss": 0.06301882117986679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.029819779098033905,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3393843173980713,
+      "rewards/margins": 5.922677516937256,
+      "rewards/rejected": -5.5832929611206055,
+      "step": 510,
+      "train_speed(iter/s)": 0.021213
+    },
+    {
+      "epoch": 1.030295379954557,
+      "grad_norm": 6.209945201873779,
+      "learning_rate": 0.00010300745431241259,
+      "logits/chosen": -1.1784838438034058,
+      "logits/rejected": -1.2575358152389526,
+      "logps/chosen": -2.9259207248687744,
+      "logps/rejected": -66.25347137451172,
+      "loss": 0.493066668510437,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34985142946243286,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2988048195838928,
+      "rewards/margins": 5.669159412384033,
+      "rewards/rejected": -5.370354652404785,
+      "step": 511,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0323150719515275,
+      "grad_norm": 14.864357948303223,
+      "learning_rate": 0.00010267337733153089,
+      "logits/chosen": -1.211972951889038,
+      "logits/rejected": -1.2457599639892578,
+      "logps/chosen": -4.108205795288086,
+      "logps/rejected": -49.29187774658203,
+      "loss": 0.7474625110626221,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5005552768707275,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1060146689414978,
+      "rewards/margins": 3.993147134780884,
+      "rewards/rejected": -3.8871326446533203,
+      "step": 512,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0343347639484979,
+      "grad_norm": 4.5455522537231445,
+      "learning_rate": 0.0001023392704896803,
+      "logits/chosen": -1.1689393520355225,
+      "logits/rejected": -1.1586272716522217,
+      "logps/chosen": -2.678110361099243,
+      "logps/rejected": -66.11392974853516,
+      "loss": 0.3088136315345764,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2110619693994522,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2811347544193268,
+      "rewards/margins": 5.262904167175293,
+      "rewards/rejected": -4.981769561767578,
+      "step": 513,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0363544559454683,
+      "grad_norm": 6.412376880645752,
+      "learning_rate": 0.00010200513751875227,
+      "logits/chosen": -1.1625449657440186,
+      "logits/rejected": -1.1948907375335693,
+      "logps/chosen": -2.3596391677856445,
+      "logps/rejected": -68.15946197509766,
+      "loss": 0.39961010217666626,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26144012808799744,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3064810037612915,
+      "rewards/margins": 5.5451202392578125,
+      "rewards/rejected": -5.2386393547058105,
+      "step": 514,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0383741479424389,
+      "grad_norm": 2.85520601272583,
+      "learning_rate": 0.00010167098215093009,
+      "logits/chosen": -1.1633456945419312,
+      "logits/rejected": -1.2228730916976929,
+      "logps/chosen": -3.7188360691070557,
+      "logps/rejected": -95.9930419921875,
+      "loss": 0.2916540801525116,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23056216537952423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3937748074531555,
+      "rewards/margins": 7.421454429626465,
+      "rewards/rejected": -7.027679443359375,
+      "step": 515,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0403938399394093,
+      "grad_norm": 5.375782012939453,
+      "learning_rate": 0.00010133680811864727,
+      "logits/chosen": -1.1782926321029663,
+      "logits/rejected": -1.2417353391647339,
+      "logps/chosen": -1.3107043504714966,
+      "logps/rejected": -53.36506652832031,
+      "loss": 0.19477064907550812,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14968222379684448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25208836793899536,
+      "rewards/margins": 4.3324995040893555,
+      "rewards/rejected": -4.080410957336426,
+      "step": 516,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0424135319363796,
+      "grad_norm": 1.9422154426574707,
+      "learning_rate": 0.00010100261915454571,
+      "logits/chosen": -1.1514760255813599,
+      "logits/rejected": -1.2427828311920166,
+      "logps/chosen": -0.6266792416572571,
+      "logps/rejected": -79.95481872558594,
+      "loss": 0.09264641255140305,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06077795848250389,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45081964135169983,
+      "rewards/margins": 6.714786529541016,
+      "rewards/rejected": -6.263967037200928,
+      "step": 517,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0444332239333503,
+      "grad_norm": 20.16422462463379,
+      "learning_rate": 0.00010066841899143425,
+      "logits/chosen": -1.1963735818862915,
+      "logits/rejected": -1.2633908987045288,
+      "logps/chosen": -6.527058124542236,
+      "logps/rejected": -39.10000991821289,
+      "loss": 0.7584791779518127,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5177947878837585,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3015112280845642,
+      "rewards/margins": 3.383577585220337,
+      "rewards/rejected": -3.082066535949707,
+      "step": 518,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0464529159303206,
+      "grad_norm": 7.269643783569336,
+      "learning_rate": 0.00010033421136224659,
+      "logits/chosen": -1.1538660526275635,
+      "logits/rejected": -1.1920374631881714,
+      "logps/chosen": -1.6170885562896729,
+      "logps/rejected": -57.34727096557617,
+      "loss": 0.1741914451122284,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11453510820865631,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2959902584552765,
+      "rewards/margins": 4.944981575012207,
+      "rewards/rejected": -4.648991584777832,
+      "step": 519,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.048472607927291,
+      "grad_norm": 2.5758349895477295,
+      "learning_rate": 0.0001,
+      "logits/chosen": -1.1641168594360352,
+      "logits/rejected": -1.2306013107299805,
+      "logps/chosen": -2.8853821754455566,
+      "logps/rejected": -45.58841323852539,
+      "loss": 0.35295766592025757,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2536754012107849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34334903955459595,
+      "rewards/margins": 3.8398725986480713,
+      "rewards/rejected": -3.49652361869812,
+      "step": 520,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.0504922999242616,
+      "grad_norm": 3.192835569381714,
+      "learning_rate": 9.966578863775344e-05,
+      "logits/chosen": -1.1690419912338257,
+      "logits/rejected": -1.2847118377685547,
+      "logps/chosen": -1.6086413860321045,
+      "logps/rejected": -75.8061294555664,
+      "loss": 0.2794322371482849,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20665356516838074,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3336547017097473,
+      "rewards/margins": 6.422711372375488,
+      "rewards/rejected": -6.089056491851807,
+      "step": 521,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.052511991921232,
+      "grad_norm": 5.373813629150391,
+      "learning_rate": 9.93315810085658e-05,
+      "logits/chosen": -1.099102258682251,
+      "logits/rejected": -1.2499687671661377,
+      "logps/chosen": -0.8937387466430664,
+      "logps/rejected": -83.5426254272461,
+      "loss": 0.22286824882030487,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1412465125322342,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34925389289855957,
+      "rewards/margins": 6.224581718444824,
+      "rewards/rejected": -5.8753275871276855,
+      "step": 522,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.0545316839182024,
+      "grad_norm": 1.588864803314209,
+      "learning_rate": 9.89973808454543e-05,
+      "logits/chosen": -1.2088499069213867,
+      "logits/rejected": -1.2908425331115723,
+      "logps/chosen": -1.425804615020752,
+      "logps/rejected": -84.73382568359375,
+      "loss": 0.24856942892074585,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17529940605163574,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33582112193107605,
+      "rewards/margins": 7.493390083312988,
+      "rewards/rejected": -7.157568454742432,
+      "step": 523,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.056551375915173,
+      "grad_norm": 11.534995079040527,
+      "learning_rate": 9.866319188135274e-05,
+      "logits/chosen": -1.155828833580017,
+      "logits/rejected": -1.226155400276184,
+      "logps/chosen": -2.9224207401275635,
+      "logps/rejected": -76.7179946899414,
+      "loss": 0.39948102831840515,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28366178274154663,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40342777967453003,
+      "rewards/margins": 6.173599720001221,
+      "rewards/rejected": -5.770172119140625,
+      "step": 524,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0585710679121434,
+      "grad_norm": 3.017298936843872,
+      "learning_rate": 9.83290178490699e-05,
+      "logits/chosen": -1.2128490209579468,
+      "logits/rejected": -1.3072235584259033,
+      "logps/chosen": -2.281188488006592,
+      "logps/rejected": -75.67351531982422,
+      "loss": 0.28667759895324707,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2029131054878235,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27607351541519165,
+      "rewards/margins": 6.455198287963867,
+      "rewards/rejected": -6.17912483215332,
+      "step": 525,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0605907599091138,
+      "grad_norm": 2.768603563308716,
+      "learning_rate": 9.799486248124775e-05,
+      "logits/chosen": -1.1740716695785522,
+      "logits/rejected": -1.267440915107727,
+      "logps/chosen": -3.626737594604492,
+      "logps/rejected": -79.38348388671875,
+      "loss": 0.2453736811876297,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15417639911174774,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.7520807385444641,
+      "rewards/margins": 7.059826850891113,
+      "rewards/rejected": -6.307745933532715,
+      "step": 526,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0626104519060844,
+      "grad_norm": 2.207165002822876,
+      "learning_rate": 9.766072951031972e-05,
+      "logits/chosen": -1.2672584056854248,
+      "logits/rejected": -1.32329261302948,
+      "logps/chosen": -1.5716861486434937,
+      "logps/rejected": -68.19523620605469,
+      "loss": 0.20522795617580414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16552022099494934,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3385736048221588,
+      "rewards/margins": 5.919947624206543,
+      "rewards/rejected": -5.581373691558838,
+      "step": 527,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0646301439030548,
+      "grad_norm": 1.6318920850753784,
+      "learning_rate": 9.732662266846912e-05,
+      "logits/chosen": -1.157897710800171,
+      "logits/rejected": -1.2902889251708984,
+      "logps/chosen": -1.1771454811096191,
+      "logps/rejected": -94.12054443359375,
+      "loss": 0.21492774784564972,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17460428178310394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3003194034099579,
+      "rewards/margins": 8.205141067504883,
+      "rewards/rejected": -7.904821395874023,
+      "step": 528,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0666498359000252,
+      "grad_norm": 5.924353122711182,
+      "learning_rate": 9.699254568758741e-05,
+      "logits/chosen": -1.2643760442733765,
+      "logits/rejected": -1.368143081665039,
+      "logps/chosen": -1.368375301361084,
+      "logps/rejected": -86.12940979003906,
+      "loss": 0.14123007655143738,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11123973876237869,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3200608789920807,
+      "rewards/margins": 7.876832008361816,
+      "rewards/rejected": -7.556771278381348,
+      "step": 529,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0686695278969958,
+      "grad_norm": 1.9186508655548096,
+      "learning_rate": 9.665850229923258e-05,
+      "logits/chosen": -1.2655577659606934,
+      "logits/rejected": -1.344153642654419,
+      "logps/chosen": -0.23585116863250732,
+      "logps/rejected": -80.97904968261719,
+      "loss": 0.11053043603897095,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.04247582331299782,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3903675675392151,
+      "rewards/margins": 7.256943702697754,
+      "rewards/rejected": -6.866576194763184,
+      "step": 530,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.0706892198939661,
+      "grad_norm": 1.6089398860931396,
+      "learning_rate": 9.632449623458718e-05,
+      "logits/chosen": -1.2310450077056885,
+      "logits/rejected": -1.306509256362915,
+      "logps/chosen": -1.039363145828247,
+      "logps/rejected": -56.28333282470703,
+      "loss": 0.18445441126823425,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09440238028764725,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4418891668319702,
+      "rewards/margins": 4.846897602081299,
+      "rewards/rejected": -4.405008792877197,
+      "step": 531,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0727089118909365,
+      "grad_norm": 4.852963447570801,
+      "learning_rate": 9.59905312244171e-05,
+      "logits/chosen": -1.2377697229385376,
+      "logits/rejected": -1.3623260259628296,
+      "logps/chosen": -0.7856588363647461,
+      "logps/rejected": -99.22467041015625,
+      "loss": 0.1534590870141983,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11053737998008728,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27947190403938293,
+      "rewards/margins": 8.624140739440918,
+      "rewards/rejected": -8.3446683883667,
+      "step": 532,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0747286038879071,
+      "grad_norm": 10.742178916931152,
+      "learning_rate": 9.565661099902952e-05,
+      "logits/chosen": -1.1463772058486938,
+      "logits/rejected": -1.2955669164657593,
+      "logps/chosen": -3.547930955886841,
+      "logps/rejected": -82.50328063964844,
+      "loss": 0.657089352607727,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4685670733451843,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0838821679353714,
+      "rewards/margins": 6.732100009918213,
+      "rewards/rejected": -6.64821720123291,
+      "step": 533,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0767482958848775,
+      "grad_norm": 1.633371114730835,
+      "learning_rate": 9.532273928823151e-05,
+      "logits/chosen": -1.2212746143341064,
+      "logits/rejected": -1.269982099533081,
+      "logps/chosen": -3.351022243499756,
+      "logps/rejected": -92.98002624511719,
+      "loss": 0.20061077177524567,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09750235080718994,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6985870599746704,
+      "rewards/margins": 7.949156284332275,
+      "rewards/rejected": -7.250568389892578,
+      "step": 534,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.078767987881848,
+      "grad_norm": 1.1351590156555176,
+      "learning_rate": 9.498891982128809e-05,
+      "logits/chosen": -1.2288845777511597,
+      "logits/rejected": -1.3357539176940918,
+      "logps/chosen": -0.435973584651947,
+      "logps/rejected": -82.69353485107422,
+      "loss": 0.06550046801567078,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.051096200942993164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3056626617908478,
+      "rewards/margins": 7.295601844787598,
+      "rewards/rejected": -6.989939212799072,
+      "step": 535,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.0807876798788185,
+      "grad_norm": 7.175545692443848,
+      "learning_rate": 9.46551563268808e-05,
+      "logits/chosen": -1.1364165544509888,
+      "logits/rejected": -1.2312272787094116,
+      "logps/chosen": -3.2798967361450195,
+      "logps/rejected": -68.39805603027344,
+      "loss": 0.40154242515563965,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.27947506308555603,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20716847479343414,
+      "rewards/margins": 5.617040634155273,
+      "rewards/rejected": -5.409872531890869,
+      "step": 536,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.082807371875789,
+      "grad_norm": 16.1717472076416,
+      "learning_rate": 9.432145253306604e-05,
+      "logits/chosen": -1.1581474542617798,
+      "logits/rejected": -1.275297999382019,
+      "logps/chosen": -6.416726112365723,
+      "logps/rejected": -84.11360931396484,
+      "loss": 0.6197578310966492,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6048212647438049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3198436498641968,
+      "rewards/margins": 7.643919944763184,
+      "rewards/rejected": -7.324074745178223,
+      "step": 537,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.0848270638727593,
+      "grad_norm": 16.336959838867188,
+      "learning_rate": 9.398781216723331e-05,
+      "logits/chosen": -1.035546898841858,
+      "logits/rejected": -1.2581911087036133,
+      "logps/chosen": -1.4019535779953003,
+      "logps/rejected": -93.62826538085938,
+      "loss": 0.3454136550426483,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23441605269908905,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2636178433895111,
+      "rewards/margins": 8.16170597076416,
+      "rewards/rejected": -7.898087501525879,
+      "step": 538,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.08684675586973,
+      "grad_norm": 4.587718963623047,
+      "learning_rate": 9.36542389560636e-05,
+      "logits/chosen": -1.0745295286178589,
+      "logits/rejected": -1.2483410835266113,
+      "logps/chosen": -5.092793941497803,
+      "logps/rejected": -87.74403381347656,
+      "loss": 0.5541177988052368,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42148298025131226,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3391857147216797,
+      "rewards/margins": 7.316706657409668,
+      "rewards/rejected": -6.977520942687988,
+      "step": 539,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.0888664478667003,
+      "grad_norm": 2.1240127086639404,
+      "learning_rate": 9.332073662548784e-05,
+      "logits/chosen": -1.1243212223052979,
+      "logits/rejected": -1.193894863128662,
+      "logps/chosen": -1.587296724319458,
+      "logps/rejected": -59.78162384033203,
+      "loss": 0.28339654207229614,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18274566531181335,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2662951350212097,
+      "rewards/margins": 5.166958808898926,
+      "rewards/rejected": -4.900664329528809,
+      "step": 540,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.090886139863671,
+      "grad_norm": 2.6355695724487305,
+      "learning_rate": 9.29873089006453e-05,
+      "logits/chosen": -1.0941351652145386,
+      "logits/rejected": -1.2066431045532227,
+      "logps/chosen": -1.4627405405044556,
+      "logps/rejected": -77.10699462890625,
+      "loss": 0.2207845151424408,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13547611236572266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3999878466129303,
+      "rewards/margins": 6.711189270019531,
+      "rewards/rejected": -6.311202049255371,
+      "step": 541,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.0929058318606413,
+      "grad_norm": 2.05720853805542,
+      "learning_rate": 9.26539595058418e-05,
+      "logits/chosen": -1.0910502672195435,
+      "logits/rejected": -1.2283904552459717,
+      "logps/chosen": -1.5382826328277588,
+      "logps/rejected": -76.15080261230469,
+      "loss": 0.25323227047920227,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1447850912809372,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3722470700740814,
+      "rewards/margins": 6.785793304443359,
+      "rewards/rejected": -6.413545608520508,
+      "step": 542,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.0949255238576117,
+      "grad_norm": 2.266726493835449,
+      "learning_rate": 9.23206921645083e-05,
+      "logits/chosen": -1.0470198392868042,
+      "logits/rejected": -1.150384545326233,
+      "logps/chosen": -3.195725679397583,
+      "logps/rejected": -61.4305534362793,
+      "loss": 0.293194055557251,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2274458259344101,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5585137605667114,
+      "rewards/margins": 5.642398834228516,
+      "rewards/rejected": -5.083885192871094,
+      "step": 543,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0969452158545823,
+      "grad_norm": 2.564605474472046,
+      "learning_rate": 9.198751059915925e-05,
+      "logits/chosen": -1.0186107158660889,
+      "logits/rejected": -1.2122492790222168,
+      "logps/chosen": -2.0936737060546875,
+      "logps/rejected": -82.15401458740234,
+      "loss": 0.34008562564849854,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2496323138475418,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31812718510627747,
+      "rewards/margins": 6.999300479888916,
+      "rewards/rejected": -6.6811723709106445,
+      "step": 544,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0989649078515527,
+      "grad_norm": 9.990952491760254,
+      "learning_rate": 9.165441853135104e-05,
+      "logits/chosen": -1.1474440097808838,
+      "logits/rejected": -1.2639394998550415,
+      "logps/chosen": -1.8011548519134521,
+      "logps/rejected": -73.41018676757812,
+      "loss": 0.36625638604164124,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29132476449012756,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21145863831043243,
+      "rewards/margins": 6.8196306228637695,
+      "rewards/rejected": -6.608172416687012,
+      "step": 545,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.100984599848523,
+      "grad_norm": 2.7831358909606934,
+      "learning_rate": 9.132141968164026e-05,
+      "logits/chosen": -1.0405611991882324,
+      "logits/rejected": -1.2216545343399048,
+      "logps/chosen": -1.1892681121826172,
+      "logps/rejected": -94.48490905761719,
+      "loss": 0.25090837478637695,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14087605476379395,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3783609867095947,
+      "rewards/margins": 8.133208274841309,
+      "rewards/rejected": -7.754847526550293,
+      "step": 546,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.1030042918454936,
+      "grad_norm": 8.550939559936523,
+      "learning_rate": 9.098851776954241e-05,
+      "logits/chosen": -1.0600169897079468,
+      "logits/rejected": -1.1879816055297852,
+      "logps/chosen": -2.9978091716766357,
+      "logps/rejected": -69.18419647216797,
+      "loss": 0.491380512714386,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32408130168914795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21779419481754303,
+      "rewards/margins": 5.817697048187256,
+      "rewards/rejected": -5.599903106689453,
+      "step": 547,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.105023983842464,
+      "grad_norm": 14.694620132446289,
+      "learning_rate": 9.065571651349015e-05,
+      "logits/chosen": -1.0587307214736938,
+      "logits/rejected": -1.1352782249450684,
+      "logps/chosen": -2.723759174346924,
+      "logps/rejected": -73.38800048828125,
+      "loss": 0.3820952773094177,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28878253698349,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25954926013946533,
+      "rewards/margins": 6.028653621673584,
+      "rewards/rejected": -5.769104480743408,
+      "step": 548,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1070436758394344,
+      "grad_norm": 3.4368412494659424,
+      "learning_rate": 9.032301963079191e-05,
+      "logits/chosen": -1.0471107959747314,
+      "logits/rejected": -1.216453194618225,
+      "logps/chosen": -2.2245757579803467,
+      "logps/rejected": -64.19511413574219,
+      "loss": 0.2849371135234833,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2175203114748001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3077858090400696,
+      "rewards/margins": 5.670999526977539,
+      "rewards/rejected": -5.363213539123535,
+      "step": 549,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.109063367836405,
+      "grad_norm": 2.0185515880584717,
+      "learning_rate": 8.999043083759017e-05,
+      "logits/chosen": -1.0703552961349487,
+      "logits/rejected": -1.2551573514938354,
+      "logps/chosen": -1.0120265483856201,
+      "logps/rejected": -85.53215026855469,
+      "loss": 0.15907928347587585,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11441606283187866,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2885012924671173,
+      "rewards/margins": 7.635853290557861,
+      "rewards/rejected": -7.347352504730225,
+      "step": 550,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1110830598333754,
+      "grad_norm": 2.8778460025787354,
+      "learning_rate": 8.965795384882021e-05,
+      "logits/chosen": -0.9114001393318176,
+      "logits/rejected": -1.166597604751587,
+      "logps/chosen": -1.139864444732666,
+      "logps/rejected": -100.16082000732422,
+      "loss": 0.30031049251556396,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2430577278137207,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24268564581871033,
+      "rewards/margins": 8.84631633758545,
+      "rewards/rejected": -8.603631019592285,
+      "step": 551,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.1131027518303458,
+      "grad_norm": 2.5478785037994385,
+      "learning_rate": 8.932559237816839e-05,
+      "logits/chosen": -0.9818710088729858,
+      "logits/rejected": -1.225954532623291,
+      "logps/chosen": -3.4108481407165527,
+      "logps/rejected": -79.83183288574219,
+      "loss": 0.4420861601829529,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3446241617202759,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22779923677444458,
+      "rewards/margins": 6.333407402038574,
+      "rewards/rejected": -6.105607509613037,
+      "step": 552,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.1151224438273164,
+      "grad_norm": 6.415960788726807,
+      "learning_rate": 8.899335013803084e-05,
+      "logits/chosen": -1.091843605041504,
+      "logits/rejected": -1.242721438407898,
+      "logps/chosen": -1.220659613609314,
+      "logps/rejected": -61.345306396484375,
+      "loss": 0.22371630370616913,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1757785677909851,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40316012501716614,
+      "rewards/margins": 5.553282737731934,
+      "rewards/rejected": -5.150122165679932,
+      "step": 553,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.1171421358242868,
+      "grad_norm": 1.3796110153198242,
+      "learning_rate": 8.866123083947182e-05,
+      "logits/chosen": -1.075179100036621,
+      "logits/rejected": -1.1752409934997559,
+      "logps/chosen": -1.5424776077270508,
+      "logps/rejected": -65.63243103027344,
+      "loss": 0.23596073687076569,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18632307648658752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34452810883522034,
+      "rewards/margins": 5.910964012145996,
+      "rewards/rejected": -5.566435813903809,
+      "step": 554,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.1191618278212572,
+      "grad_norm": 3.3275868892669678,
+      "learning_rate": 8.832923819218238e-05,
+      "logits/chosen": -1.001117467880249,
+      "logits/rejected": -1.1721633672714233,
+      "logps/chosen": -1.894881010055542,
+      "logps/rejected": -61.61512756347656,
+      "loss": 0.35188451409339905,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3188917636871338,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12245701998472214,
+      "rewards/margins": 5.278988838195801,
+      "rewards/rejected": -5.15653133392334,
+      "step": 555,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.1211815198182278,
+      "grad_norm": 1.58943510055542,
+      "learning_rate": 8.799737590443902e-05,
+      "logits/chosen": -0.9907103776931763,
+      "logits/rejected": -1.2039836645126343,
+      "logps/chosen": -3.9374332427978516,
+      "logps/rejected": -85.5752944946289,
+      "loss": 0.3835102617740631,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3536885380744934,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1798345446586609,
+      "rewards/margins": 7.102011203765869,
+      "rewards/rejected": -6.922177314758301,
+      "step": 556,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.1232012118151982,
+      "grad_norm": 4.0305495262146,
+      "learning_rate": 8.766564768306207e-05,
+      "logits/chosen": -1.1515066623687744,
+      "logits/rejected": -1.214231252670288,
+      "logps/chosen": -0.7779844999313354,
+      "logps/rejected": -56.81134796142578,
+      "loss": 0.1507856547832489,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0650799572467804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30766594409942627,
+      "rewards/margins": 5.128314971923828,
+      "rewards/rejected": -4.820648670196533,
+      "step": 557,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1252209038121685,
+      "grad_norm": 4.040610313415527,
+      "learning_rate": 8.733405723337432e-05,
+      "logits/chosen": -1.0960428714752197,
+      "logits/rejected": -1.2518525123596191,
+      "logps/chosen": -0.9766073226928711,
+      "logps/rejected": -73.12732696533203,
+      "loss": 0.26937615871429443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1882752627134323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26189538836479187,
+      "rewards/margins": 6.15379524230957,
+      "rewards/rejected": -5.891900062561035,
+      "step": 558,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1272405958091392,
+      "grad_norm": 2.961078643798828,
+      "learning_rate": 8.700260825915976e-05,
+      "logits/chosen": -0.997047483921051,
+      "logits/rejected": -1.174019455909729,
+      "logps/chosen": -1.8312853574752808,
+      "logps/rejected": -93.381591796875,
+      "loss": 0.2475411295890808,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21483805775642395,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3925812244415283,
+      "rewards/margins": 7.863485336303711,
+      "rewards/rejected": -7.470904350280762,
+      "step": 559,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1292602878061095,
+      "grad_norm": 13.051665306091309,
+      "learning_rate": 8.667130446262214e-05,
+      "logits/chosen": -0.8055945634841919,
+      "logits/rejected": -1.1168835163116455,
+      "logps/chosen": -1.7664347887039185,
+      "logps/rejected": -100.53841400146484,
+      "loss": 0.30384474992752075,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2042587250471115,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1575130969285965,
+      "rewards/margins": 8.052501678466797,
+      "rewards/rejected": -7.894989490509033,
+      "step": 560,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1312799798030801,
+      "grad_norm": 2.204517364501953,
+      "learning_rate": 8.634014954434355e-05,
+      "logits/chosen": -0.9541032314300537,
+      "logits/rejected": -1.1428359746932983,
+      "logps/chosen": -0.8130196928977966,
+      "logps/rejected": -77.63139343261719,
+      "loss": 0.11135084927082062,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06836092472076416,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31184935569763184,
+      "rewards/margins": 5.629817485809326,
+      "rewards/rejected": -5.317968368530273,
+      "step": 561,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1332996718000505,
+      "grad_norm": 0.7750061750411987,
+      "learning_rate": 8.600914720324316e-05,
+      "logits/chosen": -1.127307415008545,
+      "logits/rejected": -1.2096644639968872,
+      "logps/chosen": -0.43007510900497437,
+      "logps/rejected": -60.772300720214844,
+      "loss": 0.0971565991640091,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06539256125688553,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.367216557264328,
+      "rewards/margins": 5.615878105163574,
+      "rewards/rejected": -5.248661041259766,
+      "step": 562,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.135319363797021,
+      "grad_norm": 2.891634464263916,
+      "learning_rate": 8.567830113653597e-05,
+      "logits/chosen": -1.1409940719604492,
+      "logits/rejected": -1.1916230916976929,
+      "logps/chosen": -1.7586002349853516,
+      "logps/rejected": -63.676971435546875,
+      "loss": 0.2738097012042999,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18838034570217133,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5218551158905029,
+      "rewards/margins": 5.294318675994873,
+      "rewards/rejected": -4.772463798522949,
+      "step": 563,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.1373390557939915,
+      "grad_norm": 1.045423150062561,
+      "learning_rate": 8.534761503969136e-05,
+      "logits/chosen": -1.0010972023010254,
+      "logits/rejected": -1.1904897689819336,
+      "logps/chosen": -1.6170446872711182,
+      "logps/rejected": -74.0367660522461,
+      "loss": 0.19317412376403809,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13384878635406494,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2380320429801941,
+      "rewards/margins": 5.972710132598877,
+      "rewards/rejected": -5.734678745269775,
+      "step": 564,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.139358747790962,
+      "grad_norm": 2.1970374584198,
+      "learning_rate": 8.501709260639186e-05,
+      "logits/chosen": -1.0450353622436523,
+      "logits/rejected": -1.163125991821289,
+      "logps/chosen": -1.5424336194992065,
+      "logps/rejected": -73.43540954589844,
+      "loss": 0.21883387863636017,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1583956778049469,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27178311347961426,
+      "rewards/margins": 5.603139877319336,
+      "rewards/rejected": -5.331357002258301,
+      "step": 565,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.1413784397879323,
+      "grad_norm": 2.8700995445251465,
+      "learning_rate": 8.4686737528492e-05,
+      "logits/chosen": -1.1470935344696045,
+      "logits/rejected": -1.2003036737442017,
+      "logps/chosen": -2.7194712162017822,
+      "logps/rejected": -73.11854553222656,
+      "loss": 0.24578091502189636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14729218184947968,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32028070092201233,
+      "rewards/margins": 6.802761077880859,
+      "rewards/rejected": -6.482481002807617,
+      "step": 566,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.143398131784903,
+      "grad_norm": 1.381791114807129,
+      "learning_rate": 8.435655349597689e-05,
+      "logits/chosen": -1.0863614082336426,
+      "logits/rejected": -1.2500827312469482,
+      "logps/chosen": -0.5668889880180359,
+      "logps/rejected": -100.23611450195312,
+      "loss": 0.11013483256101608,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0759727880358696,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25675472617149353,
+      "rewards/margins": 8.975456237792969,
+      "rewards/rejected": -8.71870231628418,
+      "step": 567,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.1454178237818733,
+      "grad_norm": 1.6486469507217407,
+      "learning_rate": 8.40265441969213e-05,
+      "logits/chosen": -1.108482837677002,
+      "logits/rejected": -1.2181017398834229,
+      "logps/chosen": -0.9796234965324402,
+      "logps/rejected": -72.5177993774414,
+      "loss": 0.17747226357460022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0940045714378357,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4885254204273224,
+      "rewards/margins": 5.819605350494385,
+      "rewards/rejected": -5.331079959869385,
+      "step": 568,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.1474375157788437,
+      "grad_norm": 5.995792865753174,
+      "learning_rate": 8.369671331744798e-05,
+      "logits/chosen": -1.119094729423523,
+      "logits/rejected": -1.2429081201553345,
+      "logps/chosen": -1.075197696685791,
+      "logps/rejected": -81.46918487548828,
+      "loss": 0.21839378774166107,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15385104715824127,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.457529217004776,
+      "rewards/margins": 7.371258735656738,
+      "rewards/rejected": -6.913729667663574,
+      "step": 569,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.1494572077758143,
+      "grad_norm": 5.149206638336182,
+      "learning_rate": 8.336706454168701e-05,
+      "logits/chosen": -1.1530694961547852,
+      "logits/rejected": -1.29622220993042,
+      "logps/chosen": -1.0280964374542236,
+      "logps/rejected": -119.51797485351562,
+      "loss": 0.20643891394138336,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08810500800609589,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43384313583374023,
+      "rewards/margins": 10.894699096679688,
+      "rewards/rejected": -10.460856437683105,
+      "step": 570,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.1514768997727847,
+      "grad_norm": 1.2678836584091187,
+      "learning_rate": 8.303760155173431e-05,
+      "logits/chosen": -1.1424627304077148,
+      "logits/rejected": -1.3004976511001587,
+      "logps/chosen": -0.5466084480285645,
+      "logps/rejected": -118.68138122558594,
+      "loss": 0.12502805888652802,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1129414513707161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3263624608516693,
+      "rewards/margins": 10.448878288269043,
+      "rewards/rejected": -10.122516632080078,
+      "step": 571,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.153496591769755,
+      "grad_norm": 1.4974409341812134,
+      "learning_rate": 8.27083280276107e-05,
+      "logits/chosen": -1.1443991661071777,
+      "logits/rejected": -1.3150042295455933,
+      "logps/chosen": -0.3272748589515686,
+      "logps/rejected": -139.84246826171875,
+      "loss": 0.08136013895273209,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.050167154520750046,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4843906760215759,
+      "rewards/margins": 12.638895034790039,
+      "rewards/rejected": -12.15450382232666,
+      "step": 572,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.1555162837667257,
+      "grad_norm": 1.1129848957061768,
+      "learning_rate": 8.237924764722058e-05,
+      "logits/chosen": -1.1491975784301758,
+      "logits/rejected": -1.307010531425476,
+      "logps/chosen": -1.0129919052124023,
+      "logps/rejected": -127.45437622070312,
+      "loss": 0.11947691440582275,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10945737361907959,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2766776978969574,
+      "rewards/margins": 11.592185974121094,
+      "rewards/rejected": -11.315506935119629,
+      "step": 573,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.157535975763696,
+      "grad_norm": 12.299660682678223,
+      "learning_rate": 8.20503640863111e-05,
+      "logits/chosen": -1.2407050132751465,
+      "logits/rejected": -1.3422547578811646,
+      "logps/chosen": -0.5153883099555969,
+      "logps/rejected": -133.4427032470703,
+      "loss": 0.14274229109287262,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09612210839986801,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25790077447891235,
+      "rewards/margins": 12.522222518920898,
+      "rewards/rejected": -12.264321327209473,
+      "step": 574,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.1595556677606664,
+      "grad_norm": 4.597177505493164,
+      "learning_rate": 8.172168101843099e-05,
+      "logits/chosen": -1.2469947338104248,
+      "logits/rejected": -1.3221312761306763,
+      "logps/chosen": -2.5218143463134766,
+      "logps/rejected": -64.33858489990234,
+      "loss": 0.33642834424972534,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2502836585044861,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3977520167827606,
+      "rewards/margins": 6.087981700897217,
+      "rewards/rejected": -5.690229415893555,
+      "step": 575,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.161575359757637,
+      "grad_norm": 1.4237756729125977,
+      "learning_rate": 8.139320211488938e-05,
+      "logits/chosen": -1.20731520652771,
+      "logits/rejected": -1.3155488967895508,
+      "logps/chosen": -1.9141697883605957,
+      "logps/rejected": -84.08015441894531,
+      "loss": 0.20295384526252747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1318996697664261,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3964309096336365,
+      "rewards/margins": 8.057353973388672,
+      "rewards/rejected": -7.660923480987549,
+      "step": 576,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1635950517546074,
+      "grad_norm": 2.7795519828796387,
+      "learning_rate": 8.106493104471512e-05,
+      "logits/chosen": -1.1746073961257935,
+      "logits/rejected": -1.2740343809127808,
+      "logps/chosen": -2.3649284839630127,
+      "logps/rejected": -110.2456283569336,
+      "loss": 0.2869383990764618,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21996352076530457,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37983080744743347,
+      "rewards/margins": 9.394953727722168,
+      "rewards/rejected": -9.015122413635254,
+      "step": 577,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1656147437515778,
+      "grad_norm": 2.63078236579895,
+      "learning_rate": 8.073687147461547e-05,
+      "logits/chosen": -1.1673715114593506,
+      "logits/rejected": -1.330165147781372,
+      "logps/chosen": -2.938627004623413,
+      "logps/rejected": -126.129638671875,
+      "loss": 0.36669260263442993,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3255380392074585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19922614097595215,
+      "rewards/margins": 10.992043495178223,
+      "rewards/rejected": -10.792817115783691,
+      "step": 578,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1676344357485484,
+      "grad_norm": 3.1014156341552734,
+      "learning_rate": 8.04090270689354e-05,
+      "logits/chosen": -1.2527494430541992,
+      "logits/rejected": -1.3397026062011719,
+      "logps/chosen": -2.1539173126220703,
+      "logps/rejected": -119.85777282714844,
+      "loss": 0.35629451274871826,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26880595088005066,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17089465260505676,
+      "rewards/margins": 10.79810619354248,
+      "rewards/rejected": -10.62721061706543,
+      "step": 579,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1696541277455188,
+      "grad_norm": 2.183499813079834,
+      "learning_rate": 8.008140148961641e-05,
+      "logits/chosen": -1.2131707668304443,
+      "logits/rejected": -1.3254077434539795,
+      "logps/chosen": -1.9817386865615845,
+      "logps/rejected": -97.93504333496094,
+      "loss": 0.2147722840309143,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13111397624015808,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3702988922595978,
+      "rewards/margins": 8.80711555480957,
+      "rewards/rejected": -8.436817169189453,
+      "step": 580,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1716738197424892,
+      "grad_norm": 1.777718186378479,
+      "learning_rate": 7.975399839615588e-05,
+      "logits/chosen": -1.2131725549697876,
+      "logits/rejected": -1.3256409168243408,
+      "logps/chosen": -1.6898517608642578,
+      "logps/rejected": -143.40451049804688,
+      "loss": 0.10389101505279541,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07294470816850662,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7127897143363953,
+      "rewards/margins": 13.449665069580078,
+      "rewards/rejected": -12.736875534057617,
+      "step": 581,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1736935117394598,
+      "grad_norm": 3.9212543964385986,
+      "learning_rate": 7.942682144556604e-05,
+      "logits/chosen": -1.1815121173858643,
+      "logits/rejected": -1.2933522462844849,
+      "logps/chosen": -1.2875924110412598,
+      "logps/rejected": -102.97988891601562,
+      "loss": 0.25881722569465637,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17525863647460938,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23393197357654572,
+      "rewards/margins": 9.580009460449219,
+      "rewards/rejected": -9.346076011657715,
+      "step": 582,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1757132037364302,
+      "grad_norm": 10.820318222045898,
+      "learning_rate": 7.909987429233317e-05,
+      "logits/chosen": -1.1671289205551147,
+      "logits/rejected": -1.2844188213348389,
+      "logps/chosen": -3.267451047897339,
+      "logps/rejected": -142.51571655273438,
+      "loss": 0.3873688876628876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2906343936920166,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.45434269309043884,
+      "rewards/margins": 12.700260162353516,
+      "rewards/rejected": -12.245917320251465,
+      "step": 583,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1777328957334006,
+      "grad_norm": 8.162153244018555,
+      "learning_rate": 7.877316058837674e-05,
+      "logits/chosen": -1.0726277828216553,
+      "logits/rejected": -1.2778667211532593,
+      "logps/chosen": -2.711777687072754,
+      "logps/rejected": -147.33900451660156,
+      "loss": 0.43292930722236633,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37318992614746094,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31741955876350403,
+      "rewards/margins": 13.22387981414795,
+      "rewards/rejected": -12.906461715698242,
+      "step": 584,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1797525877303712,
+      "grad_norm": 8.373458862304688,
+      "learning_rate": 7.844668398300865e-05,
+      "logits/chosen": -1.1396088600158691,
+      "logits/rejected": -1.260819435119629,
+      "logps/chosen": -2.466878890991211,
+      "logps/rejected": -117.13096618652344,
+      "loss": 0.3733997344970703,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3299589157104492,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13099858164787292,
+      "rewards/margins": 10.681720733642578,
+      "rewards/rejected": -10.550722122192383,
+      "step": 585,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1817722797273416,
+      "grad_norm": 20.348384857177734,
+      "learning_rate": 7.812044812289249e-05,
+      "logits/chosen": -1.2010091543197632,
+      "logits/rejected": -1.3245834112167358,
+      "logps/chosen": -3.3453171253204346,
+      "logps/rejected": -117.00688171386719,
+      "loss": 0.6515809893608093,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3382578492164612,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15548963844776154,
+      "rewards/margins": 10.246532440185547,
+      "rewards/rejected": -10.091042518615723,
+      "step": 586,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.183791971724312,
+      "grad_norm": 14.155014991760254,
+      "learning_rate": 7.779445665200284e-05,
+      "logits/chosen": -1.1876106262207031,
+      "logits/rejected": -1.2901134490966797,
+      "logps/chosen": -3.768019437789917,
+      "logps/rejected": -83.8199691772461,
+      "loss": 0.7886047959327698,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5176796913146973,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16025924682617188,
+      "rewards/margins": 7.629866600036621,
+      "rewards/rejected": -7.469607353210449,
+      "step": 587,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.1858116637212825,
+      "grad_norm": 2.6726722717285156,
+      "learning_rate": 7.746871321158434e-05,
+      "logits/chosen": -1.2011233568191528,
+      "logits/rejected": -1.2838491201400757,
+      "logps/chosen": -1.551110863685608,
+      "logps/rejected": -74.54078674316406,
+      "loss": 0.3482269048690796,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20897427201271057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13612668216228485,
+      "rewards/margins": 6.496390342712402,
+      "rewards/rejected": -6.360263824462891,
+      "step": 588,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.187831355718253,
+      "grad_norm": 11.104055404663086,
+      "learning_rate": 7.71432214401114e-05,
+      "logits/chosen": -1.184781551361084,
+      "logits/rejected": -1.367568850517273,
+      "logps/chosen": -2.4659273624420166,
+      "logps/rejected": -158.41024780273438,
+      "loss": 0.46998411417007446,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37064406275749207,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14929750561714172,
+      "rewards/margins": 14.278914451599121,
+      "rewards/rejected": -14.129616737365723,
+      "step": 589,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1898510477152233,
+      "grad_norm": 1.3642657995224,
+      "learning_rate": 7.681798497324716e-05,
+      "logits/chosen": -1.1494981050491333,
+      "logits/rejected": -1.2755074501037598,
+      "logps/chosen": -3.920079231262207,
+      "logps/rejected": -99.15157318115234,
+      "loss": 0.1694917380809784,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1302870661020279,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6689797043800354,
+      "rewards/margins": 8.986592292785645,
+      "rewards/rejected": -8.317612648010254,
+      "step": 590,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.191870739712194,
+      "grad_norm": 2.9878756999969482,
+      "learning_rate": 7.649300744380328e-05,
+      "logits/chosen": -1.2234166860580444,
+      "logits/rejected": -1.3413805961608887,
+      "logps/chosen": -3.245425224304199,
+      "logps/rejected": -97.82754516601562,
+      "loss": 0.6991743445396423,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5142159461975098,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.04035697877407074,
+      "rewards/margins": 8.96734619140625,
+      "rewards/rejected": -8.926989555358887,
+      "step": 591,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1938904317091643,
+      "grad_norm": 3.8199944496154785,
+      "learning_rate": 7.616829248169891e-05,
+      "logits/chosen": -1.168949842453003,
+      "logits/rejected": -1.2326381206512451,
+      "logps/chosen": -2.8712589740753174,
+      "logps/rejected": -63.083431243896484,
+      "loss": 0.4011121094226837,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2862622141838074,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.291637659072876,
+      "rewards/margins": 5.633680820465088,
+      "rewards/rejected": -5.342043399810791,
+      "step": 592,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1959101237061347,
+      "grad_norm": 2.4625227451324463,
+      "learning_rate": 7.584384371392055e-05,
+      "logits/chosen": -1.2142324447631836,
+      "logits/rejected": -1.2614374160766602,
+      "logps/chosen": -1.5779057741165161,
+      "logps/rejected": -86.51065826416016,
+      "loss": 0.1223965585231781,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09837526828050613,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9265052676200867,
+      "rewards/margins": 8.553476333618164,
+      "rewards/rejected": -7.62697172164917,
+      "step": 593,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.1979298157031053,
+      "grad_norm": 3.092069149017334,
+      "learning_rate": 7.55196647644814e-05,
+      "logits/chosen": -1.1460374593734741,
+      "logits/rejected": -1.325160264968872,
+      "logps/chosen": -0.844829261302948,
+      "logps/rejected": -130.2720184326172,
+      "loss": 0.1258152425289154,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09725081920623779,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22555547952651978,
+      "rewards/margins": 11.706364631652832,
+      "rewards/rejected": -11.480809211730957,
+      "step": 594,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.1999495077000757,
+      "grad_norm": 4.890739440917969,
+      "learning_rate": 7.519575925438067e-05,
+      "logits/chosen": -1.1363235712051392,
+      "logits/rejected": -1.3047442436218262,
+      "logps/chosen": -3.450829267501831,
+      "logps/rejected": -122.64608001708984,
+      "loss": 0.40785038471221924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2815420627593994,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.30323055386543274,
+      "rewards/margins": 10.626569747924805,
+      "rewards/rejected": -10.323339462280273,
+      "step": 595,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.201969199697046,
+      "grad_norm": 9.498205184936523,
+      "learning_rate": 7.487213080156355e-05,
+      "logits/chosen": -1.2033909559249878,
+      "logits/rejected": -1.2762019634246826,
+      "logps/chosen": -1.286055326461792,
+      "logps/rejected": -79.14070129394531,
+      "loss": 0.24982167780399323,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14821632206439972,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2594597637653351,
+      "rewards/margins": 7.015608787536621,
+      "rewards/rejected": -6.756147861480713,
+      "step": 596,
+      "train_speed(iter/s)": 0.02123
+    },
+    {
+      "epoch": 1.2039888916940167,
+      "grad_norm": 5.124003887176514,
+      "learning_rate": 7.454878302088039e-05,
+      "logits/chosen": -1.1855096817016602,
+      "logits/rejected": -1.2832536697387695,
+      "logps/chosen": -2.1323423385620117,
+      "logps/rejected": -77.67525482177734,
+      "loss": 0.3236369490623474,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22611530125141144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2844330668449402,
+      "rewards/margins": 6.800313472747803,
+      "rewards/rejected": -6.515880584716797,
+      "step": 597,
+      "train_speed(iter/s)": 0.02123
+    },
+    {
+      "epoch": 1.206008583690987,
+      "grad_norm": 7.811437606811523,
+      "learning_rate": 7.422571952404663e-05,
+      "logits/chosen": -1.1619443893432617,
+      "logits/rejected": -1.2858407497406006,
+      "logps/chosen": -3.6687519550323486,
+      "logps/rejected": -89.48348236083984,
+      "loss": 0.6337468028068542,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.430719256401062,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09281295537948608,
+      "rewards/margins": 7.673408508300781,
+      "rewards/rejected": -7.58059549331665,
+      "step": 598,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.2080282756879577,
+      "grad_norm": 6.1173577308654785,
+      "learning_rate": 7.390294391960217e-05,
+      "logits/chosen": -1.2435667514801025,
+      "logits/rejected": -1.3407572507858276,
+      "logps/chosen": -0.851167619228363,
+      "logps/rejected": -90.95581817626953,
+      "loss": 0.1322077065706253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07706882804632187,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3039519786834717,
+      "rewards/margins": 8.613190650939941,
+      "rewards/rejected": -8.309237480163574,
+      "step": 599,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.210047967684928,
+      "grad_norm": 1.1325865983963013,
+      "learning_rate": 7.358045981287141e-05,
+      "logits/chosen": -1.1153496503829956,
+      "logits/rejected": -1.2983583211898804,
+      "logps/chosen": -0.8740055561065674,
+      "logps/rejected": -100.239990234375,
+      "loss": 0.1091877669095993,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0765417069196701,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24975469708442688,
+      "rewards/margins": 8.55569076538086,
+      "rewards/rejected": -8.305935859680176,
+      "step": 600,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.210047967684928,
+      "eval_logits/chosen": -1.1828652620315552,
+      "eval_logits/rejected": -1.3658164739608765,
+      "eval_logps/chosen": -1.8309376239776611,
+      "eval_logps/rejected": -125.0011215209961,
+      "eval_loss": 0.24665305018424988,
+      "eval_nll_loss": 0.17694206535816193,
+      "eval_rewards/accuracies": 0.9750000238418579,
+      "eval_rewards/chosen": 0.26265937089920044,
+      "eval_rewards/margins": 11.061813354492188,
+      "eval_rewards/rejected": -10.799153327941895,
+      "eval_runtime": 92.2944,
+      "eval_samples_per_second": 0.867,
+      "eval_steps_per_second": 0.433,
+      "step": 600
     }
   ],
   "logging_steps": 1,
-  "max_steps": 360,
+  "max_steps": 990,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
-  "save_steps": 5,
+  "num_train_epochs": 2,
+  "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -1183,8 +10861,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 0.0,
-  "train_batch_size": 8,
+  "total_flos": 1.7337917669638144e+17,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }